Upload 4 files

Updated the client service with History of Dialogues (Database usage)

WarClient.py

@@ -1,7 +1,6 @@
 import socket
+import config # Here the constants are stored
 
-HOST = '129.159.146.88'
-PORT = 5000
 
 def getReply(message):
     # Returns a reply from the server
@@ -15,7 +14,7 @@ def getReply(message):
 
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as client_socket:
         try:
-            client_socket.connect((HOST, PORT))
+            client_socket.connect((config.HOST, config.PORT))
             client_socket.sendall(message.encode())
             print('Wait...')
             data = client_socket.recv(1024)
@@ -28,4 +27,7 @@ def getReply(message):
                 client_socket.shutdown(socket.SHUT_RDWR)
                 client_socket.close()
         except:
-            return ""
+            return ""
+
+if __name__ == '__main__':
+    pass

WarOnline_Chat.py

@@ -1,24 +1,16 @@
 # This is an quote and post library for a specific thread in the WarOnline forum.
 
 import WarClient
+import conversationDB
 import requests
 import re
 from bs4 import BeautifulSoup
 import urllib.request as urllib
 import warnings
-#import schedule
 import time
-#from tqdm import tqdm
-warnings.filterwarnings("ignore")
-
-# Define the login URL and the thread URL
-login_url = 'https://waronline.org/fora/index.php?login/login'
-thread_url = 'https://waronline.org/fora/index.php?threads/warbot-playground.17636/'
-post_url = "https://waronline.org/fora/index.php?threads/warbot-playground.17636/add-reply"
+import config # Here the constants are stored
 
-# Define the login credentials
-username = 'WarBot'
-password = 'naP2tion'
+warnings.filterwarnings("ignore")
 
 # Start a session to persist the login cookie across requests
 session = requests.Session()
@@ -30,6 +22,9 @@ def fixString(S):
     S = S.replace(".?", "?")
     S = S.replace(",,", ",")
     S = S.replace("?.", "?")
+    S = S.replace("??", "?")
+    S = S.replace(" ?", "?")
+    S = S.replace(" .", ".")
     S = S.replace(",!", "!")
     S = S.replace(",.", ",")
     S = S.replace(".]", ".")
@@ -61,7 +56,7 @@ def remove_extra_spaces(s):
     s = re.sub(r"\s+([.,])", r"\1", s)  # remove spaces before period or comma
     return(s)
 
-def getLastPage(thread_url=thread_url):
+def getLastPage(thread_url=config.thread_url):
     # Returns the number of the last page
     page = 1  # Starting page
     lastPage = False
@@ -74,11 +69,11 @@ def getLastPage(thread_url=thread_url):
     return page
 
 
-def login(username=username, password=password, thread_url=thread_url):
+def login(username=config.username, password=config.password, thread_url=config.thread_url):
     # Log-In to the forum and redirect to thread
 
     # Retrieve the login page HTML to get the CSRF token
-    login_page_response = session.get(login_url)
+    login_page_response = session.get(config.login_url)
     soup = BeautifulSoup(login_page_response.text, 'html.parser')
     csrf_token = soup.find('input', {'name': '_xfToken'})['value']
 
@@ -90,14 +85,14 @@ def login(username=username, password=password, thread_url=thread_url):
         '_xfRedirect': thread_url,
         '_xfToken': csrf_token
     }
-    response = session.post(login_url, data=login_data)
+    response = session.post(config.login_url, data=login_data)
 
     # Check if the login was successful
     if 'Invalid login' in response.text:
         print('Login failed!')
         exit()
 
-def post(message="", thread_url=thread_url, post_url=post_url, quoted_by="",quote_text="",quote_source=""):
+def post(message="", thread_url=config.thread_url, post_url=config.post_url, quoted_by="",quote_text="",quote_source=""):
     #Post a message to the forum (with or without the quote
     #quote_source is in format 'post-3920992'
     quote_source = quote_source.split('-')[-1] # Take the numbers only
@@ -134,7 +129,7 @@ def post(message="", thread_url=thread_url, post_url=post_url, quoted_by="",quot
 
     print('Post submitted successfully.')
 
-def getMessages(thread_url=thread_url, quotedUser="", startingPage=1):
+def getMessages(thread_url=config.thread_url, quotedUser="", startingPage=1):
     # Returns all the quotes for #username in the specific multi-page thread url
     allquotes =[]
 
@@ -182,6 +177,7 @@ def getMessages(thread_url=thread_url, quotedUser="", startingPage=1):
                     if matchID:
                         messageID = matchID.group(1)
 
+                    # Match the QuotedName
                     matchQuotedName = quotedNamePattern.search(str(data))
                     if matchQuotedName:
                         quotedName = matchQuotedName.group(1)
@@ -216,28 +212,28 @@ def getMessages(thread_url=thread_url, quotedUser="", startingPage=1):
 def WarOnlineBot():
 
     lookUpPages = 5 # How many pages back to look in the thread
-    startingPage = getLastPage(thread_url=thread_url) - lookUpPages
+    startingPage = getLastPage(thread_url=config.thread_url) - lookUpPages
     if startingPage < 1:
         startingPage = 1 # Starting page cannot be less than 1
 
-    login(username=username, password=password, thread_url=thread_url)
+    login(username=config.username, password=config.password, thread_url=config.thread_url)
     #print("logged in")
 
     # All messages (with quotes) by ALL users:
-    allMessages = getMessages(thread_url=thread_url, quotedUser='', startingPage=startingPage)
+    allMessages = getMessages(thread_url=config.thread_url, quotedUser='', startingPage=startingPage)
 
     # IDs of the quoted messages, replied by the bot:
     messages_by_bot_IDs = []
 
     for msg in allMessages:
         # Set a list of replied messages IDs
-        if msg['messengerName'] == username: #message posted by the WarBot
+        if msg['messengerName'] == config.username: #message posted by the WarBot
             messages_by_bot_IDs.append(msg['quotedID'].split(': ')[-1])
     # remove empty and repeated elements
     messages_by_bot_IDs = list(set([elem for elem in messages_by_bot_IDs if elem]))
 
     # All messages (with quotes) sent _FOR_ the Bot:
-    messagesForBot = getMessages(thread_url=thread_url, quotedUser=username, startingPage=startingPage)
+    messagesForBot = getMessages(thread_url=config.thread_url, quotedUser=config.username, startingPage=startingPage)
 
     # IDs of the messages, quoting the bot:
     messages_for_bot_IDs = []
@@ -262,10 +258,27 @@ def WarOnlineBot():
                 quote = remove_non_english_russian_chars(msg['reply'])
                 quote = remove_extra_spaces(quote)
 
-            message = ""
+            message = "" #Initiating the reply message by Bot
+            previous_dialogue = "" #Initiating the previous dialogue
 
             print('Quote: ', originalQuote)
 
+            # Init Connection
+            db = conversationDB.DataBase()
+            # Get the previous dialogue from the database
+            dbmessages = db.getmessages(msg['messengerName'])
+            for dbmessage in dbmessages:
+                previous_dialogue += dbmessage[0]+' '+dbmessage[1]+' '
+            # Update the string and preprocess it
+            quote = previous_dialogue + quote
+            quote = remove_non_english_russian_chars(quote)
+            quote = remove_extra_spaces(quote)
+            # Truncate the quote to return only the last MaxWords of words:
+            quote = " ".join(quote.split()[-config.MaxWords:])
+
+            # Fix the quote string, to eliminate errors:
+            quote = fixString(quote)
+
             FailureCounter = 0 # In case there is a bug in the model
             while (not message) and (FailureCounter<3):
                 message = WarClient.getReply(message=quote)
@@ -281,20 +294,27 @@ def WarOnlineBot():
             message = fixString(message)
             print('Reply: ', message)
 
-            login(username=username, password=password, thread_url=thread_url)
+            # Add the new conversation pair to the database
+            db.setmessages(username=msg['messengerName'], message_text=originalQuote, bot_reply=message)
+            # Clean up the excessive records, leaving only the remaining messages
+            db.cleanup(username=msg['messengerName'], remaining_messages=config.remaining_messages)
+            # Delete the duplicate records
+            db.deleteDuplicates()
+
+            login(username=config.username, password=config.password, thread_url=config.thread_url)
             time.sleep(1)
-            post(message=message, thread_url=thread_url, post_url=post_url, quoted_by=msg['messengerName'], quote_text=originalQuote, quote_source=msg['messageID'])
+            post(message=message, thread_url=config.thread_url, post_url=config.post_url, quoted_by=msg['messengerName'], quote_text=originalQuote, quote_source=msg['messageID'])
 
             time.sleep(10)  # Standby time for server load release
 
 
 if __name__ == '__main__':
-    timeout = 5 # min
 
     # Start the scheduler
     while True:
+        print('Starting Session')
         WarOnlineBot()
 
-        timer = range(60 * timeout)
+        timer = range(60 * config.timeout)
         for t in timer:
             time.sleep(1)

config.py

@@ -0,0 +1,38 @@
+# Configuration File with stored constants
+
+# Define the login URL and the thread URL:
+login_url = 'https://waronline.org/fora/index.php?login/login'
+thread_url = 'https://waronline.org/fora/index.php?threads/warbot-playground.17636/'
+post_url = "https://waronline.org/fora/index.php?threads/warbot-playground.17636/add-reply"
+
+# SSH settings
+ssh_host = '129.159.146.88'
+ssh_user = 'ubuntu'
+ssh_key_path = 'C:/Users/kerts/OneDrive/Documents/Keys/Ubuntu_Oracle/ssh-key-2023-02-12.key'
+#ssh_key_path = 'ssh-key-2023-02-12.key'
+
+# MySQL settings:
+mysql_host = 'localhost'  # because we will connect through the SSH tunnel
+mysql_port = 3306  # the default MySQL port
+mysql_user = 'root'
+mysql_password = 'naP2tion'
+mysql_db = 'warbot'
+ssh_port = 22
+dbTableName = 'conversations' # messages data table
+
+# WarBot Server Settings:
+HOST = '129.159.146.88'
+PORT = 5000
+
+# Define the login credentials:
+username = 'WarBot'
+password = 'naP2tion'
+
+# Maximum number of words in message:
+MaxWords = 50 # The server is relatively weak to fast-process the long messages
+
+# Maximum conversation pairs in the Database
+remaining_messages = 2
+
+# Time between the reply sessions:
+timeout = 5 # min

conversationDB.py

@@ -1,20 +1,21 @@
 import paramiko
 import pymysql
 from sshtunnel import SSHTunnelForwarder
+import config # Here the constants are stored
 
 # SSH settings
-ssh_host = '129.159.146.88'
-ssh_user = 'ubuntu'
-ssh_key_path = 'C:/Users/kerts/OneDrive/Documents/Keys/Ubuntu_Oracle/ssh-key-2023-02-12.key'
+ssh_host = config.ssh_host
+ssh_user = config.ssh_user
+ssh_key_path = config.ssh_key_path
 
 # MySQL settings
-mysql_host = 'localhost'  # because we will connect through the SSH tunnel
-mysql_port = 3306  # the default MySQL port
-mysql_user = 'root'
-mysql_password = 'naP2tion'
-mysql_db = 'warbot'
-ssh_port = 22
-dbTableName = 'conversations' # messages data table
+mysql_host = config.mysql_host
+mysql_port = config.mysql_port
+mysql_user = config.mysql_user
+mysql_password = config.mysql_password
+mysql_db = config.mysql_db
+ssh_port = config.ssh_port
+dbTableName = config.dbTableName
 
 class DataBase():
     # manages mySQL connection and send-receive
@@ -92,7 +93,7 @@ class DataBase():
 
         # close the SSH client
         db.ssh_client.close()
-    def cleanup(db, username = "", remaining_messages = 3):
+    def cleanup(db, username = "", remaining_messages = config.remaining_messages):
         # Cleanup the records, except the last N rows
         with SSHTunnelForwarder(
                 (db.ssh_host, db.ssh_port),
@@ -124,13 +125,47 @@ class DataBase():
         # close the SSH client
         db.ssh_client.close()
 
+    def deleteDuplicates(db):
+        # Deletes all the duplicate records
+        with SSHTunnelForwarder(
+                (db.ssh_host, db.ssh_port),
+                ssh_username=db.ssh_user,
+                ssh_pkey=db.private_key,
+                remote_bind_address=(db.mysql_host, db.mysql_port)) as tunnel:
+            # connect to the MySQL server through the SSH tunnel
+            mysql_conn = pymysql.connect(
+                host='localhost',  # will be due to the SSH tunneling issue
+                port=tunnel.local_bind_port,
+                user=db.mysql_user,
+                password=db.mysql_password,
+                db=db.mysql_db
+            )
 
-if __name__ == '__main__':
-    # This is for testing purpose only:
-    username = 'user1'
+            # send a query to the MySQL database to delete the records except the last ones
+            with mysql_conn.cursor() as cursor:
 
+                query = f"DELETE c1 FROM {dbTableName} c1, {dbTableName} c2 WHERE " \
+                        f"c1.id > c2.id AND c1.username = c2.username " \
+                        f"AND c1.message_text = c2.message_text;"
+                cursor.execute(query)
+                mysql_conn.commit()
+
+            # close the MySQL connection
+            mysql_conn.close()
+
+        # close the SSH client
+        db.ssh_client.close()
+
+if __name__ == '__main__':
+    pass
+    """
+    #This is for testing purpose only:
+    username = 'u1'
     db = DataBase()
     db.setmessages(username='user2',message_text='some message',bot_reply='some reply')
-    #db.cleanup(username='user2',remaining_messages=1)
+    db.cleanup(username='user2',remaining_messages=1)
+    db.deleteDuplicates()
     messages = db.getmessages(username)
-    print(messages)
+    for message in messages:
+        print(message[0]+' '+message[0]+' ')
+    """