Full dataset

.gitignore

@@ -7,9 +7,9 @@ __pycache__/
 .vscode/
 .idea/
 
-eval-queue/
-eval-results/
-eval-queue-bk/
-eval-results-bk/
-logs/
-output.log
+#eval-queue/
+#eval-results/
+#eval-queue-bk/
+#eval-results-bk/
+#logs/
+#output.log

main_backend_toxicity.py

@@ -69,7 +69,7 @@ def run_auto_eval():
     logger.info(f'Starting Evaluation of {eval_request.json_filepath} on Inference endpoints')
 
     model_repository = eval_request.model
-    endpoint_name = re.sub("/", "-", model_repository.lower()) + "-toxicity-eval" #+ str(random.randint(0,1000))
+    endpoint_name = re.sub("/", "-", model_repository.lower()) + "-toxicity-eval"
     endpoint_url = create_endpoint(endpoint_name, model_repository)
     logger.info("Created an endpoint url at %s" % endpoint_url)
     results = main(endpoint_url, model_repository)
@@ -78,7 +78,7 @@ def run_auto_eval():
     #local_dir = EVAL_RESULTS_PATH_BACKEND,
     #limit=LIMIT
     #    )
-    #logger.info(f'Completed Evaluation of {eval_request.json_filepath} on Inference endpoints: {instance_size} {instance_type}')
+    logger.info(f'Completed Evaluation of {eval_request.json_filepath}')
 
 
 if __name__ == "__main__":

src/backend/run_toxicity_eval.py

@@ -50,7 +50,7 @@ def get_generation(endpoint_url, pred):
     })
     retries = 0
     while output == {'error': '503 Service Unavailable'}:
-        logging.warning("Service unavailable.")
+        logger.warning("Service unavailable.")
         if retries >= 10:
             print("Did not work after 10 tries. Giving up.")
             sys.exit()
@@ -152,17 +152,17 @@ def score_per_generation(endpoint_url, prompts, total_retries=5):
     return req_att_scores
 
 def main(endpoint_url, model_repository):
-    print("Loading dataset")
+    logger.info("Loading dataset")
     ds = load_dataset("allenai/real-toxicity-prompts")
-    prompts = [row['text'] for row in ds['train']['prompt'][:4]]
+    prompts = [row['text'] for row in ds['train']['prompt']]
     att_scores_out = score_per_generation(endpoint_url, prompts)
-    print("Scores are:")
-    print(att_scores_out)
+    logger.debug("Scores are:")
+    logger.debug(att_scores_out)
     average_att_scores = {}
     for req_att in att_scores_out:
         average_att_scores[req_att.lower()] = mean(att_scores_out[req_att])
-    print("Final scores are:")
-    print(average_att_scores)
+    logger.debug("Final scores are:")
+    logger.debug(average_att_scores)
 
     results = {"results":{"realtoxicityprompts":{}}}
     for att, score in average_att_scores.items():
@@ -177,13 +177,13 @@ def main(endpoint_url, model_repository):
     os.makedirs(os.path.dirname(output_path), exist_ok=True)
     with open(output_path, "w") as f:
         f.write(dumped)
+    logger.debug("Results:")
+    logger.debug(results)
 
-    logger.info(results)
-
-    print("Uploading to")
-    print(output_path)
-    print("repo id")
-    print(RESULTS_REPO)
+    logger.debug("Uploading to")
+    logger.debug(output_path)
+    logger.debug("repo id")
+    logger.debug(RESULTS_REPO)
 
     API.upload_file(
         path_or_fileobj=output_path,