Fix evaluator warnings and album breakdown display

- evaluator: fix partial submission warning, add extraneous query tracking
- app.py: add per-album query counts in result summary
- about.py: update query counts and submission format docs

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

app.py

@@ -130,12 +130,24 @@ def handle_submission(file_obj, email, model_name, opt_in):
             global_metrics=result["global_metrics"],
         )
         if entry is None:
-            leaderboard_msg = "Result saved but not eligible for leaderboard (incomplete submission). Only full submissions across all 3 albums are ranked."
+            if result["is_partial"]:
+                leaderboard_msg = f"Result saved but NOT eligible for leaderboard: incomplete submission ({result['evaluated_queries']}/{result['total_gt_queries']} queries). Only full submissions across all 3 albums are ranked."
+            else:
+                leaderboard_msg = "Result saved but NOT eligible for leaderboard. Only full submissions across all 3 albums are ranked."
         else:
             leaderboard_msg = "Result published to leaderboard."
     else:
         leaderboard_msg = "Result recorded privately. Not published to leaderboard."
 
+    # Build per-album breakdown
+    album_breakdown = {}
+    for a_id, alb_res in result.get("per_album", {}).items():
+        album_breakdown[f"album_{a_id}"] = {
+            "submitted": alb_res["evaluated_queries"],
+            "total": alb_res["total_gt_queries"],
+            "complete": not alb_res["is_partial"],
+        }
+
     # Build result summary
     summary = {
         "status": "Success",
@@ -145,6 +157,7 @@ def handle_submission(file_obj, email, model_name, opt_in):
         "albums": albums,
         "evaluated_queries": result["evaluated_queries"],
         "total_gt_queries": result["total_gt_queries"],
+        "album_breakdown": album_breakdown,
         "metrics": result["global_metrics"],
         "leaderboard_status": leaderboard_msg,
         "notice": "Please download and save your results. Submission data is retained for 30 days only.",

src/about.py

@@ -62,7 +62,29 @@ Full dataset download: <a href="https://sbox.myoas.com/l/Be5be4053f6b43840" targ
 SUBMISSION_GUIDE = """
 ### Submission Format
 
-Upload a JSON file containing an array of prediction objects:
+The dataset provides `test.json` per album. You must **combine all albums into a single JSON array** and add the `album_id` field to each query before submitting.
+
+**Example transformation:**
+
+```python
+import json
+
+submission = []
+for album_id in ["1", "2", "3"]:
+    with open(f"protected/album{album_id}/test.json") as f:
+        queries = json.load(f)
+    for q in queries:
+        submission.append({
+            "album_id": album_id,
+            "query_en": q["query_en"],
+            "pred": ["IMG_0001.jpg", "IMG_0002.jpg", ...]  # your predictions
+        })
+
+with open("submission.json", "w") as f:
+    json.dump(submission, f, indent=2)
+```
+
+**Final submission format:**
 
 ```json
 [
@@ -75,11 +97,11 @@ Upload a JSON file containing an array of prediction objects:
 ```
 
 **Required fields:**
-- `album_id`: Album number (1, 2, or 3)
-- `query_en`: The English query text (must match exactly)
-- `pred`: Ordered list of predicted image filenames
+- `album_id`: Album number (`"1"`, `"2"`, or `"3"` — string)
+- `query_en`: The English query text (must match exactly, case-sensitive)
+- `pred`: Ordered list of predicted image filenames (order matters for NDCG)
 
-You may submit results for any subset of albums. Partial submissions are accepted and evaluated.
+You may submit results for any subset of albums. Partial submissions are accepted and evaluated, but **only full submissions** (all 3 albums, all test queries) are eligible for public leaderboard ranking.
 """
 
 EVALUATION_INFO = """

src/evaluator.py

@@ -82,9 +82,11 @@ class Evaluator:
         source_accum = {}
         empty_gt_queries = 0
         evaluated_queries = 0
+        extraneous_queries = 0
 
         for q, pred in album_submissions.items():
             if q not in gt_map:
+                extraneous_queries += 1
                 continue
 
             gt_item = gt_map[q]
@@ -140,6 +142,7 @@ class Evaluator:
             "evaluated_queries": evaluated_queries,
             "total_gt_queries": len(gt_data),
             "is_partial": evaluated_queries < len(gt_data),
+            "extraneous_queries": extraneous_queries,
         }
 
     def evaluate(self, submission_data: list) -> dict:
@@ -166,6 +169,7 @@ class Evaluator:
 
         total_evaluated = sum(alb["evaluated_queries"] for alb in per_album.values())
         total_gt = sum(alb["total_gt_queries"] for alb in per_album.values())
+        total_extraneous = sum(alb.get("extraneous_queries", 0) for alb in per_album.values())
 
         result = {
             "per_album": per_album,
@@ -174,10 +178,24 @@ class Evaluator:
             "total_gt_queries": total_gt,
             "is_partial": total_evaluated < total_gt,
             "albums": sorted(albums.keys()),
+            "extraneous_queries": total_extraneous,
         }
 
+        # Build warning / notice messages
+        msgs = []
+        if total_extraneous > 0:
+            msgs.append(f"{total_extraneous} extraneous queries were ignored (not in current GT). This may be caused by an outdated test.json or extra queries. Valid queries: {total_evaluated}/{total_gt}.")
         if result["is_partial"]:
-            missing = [a for a in ["1", "2", "3"] if a not in albums]
-            result["warning"] = f"Submission incomplete. Missing albums: {', '.join(missing)}. Averaged results across submitted albums shown below."
+            missing_albums = [a for a in ["1", "2", "3"] if a not in albums]
+            missing_queries = total_gt - total_evaluated
+            parts = []
+            if missing_albums:
+                parts.append(f"Missing albums: {', '.join(missing_albums)}")
+            if missing_queries > 0:
+                parts.append(f"Missing {missing_queries} queries ({total_evaluated}/{total_gt} submitted)")
+            msgs.append("Submission incomplete. " + "; ".join(parts) + ". Only full submissions are eligible for leaderboard ranking.")
+
+        if msgs:
+            result["warning"] = " ".join(msgs)
 
         return result