Update app.py

app.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # صفحتان ثابتتان + Submit لكل سؤال يعمل فعليًا + منع تغيّر أبعاد صفحة الإدخال
 
-import os, json, uuid, random, unicodedata
+import os, json, uuid, random, unicodedata, difflib
 from dataclasses import dataclass
 from pathlib import Path
 from typing import List, Tuple, Optional
@@ -93,7 +93,7 @@ def norm_ar(t:str)->str:
     t = re2.sub(AR_DIAC, "", t)
     t = re2.sub(r"[إأآا]", "ا", t)
     t = re2.sub(r"[يى]", "ي", t)
-    t = re2.sub(r"\s+", " ", t)
+    t = re2.sub(r"\س+", " ", t) if False else re2.sub(r"\s+", " ", t)
     t = re2.sub(r'(\p{L})\1{2,}', r'\1', t)
     t = re2.sub(r'(\p{L})\1', r'\1', t)
     return t.strip()
@@ -138,7 +138,6 @@ def yake_keyphrases(t: str, top_k: int = 180) -> List[str]:
                 continue
             if 2 <= len(w) <= 42:
                 phrases.append(w); seen.add(w)
-    # إزالة العبارات التي هي جزء من أطول
     phrases_sorted = sorted(phrases, key=lambda x: (-len(x), x))
     kept=[]
     for p in phrases_sorted:
@@ -191,7 +190,6 @@ def mlm_fill(sentence_with_blank: str, correct: str, k: int = 20) -> List[str]:
             tok = o["token_str"].strip()
             if tok and tok != correct and len(tok) >= 2 and not re2.match(r"^[\p{P}\p{S}\d_]+$", tok):
                 cands.append(tok)
-        # فريد مع الحفاظ على الترتيب
         seen=set(); uniq=[]
         for w in cands:
             if w not in seen:
@@ -269,102 +267,158 @@ def sentence_score(s: str) -> float:
     bonus = 0.2 if ("،" in s or ":" in s) else 0.0
     return base + bonus + penalties
 
-# --- (H) ترتيب المرشّحات بالانسجام مع الجملة ---
-def rank_by_sentence_coherence(sentence_with_blank: str, correct: str, candidates: List[str], topk: int=3) -> List[str]:
-    emb = get_embedder()
-    if not emb or not candidates:
-        return candidates[:topk]
-    filled = [sentence_with_blank.replace("_____", c) for c in candidates]
-    ref = sentence_with_blank.replace("_____", correct)
-    vecs = embed_texts([ref] + filled)
-    if vecs is None:
-        return candidates[:topk]
-    import numpy as np
-    ref_vec = vecs[0]
-    cand_vecs = vecs[1:]
-    sims = cand_vecs @ ref_vec
-    order = list(reversed(sorted(range(len(candidates)), key=lambda i: sims[i])))
-    ranked = [candidates[i] for i in order]
-    return ranked[:topk]
+# ================== (NEW) جودة المشتِّتات والتطويل ==================
 
-# --- (I) حصاد مصطلحات احتياطية عالية التكرار من النص كله ---
-def harvest_backup_terms(text: str, limit: int = 400) -> List[str]:
-    toks = re2.findall(r"[\p{L}][\p{L}\p{N}_\-]{1,}", text)
-    stats = {}
-    for t in toks:
-        tt = norm_ar(t)
-        if not good_kw(tt): 
-            continue
-        stats[tt] = stats.get(tt, 0) + 1
-    top = [w for w,_ in sorted(stats.items(), key=lambda kv: -kv[1])]
-    return top[:limit]
-
-# ================== (NEW) موازنة الطول والتطويل ==================
-
-# كاش صغير لعبارة الصحيحة لاستخدامها أثناء التطويل
+# كاش نص كامل لتحسين تقييم الجودة
+global_full_text_cache = ""
+# كاش عبارة صحيحة لتجنّب التطابق بعد التطويل
 ref_phrase_cache = {}
 
+ADJ_WHITELIST = {"التعليمية","الذكية","الرقمية","الافتراضية","التكيفية","الحديثة","المتقدمة"}
+NOUN_PREFIXES = {"مجال","تقنيات","أنظمة","مفاهيم","نماذج","ممارسات","آليات","تطبيقات"}
+
+def is_arabic_word(w:str)->bool:
+    return bool(re2.match(r"^[\p{Arabic}]+$", w))
+
+def clean_spaces(s:str)->str:
+    s = re2.sub(r"\s+", " ", s).strip()
+    s = re2.sub(r"\bال\s+ال\b", "ال", s)
+    return s
+
+def bad_token(w:str)->bool:
+    return (not is_arabic_word(w)) or (len(w) < 2 or len(w) > 18)
+
+def looks_weird(phrase:str)->bool:
+    toks = [t for t in re2.split(r"\s+", phrase.strip()) if t]
+    if len(toks) == 0: return True
+    if any(bad_token(t) for t in toks): return True
+    for i in range(1, len(toks)):
+        if toks[i] == toks[i-1]:
+            return True
+    if len(set(toks)) <= len(toks) - 1:
+        if any(toks.count(t) > 1 for t in toks):
+            return True
+    pos = [phrase_pos(t) or "" for t in toks]
+    streak = 0
+    for p in pos:
+        if p.startswith("ADJ"):
+            streak += 1
+            if streak > 2: return True
+        else:
+            streak = 0
+    return False
+
+def quality_score(phrase:str, sentence:str, full_text:str)->float:
+    phrase = clean_spaces(phrase)
+    if looks_weird(phrase):
+        return 0.0
+    hits = sum(1 for t in set(phrase.split()) if t in full_text)
+    toks = phrase.split()
+    pos0 = phrase_pos(toks[0]) or ""
+    pos1 = phrase_pos(toks[1]) if len(toks)>1 else ""
+    nominal_bonus = 0.2 if (pos0.startswith("N") and (not pos1 or pos1.startswith("ADJ"))) else 0.0
+    return min(1.0, 0.3 + 0.1*hits + nominal_bonus)
+
 def word_len(s: str) -> int:
     return len([w for w in re2.split(r"\s+", s.strip()) if w])
 
-def within_ratio(cand: str, target_len: int, tol: float = 0.2) -> bool:
+def within_ratio(cand: str, target_len: int, tol: float = 0.15) -> bool:
     L = word_len(cand)
     return (target_len*(1-tol) <= L <= target_len*(1+tol))
 
-# قوالب عربية عامة للتطويل عند غياب الـMLM أو فشل توقع مناسب
-GENERIC_PREFIXES = ["تقنيات", "مجال", "أنظمة", "تطبيقات", "مفاهيم", "ممارسات", "نماذج", "آليات"]
-GENERIC_SUFFIXES = ["الذكية", "التعليمية", "الحديثة", "المتقدمة", "المبتكرة", "الرقمية"]
-
 def shape_phrase_like(ref: str, cand: str) -> str:
-    """مواءمة التعريف/التنكير لتقارب الشكل العام."""
     return with_same_definiteness(ref, cand)
 
 def try_mlm_expand(cand: str, sentence_with_blank: str, target_len: int) -> Optional[str]:
-    """توسيع المشتّت عبر MLم بإضافة كلمة قبل/بعد ليقترب الطول من الصحيحة."""
     masker = get_masker()
     if not masker:
         return None
-    trials = []
-    trials.append(sentence_with_blank.replace("_____", f"{masker.tokenizer.mask_token} {cand}"))
-    trials.append(sentence_with_blank.replace("_____", f"{cand} {masker.tokenizer.mask_token}"))
-
-    for masked_sent in trials:
+    trials = [
+        sentence_with_blank.replace("_____", f"{masker.tokenizer.mask_token} {cand}"),
+        sentence_with_blank.replace("_____", f"{cand} {masker.tokenizer.mask_token}")
+    ]
+    for masked in trials:
         try:
-            outs = masker(masked_sent, top_k=8)
+            outs = masker(masked, top_k=12)
         except Exception:
             continue
         for o in outs:
             tok = o["token_str"].strip()
-            if not tok or re2.match(r"^[\p{P}\p{S}\d_]+$", tok):
+            if not is_arabic_word(tok):
                 continue
-            if masked_sent.strip().startswith(masker.tokenizer.mask_token):
+            if masked.startswith(masker.tokenizer.mask_token):
+                if tok not in NOUN_PREFIXES:
+                    continue
                 phrase = f"{tok} {cand}"
             else:
+                if tok not in ADJ_WHITELIST:
+                    continue
                 phrase = f"{cand} {tok}"
-            # تجنب التطابق مع الصحيحة بعد التطبيع
-            if within_ratio(phrase, target_len) and norm_ar(phrase) != norm_ar(ref_phrase_cache.get("correct","")):
+            phrase = clean_spaces(phrase)
+            if within_ratio(phrase, target_len, tol=0.15) and norm_ar(phrase) != norm_ar(ref_phrase_cache.get("correct","")) and not looks_weird(phrase):
                 return phrase
     return None
 
 def fallback_expand(cand: str, target_len: int) -> str:
-    """تطويل بسيط بقوالب عامة إذا فشل الـMLM."""
-    for p in GENERIC_PREFIXES:
+    for p in NOUN_PREFIXES:
         phrase = f"{p} {cand}"
-        if within_ratio(phrase, target_len):
-            return phrase
-    for sfx in GENERIC_SUFFIXES:
+        if within_ratio(phrase, target_len, tol=0.15):
+            return clean_spaces(phrase)
+    for sfx in ADJ_WHITELIST:
         phrase = f"{cand} {sfx}"
-        if within_ratio(phrase, target_len):
-            return phrase
-    phrase = f"{random.choice(GENERIC_PREFIXES)} {cand} {random.choice(GENERIC_SUFFIXES)}"
-    return phrase
+        if within_ratio(phrase, target_len, tol=0.15):
+            return clean_spaces(phrase)
+    candidates = [f"{p} {cand}" for p in NOUN_PREFIXES] + [f"{cand} {sfx}" for sfx in ADJ_WHITELIST]
+    candidates = sorted(candidates, key=lambda ph: abs(word_len(ph) - target_len))
+    return clean_spaces(candidates[0])
+
+# --- (H*) ترتيب المرشّحات بالانسجام + الجودة + منع التشابه ---
+def rank_by_sentence_coherence(sentence_with_blank: str, correct: str, candidates: List[str], topk: int=3, full_text: str="") -> List[str]:
+    emb = get_embedder()
+    if not candidates:
+        return []
+    coherence = {}
+    if emb:
+        filled = [sentence_with_blank.replace("_____", c) for c in candidates]
+        ref = sentence_with_blank.replace("_____", correct)
+        vecs = embed_texts([ref] + filled)
+        if vecs is not None:
+            import numpy as np
+            ref_vec = vecs[0]; cand_vecs = vecs[1:]
+            sims = cand_vecs @ ref_vec
+            for i, c in enumerate(candidates):
+                coherence[c] = float(sims[i])
+    qscore = {c: quality_score(c, sentence_with_blank, full_text) for c in candidates}
+    def final_score(c):
+        coh = coherence.get(c, 0.0)
+        return 0.7*coh + 0.3*qscore.get(c, 0.0)
+    ranked = sorted(candidates, key=lambda c: final_score(c), reverse=True)
+
+    kept = []
+    for c in ranked:
+        if all(difflib.SequenceMatcher(None, c, x).ratio() < 0.90 for x in kept):
+            kept.append(c)
+        if len(kept) >= topk:
+            break
+    return kept[:topk]
+
+# --- (I) حصاد مصطلحات احتياطية عالية التكرار من النص كله ---
+def harvest_backup_terms(text: str, limit: int = 400) -> List[str]:
+    toks = re2.findall(r"[\p{L}][\p{L}\p{N}_\-]{1,}", text)
+    stats = {}
+    for t in toks:
+        tt = norm_ar(t)
+        if not good_kw(tt): 
+            continue
+        stats[tt] = stats.get(tt, 0) + 1
+    top = [w for w,_ in sorted(stats.items(), key=lambda kv: -kv[1])]
+    return top[:limit]
 
 # --- (J) مشتّتات ذكية تضمن دائمًا ≥3 خيارات فعلية + موازنة الطول ---
 def smart_distractors(correct: str, phrase_pool: List[str], sentence_with_blank: str, backup_terms: List[str], k: int = 3) -> List[str]:
     target = correct.strip()
-    ref_phrase_cache["correct"] = target  # لتجنّب مطابقة بعد التطويل
+    ref_phrase_cache["correct"] = target
 
-    # 1) مصادر متعددة
     neigh = nearest_terms(target, phrase_pool, k=48)
     mlm  = mlm_fill(sentence_with_blank, target, k=24)
 
@@ -379,7 +433,6 @@ def smart_distractors(correct: str, phrase_pool: List[str], sentence_with_blank:
         if w not in seen:
             seen.add(w); raw_pool.append(w)
 
-    # 2) إن لم يكفِ، أضف من backup_terms (من النص كله)
     for w in backup_terms:
         if len(raw_pool) >= max(60, k*10): break
         if not w or norm_ar(w) == norm_ar(target): 
@@ -389,7 +442,6 @@ def smart_distractors(correct: str, phrase_pool: List[str], sentence_with_blank:
         if w not in seen:
             seen.add(w); raw_pool.append(w)
 
-    # 3) فلترة POS إن توفّر
     filtered = []
     for w in raw_pool:
         if same_pos(target, w):
@@ -399,39 +451,35 @@ def smart_distractors(correct: str, phrase_pool: List[str], sentence_with_blank:
     if not filtered:
         filtered = raw_pool[:max(24, k*6)]
 
-    # 4) موازنة الطول (أساسي): اجعل المشتّت قريب طولًا من الصحيحة
     target_words = word_len(target)
     shaped = []
     for w in filtered:
-        cand = shape_phrase_like(target, w)  # مواءمة "الـ"
-        if within_ratio(cand, target_words, tol=0.2):
-            shaped.append(cand)
+        cand = shape_phrase_like(target, w)
+        if within_ratio(cand, target_words, tol=0.15) and not looks_weird(cand):
+            shaped.append(clean_spaces(cand))
             continue
-        # جرّب توسيع بالـMLM
         expanded = try_mlm_expand(cand, sentence_with_blank, target_words)
-        if expanded and within_ratio(expanded, target_words, tol=0.2):
-            shaped.append(expanded)
+        if expanded and within_ratio(expanded, target_words, tol=0.15) and not looks_weird(expanded):
+            shaped.append(clean_spaces(expanded))
             continue
-        # fallback بقوالب عامة
         fb = fallback_expand(cand, target_words)
-        shaped.append(fb)
+        if not looks_weird(fb):
+            shaped.append(clean_spaces(fb))
 
-    # إزالة أي تطويل خرج متطابقًا مع الصحيحة بعد التطبيع
     shaped = [s for s in shaped if norm_ar(s) != norm_ar(target)]
 
-    # 5) ترتيب بالانسجام
-    ranked = rank_by_sentence_coherence(sentence_with_blank, target, shaped, topk=max(k, 12))
+    ranked = rank_by_sentence_coherence(
+        sentence_with_blank, target, shaped, topk=max(k, 12), full_text=global_full_text_cache
+    )
 
-    # 6) خذ أفضل k؛ وإن لم يكفِ، أكمل من shaped ثم filtered ثم raw_pool ثم backup_terms
     out = []
     for src in [ranked, shaped, filtered, raw_pool, backup_terms]:
         for w in src:
             if len(out) >= k: break
-            if w and norm_ar(w) != norm_ar(target) and w not in out:
+            if w and norm_ar(w) != norm_ar(target) and w not in out and not looks_weird(w):
                 out.append(w)
         if len(out) >= k: break
 
-    # ضمان العدد بدون أي placeholders
     if len(out) < k:
         while len(out) < k and ranked:
             out.append(ranked[len(out) % len(ranked)])
@@ -440,6 +488,9 @@ def smart_distractors(correct: str, phrase_pool: List[str], sentence_with_blank:
 
 # ------------------ مُولِّد الأسئلة ------------------
 def make_mcqs(text:str, n:int=6)->List[MCQ]:
+    global global_full_text_cache
+    global_full_text_cache = text
+
     sents = split_sents(text)
     if not sents: 
         raise ValueError("النص قصير أو غير صالح.")
@@ -450,7 +501,6 @@ def make_mcqs(text:str, n:int=6)->List[MCQ]:
         freq = [w for w,_ in sorted(((t, text.count(t)) for t in tokens), key=lambda x:-x[1])]
         keyphrases = freq[:160]
 
-    # مصطلحات احتياطية عامة من النص لضمان تعبئة المشتتات دائمًا
     backup_terms = harvest_backup_terms(text, limit=400)
 
     kp2best_sent = {}
@@ -479,12 +529,18 @@ def make_mcqs(text:str, n:int=6)->List[MCQ]:
 
         q = re2.sub(rf"(?<!\p{{L}}){re2.escape(kp)}(?!\p{{L}})", "_____", s, count=1)
 
-        # مشتّتات ذكية تضمن دائمًا ≥3 ومتوازنة الطول
         pool = [x for x in keyphrases if x != kp]
         distracts = smart_distractors(kp, pool, q, backup_terms, k=3)
 
         ch = distracts + [kp]
-        random.shuffle(ch)
+
+        # ترتيب غير عشوائي: تدوير حتمي لموضع الصحيحة
+        # 1) ضع الصحيحة مؤقتًا في النهاية
+        ch_sorted = sorted(ch, key=lambda c: c != kp)
+        # 2) تدوير بناءً على رقم السؤال (طول القائمة الحالية) وهاش العبارة
+        rot = (len(items) + (hash(kp) & 3)) % 4
+        ch = ch_sorted[-rot:] + ch_sorted[:-rot]
+
         ans = ch.index(kp)
 
         items.append(MCQ(id=str(uuid.uuid4())[:8], question=q, choices=ch, answer_index=ans))
@@ -501,7 +557,6 @@ def to_records(items:List[MCQ])->List[dict]:
         for i,lbl in enumerate(["A","B","C","D"]):
             txt=(it.choices[i] if i<len(it.choices) else "—").strip()
             txt=txt.replace(",", "،").replace("?", "؟").replace(";", "؛")
-            # منع أي Placeholder يظهر للمستخدم
             if txt == "—" or not txt:
                 txt = "خيار"
             opts.append({"id":lbl,"text":txt or "خيار","is_correct":(i==it.answer_index)})
@@ -587,7 +642,7 @@ textarea{min-height:120px}
 .q-badge.ok{background:#0f2f22;color:#b6f4db;border:1px solid #145b44}
 .q-badge.err{background:#3a0d14;color:#ffd1d6;border:1px solid #6a1e2b}
 
-.q-text{color:var(--text);font-size:1.06rem;line-height:1.8;margin:8px 0 12px}
+.q-text{color:#الtext;font-size:1.06rem;line-height:1.8;margin:8px 0 12px}
 .opts{display:flex;flex-direction:column;gap:8px}
 .opt{display:flex;gap:10px;align-items:center;background:#14161c;border:1px solid #2a2d3a;border-radius:12px;padding:10px;transition:background .15s,border-color .15s}
 .opt input{accent-color:var(--accent2)}