Update app.py

app.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # صفحتان ثابتتان + Submit لكل سؤال يعمل فعليًا + منع تغيّر أبعاد صفحة الإدخال
 # + طور اختياري لأسئلة فهم مباشر باستخدام mT5 (تحميل كسول + fallback)
+# + تحكم بدرجة الصعوبة + مشتّتات أقوى (BM25) + فلاتر POS/NER وCross-Encoder اختياريين
 
 import os, json, uuid, random, unicodedata
 from dataclasses import dataclass
@@ -19,17 +20,29 @@ random.seed(42)
 DEFAULT_NUM_QUESTIONS = 6
 DEFAULT_TROCR_MODEL = "microsoft/trocr-base-printed"
 DEFAULT_TROCR_ZOOM   = 2.6
-QUESTION_MODES = ["فراغ", "فهم مباشر"]  # جديد
+QUESTION_MODES = ["فراغ", "فهم مباشر"]
+DIFFICULTY_MODES = ["سهل", "متوسط", "صعب"]
+
+# NEW: BM25 (اختياري)
+try:
+    from rank_bm25 import BM25Okapi
+    _HAS_BM25 = True
+except Exception:
+    _HAS_BM25 = False
 
 # ------------------ OCR (تحميل كسول) ------------------
 _OCR = {}
 def get_ocr(model_id: str):
-    from transformers import pipeline
-    import torch
-    dev = 0 if torch.cuda.is_available() else -1
-    if model_id not in _OCR:
-        _OCR[model_id] = pipeline("image-to-text", model=model_id, device=dev)
-    return _OCR[model_id]
+    try:
+        from transformers import pipeline
+        import torch
+        dev = 0 if torch.cuda.is_available() else -1
+        if model_id not in _OCR:
+            _OCR[model_id] = pipeline("image-to-text", model=model_id, device=dev)
+        return _OCR[model_id]
+    except Exception:
+        # ارجعي دالة وهمية تعيد نصًا فارغًا بدل التعطّل
+        return lambda im: [{"generated_text": ""}]
 
 # ------------------ PDF/TXT → نص ------------------
 def extract_text_with_pypdf(path: str) -> str:
@@ -123,7 +136,7 @@ def split_sents(t:str)->List[str]:
     return [x for x in s if len(x)>=25]
 
 # ====== (1) عبارات مفتاحية أذكى: n=3 ثم 2 ثم 1، مع فلترة ======
-def yake_keywords(t: str, k: int = 160) -> List[str]:
+def yake_keywords(t: str, k: int = 200) -> List[str]:
     phrases = []
     seen = set()
     for n in [3, 2, 1]:
@@ -208,7 +221,7 @@ def get_embedder():
             _EMB = False
     return _EMB
 
-def nearest_terms(target: str, pool: List[str], k: int = 12) -> List[Tuple[str, float]]:
+def nearest_terms(target: str, pool: List[str], k: int = 24) -> List[Tuple[str, float]]:
     emb = get_embedder()
     if not emb:
         return []
@@ -218,7 +231,7 @@ def nearest_terms(target: str, pool: List[str], k: int = 12) -> List[Tuple[str,
     vecs = emb.encode([target] + cand, normalize_embeddings=True)
     t, C = vecs[0], vecs[1:]
     import numpy as np
-    sims = (C @ t)
+    sims = (C @ t)  # cosine لأن المتجهات مُطبّعة
     idx = np.argsort(-sims)[:k]
     return [(cand[i], float(sims[i])) for i in idx]
 
@@ -233,19 +246,19 @@ def get_masker():
             _MLM = False
     return _MLM
 
-def mlm_distractors(sentence_with_blank: str, correct: str, k: int = 8) -> List[str]:
+def mlm_distractors(sentence_with_blank: str, correct: str, k: int = 18) -> List[str]:
     masker = get_masker()
     if not masker:
         return []
     masked = sentence_with_blank.replace("_____", masker.tokenizer.mask_token)
     try:
-        outs = masker(masked, top_k=max(15, k+5))
+        outs = masker(masked, top_k=max(25, k+7))
         cands = []
         for o in outs:
             tok = o["token_str"].strip()
             if tok and tok != correct and len(tok) >= 2 and not re2.match(r"^[\p{P}\p{S}\d_]+$", tok):
                 cands.append(tok)
-        seen = set(); uniq = []
+        uniq, seen = [], set()
         for w in cands:
             if w not in seen:
                 uniq.append(w); seen.add(w)
@@ -298,21 +311,94 @@ def rank_by_ce(sentence_with_blank: str, candidates: List[str]) -> List[str]:
     except Exception:
         return candidates
 
-def smart_distractors(correct: str, phrase_pool: List[str], sentence: str, k: int = 3) -> List[str]:
+# --------- أدوات BM25 للصعوبة ---------
+def tokenize_ar(s: str) -> List[str]:
+    s = norm_ar(s)
+    toks = re2.findall(r"\p{L}+", s)
+    return [t for t in toks if len(t) >= 2 and t not in AR_STOP]
+
+def bm25_build(sentences: List[str]):
+    if not _HAS_BM25 or not sentences:
+        return None, []
+    corpus_tokens = [tokenize_ar(s) for s in sentences]
+    bm = BM25Okapi(corpus_tokens)
+    return bm, corpus_tokens
+
+def bm25_candidates(correct: str, sentences: List[str], bm, corpus_tokens, top: int = 20) -> List[str]:
+    if not bm: return []
+    q = tokenize_ar(correct)
+    scores = bm.get_scores(q)
+    idxs = sorted(range(len(scores)), key=lambda i: -scores[i])[:min(top, len(scores))]
+    pool = set()
+    for i in idxs:
+        for tok in corpus_tokens[i]:
+            if tok != correct and good_kw(tok):
+                pool.add(tok)
+    return list(pool)
+
+def pick_clean_sentences(sents: List[str], difficulty: str) -> List[str]:
+    out = []
+    for s in sents:
+        if not is_clean_sentence(s): 
+            continue
+        L = len(s)
+        if difficulty == "سهل" and not (70 <= L <= 180):
+            continue
+        if difficulty == "متوسط" and not (70 <= L <= 220):
+            continue
+        if difficulty == "صعب" and not (60 <= L <= 240):
+            continue
+        out.append(s)
+    return out
+
+def similarity_caps(difficulty: str):
+    if difficulty == "سهل":
+        return 0.88
+    if difficulty == "صعب":
+        return 0.95
+    return 0.92
+
+# ====== مشتّتات ذكية مع الصعوبة وBM25 ======
+def smart_distractors(correct: str, phrase_pool: List[str], sentence: str, k: int = 3,
+                      all_sentences: Optional[List[str]] = None, difficulty: str = "متوسط") -> List[str]:
     base = []
-    base.extend([w for w,_ in nearest_terms(correct, phrase_pool, k=20)])
-    for w in mlm_distractors(sentence.replace(correct, "_____"), correct, k=15):
-        if w not in base: base.append(w)
+
+    # (أ) جيران دلاليين
+    base.extend([w for w,_ in nearest_terms(correct, phrase_pool, k=24)])
+
+    # (ب) FILL-MASK
+    for w in mlm_distractors(sentence.replace(correct, "_____"), correct, k=18):
+        if w not in base:
+            base.append(w)
+
+    # (ج) BM25 من النص
+    if all_sentences:
+        bm, corp = bm25_build(all_sentences)
+        for w in bm25_candidates(correct, all_sentences, bm, corp, top=18):
+            if w not in base:
+                base.append(w)
+
+    # فلترة POS/NER وطول وتطبيع
     clean = []
     for w in base:
         w = w.strip()
-        if not w or w == correct: continue
-        if is_named_entity(w): continue
-        if not pos_compatible(w, correct): continue
-        if not length_close(w, correct): continue
-        if norm_ar(w) == norm_ar(correct): continue
+        if not w or w == correct: 
+            continue
+        if is_named_entity(w):
+            continue
+        if not pos_compatible(w, correct):
+            continue
+        if not length_close(w, correct):
+            continue
+        if norm_ar(w) == norm_ar(correct):
+            continue
         clean.append(w)
-    clean = rank_by_ce(sentence.replace(correct, "_____"), clean)[:max(k*2, k)]
+
+    # ترتيب Cross-Encoder (اختياري)
+    clean = rank_by_ce(sentence.replace(correct, "_____"), clean)[:max(k*3, k)]
+
+    # حذف المشتّت شديد القرب دلالياً
+    cap = similarity_caps(difficulty)
     try:
         emb = get_embedder()
         if emb and clean:
@@ -320,11 +406,12 @@ def smart_distractors(correct: str, phrase_pool: List[str], sentence: str, k: in
             c, others = vecs[0], vecs[1:]
             import numpy as np
             sims = others @ c
-            filtered = [w for w, s in zip(clean, sims) if s < 0.92]
+            filtered = [w for w, s in zip(clean, sims) if s < cap]
             if len(filtered) >= k:
                 clean = filtered
     except Exception:
         pass
+
     out = clean[:k]
     while len(out) < k:
         extra = [w for w in phrase_pool if w not in out and w != correct and length_close(w, correct)]
@@ -334,43 +421,55 @@ def smart_distractors(correct: str, phrase_pool: List[str], sentence: str, k: in
         out.extend(legacy_distractors(correct, phrase_pool, k=k-len(out)))
     return out[:k]
 
-# ====== (4-أ) مُولِّد أسئلة "فراغ" (القائم) ======
-def make_mcqs(text:str, n:int=6)->List[MCQ]:
-    sents=split_sents(text)
-    if not sents: 
+# ====== (4-أ) مُولِّد أسئلة "فراغ" ======
+def make_mcqs(text:str, n:int=6, difficulty: str = "متوسط")->List[MCQ]:
+    all_sents = split_sents(text)
+    sents = pick_clean_sentences(all_sents, difficulty)
+    if not sents:
         raise ValueError("النص قصير أو غير صالح.")
-    keyphrases = yake_keywords(text, k=160)
+
+    keyphrases = yake_keywords(text, k=200)
     keyphrases = [kp for kp in keyphrases if safe_keyword(kp) and 2 <= len(kp) <= 40]
+
+    # ربط العبارة بجملة مناسبة (ظهور وحيد)
     sent_for={}
     for s in sents:
-        if not is_clean_sentence(s): continue
         for kp in keyphrases:
             if kp in sent_for: continue
             hits = re2.findall(rf"(?<!\p{{L}}){re2.escape(kp)}(?!\p{{L}})", s)
             if len(hits) == 1:
                 sent_for[kp]=s
-        if len(sent_for)>=n*3: break
+        if len(sent_for)>=n*3:
+            break
+
     if not sent_for:
         tokens = [t for t in re2.findall(r"[\p{L}\p{N}_]+", text) if good_kw(t)]
         freq = [w for w,_ in sorted(((t, text.count(t)) for t in tokens), key=lambda x:-x[1])]
-        keyphrases = [w for w in freq if safe_keyword(w)][:120]
+        keyphrases = [w for w in freq if safe_keyword(w)][:150]
         for s in sents:
-            if not is_clean_sentence(s): continue
             for kp in keyphrases:
                 if kp in sent_for: continue
                 hits = re2.findall(rf"(?<!\p{{L}}){re2.escape(kp)}(?!\p{{L}})", s)
                 if len(hits) == 1: sent_for[kp]=s
-            if len(sent_for)>=n*2: break
+            if len(sent_for)>=n*2:
+                break
+
     if not sent_for:
         raise RuntimeError("تعذّر توليد أسئلة من هذا النص.")
+
+    # أولوية للعبارات الأطول (أعلميّة أعلى)
     items=[]; used_sents=set(); used_keys=set()
     for kp in sorted(sent_for.keys(), key=lambda x: (-len(x), x)):
         if len(items)>=n: break
         s=sent_for[kp]
         if s in used_sents or kp in used_keys: continue
+
         q=re2.sub(rf"(?<!\p{{L}}){re2.escape(kp)}(?!\p{{L}})", "_____", s, count=1)
+
         pool = [x for x in keyphrases if x != kp]
-        ch = smart_distractors(kp, pool, s, k=3) + [kp]
+        ch = smart_distractors(kp, pool, s, k=3, all_sentences=all_sents, difficulty=difficulty) + [kp]
+
+        # تنظيف ومنع تكرار وضمان أربع خيارات
         clean_choices=[]; seen=set()
         for c in ch:
             c = c.strip()
@@ -379,13 +478,15 @@ def make_mcqs(text:str, n:int=6)->List[MCQ]:
         ch = clean_choices[:4]
         while len(ch)<4: ch.append("…")
         random.shuffle(ch); ans=ch.index(kp) if kp in ch else 3
+
         items.append(MCQ(id=str(uuid.uuid4())[:8], question=q, choices=ch, answer_index=ans))
         used_sents.add(s); used_keys.add(kp)
-    if not items: 
+
+    if not items:
         raise RuntimeError("تعذّر توليد أسئلة.")
     return items
 
-# ====== (4-ب) مُولِّد أسئلة "فهم مباشر" (توليدي mT5) ======
+# ====== (4-ب) مُولِّد أسئلة "فهم مباشر" (mT5) ======
 _MT5 = {"tok": None, "model": None, "ok": False}
 def get_mt5():
     if _MT5["tok"] is not None or _MT5["model"] is not None or _MT5["ok"]:
@@ -400,13 +501,10 @@ def get_mt5():
     return _MT5["tok"], _MT5["model"], _MT5["ok"]
 
 def parse_json_block(s: str) -> Optional[dict]:
-    # حاول التقاط أول كائن JSON صالح
     try:
-        # إن وُجد JSON مباشر
         return json.loads(s)
     except Exception:
         pass
-    # التقط أقواس {} الأولى والأخيرة
     m = re2.search(r"\{.*\}", s, flags=re2.DOTALL)
     if m:
         try:
@@ -416,7 +514,6 @@ def parse_json_block(s: str) -> Optional[dict]:
     return None
 
 def comp_prompt(sentence: str) -> str:
-    # تعليمات قصيرة ومحددة مع تنسيق JSON
     return (
         "أنت منشئ أسئلة متعددة الخيارات باللغة العربية.\n"
         "من الجملة التالية، أنشئ سؤال فهم مباشر واحدًا مع أربع خيارات وإشارة للجواب الصحيح.\n"
@@ -457,15 +554,18 @@ def gen_one_comp_q(sentence: str, tok, model, max_new_tokens=128) -> Optional[MC
     except Exception:
         return None
 
-def make_comp_mcqs(text: str, n: int = 6) -> List[MCQ]:
+def make_comp_mcqs(text: str, n: int = 6, difficulty: str = "متوسط") -> List[MCQ]:
     tok, model, ok = get_mt5()
     if not ok:
-        # لو ما توفر mT5 نرجع للفراغ
-        return make_mcqs(text, n)
-    sents = [s for s in split_sents(text) if is_clean_sentence(s)]
+        return make_mcqs(text, n, difficulty=difficulty)
+
+    sents_all = split_sents(text)
+    sents = pick_clean_sentences(sents_all, difficulty)
     if not sents:
-        return make_mcqs(text, n)
+        return make_mcqs(text, n, difficulty=difficulty)
+
     random.shuffle(sents)
+
     items: List[MCQ] = []
     tried = 0
     for s in sents:
@@ -473,23 +573,24 @@ def make_comp_mcqs(text: str, n: int = 6) -> List[MCQ]:
         mcq = gen_one_comp_q(s, tok, model)
         tried += 1
         if mcq:
-            # تنظيف بسيط للخيار والنص
-            mcq.question = re2.sub(r"\s+", " ", mcq.question).strip()
-            mcq.choices = [re2.sub(r"\s+", " ", c).strip() or "…" for c in mcq.choices]
-            items.append(mcq)
-        if tried >= n * 6:  # سقف محاولات معقول
+            q = re2.sub(r"\s+", " ", mcq.question).strip()
+            if not (12 <= len(q) <= 220): 
+                continue
+            choices = [re2.sub(r"\s+", " ", c).strip() for c in mcq.choices]
+            seen=set(); clean=[]
+            for c in choices:
+                if c and c not in seen:
+                    seen.add(c); clean.append(c)
+            clean = (clean + ["…","…","…","…"])[:4]
+            ai = mcq.answer_index if isinstance(mcq.answer_index,int) and 0<=mcq.answer_index<4 else 0
+
+            items.append(MCQ(id=str(uuid.uuid4())[:8], question=q, choices=clean, answer_index=ai))
+        if tried >= n * 7:
             break
+
     if not items:
-        # fallback احتياطي
-        return make_mcqs(text, n)
-    # توحيد البنية (A..D) بنفس الشكل
-    normed=[]
-    for it in items[:n]:
-        # القص إلى 4 خيارات وتأمين الفهارس
-        ch = (it.choices + ["…","…","…","…"])[:4]
-        ai = it.answer_index if 0 <= it.answer_index < 4 else 0
-        normed.append(MCQ(id=it.id, question=it.question, choices=ch, answer_index=ai))
-    return normed
+        return make_mcqs(text, n, difficulty=difficulty)
+    return items[:n]
 
 # ------------------ تحويل إلى سجلات العرض ------------------
 def clean_option_text(t: str) -> str:
@@ -548,30 +649,25 @@ def render_quiz_html(records: List[dict]) -> str:
     return f"""<div id="quiz" class="quiz-wrap">{''.join(parts)}</div>"""
 
 # ------------------ توليد الامتحان وتبديل الصفحات ------------------
-def build_quiz(text_area, file_path, n, model_id, zoom, mode):
+def build_quiz(text_area, file_path, n, model_id, zoom, mode, difficulty):
     text_area = (text_area or "").strip()
     if not text_area and not file_path:
         return "", gr.update(visible=True), gr.update(visible=False), "🛈 الصق نصًا أو ارفع ملفًا أولًا."
-    if text_area:
-        raw = text_area
-    else:
-        raw, _ = file_to_text(file_path, model_id=model_id, zoom=float(zoom))
+    raw = text_area if text_area else file_to_text(file_path, model_id=model_id, zoom=float(zoom))[0]
     cleaned = postprocess(raw)
 
-    # اختيار الطور
     try:
         if mode == "فهم مباشر":
-            items = make_comp_mcqs(cleaned, n=int(n))
+            items = make_comp_mcqs(cleaned, n=int(n), difficulty=difficulty)
         else:
-            items = make_mcqs(cleaned, n=int(n))
-    except Exception as e:
-        # fallback النهائي
-        items = make_mcqs(cleaned, n=int(n))
+            items = make_mcqs(cleaned, n=int(n), difficulty=difficulty)
+    except Exception:
+        items = make_mcqs(cleaned, n=int(n), difficulty=difficulty)
 
-    recs    = to_records(items)
+    recs = to_records(items)
     return render_quiz_html(recs), gr.update(visible=False), gr.update(visible=True), ""
 
-# ------------------ CSS (كما هو) ------------------
+# ------------------ CSS ------------------
 CSS = """
 :root{
   --bg:#0e0e11; --panel:#15161a; --card:#1a1b20; --muted:#a7b0be;
@@ -618,10 +714,10 @@ textarea{min-height:120px}
 }
 .q-actions .q-submit:disabled{opacity:.5;cursor:not-allowed}
 .q-note{color:#ffd1d6}
-.q-note.warn{color:#ffd1د6}
+.q-note.warn{color:#ffd1d6}
 """
 
-# ------------------ JS: ربط Submit بعد الرندر (كما هو مع إبراز الصح) ------------------
+# ------------------ JS: ربط Submit بعد الرندر (مع إبراز الصح لأسئلة الفراغ) ------------------
 ATTACH_LISTENERS_JS = """
 () => {
   if (window.__q_submit_bound_multi2) { return 'already'; }
@@ -676,7 +772,7 @@ ATTACH_LISTENERS_JS = """
 }
 """
 
-# ------------------ واجهة Gradio (نفس الصفحتين + اختيار نوع السؤال) ------------------
+# ------------------ واجهة Gradio ------------------
 with gr.Blocks(title="Question Generator", css=CSS) as demo:
     gr.Markdown("<h2 class='top'>Question Generator</h2>")
 
@@ -688,8 +784,10 @@ with gr.Blocks(title="Question Generator", css=CSS) as demo:
         file_comp = gr.File(label="أو ارفع ملف (PDF / TXT)", file_count="single",
                             file_types=[".pdf",".txt"], type="filepath", elem_classes=["upload-like"])
         num_q = gr.Slider(4, 20, value=DEFAULT_NUM_QUESTIONS, step=1, label="عدد الأسئلة")
-        # جديد: اختيار نوع السؤال دون تغيير بنية الصفحة
+
+        # خيارات إضافية بدون تغيير البنية العامة
         mode_radio = gr.Radio(choices=QUESTION_MODES, value="فراغ", label="نوع السؤال")
+        difficulty_radio = gr.Radio(choices=DIFFICULTY_MODES, value="متوسط", label="درجة الصعوبة")
 
         with gr.Accordion("خيارات PDF المصوّر (اختياري)", open=False):
             trocr_model = gr.Dropdown(
@@ -702,6 +800,7 @@ with gr.Blocks(title="Question Generator", css=CSS) as demo:
                 value=DEFAULT_TROCR_MODEL, label="نموذج TrOCR"
             )
             trocr_zoom = gr.Slider(2.0, 3.5, value=DEFAULT_TROCR_ZOOM, step=0.1, label="Zoom OCR")
+
         btn_build = gr.Button("generate quistion", elem_classes=["button-primary"])
         warn = gr.Markdown("", elem_classes=["small"])
 
@@ -714,7 +813,7 @@ with gr.Blocks(title="Question Generator", css=CSS) as demo:
     # بناء الامتحان + تبديل الصفحات + ربط الـJS
     btn_build.click(
         build_quiz,
-        inputs=[text_area, file_comp, num_q, trocr_model, trocr_zoom, mode_radio],
+        inputs=[text_area, file_comp, num_q, trocr_model, trocr_zoom, mode_radio, difficulty_radio],
         outputs=[quiz_html, page1, page2, warn]
     ).then(
         None, inputs=None, outputs=[js_wired], js=ATTACH_LISTENERS_JS