"""Deep-learning style publication figures. Produces additional academic figures beyond the 8 base charts: - training_history.png — XGBoost learning curve across boosting rounds - per_class_metrics.png — precision/recall/F1 per class (Human/AI) - learning_curve.png — train vs CV score vs training set size - threshold_sweep.png — precision/recall/F1 across thresholds - score_distribution.png — predicted-probability histogram by true class - per_source_performance.png — breakdown by dataset source - classification_report.png — styled report table Usage: python -m app.training.generate_deep_figures """ from __future__ import annotations import csv import json import pickle from pathlib import Path import numpy as np import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import matplotlib.patches as mpatches from sklearn.metrics import ( precision_recall_fscore_support, precision_score, recall_score, f1_score, ) from sklearn.model_selection import ( StratifiedKFold, cross_val_predict, learning_curve, ) from sklearn.base import clone BACKEND = Path(__file__).resolve().parents[2] MODELS_DIR = BACKEND / "models" DATASET_DIR = BACKEND.parent / "DataSet" FIGURES_DIR = BACKEND.parent / "docs" / "academic" / "figures" FEATURES_CSV = DATASET_DIR / "features.csv" METADATA_CSV = DATASET_DIR / "metadata.csv" PALETTE = { "bg": "#faf6ed", "fg": "#3d2817", "primary": "#c99347", "secondary": "#7fb069", "error": "#a64b3c", "grid": "#d8c9a8", "accent": "#e7c77a", "human": "#7fb069", "ai": "#a64b3c", } plt.rcParams.update({ "figure.facecolor": PALETTE["bg"], "axes.facecolor": PALETTE["bg"], "axes.edgecolor": PALETTE["fg"], "axes.labelcolor": PALETTE["fg"], "xtick.color": PALETTE["fg"], "ytick.color": PALETTE["fg"], "text.color": PALETTE["fg"], "font.family": "DejaVu Sans", "font.size": 11, "axes.grid": True, "grid.color": PALETTE["grid"], "grid.alpha": 0.4, "savefig.dpi": 150, "savefig.bbox": "tight", "figure.dpi": 100, }) def _load(): with open(MODELS_DIR / "auris_classifier_v1.pkl", "rb") as f: model = pickle.load(f) with open(MODELS_DIR / "feature_scaler_v1.pkl", "rb") as f: scaler = pickle.load(f) with open(MODELS_DIR / "feature_columns_v1.json", "r") as f: feature_cols = json.load(f) with open(MODELS_DIR / "training_results.json", "r") as f: results = json.load(f) return model, scaler, feature_cols, results def _load_data(feature_cols): with open(FEATURES_CSV, "r", encoding="utf-8") as f: rows = list(csv.DictReader(f)) X = np.array([[float(r[c]) for c in feature_cols] for r in rows]) X = np.nan_to_num(X, nan=0.0, posinf=1.0, neginf=-1.0) y = np.array([int(r["label_int"]) for r in rows]) paths = [r.get("path", "") for r in rows] return X, y, paths, rows def _cv_predict(model, X_scaled, y): cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42) y_prob = cross_val_predict( clone(model), X_scaled, y, cv=cv, method="predict_proba", n_jobs=-1, )[:, 1] y_pred = (y_prob > 0.5).astype(int) return y_pred, y_prob # ── 1. Training history (XGBoost boosting-round learning curve) ────────── def fig_training_history(model, scaler, X, y): """Retrain with staged_predict to capture boosting progression.""" from sklearn.ensemble import GradientBoostingClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import log_loss, roc_auc_score X_scaled = scaler.transform(X) X_tr, X_val, y_tr, y_val = train_test_split( X_scaled, y, test_size=0.2, stratify=y, random_state=42, ) clf = clone(model) clf.fit(X_tr, y_tr) n_est = clf.n_estimators_ if hasattr(clf, 'n_estimators_') else clf.n_estimators tr_loss, vl_loss = [], [] tr_err, vl_err = [], [] tr_auc, vl_auc = [], [] for i, (tr_prob, vl_prob) in enumerate( zip(clf.staged_predict_proba(X_tr), clf.staged_predict_proba(X_val)) ): tr_loss.append(log_loss(y_tr, tr_prob)) vl_loss.append(log_loss(y_val, vl_prob)) tr_err.append(1.0 - (tr_prob.argmax(1) == y_tr).mean()) vl_err.append(1.0 - (vl_prob.argmax(1) == y_val).mean()) tr_auc.append(roc_auc_score(y_tr, tr_prob[:, 1])) vl_auc.append(roc_auc_score(y_val, vl_prob[:, 1])) fig, axes = plt.subplots(1, 3, figsize=(16, 5)) x = np.arange(1, len(tr_loss) + 1) panels = [ (axes[0], tr_loss, vl_loss, "Log Loss", True), (axes[1], tr_err, vl_err, "Error Rate", True), (axes[2], tr_auc, vl_auc, "ROC-AUC", False), ] for ax, tr_vals, vl_vals, title, lower_better in panels: ax.plot(x, tr_vals, color=PALETTE["primary"], lw=2.2, label="Eğitim / Train") ax.plot(x, vl_vals, color=PALETTE["error"], lw=2.2, linestyle="--", label="Doğrulama / Validation") ax.set_xlabel("Boosting Round") ax.set_ylabel(title) ax.set_title(f"{title} — Boosting İlerlemesi", fontweight="bold") ax.legend(framealpha=0.85) best_idx = int(np.argmin(vl_vals)) if lower_better else int(np.argmax(vl_vals)) ax.axvline(best_idx + 1, color=PALETTE["accent"], linestyle=":", alpha=0.7) ax.annotate( f"en iyi: {best_idx + 1}", xy=(best_idx + 1, vl_vals[best_idx]), xytext=(12, -12), textcoords="offset points", fontsize=9, color=PALETTE["fg"], ) model_name = type(model).__name__ fig.suptitle(f"{model_name} Eğitim Geçmişi — Train vs Validation", fontsize=14, fontweight="bold") plt.tight_layout() plt.savefig(FIGURES_DIR / "training_history.png") plt.close() print(" ✓ training_history.png") # ── 2. Per-class precision/recall/F1 ───────────────────────────────────── def fig_per_class_metrics(y_true, y_pred): p, r, f, support = precision_recall_fscore_support(y_true, y_pred) classes = ["İnsan / Human", "AI / Yapay"] metrics = {"Precision": p, "Recall": r, "F1 Score": f} fig, ax = plt.subplots(figsize=(9, 6)) x = np.arange(len(classes)) width = 0.25 colors = [PALETTE["primary"], PALETTE["secondary"], PALETTE["error"]] for i, (name, vals) in enumerate(metrics.items()): bars = ax.bar(x + (i - 1) * width, vals, width, label=name, color=colors[i], edgecolor=PALETTE["fg"], linewidth=0.5) for bar, v in zip(bars, vals): ax.text(bar.get_x() + bar.get_width() / 2, v + 0.01, f"{v:.3f}", ha="center", va="bottom", fontsize=10, fontweight="bold") ax.set_xticks(x) ax.set_xticklabels([f"{c}\n(n={s})" for c, s in zip(classes, support)]) ax.set_ylabel("Skor / Score") ax.set_title("Sınıf Başına Performans — Precision / Recall / F1", fontsize=13, fontweight="bold") ax.set_ylim([0, 1.08]) ax.legend(loc="lower right", framealpha=0.85) plt.savefig(FIGURES_DIR / "per_class_metrics.png") plt.close() print(" ✓ per_class_metrics.png") # ── 3. Learning curve (score vs training set size) ─────────────────────── def fig_learning_curve(model, scaler, X, y): X_scaled = scaler.transform(X) train_sizes = np.linspace(0.1, 1.0, 6) cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42) sizes, tr_scores, val_scores = learning_curve( clone(model), X_scaled, y, train_sizes=train_sizes, cv=cv, scoring="roc_auc", n_jobs=-1, random_state=42, ) tr_mean, tr_std = tr_scores.mean(1), tr_scores.std(1) val_mean, val_std = val_scores.mean(1), val_scores.std(1) fig, ax = plt.subplots(figsize=(9, 6.5)) ax.plot(sizes, tr_mean, "o-", color=PALETTE["primary"], lw=2.5, label="Eğitim / Train") ax.fill_between(sizes, tr_mean - tr_std, tr_mean + tr_std, alpha=0.18, color=PALETTE["primary"]) ax.plot(sizes, val_mean, "s-", color=PALETTE["error"], lw=2.5, label="Çapraz Doğrulama / Cross-Validation") ax.fill_between(sizes, val_mean - val_std, val_mean + val_std, alpha=0.18, color=PALETTE["error"]) ax.set_xlabel("Eğitim Örneği Sayısı / Training Examples") ax.set_ylabel("ROC-AUC") ax.set_title("Öğrenme Eğrisi — Model Veri ile Öğreniyor mu?", fontsize=13, fontweight="bold") ax.legend(loc="lower right", framealpha=0.85) gap = tr_mean[-1] - val_mean[-1] if gap > 0.05: diagnosis = "yüksek varyans — regularizasyon gerekli" elif gap > 0.03: diagnosis = "orta varyans — kabul edilebilir" else: diagnosis = "düşük varyans (iyi)" ax.annotate( f"Train-Val Gap: {gap:.4f}\n→ {diagnosis}\n" f"Not: Tree ensemble train score\n" f"yapısal olarak ~1.0 olur", xy=(0.42, 0.05), xycoords="axes fraction", fontsize=9, bbox=dict(boxstyle="round,pad=0.5", facecolor=PALETTE["bg"], edgecolor=PALETTE["primary"], alpha=0.85), ) plt.savefig(FIGURES_DIR / "learning_curve.png") plt.close() print(" ✓ learning_curve.png") # ── 4. Threshold sweep ─────────────────────────────────────────────────── def fig_threshold_sweep(y_true, y_prob): thresholds = np.linspace(0.05, 0.95, 91) precisions, recalls, f1s = [], [], [] for t in thresholds: pred = (y_prob > t).astype(int) precisions.append(precision_score(y_true, pred, zero_division=0)) recalls.append(recall_score(y_true, pred, zero_division=0)) f1s.append(f1_score(y_true, pred, zero_division=0)) precisions, recalls, f1s = np.array(precisions), np.array(recalls), np.array(f1s) best_idx = int(np.argmax(f1s)) best_t = thresholds[best_idx] fig, ax = plt.subplots(figsize=(10, 6)) ax.plot(thresholds, precisions, color=PALETTE["primary"], lw=2.5, label="Precision") ax.plot(thresholds, recalls, color=PALETTE["secondary"], lw=2.5, label="Recall") ax.plot(thresholds, f1s, color=PALETTE["error"], lw=2.8, label="F1 Score") ax.axvline(0.5, color=PALETTE["fg"], linestyle=":", alpha=0.5, label="Varsayılan 0.5") ax.axvline(best_t, color=PALETTE["accent"], linestyle="--", lw=2, label=f"En iyi F1 @ {best_t:.2f}") ax.scatter([best_t], [f1s[best_idx]], color=PALETTE["accent"], s=100, zorder=5, edgecolor=PALETTE["fg"]) ax.set_xlabel("Karar Eşiği / Decision Threshold") ax.set_ylabel("Skor") ax.set_title("Eşik Taraması — Precision / Recall / F1 vs Threshold", fontsize=13, fontweight="bold") ax.legend(loc="lower left", framealpha=0.85) ax.set_xlim([0, 1]) ax.set_ylim([0, 1.02]) plt.savefig(FIGURES_DIR / "threshold_sweep.png") plt.close() print(" ✓ threshold_sweep.png") # ── 5. Score distribution histogram ────────────────────────────────────── def fig_score_distribution(y_true, y_prob): fig, ax = plt.subplots(figsize=(10, 6)) bins = np.linspace(0, 1, 41) human_probs = y_prob[y_true == 0] ai_probs = y_prob[y_true == 1] ax.hist(human_probs, bins=bins, alpha=0.65, color=PALETTE["human"], label=f"İnsan (n={len(human_probs)})", edgecolor=PALETTE["fg"], linewidth=0.3) ax.hist(ai_probs, bins=bins, alpha=0.65, color=PALETTE["ai"], label=f"AI (n={len(ai_probs)})", edgecolor=PALETTE["fg"], linewidth=0.3) ax.axvline(0.5, color=PALETTE["fg"], linestyle="--", alpha=0.7, lw=2, label="Karar Eşiği") ax.set_xlabel("Tahmin Olasılığı P(AI) / Predicted Probability") ax.set_ylabel("Örnek Sayısı / Count") ax.set_title("Tahmin Olasılığı Dağılımı — Sınıf Bazlı", fontsize=13, fontweight="bold") ax.legend(framealpha=0.85) plt.savefig(FIGURES_DIR / "score_distribution.png") plt.close() print(" ✓ score_distribution.png") # ── 6. Per-source breakdown ────────────────────────────────────────────── def fig_per_source_performance(y_true, y_pred, paths, rows): # Join features.csv by path with metadata.csv source info if not METADATA_CSV.exists(): print(" ! metadata.csv missing, skipping per_source_performance") return with open(METADATA_CSV, "r", encoding="utf-8") as f: meta_rows = list(csv.DictReader(f)) # normalize paths for join (forward slashes) path_to_source = { r["path"].replace("\\", "/"): r.get("source", "unknown") for r in meta_rows } sources_hits: dict[str, dict[str, int]] = {} for yt, yp, path in zip(y_true, y_pred, paths): key = path.replace("\\", "/") src = path_to_source.get(key, "unknown") d = sources_hits.setdefault(src, {"total": 0, "correct": 0, "ai": 0, "human": 0}) d["total"] += 1 if yt == yp: d["correct"] += 1 d["ai" if yt == 1 else "human"] += 1 sources = [s for s in sources_hits if sources_hits[s]["total"] >= 20] sources.sort(key=lambda s: -sources_hits[s]["total"]) if not sources: print(" ! no source has >=20 samples, skipping") return accs = [sources_hits[s]["correct"] / sources_hits[s]["total"] for s in sources] totals = [sources_hits[s]["total"] for s in sources] fig, ax = plt.subplots(figsize=(10, max(4, len(sources) * 0.45))) y_pos = np.arange(len(sources)) colors = plt.cm.copper(np.linspace(0.3, 0.9, len(sources))) ax.barh(y_pos, accs, color=colors, edgecolor=PALETTE["fg"], linewidth=0.5) ax.set_yticks(y_pos) ax.set_yticklabels([f"{s} (n={n})" for s, n in zip(sources, totals)]) ax.invert_yaxis() ax.set_xlabel("Accuracy") ax.set_title("Veri Kaynağı Bazlı Performans", fontsize=13, fontweight="bold") ax.set_xlim([0, 1.0]) for i, v in enumerate(accs): ax.text(v + 0.005, i, f"{v:.3f}", va="center", fontsize=9) plt.savefig(FIGURES_DIR / "per_source_performance.png") plt.close() print(" ✓ per_source_performance.png") # ── 7. Classification report as styled table ───────────────────────────── def fig_classification_report(y_true, y_pred): from sklearn.metrics import classification_report report = classification_report( y_true, y_pred, target_names=["Human (İnsan)", "AI (Yapay)"], digits=4, output_dict=True, ) fig, ax = plt.subplots(figsize=(10, 4.5)) ax.axis("off") classes = ["Human (İnsan)", "AI (Yapay)", "accuracy", "macro avg", "weighted avg"] header = ["Class", "Precision", "Recall", "F1", "Support"] data = [header] for c in classes: r = report.get(c, {}) if c == "accuracy": data.append([c, "", "", f"{report['accuracy']:.4f}", f"{len(y_true)}"]) else: data.append([ c, f"{r.get('precision', 0):.4f}", f"{r.get('recall', 0):.4f}", f"{r.get('f1-score', 0):.4f}", f"{int(r.get('support', 0))}", ]) table = ax.table( cellText=data, cellLoc="center", loc="center", colWidths=[0.25, 0.18, 0.18, 0.18, 0.18], ) table.auto_set_font_size(False) table.set_fontsize(11) table.scale(1, 1.8) # header styling for i in range(len(header)): table[(0, i)].set_facecolor(PALETTE["primary"]) table[(0, i)].set_text_props(weight="bold", color=PALETTE["bg"]) # row stripes for r in range(1, len(data)): for c in range(len(header)): table[(r, c)].set_facecolor( PALETTE["bg"] if r % 2 else "#f0e6d0", ) table[(r, c)].set_edgecolor(PALETTE["grid"]) ax.set_title("Sınıflandırma Raporu — 5-fold Cross-Validation", fontsize=13, fontweight="bold", pad=18) plt.savefig(FIGURES_DIR / "classification_report.png") plt.close() print(" ✓ classification_report.png") def main(): FIGURES_DIR.mkdir(parents=True, exist_ok=True) print(f"Output: {FIGURES_DIR}") print("Loading...") model, scaler, feature_cols, results = _load() X, y, paths, rows = _load_data(feature_cols) print("CV predictions (5-fold)...") X_scaled = scaler.transform(X) y_pred, y_prob = _cv_predict(model, X_scaled, y) print("\nGenerating deep figures...") fig_per_class_metrics(y, y_pred) fig_threshold_sweep(y, y_prob) fig_score_distribution(y, y_prob) fig_per_source_performance(y, y_pred, paths, rows) fig_classification_report(y, y_pred) fig_training_history(model, scaler, X, y) print("Learning curve (may take ~30s)...") fig_learning_curve(model, scaler, X, y) total = len(list(FIGURES_DIR.glob("*.png"))) print(f"\nDone. Total figures in {FIGURES_DIR}: {total}") if __name__ == "__main__": main()