Spaces:
Sleeping
Sleeping
File size: 3,214 Bytes
575dde4 edc551d 575dde4 7e3e9a6 575dde4 7e3e9a6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | """Compute per-feature population statistics from training CSV.
Produces feature_stats_v1.json with mean, std, min, max, median, q25, q75
for each feature column. Used by XAI service for z-score computation.
Usage:
python -m app.training.compute_feature_stats DataSet/features.csv
"""
from __future__ import annotations
import csv
import json
import sys
from pathlib import Path
import numpy as np
def compute_stats(csv_path: str | Path, output_path: str | Path) -> None:
csv_path = Path(csv_path)
output_path = Path(output_path)
with open(csv_path, "r", encoding="utf-8") as f:
reader = csv.DictReader(f)
rows = list(reader)
fieldnames = reader.fieldnames or []
excluded = {"file_path", "label_int", "duration_sec", "sample_rate"}
feature_cols = [c for c in fieldnames if c not in excluded]
stats: dict[str, dict[str, float]] = {}
for col in feature_cols:
values = []
for row in rows:
try:
v = float(row[col])
if not (np.isnan(v) or np.isinf(v)):
values.append(v)
except (ValueError, TypeError):
continue
if not values:
stats[col] = {
"mean": 0.0, "std": 1.0,
"min": 0.0, "max": 1.0,
"median": 0.0, "q25": 0.0, "q75": 0.0,
"count": 0,
}
continue
arr = np.array(values)
stats[col] = {
"mean": float(np.mean(arr)),
"std": float(np.std(arr)) or 1.0,
"min": float(np.min(arr)),
"max": float(np.max(arr)),
"median": float(np.median(arr)),
"q25": float(np.percentile(arr, 25)),
"q75": float(np.percentile(arr, 75)),
"count": len(values),
}
# Also split by class for AI vs Human distributions
by_class: dict[str, dict[str, dict[str, float]]] = {"ai": {}, "human": {}}
for cls_label, cls_int in (("ai", "1"), ("human", "0")):
for col in feature_cols:
values = []
for row in rows:
if row.get("label_int") != cls_int:
continue
try:
v = float(row[col])
if not (np.isnan(v) or np.isinf(v)):
values.append(v)
except (ValueError, TypeError):
continue
if values:
arr = np.array(values)
by_class[cls_label][col] = {
"mean": float(np.mean(arr)),
"std": float(np.std(arr)) or 1.0,
"median": float(np.median(arr)),
}
output = {**stats, "_by_class": by_class}
output_path.parent.mkdir(parents=True, exist_ok=True)
with open(output_path, "w", encoding="utf-8") as f:
json.dump(output, f, indent=2)
print(f"Wrote {output_path} ({len(stats)} features, {len(rows)} samples)")
if __name__ == "__main__":
csv_in = sys.argv[1] if len(sys.argv) > 1 else "../DataSet/features.csv"
out = sys.argv[2] if len(sys.argv) > 2 else "models/feature_stats_v1.json"
compute_stats(csv_in, out)
|