|
|
|
|
|
""" |
|
|
Simple test that only tests data loading and GPU monitoring without model downloads |
|
|
""" |
|
|
|
|
|
import sys |
|
|
import os |
|
|
sys.path.append('src') |
|
|
|
|
|
def test_data_only(): |
|
|
"""Test only data loading functionality""" |
|
|
try: |
|
|
import pandas as pd |
|
|
from tevatron.utils.gpu_monitor import GPUMemoryMonitor |
|
|
|
|
|
print("Testing data loading...") |
|
|
df = pd.read_csv("data/the_vault/DOC_VAULT_train.tsv", sep='\t', nrows=5) |
|
|
print(f"Loaded {len(df)} samples") |
|
|
print(f"Columns: {list(df.columns)}") |
|
|
|
|
|
print("Testing GPU monitor...") |
|
|
monitor = GPUMemoryMonitor(memory_threshold=0.8, check_interval=10) |
|
|
stats = monitor.get_memory_stats() |
|
|
print(f"GPU monitor initialized: {stats}") |
|
|
|
|
|
print("Testing tevatron imports...") |
|
|
from tevatron.arguments import GLENP1ModelArguments, GLENP1DataArguments |
|
|
print("Arguments imported successfully") |
|
|
|
|
|
print("Basic functionality test PASSED!") |
|
|
return True |
|
|
|
|
|
except Exception as e: |
|
|
print(f"Test failed: {e}") |
|
|
import traceback |
|
|
traceback.print_exc() |
|
|
return False |
|
|
|
|
|
if __name__ == "__main__": |
|
|
success = test_data_only() |
|
|
sys.exit(0 if success else 1) |
|
|
|
