| | """debug_downloaded_data.py - Inspect the downloaded conversation format"""
|
| |
|
| | import json
|
| |
|
| | def inspect_downloaded_data():
|
| | """Inspect the first few records to understand the format"""
|
| |
|
| | data_path = "data/conversation_raw/OpenAssistant_oasst1_raw.jsonl"
|
| |
|
| | print("🔍 Inspecting downloaded OpenAssistant data...")
|
| | print("="*50)
|
| |
|
| | try:
|
| | with open(data_path, 'r', encoding='utf-8') as f:
|
| | for i in range(5):
|
| | line = f.readline().strip()
|
| | if line:
|
| | record = json.loads(line)
|
| | print(f"\nRecord {i+1}:")
|
| | print(f"Top-level keys: {list(record.keys())}")
|
| |
|
| |
|
| | for key, value in record.items():
|
| | if isinstance(value, str) and len(value) > 100:
|
| | value = value[:100] + "..."
|
| | elif isinstance(value, dict):
|
| | value = f"Dict with keys: {list(value.keys())}"
|
| | elif isinstance(value, list):
|
| | value = f"List with {len(value)} items"
|
| |
|
| | print(f" {key}: {value}")
|
| |
|
| |
|
| | for key in ['prompt', 'conversation', 'messages']:
|
| | if key in record and isinstance(record[key], (dict, list)):
|
| | print(f"\n Exploring {key}:")
|
| | nested = record[key]
|
| | if isinstance(nested, dict):
|
| | print(f" Keys: {list(nested.keys())}")
|
| | for nkey, nvalue in list(nested.items())[:3]:
|
| | if isinstance(nvalue, str) and len(nvalue) > 50:
|
| | nvalue = nvalue[:50] + "..."
|
| | print(f" {nkey}: {nvalue}")
|
| | elif isinstance(nested, list) and nested:
|
| | print(f" First item type: {type(nested[0])}")
|
| | if isinstance(nested, dict):
|
| | print(f" First item keys: {list(nested.keys())}")
|
| |
|
| | except Exception as e:
|
| | print(f"Error reading file: {e}")
|
| |
|
| | if __name__ == "__main__":
|
| | inspect_downloaded_data()
|
| |
|
