Neon-tech commited on
Commit
973f84b
Β·
verified Β·
1 Parent(s): 3e98f99

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -6
app.py CHANGED
@@ -49,7 +49,6 @@ def claim_shard(state):
49
  for name, info in state["shards"].items():
50
  if info["status"] == "pending":
51
  raw_path = Path(RAW_DIR) / name
52
- print(f" [{WORKER_ID}] Checking: {raw_path} | exists: {raw_path.exists()}")
53
  if raw_path.exists():
54
  info["status"] = "claimed"
55
  info["worker"] = WORKER_ID
@@ -160,11 +159,8 @@ def flush_memory():
160
 
161
  # ── Worker loop ───────────────────────────────────────────────────────────────
162
  def worker_loop():
163
- print(f"βœ“ [{WORKER_ID}] worker_loop started")
164
- print(f" STATE_FILE exists: {os.path.exists(STATE_FILE)}")
165
- print(f" RAW_DIR contents: {list(Path(RAW_DIR).iterdir())[:5]}")
166
- pool = mp.Pool(processes=2, initializer=init_worker, initargs=(TOK_PATH,),
167
- context=mp.get_context("spawn"))
168
  print(f"βœ“ [{WORKER_ID}] 2-core tokenizer pool ready")
169
 
170
  try:
 
49
  for name, info in state["shards"].items():
50
  if info["status"] == "pending":
51
  raw_path = Path(RAW_DIR) / name
 
52
  if raw_path.exists():
53
  info["status"] = "claimed"
54
  info["worker"] = WORKER_ID
 
159
 
160
  # ── Worker loop ───────────────────────────────────────────────────────────────
161
  def worker_loop():
162
+ print(f"βœ“ [{WORKER_ID}] Starting worker...")
163
+ pool = mp.Pool(processes=2, initializer=init_worker, initargs=(TOK_PATH,))
 
 
 
164
  print(f"βœ“ [{WORKER_ID}] 2-core tokenizer pool ready")
165
 
166
  try: