Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
|
@@ -49,7 +49,6 @@ def claim_shard(state):
|
|
| 49 |
for name, info in state["shards"].items():
|
| 50 |
if info["status"] == "pending":
|
| 51 |
raw_path = Path(RAW_DIR) / name
|
| 52 |
-
print(f" [{WORKER_ID}] Checking: {raw_path} | exists: {raw_path.exists()}")
|
| 53 |
if raw_path.exists():
|
| 54 |
info["status"] = "claimed"
|
| 55 |
info["worker"] = WORKER_ID
|
|
@@ -160,11 +159,8 @@ def flush_memory():
|
|
| 160 |
|
| 161 |
# ββ Worker loop βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 162 |
def worker_loop():
|
| 163 |
-
print(f"β [{WORKER_ID}]
|
| 164 |
-
|
| 165 |
-
print(f" RAW_DIR contents: {list(Path(RAW_DIR).iterdir())[:5]}")
|
| 166 |
-
pool = mp.Pool(processes=2, initializer=init_worker, initargs=(TOK_PATH,),
|
| 167 |
-
context=mp.get_context("spawn"))
|
| 168 |
print(f"β [{WORKER_ID}] 2-core tokenizer pool ready")
|
| 169 |
|
| 170 |
try:
|
|
|
|
| 49 |
for name, info in state["shards"].items():
|
| 50 |
if info["status"] == "pending":
|
| 51 |
raw_path = Path(RAW_DIR) / name
|
|
|
|
| 52 |
if raw_path.exists():
|
| 53 |
info["status"] = "claimed"
|
| 54 |
info["worker"] = WORKER_ID
|
|
|
|
| 159 |
|
| 160 |
# ββ Worker loop βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 161 |
def worker_loop():
|
| 162 |
+
print(f"β [{WORKER_ID}] Starting worker...")
|
| 163 |
+
pool = mp.Pool(processes=2, initializer=init_worker, initargs=(TOK_PATH,))
|
|
|
|
|
|
|
|
|
|
| 164 |
print(f"β [{WORKER_ID}] 2-core tokenizer pool ready")
|
| 165 |
|
| 166 |
try:
|