File size: 3,534 Bytes
72c0672 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 | #!/bin/bash
DATA=${1:-"ocp_subsampled_5G"}
if [[ $DATA == "ocp_subsampled_5G" ]]; then
input_dir="ocp_subsampled_5G"
splits_dir="ocp_subsampled_5G_entropy87_splits"
compression_dir="ocp_subsampled_5G_entropy87_enumerative_compression"
#### Step 1: setup subsampled data for evaluation
if [[ ! -d "$input_dir" ]]; then
mkdir $input_dir
head -n 1050000 /mnt/hdfs/user/linzheng/data/opencoder_python/opencoder_python.chunk.1.jsonl > temp.jsonl
split -n r/8 --suffix-length=1 --numeric-suffixes=1 --additional-suffix=.jsonl temp.jsonl ${input_dir}/ocp.chunk.
rm temp.jsonl
else
echo "Directory '$input_dir' already exists."
fi
elif [[ $DATA == "opencoder_subsampled_5G" ]]; then
input_dir="opencoder_subsampled_5G"
splits_dir="opencoder_subsampled_5G_entropy87_splits"
compression_dir="opencoder_subsampled_5G_entropy87_enumerative_compression"
#### Step 1: setup subsampled data for evaluation
if [[ ! -d "$input_dir" ]]; then
mkdir $input_dir
head -n 1050000 /mnt/hdfs/user/linzheng/data/opencoder/chunk.1.jsonl > temp.jsonl
split -n r/8 --suffix-length=1 --numeric-suffixes=1 --additional-suffix=.jsonl temp.jsonl ${input_dir}/ocp.chunk.
rm temp.jsonl
else
echo "Directory '$input_dir' already exists."
fi
elif [[ $DATA == "ocp_subsampled_50G" ]]; then
input_dir="/mnt/hdfs/user/linzheng/data/ocpython_subsampled_50G"
splits_dir="ocpython_subsampled_50G_entropy87_splits"
compression_dir="ocpython_subsampled_50G_entropy87_enumerative_compression"
else
echo "Unknown $DATA."
exit 0
fi
entropy_model_path=/mnt/bn/tiktok-mm-5/aiic/users/linzheng/artifacts/m1_checkpoints/m1_40M_lr1e-3_steps200k_bs8_seqlen2048_full/checkpoints/0000200000
compression_model_path=/mnt/bn/tiktok-mm-5/aiic/users/linzheng/artifacts/m1_checkpoints/m1_40M_lr1e-3_steps200k_bs8_seqlen2048_full/checkpoints/0000200000
NUM_GPUS=8
total_jsonls=8
total_jobs=1
wait
for JSONL_IDX in $(seq 1 $total_jsonls); do
for index in $(seq 0 $((total_jobs - 1))); do
echo "Starting job $index..."
GPU_IDX=$(( (JSONL_IDX - 1) % NUM_GPUS ))
CUDA_VISIBLE_DEVICES=${GPU_IDX} python3 offline_entropy_window_split.py \
--input_file ${input_dir}/ocp.chunk.${JSONL_IDX}.jsonl \
--output_dir $splits_dir \
--entropy_model_path $entropy_model_path --compression_model_path $compression_model_path \
--data_batch_size 256 --max_entropy_batch_size 256 \
--num_workers 1 --process_id $index --num_processes $total_jobs \
--base_global_quantile 0.87 --base_monotonic_quantile 0.87 \
--chunk_size 2048 --apply_line_split > split_jsonl${JSONL_IDX}_process${index}_total${total_jobs}_lines.log 2>&1 &
done
done
wait
NUM_GPUS=8
total_jsonls=8
total_jobs=3
for JSONL_IDX in $(seq 1 $total_jsonls); do
for index in $(seq 0 $((total_jobs - 1))); do
echo "Starting job $index..."
GPU_IDX=$(( (JSONL_IDX - 1) % NUM_GPUS ))
CUDA_VISIBLE_DEVICES=${GPU_IDX} python3 offline_entropy_window_compress.py \
--input_file ${splits_dir}/ocp.chunk.${JSONL_IDX}_out_0.jsonl \
--output_dir $compression_dir \
--entropy_model_path $entropy_model_path --compression_model_path $compression_model_path \
--data_batch_size 1024 --max_compression_batch_size 3072 \
--num_workers 1 --process_id $index --num_processes $total_jobs > compress_jsonl${JSONL_IDX}_process${index}_total${total_jobs}.log 2>&1 &
done
done
|