Training in progress, step 300

archive_checkpoints.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Path to the parent directory containing checkpoint folders
+SOURCE_DIR="/disk2/bhanu/research/open-r1-main/data/gemma-3-12b-it-sft"
+
+# Azure destination
+AZURE_DEST="azai:private-model-checkpoints/gemma-3-12b-it-sft-graphllm"
+
+# Log file
+LOG_FILE="rclone_archive.log"
+
+# Get current date and time for logging
+CURRENT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
+
+echo "[$CURRENT_TIME] Starting checkpoint archiving" >> $LOG_FILE
+
+# Use rclone copy with optimizations to avoid unnecessary transfers
+rclone copy "$SOURCE_DIR" "$AZURE_DEST" \
+	  --progress \
+	    --include "checkpoint-*/**" \
+	      --log-file=$LOG_FILE \
+	        --stats 10s \
+		  --checksum \
+		    --skip-links \
+		      --transfers=4 \
+		        --checkers=8
+
+# Log completion
+CURRENT_TIME=$(date "+%Y-%m-%d %H:%M:%S")
+echo "[$CURRENT_TIME] Checkpoint archiving completed" >> $LOG_FILE
+
+# Optional: List all checkpoints currently in Azure for verification
+echo "[$CURRENT_TIME] Current checkpoints in Azure:" >> $LOG_FILE
+rclone lsf "$AZURE_DEST" --include "checkpoint-*/" >> $LOG_FILE

model-00001-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:32cac6b328e0e8196e5c7bbeeaba72f60cf0fd5959a10c68e5cf4692558ba1e6
+oid sha256:eddea002f1ec84ae617382569db432bfecf203e7bede1ef6e4bba93f64e3b3b4
 size 4979902192

model-00002-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e15276f11f27f85fc5d5853be1f9e51fd06d13ab6722517a5cbeeb7a15defb02
+oid sha256:2810b71a7604101b75c0161078638fab875984009617064e0165499129f5d3d9
 size 4931296592

model-00003-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9804775a6658945ff1644205c9db0b1da74281909270415df7701be2b8a6c3c
+oid sha256:dd1365ef853cd19234ce2fa5f35a1719325e300a1f21e36fb0fc161e90c43049
 size 4931296656

model-00004-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c65777f816d69020abd67c9a5639b3165946db6e6a32e6699e6afd95f729ae28
+oid sha256:966a6bee515170dcdf1808412da0417d30fbee4f6926c0a64f078b1f0b79ff64
 size 4931296656

model-00005-of-00005.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8bb76249c58cacba5c0226a563adcbe0a3f6f345bf1c58c5b9f208bc04679720
+oid sha256:bb073c0d7157ede6f8aa89db33647c06cfccc8dfee589fffe2e821b6855b65d8
 size 4601000928

rclone_archive.log

@@ -0,0 +1,8 @@
+[2025-04-30 23:01:42] Starting checkpoint archiving
+2025/04/30 23:01:42 Failed to create file system for "azure:private-model-checkpoints/gemma-3-12b-it-sft-graphllm": didn't find section in config file
+[2025-04-30 23:01:42] Checkpoint archiving completed
+[2025-04-30 23:01:42] Current checkpoints in Azure:
+[2025-04-30 23:02:27] Starting checkpoint archiving
+[2025-04-30 23:56:48] Checkpoint archiving completed
+[2025-04-30 23:56:48] Current checkpoints in Azure:
+checkpoint-200/