update

.gitignore

@@ -0,0 +1,2 @@
+LLM4Music/
+repo/

script/fetch_data.py

@@ -4,41 +4,28 @@ import multiprocessing
 from functools import partial
 import time
 
-save_dir = "./dataset"
-urls = [
-    "https://huggingface.co/datasets/seungheondoh/cmd-moisesdb-metadata/resolve/main/moisesdb.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-musicnet-metadata/resolve/main/musicnet.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-medleydb-metadata/resolve/main/medleydb.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-fma-metadata/resolve/main/fma.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-mtg_jamendo-metadata/resolve/main/mtg_jamendo/0.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-mtg_jamendo-metadata/resolve/main/mtg_jamendo/1.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-mtg_jamendo-metadata/resolve/main/mtg_jamendo/2.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-mtg_jamendo-metadata/resolve/main/mtg_jamendo/3.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-mtg_jamendo-metadata/resolve/main/mtg_jamendo/4.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-mtg_jamendo-metadata/resolve/main/mtg_jamendo/5.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-mtg_jamendo-metadata/resolve/main/mtg_jamendo/6.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-mtg_jamendo-metadata/resolve/main/mtg_jamendo/7.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-mtg_jamendo-metadata/resolve/main/mtg_jamendo/8.tar.gz",
-    "https://huggingface.co/datasets/seungheondoh/cmd-mtg_jamendo-metadata/resolve/main/mtg_jamendo/9.tar.gz",
-]
+save_dir = "/workspace/seungheon/dataset"
+os.makedirs(save_dir, exist_ok=True)
 
-def download_file(url):
-    wget.download(url)
-
-def unzip_file(file_path):
-    with tarfile.open(file_path, 'r:gz') as tar:
+urls = []
+db_config = {"fma": 34, "mtg_jamendo": 134, "medleydb": 100, "moisesdb": 8, "musicnet": 21}
+for db_name, num_files in db_config.items():
+    for i in range(num_files):
+        urls.append(f"https://huggingface.co/datasets/seungheondoh/cmd-audio-dump/resolve/main/{db_name}{i}.tar.gz")
+def download_and_unzip(url):
+    # Download file
+    filename = wget.download(url)
+    # Unzip file
+    with tarfile.open(filename, 'r:gz') as tar:
         tar.extractall(path=save_dir)
-
+    
 if __name__ == "__main__":    
     os.makedirs(save_dir, exist_ok=True)
     # Start timing
     start_time = time.time()
     num_processes = min(multiprocessing.cpu_count(), len(urls))
     with multiprocessing.Pool(processes=num_processes) as pool:
-        pool.map(download_file, urls)
-    
-    with multiprocessing.Pool(processes=num_processes) as pool:
-        pool.map(unzip_file, [f for f in os.listdir(save_dir) if f.endswith('.tar.gz')])    
+        pool.map(download_and_unzip, urls)
     # Calculate and display total time
     end_time = time.time()
     elapsed = end_time - start_time

script/upload_model.py

@@ -5,8 +5,8 @@ from huggingface_hub import HfApi
 # Initialize Hugging Face API
 api = HfApi()
 # Define paths and filenames
-source_dir = "/workspace/logs/llm4music_diffusion/full_channel_concat"
-output_filename = "full_channel_concat.tar.gz"
+source_dir = "/workspace/logs/codebooks/kmeans/stable_vae_16384"
+output_filename = "vq_codebook.tar.gz"
 print(f"Creating tar.gz archive from {source_dir}...")
 # Create tar.gz archive
 with tarfile.open(output_filename, "w:gz") as tar: