BGM生成対応

app.py

@@ -9,6 +9,7 @@ import requests
 import argparse
 import markdown2
 import uuid
+import traceback
 from pathlib import Path
 
 from dotenv import load_dotenv
@@ -16,6 +17,8 @@ from IPython.display import Image
 from moviepy.editor import VideoFileClip, concatenate_videoclips, ImageClip
 from moviepy.video.fx.all import fadein, fadeout
 from PIL import Image as PIL_Image
+from pydub import AudioSegment
+from moviepy.editor import VideoFileClip, AudioFileClip
 
 from jinja2 import Template
 
@@ -28,28 +31,23 @@ openai.api_key = os.getenv('OPENAI_API_KEY')
 REPLICATE_API_TOKEN_LIST = os.getenv("REPLICATE_API_TOKEN_LIST").split(',')
 NUMBER_OF_SCENES = os.getenv("NUMBER_OF_SCENES")
 
-if ENV == "PRODUCTION":
-    import replicate
-    from replicate.client import Client
-else:
-    # from stub import replicate
-    pass
+import replicate
+from replicate.client import Client
 
-class Video:
-    def __init__(self, scene, index, client: Client):
+class Replicate:
+    def __init__(self, id, client: Client, args, index=None):
+        self.id = id
         self.client = client
-        self.scene = scene
-        self.prompt = "masterpiece, awards, best quality, dramatic-lighting, "
-        self.prompt = self.prompt + scene.get("visual_prompt_in_en")
-        self.prompt = self.prompt + ", cinematic-angles-" + scene.get("cinematic_angles") 
-        self.nagative_prompt = "badhandv4, easynegative, ng_deepnegative_v1_75t, verybadimagenegative_v1.3, bad-artist, bad_prompt_version2-neg, nsfw, "
+        self.args = args
         self.index = index
+        self.prompt = ""
+        self.file_path_format = ""
+        self.REPLICATE_MODEL_PATH = ""
+        self.REPLICATE_MODEL_VERSION = ""
+        self.input={}
         self.output_url = None
-        self.video_id = uuid.uuid4()
-        self.file_path = f"assets/thread_{index}_request_{self.video_id}_video.mp4"
-
-    REPLICATE_MODEL_PATH = "lucataco/animate-diff"
-    REPLICATE_MODEL_VERSION = "1531004ee4c98894ab11f8a4ce6206099e732c1da15121987a8eef54828f0663"
+        self.response = None
+        self.prediction_id = None
 
     def run_replicate(self, retries=0):
         try:
@@ -64,12 +62,7 @@ class Video:
             version = model.versions.get(self.REPLICATE_MODEL_VERSION)
             self.prediction = self.client.predictions.create(
                 version=version,
-                input={
-                    "motion_module": "mm_sd_v14", 
-                    "prompt": self.prompt,
-                    "n_prompt": self.nagative_prompt,
-                    "seed": 0,
-                    },
+                input=self.input
             )
             
             self.prediction_id = self.prediction.id
@@ -89,13 +82,17 @@ class Video:
             else:
                 self.output_url = None
             
+            self.file_path = self.file_path_format.format(id=self.id, class_name=self.__class__.__name__, index=self.index, prediction_id=self.prediction_id)
             end_time = time.time()
             duration = end_time - start_time
 
             self.download_and_save(url=self.output_url, file_path=self.file_path)
             self.print_thread_info(start_time, end_time, duration)
         except replicate.exceptions.ReplicateError as e:
-            if self.prediction and str(e) == "The requested resource could not be found.":
+            print(f"Error fetching model or version: {e}")
+            print(f"Model Path: {self.REPLICATE_MODEL_PATH}")
+            print(f"Model Version: {self.REPLICATE_MODEL_VERSION}")
+            if self.prediction_id and str(e) == "The requested resource could not be found.":
                 predictions = self.client.predictions.list()
                 self.prediction = next((p for p in predictions if p.id == self.prediction_id), None)
 
@@ -111,7 +108,10 @@ class Video:
                     print(f"Thread {self.index} token: {self.client.api_token[:10]} prediction.output: {self.prediction.output}")
                 else:
                     self.output_url = None
+                    print(f"Thread {self.index} token: {self.client.api_token[:10]} prediction.output: Error")
+                    print(f"Thread {self.index} token: {self.client.api_token[:10]} prediction.output: {self.prediction.output}")
                 
+                self.file_path = self.file_path_format.format(id=self.id, class_name=self.__class__.__name__, index=self.index, prediction_id=self.prediction_id)
                 end_time = time.time()
                 duration = end_time - start_time
 
@@ -119,11 +119,14 @@ class Video:
                 self.print_thread_info(start_time, end_time, duration)
             else:
                 print(f"Error in thread {self.index}: {e}")
+                print(traceback.format_exc())
                 print("予期しないエラーが発生しました。スレッドを終了します。")
                 # 予期しないエラーが発生した場合の追加処理
+                raise e
         except Exception as e:
             print(f"Error in thread {self.index}: {e}")
-
+            print(traceback.format_exc())
+            
     def download_and_save(self, url, file_path):
         response = requests.get(url)
         with open(file_path, "wb") as f:
@@ -135,37 +138,121 @@ class Video:
         print(f"Thread {self.index} end time: {end_time}")
         print(f"Thread {self.index} duration: {duration}")
 
+class Video(Replicate):
+
+    def __init__(self, id, client: Client, args, scene, index=None):
+        super().__init__(id, client, args, index)
+        self.REPLICATE_MODEL_PATH = "lucataco/animate-diff"
+        self.REPLICATE_MODEL_VERSION = "1531004ee4c98894ab11f8a4ce6206099e732c1da15121987a8eef54828f0663"
+        self.scene = scene
+        self.prompt = "masterpiece, awards, best quality, dramatic-lighting, "
+        self.prompt = self.prompt + scene.get("visual_prompt_in_en")
+        self.prompt = self.prompt + ", cinematic-angles-" + scene.get("cinematic_angles") 
+        self.nagative_prompt = "badhandv4, easynegative, ng_deepnegative_v1_75t, verybadimagenegative_v1.3, bad-artist, bad_prompt_version2-neg, nsfw, "
+        self.file_path_format = "assets/{id}/{class_name}_thread_{index}_request_{prediction_id}.mp4"
+        self.file_path = None
+        self.input={
+            "motion_module": "mm_sd_v14", 
+            "prompt": self.prompt,
+            "n_prompt": self.nagative_prompt,
+            "seed": 0, # random
+            }
+
+    def run_replicate(self, retries=0):
+        self.response = super().run_replicate()
+        self.file_path = self.file_path_format.format(id=self.id, class_name=self.__class__.__name__, index=self.index, prediction_id=self.prediction_id)
+        return self.response
+
+class Music(Replicate):
+
+    def __init__(self, id, client: Client, args):
+        super().__init__(id, client, args)
+        self.REPLICATE_MODEL_PATH = "facebookresearch/musicgen"
+        self.REPLICATE_MODEL_VERSION = "f8578df960c345df7bc1f85dd152c5ae0b57ce45a6fc09511c467a62ad820ba3",
+        self.prompt = "innovative, exceptional, captivating, " \
+            + args.get("bgm_prompt_in_en")
+        
+        self.file_path_format = "assets/{id}/{class_name}_{index}_request_{prediction_id}.mp3"
+        self.file_path = None
+        self.duration = args.get("")
+        self.input = {
+            "model_version": "large", 
+            "prompt": self.prompt,
+            "duration": self.duration,
+            "output_format": "mp3",
+            "seed": -1, # random
+            }
+
+    def run_replicate(self, retries=0):
+        
+        start_time = time.time()
+        print(f"Thread {self.index} token: {self.client.api_token[:10]}")
+        
+        os.environ['REPLICATE_API_TOKEN'] = self.client.api_token
+        output = replicate.run(
+            "facebookresearch/musicgen:7a76a8258b23fae65c5a22debb8841d1d7e816b75c2f24218cd2bd8573787906",
+            input={
+                "model_version": "large",
+                # "prompt": "The sound of samurai's footsteps marching across the field, the echo of the mountain, the fierce battle sound, and finally the triumphant fanfare as they claim victory."
+                "prompt": self.prompt,
+                "duration": self.duration,
+                "output_format": "mp3",
+                "seed": -1, # random
+                }
+        )
+        print(output)
+        self.output_url = output
+        self.response = output
+        
+        self.file_path = self.file_path_format.format(id=self.id, class_name=self.__class__.__name__, index=self.index, prediction_id=self.prediction_id)
+        end_time = time.time()
+        duration = end_time - start_time
+        self.download_and_save(url=self.output_url, file_path=self.file_path)
+        self.print_thread_info(start_time, end_time, duration)
+        
+        return self.response
+
 class ThreadController:
     def __init__(self, args):
+        self.id = uuid.uuid4()
         self.args = args
         scenes = args.get("scenes")
+        self.music = None
         self.videos = []
         self.threads = []
         self.lock = threading.Lock()
         self.replicate_client_list = {}
-        for token in REPLICATE_API_TOKEN_LIST:
-
+        self.duration = int(2.1 * len(scenes) * len(REPLICATE_API_TOKEN_LIST)) # 2.1秒 * シーン数 * APIトークン数
+        
+        os.makedirs(f"assets/{self.id}", exist_ok=True)
+        
+        for token_index, token in enumerate(REPLICATE_API_TOKEN_LIST):
             client = Client()
             client.api_token = token
             self.replicate_client_list[token] = client
-            
+            if token_index == 0:
+                self.music = Music(self.id, client, args)
+                self.music.duration = self.duration
+
             for index, scene in enumerate(scenes):
-                # token = REPLICATE_API_TOKEN_LIST[self.client.api_token_index]
-                video = Video(scene, index, client)
+                token = REPLICATE_API_TOKEN_LIST[token_index]
+                video = Video(self.id, client, args, scene, index)
                 self.videos.append(video)
             
-            # self.client.api_token_index = (self.client.api_token_index + 1) % len(REPLICATE_API_TOKEN_LIST)
-            
+            # client.api_token_index = (token_index + 1) % len(REPLICATE_API_TOKEN_LIST)
 
     def run_threads(self):
-        os.makedirs("assets", exist_ok=True)
 
-        token = None
+        thread = threading.Thread(target=self.music.run_replicate)
+        self.threads.append(thread)
+        thread.start()
+        token = self.music.client.api_token
+        
         for video in self.videos:
             if token is not None and video.client.api_token != token:
-                # tokenが異なる場合、1秒待ってから次を実行
-                print(f"Thread {video.index} token changed. Waiting 3 seconds.")
-                time.sleep(5)
+                # tokenが異なる場合、4秒待ってから次を実行
+                print(f"Thread {video.index} token changed. Waiting 4 seconds.")
+                time.sleep(4)
             
             thread = threading.Thread(target=video.run_replicate)
             self.threads.append(thread)
@@ -186,12 +273,54 @@ class ThreadController:
                 print(f"Error: Video file {video.file_path} could not be found! Skipping this file.")
                 # 他のログ出力方法も使用可能、例: loggingモジュール
 
+        output_path = f"assets/{self.id}/concatenated_video_{self.id}.mp4"
+
         final_clip = concatenate_videoclips(clips)
+        final_clip.write_videofile(output_path, codec='libx264', fps=24)
+        
+        # Load the video file using MoviePy
+        video_clip = VideoFileClip(output_path)
+        video_duration = video_clip.duration
 
-        os.makedirs("videos", exist_ok=True)
-        output_path = f"videos/final_concatenated_video_{uuid.uuid4()}.mp4"
+        # Re-loading the audio file using pydub
+        audio_segment = AudioSegment.from_mp3(self.music.file_path)
 
-        final_clip.write_videofile(output_path, codec='libx264', fps=24)
+        # Calculating the number of loops needed to match the video duration
+        num_loops = int(video_duration * 1000) // len(audio_segment) + 1
+
+        # Creating an audio segment that has the same duration as the video by looping the original audio
+        final_audio_segment = audio_segment * num_loops
+
+        # Trimming the final audio segment to match the video duration exactly
+        final_audio_segment = final_audio_segment[:int(video_duration * 1000)]
+
+        temp_audio_path = "/tmp/temp_audio.mp3"
+
+        # Saving the final audio as a temporary WAV file
+        final_audio_segment.export(temp_audio_path, format="mp3")
+
+        # Loading the temporary audio file as a MoviePy AudioFileClip
+        final_audio_clip = AudioFileClip(temp_audio_path)
+
+        # Setting the audio to the video
+        final_video_clip = video_clip.set_audio(final_audio_clip)
+
+        # Path to save the final video with audio (different name to avoid confusion)
+        output_path_with_audio_fixed = "/tmp/final_video_with_audio_fixed.mp4"
+
+        # Saving the final video with audio
+        final_video_clip.write_videofile(output_path_with_audio_fixed, codec="libx264", audio_codec="aac")
+
+        # Path to the final video with audio (fixed version)
+        output_path_with_audio_fixed
+
+        os.makedirs(f"videos/{self.id}/", exist_ok=True)
+        output_path = f"videos/{self.id}/final_concatenated_video_{self.id}.mp4"
+
+        # final_clip.write_videofile(output_path, codec='libx264', fps=24)
+        
+        import shutil
+        shutil.move(output_path_with_audio_fixed, output_path)
 
         return output_path
 
@@ -358,3 +487,13 @@ if __name__ == "__main__":
         )
         iface.launch()
 
+# import replicate
+# import os
+# token = os.environ.get("REPLICATE_API_TOKEN")
+# os.environ["REPLICATE_API_TOKEN"] = token
+# print(f"token: {token}")
+# output = replicate.run(
+#     "facebookresearch/musicgen:7a76a8258b23fae65c5a22debb8841d1d7e816b75c2f24218cd2bd8573787906",
+#     input={"model_version": "melody"}
+# )
+# print(output)

schema.json

@@ -6,14 +6,20 @@
 			"parameters": {
 				"type": "object",
 				"required": [
+					"lang",
 					"title",
 					"story",
 					"visual_style",
 					"visual_prompt_in_en",
+					"bgm_prompt_in_en",
 					"scene_count",
 					"scenes"
 				],
 				"properties": {
+					"lang": {
+						"type": "string",
+						"description": "The language of the user input in ISO 639-1 format."
+					},
 					"title": {
 						"type": "string",
 						"description": "映画のタイトル"
@@ -30,6 +36,10 @@
 						"type": "string",
 						"description": "映像に関連する簡単な説明"
 					},
+					"bgm_prompt_in_en": {
+						"type": "string",
+						"description": "シーンBGMの聴覚的表現を文学的に説明する"
+					},
 					"negative_visual_prompt_in_en": {
 						"type": "string",
 						"description": "映像に含めないでほしい視覚的説明"

template.md

@@ -1,4 +1,4 @@
-# Movie Title: {{ args.title }}
+# Story Title: {{ args.title }}
 
 ## Story
 
@@ -8,8 +8,16 @@
 
 {{ generation_time }}
 
-## Prompts
+## BGM Prompt
+
+{{ args.bgm_prompt_in_en }}
+
+## Visual Prompts
 
 | Scene | visual_prompt_in_en | negative_visual_prompt_in_en | cinematic_angles |
 |----:|----|----|----|{% for item in args.scenes %}
 |{{ item.scene }}|{{ item.visual_prompt_in_en }}|{{ item.negative_visual_prompt_in_en }}|{{ item.cinematic_angles}}|{% endfor %}
+
+## Language
+
+{{ args.lang }}