Update youtube_utils.py

youtube_utils.py

@@ -1,3 +1,4 @@
+# youtube_utils.py
 import re
 import torch
 from transformers import BartForConditionalGeneration, BartTokenizer
@@ -5,17 +6,14 @@ from youtube_transcript_api import YouTubeTranscriptApi
 from nltk.tokenize import sent_tokenize
 import nltk
 
-# Ensure NLTK data is downloaded during the first run
 nltk.download('punkt')
 
 def clean_text(text):
-    """Clean up text by removing extra whitespace and quotes."""
     cleaned_text = re.sub(r'\s+', ' ', text)
     cleaned_text = cleaned_text.replace("'", "")
     return cleaned_text
 
 def get_youtube_captions(video_id):
-    """Fetch captions for a YouTube video, translating to English if needed."""
     try:
         transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
         full_transcript = ""
@@ -36,25 +34,19 @@ def get_youtube_captions(video_id):
         return None
 
 def summarize_large_text_with_bart(input_text):
-    """Summarize large text using BART model."""
     model_name = "facebook/bart-large-cnn"
-    
-    # Load tokenizer and model
-    tokenizer = BartTokenizer.from_pretrained(model_name)
     model = BartForConditionalGeneration.from_pretrained(model_name)
+    tokenizer = BartTokenizer.from_pretrained(model_name)
     
-    # Use GPU if available
     device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
     model.to(device)
     
-    # Tokenize input and calculate summary lengths
     input_tokens = tokenizer.encode(input_text, add_special_tokens=False)
     total_input_length = len(input_tokens)
     
     desired_min_length = int(total_input_length * 0.28)
     desired_max_length = int(total_input_length * 0.40)
     
-    # Split input into chunks of <= 1024 tokens with overlap
     sentences = sent_tokenize(input_text)
     max_chunk_length = 1024
     overlap = 2
@@ -79,7 +71,6 @@ def summarize_large_text_with_bart(input_text):
         
         chunks.append(' '.join(current_chunk))
     
-    # Generate summaries for each chunk
     summaries = []
     for chunk in chunks:
         inputs = tokenizer.encode(chunk, return_tensors='pt', max_length=1024, truncation=True).to(device)
@@ -97,4 +88,4 @@ def summarize_large_text_with_bart(input_text):
         
         summaries.append(tokenizer.decode(summary_ids[0], skip_special_tokens=True))
     
-    return ' '.join(summaries)
+    return ' '.join(summaries)