Upload 26 files

.env

@@ -0,0 +1,4 @@
+AZURE_OPENAI_ENDPOINT=https://test111222333.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2025-01-01-preview
+AZURE_OPENAI_API_KEY=8iMbEYWnZI0tYrCLDk1GNUJXPp3VMCMUvl8tdbiVxi1v34vhnI7sJQQJ99AKACfhMk5XJ3w3AAABACOGNIPC
+AZURE_OPENAI_API_VERSION=2024-05-01-preview
+AZURE_OPENAI_DEPLOYMENT=gpt-4o

.streamlit/config.toml

@@ -0,0 +1,5 @@
+[server]
+fileWatcherType = "none"
+
+[logger]
+level = "info" 

app.py

@@ -0,0 +1,30 @@
+import streamlit as st
+import os
+import sys
+from pathlib import Path
+
+# 确保能找到项目模块
+sys.path.append(str(Path(__file__).parent))
+
+from pages import emotion_analyzer, chatbot  # 导入情绪分析页面和 Chatbot 页面
+
+def main():
+    st.set_page_config(
+        page_title="Audio Emotion Recognition System",
+        page_icon="🎵",
+        layout="wide"
+    )
+    
+    st.title("Audio Emotion Recognition System")
+    st.write("This is a web application for audio emotion recognition.")
+    
+    # 选择页面
+    page = st.sidebar.selectbox("Select a page", ["Emotion Analyzer", "Chatbot"])
+    
+    if page == "Emotion Analyzer":
+        emotion_analyzer.show()
+    elif page == "Chatbot":
+        chatbot.show_chatbot()
+
+if __name__ == "__main__":
+    main() 

components/__init__.py

@@ -0,0 +1 @@
+# 空文件，使components成为一个Python包 

components/audio_player.py

@@ -0,0 +1,10 @@
+import streamlit as st
+
+def play_audio(audio_file):
+    """
+    显示音频播放器组件
+    
+    Args:
+        audio_file: 上传的音频文件
+    """
+    st.audio(audio_file) 

components/debug_tools.py

@@ -0,0 +1,56 @@
+import streamlit as st
+import sys
+import os
+import platform
+from datetime import datetime
+
+class DebugTools:
+    @staticmethod
+    def show_debug_info():
+        """显示调试信息的可折叠部分"""
+        with st.expander("Debug Information", expanded=False):
+            # 系统信息
+            st.subheader("System Information")
+            st.text(f"System: {platform.system()} {platform.version()}")
+            st.text(f"Python Version: {sys.version}")
+            
+            # 内存使用
+            try:
+                import psutil
+                process = psutil.Process(os.getpid())
+                st.text(f"Memory Usage: {process.memory_info().rss / 1024 / 1024:.2f} MB")
+            except ImportError:
+                st.text("Memory Usage: Unable to get (requires psutil)")
+            
+            # GPU信息
+            try:
+                import torch
+                if torch.cuda.is_available():
+                    st.text(f"GPU: {torch.cuda.get_device_name(0)}")
+                    st.text(f"GPU Memory: {torch.cuda.memory_allocated(0)/1024/1024:.2f}MB / "
+                           f"{torch.cuda.memory_reserved(0)/1024/1024:.2f}MB")
+                else:
+                    st.text("GPU: Not Available")
+            except Exception as e:
+                st.text("GPU Information Retrieval Failed")
+    
+    @staticmethod
+    def log_error(error, context=None):
+        """记录错误信息"""
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        error_msg = f"[{timestamp}] Error: {str(error)}"
+        if context:
+            error_msg += f"\nContext: {context}"
+        
+        st.error(error_msg)
+        # 可以添加日志文件记录
+        print(error_msg, file=sys.stderr)
+    
+    @staticmethod
+    def show_audio_info(audio_file):
+        """显示音频文件信息"""
+        if audio_file is not None:
+            st.write("Audio File Information:")
+            st.text(f"File Name: {audio_file.name}")
+            st.text(f"File Size: {audio_file.size/1024:.2f} KB")
+            st.text(f"File Type: {audio_file.type}") 

components/progress_bar.py

@@ -0,0 +1,7 @@
+import streamlit as st
+
+def show_progress():
+    """
+    返回一个streamlit进度条组件
+    """
+    return st.progress(0) 

components/visualizations.py

@@ -0,0 +1,35 @@
+import streamlit as st
+import plotly.graph_objects as go
+import plotly.express as px
+
+def plot_emotion_distribution(emotion_dict):
+    """
+    绘制情绪概率分布图（雷达图）
+    
+    Args:
+        emotion_dict: 包含情绪标签和对应概率的字典
+    """
+    emotions = list(emotion_dict.keys())
+    probabilities = [float(emotion_dict[emotion].strip('%')) / 100 for emotion in emotions]  # 转换为浮点数
+
+    # 创建雷达图
+    fig = go.Figure()
+
+    fig.add_trace(go.Scatterpolar(
+        r=probabilities + [probabilities[0]],  # 闭合图形
+        theta=emotions + [emotions[0]],  # 闭合图形
+        fill='toself',
+        name='Emotion Distribution'
+    ))
+
+    fig.update_layout(
+        title="Emotion Distribution",
+        polar=dict(
+            radialaxis=dict(
+                visible=True,
+                range=[0, 1]  # 设置范围
+            )),
+        showlegend=False
+    )
+
+    st.plotly_chart(fig, use_container_width=True) 

pages/__init__.py

@@ -0,0 +1 @@
+# 空文件，使pages成为一个Python包 

pages/chatbot.py

@@ -0,0 +1,97 @@
+import streamlit as st
+import torch
+import torchaudio
+import json
+from openai import AzureOpenAI
+from openai.types.beta.threads import Message
+from safetensors.torch import load_file
+from transformers import AutoTokenizer, Wav2Vec2Processor, BertModel, Wav2Vec2Model
+from huggingface_hub import hf_hub_download
+from dotenv import load_dotenv
+from utils import model_inference
+import os
+
+# 加载环境变量
+load_dotenv(".env")
+api_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
+api_key = os.getenv("AZURE_OPENAI_API_KEY")
+api_version = os.getenv("AZURE_OPENAI_API_VERSION")
+api_deployment_name = os.getenv("AZURE_OPENAI_DEPLOYMENT")
+
+# 初始化 OpenAI 客户端
+client = AzureOpenAI(api_key=api_key, api_version=api_version, azure_endpoint=api_endpoint)
+
+# 设定 Chatbot 角色
+instruction = (
+    "You are a psychiatrist talking to a patient who may be depressed. "
+    "You'll receive their emotional state and conversation text. "
+    "Your goal is to help them open up and guide them to a positive path. "
+    "Be friendly, professional, empathetic, and supportive."
+)
+
+# 设定 Chatbot 线程和助手
+if "thread" not in st.session_state:
+    st.session_state.thread = client.beta.threads.create()
+
+if "assistant" not in st.session_state:
+    assistant_id = "asst_Sb1W9jVTeL1iyzu6N5MilgA1"
+    try:
+        st.session_state.assistant = client.beta.assistants.retrieve(assistant_id=assistant_id)
+    except:
+        st.session_state.assistant = client.beta.assistants.create(
+            name="Depression Chatbot",
+            instructions=instruction,
+            model=api_deployment_name,
+        )
+
+# 发送消息到 Azure Chatbot
+def send_message_to_chatbot(user_input, emotion):
+    chat_history = client.beta.threads.messages.list(thread_id=st.session_state.thread.id)
+    messages = [{"role": msg.role, "content": msg.content} for msg in chat_history]
+    
+    messages.append({"role": "user", "content": f"Emotion: {emotion}. {user_input}"})
+
+    client.beta.threads.messages.create(
+        thread_id=st.session_state.thread.id,
+        role="user",
+        content=f"Emotion: {emotion}. {user_input}",
+    )
+
+    run = client.beta.threads.runs.create(
+        thread_id=st.session_state.thread.id,
+        assistant_id=st.session_state.assistant.id,
+    )
+
+    while run.status in ["queued", "in_progress"]:
+        run = client.beta.threads.runs.retrieve(run.id)
+
+    response_messages = client.beta.threads.messages.list(thread_id=st.session_state.thread.id)
+    return response_messages[-1].content if response_messages else "No response."
+
+# Streamlit 界面
+st.title("🧠 AI Depression Chatbot")
+
+if "chat_history" not in st.session_state:
+    st.session_state.chat_history = []
+
+# 用户输入
+user_input = st.text_input("Enter your message:")
+audio_file = st.file_uploader("Upload audio file", type=["wav", "mp3"])
+
+if st.button("Send"):
+    if user_input or audio_file:
+        emotion_probabilities = model_inference.predict_emotion(user_input, audio_file)
+        dominant_emotion = max(emotion_probabilities, key=emotion_probabilities.get)
+
+        chatbot_response = send_message_to_chatbot(user_input, dominant_emotion)
+        
+        # 保存聊天记录
+        st.session_state.chat_history.append({"role": "user", "content": user_input})
+        st.session_state.chat_history.append({"role": "assistant", "content": chatbot_response})
+
+        # 显示聊天记录
+        for chat in st.session_state.chat_history:
+            st.write(f"**{chat['role'].capitalize()}**: {chat['content']}")
+
+    else:
+        st.warning("Please enter a message or upload an audio file.")

pages/emotion_analyzer.py

@@ -0,0 +1,80 @@
+import streamlit as st
+from components.visualizations import plot_emotion_distribution
+from utils import model_inference
+from components.audio_player import play_audio
+from components.debug_tools import DebugTools
+import json
+import os
+
+def show_history():
+    """显示历史记录"""
+    history_file = "history.json"
+    
+    if os.path.exists(history_file):
+        with open(history_file, 'r') as f:
+            history = json.load(f)
+        
+        if history:
+            st.subheader("History")
+            for record in history:
+                st.write(f"Audio File: {record['audio_file']}")
+                st.write(f"Transcript: {record['transcript']}")
+                st.write(f"Emotions: {record['emotions']}")
+                st.write(f"Probabilities: {record['probabilities']}")
+                st.write("---")
+        else:
+            st.write("No history records.")
+    else:
+        st.write("No history file.")
+
+def show():
+    st.header("Emotion Analyzer")
+    
+    # 显示历史记录
+    show_history()
+    
+    # 初始化调试工具
+    debug = DebugTools()
+    
+    # 显示系统调试信息
+    debug.show_debug_info()
+    
+    # 文件上传
+    audio_file = st.file_uploader("Upload audio file", type=['wav', 'mp3'])
+    text_input = st.text_input("Enter text input")
+
+    if audio_file is not None and text_input:
+        # 显示音频文件信息
+        debug.show_audio_info(audio_file)
+        
+        # 使用audio_player组件
+        play_audio(audio_file)
+        
+        if st.button("Analyse Your Emotion!😊"):
+            # 显示进度条
+            progress_bar = st.progress(0)
+            
+            try:
+                # 直接使用用户输入的文本作为转写
+                transcript = text_input
+                st.write("Audio transcript:", transcript)
+                
+                # 2. 情绪分析
+                with st.spinner("Analysing emotion..."):
+                    progress_bar.progress(30)
+                    emotions = model_inference.predict_emotion(text_input, audio_file)
+                
+                # 3. 显示结果
+                progress_bar.progress(30)
+                
+                # 显示预测结果
+                st.success(f"Predict: {emotions}")
+                
+                # 显示情绪概率分布图
+                plot_emotion_distribution(emotions)
+                
+                # 保存历史记录
+                model_inference.save_history(audio_file, transcript, emotions, None)  # 这里可以根据需要调整
+                
+            except Exception as e:
+                debug.log_error(e, context=f"Processing file: {audio_file.name}")

requirements.txt

@@ -0,0 +1,14 @@
+streamlit>=1.0.0
+plotly>=5.0.0
+librosa>=0.9.0
+scipy>=1.7.0
+numpy>=1.21.0
+torch>=1.9.0
+transformers>=4.11.0
+soundfile>=0.10.3
+psutil>=5.8.0
+huggingface-hub>=0.0.12
+safetensors>=0.0.3
+torchaudio>=0.9.0
+openai
+dotenv

utils/__init__.py

@@ -0,0 +1 @@
+# 空文件，使utils成为一个Python包 

utils/audio_processing.py

@@ -0,0 +1,31 @@
+import librosa
+import numpy as np
+from scipy.signal import butter, filtfilt
+
+class AudioProcessor:
+    @staticmethod
+    def load_and_process_audio(file_path, target_sr=16000):
+        """加载并处理音频文件"""
+        # 加载音频文件
+        audio_data, sr = librosa.load(file_path, sr=target_sr)
+        
+        # 归一化音频数据
+        audio_data = librosa.util.normalize(audio_data)
+        
+        return audio_data
+
+    @staticmethod
+    def resample(audio_data, orig_sr, target_sr=16000):
+        """重采样音频"""
+        return librosa.resample(audio_data, orig_sr=orig_sr, target_sr=target_sr)
+    
+    @staticmethod
+    def denoise(audio_data, sr):
+        """音频降噪"""
+        # 实现降噪逻辑
+        return audio_data
+    
+    @staticmethod
+    def normalize(audio_data):
+        """音频归一化"""
+        return librosa.util.normalize(audio_data) 

utils/logger.py

@@ -0,0 +1,37 @@
+import logging
+import os
+from datetime import datetime
+
+class Logger:
+    def __init__(self):
+        # 创建logs目录
+        log_dir = "logs"
+        if not os.path.exists(log_dir):
+            os.makedirs(log_dir)
+            
+        # 设置日志文件名
+        log_file = os.path.join(
+            log_dir, 
+            f"app_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
+        )
+        
+        # 配置日志
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            handlers=[
+                logging.FileHandler(log_file),
+                logging.StreamHandler()
+            ]
+        )
+        
+        self.logger = logging.getLogger('emotion_recognition')
+    
+    def info(self, message):
+        self.logger.info(message)
+    
+    def error(self, message, exc_info=True):
+        self.logger.error(message, exc_info=exc_info)
+    
+    def debug(self, message):
+        self.logger.debug(message) 

utils/model_inference.py

@@ -0,0 +1,89 @@
+import torch
+import json
+import os
+from transformers import AutoTokenizer, BertModel, Wav2Vec2Model
+from utils.audio_processing import AudioProcessor
+import torchaudio
+import torch.nn.functional as F
+from huggingface_hub import hf_hub_download
+from safetensors.torch import load_file
+
+# 下载模型
+model_path = hf_hub_download(repo_id="liloge/Group7_model_test", filename="model.safetensors")
+
+class MultimodalClassifier(torch.nn.Module):
+    def __init__(self):
+        super(MultimodalClassifier, self).__init__()
+        self.bert = BertModel.from_pretrained("bert-base-uncased")
+        self.wav2vec2 = Wav2Vec2Model.from_pretrained("facebook/wav2vec2-base-960h")
+        self.classifier = torch.nn.Sequential(
+            torch.nn.Linear(self.bert.config.hidden_size + self.wav2vec2.config.hidden_size, 256),
+            torch.nn.ReLU(),
+            torch.nn.Dropout(0.7),
+            torch.nn.Linear(256, 7)  # 7分类任务
+        )
+
+    def forward(self, text_input, audio_input):
+        text_outputs = self.bert(**text_input, output_hidden_states=True)
+        text_features = text_outputs.hidden_states[-1][:, 0, :]  # [CLS] token
+        audio_outputs = self.wav2vec2(audio_input, output_hidden_states=True)
+        audio_features = audio_outputs.hidden_states[-1][:, 0, :]
+        combined_features = torch.cat((text_features, audio_features), dim=-1)
+        logits = self.classifier(combined_features)
+        return logits
+
+# 加载模型
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = MultimodalClassifier().to(device)
+
+# 加载 SafeTensors 权重
+state_dict = load_file(model_path)
+model.load_state_dict(state_dict)  
+model.eval()  # 设置为评估模式
+
+tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+
+def preprocess_text(text):
+    return tokenizer(text, return_tensors="pt", truncation=True, padding=True, max_length=128).to(device)
+
+def preprocess_audio(audio_path):
+    waveform, sample_rate = torchaudio.load(audio_path)
+    waveform = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=16000)(waveform)
+    return waveform.to(device)
+
+labels = ["Neutral", "Happy", "Sad", "Angry", "Fearful", "Disgusted", "Surprised"]
+
+def predict_emotion(text, audio):
+    text_inputs = preprocess_text(text)
+    audio_inputs = preprocess_audio(audio)
+
+    with torch.no_grad():
+        output = model(text_inputs, audio_inputs)  # (1, 7) logits
+        probabilities = F.softmax(output, dim=1).squeeze().tolist()  # 归一化为概率
+
+    return {labels[i]: f"{probabilities[i]*100:.2f}%" for i in range(len(labels))}
+
+def generate_transcript(audio_file):
+    """生成音频的文字转写"""
+    return audio_file.name  # 直接返回音频文件的名称
+
+def save_history(audio_file, transcript, emotions, probabilities):
+    """保存分析历史记录到文件"""
+    history_file = "history.json"
+    
+    if not os.path.exists(history_file):
+        with open(history_file, 'w') as f:
+            json.dump([], f)
+    
+    with open(history_file, 'r') as f:
+        history = json.load(f)
+    
+    history.append({
+        "audio_file": audio_file.name,
+        "transcript": transcript,
+        "emotions": emotions,
+        "probabilities": probabilities
+    })
+    
+    with open(history_file, 'w') as f:
+        json.dump(history, f, indent=4)