Upload 28 files

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+models/res10_300x300_ssd_iter_140000.caffemodel filter=lfs diff=lfs merge=lfs -text
+models/shape_predictor_68_face_landmarks.dat filter=lfs diff=lfs merge=lfs -text
+models/speech.keras filter=lfs diff=lfs merge=lfs -text

functions/eye_track.py

@@ -0,0 +1,143 @@
+import math
+import mediapipe
+import time
+import cv2
+from tqdm import tqdm
+import numpy as np 
+def EuclideanDistance(point1, point2):
+        x1, y1 = point1
+        x2, y2 = point2
+
+        distance = math.sqrt((x2 - x1)**2 + (y2 - y1)**2)
+
+        return distance
+
+class BlinkDetector:
+    def __init__(self):
+        self.COUNTER = 0
+        self.TOTAL_BLINKS = 0   # put 1 when divide 
+        self.blink_start_time = 0
+        self.blink_durations = []
+        self.LEFT_EYE = [362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385, 384, 398]
+        self.RIGHT_EYE = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246]
+
+    
+    def FaceMeshInitialiser(self, 
+                            max_num_faces,
+                            min_detection_confidence,
+                            min_tracking_confidence):
+        
+        face_mesh = mediapipe.solutions.face_mesh.FaceMesh(max_num_faces=max_num_faces,
+                                        min_detection_confidence=min_detection_confidence,
+                                        min_tracking_confidence=min_tracking_confidence)
+        
+        return face_mesh
+    
+    def LandmarksDetector(self, 
+                          frame,
+                          face_mesh_results,
+                          draw: bool=False
+                          ):
+        
+        image_height, image_width = frame.shape[:2]
+        mesh_coordinates = [(int(point.x * image_width), int(point.y * image_height)) for point in face_mesh_results.multi_face_landmarks[0].landmark]
+
+        if draw:
+            [cv2.circle(frame, i, 2, (0, 255, 0), -1) for i in mesh_coordinates]
+        
+        return mesh_coordinates
+    
+    def BlinkRatioCalculator(self, 
+                             landmarks):
+        
+        right_eye_landmark1 = landmarks[self.RIGHT_EYE[0]]
+        right_eye_landmark2 = landmarks[self.RIGHT_EYE[8]]
+        right_eye_landmark3 = landmarks[self.RIGHT_EYE[12]]
+        right_eye_landmark4 = landmarks[self.RIGHT_EYE[4]]
+
+        left_eye_landmark1 = landmarks[self.LEFT_EYE[0]]
+        left_eye_landmark2 = landmarks[self.LEFT_EYE[8]]
+        left_eye_landmark3 = landmarks[self.LEFT_EYE[12]]
+        left_eye_landmark4 = landmarks[self.LEFT_EYE[4]]
+
+        right_eye_horizontal_distance = EuclideanDistance(right_eye_landmark1, right_eye_landmark2)
+        right_eye_verticle_distance = EuclideanDistance(right_eye_landmark3, right_eye_landmark4)
+
+        left_eye_horizontal_distance = EuclideanDistance(left_eye_landmark1, left_eye_landmark2)
+        left_eye_verticle_distance = EuclideanDistance(left_eye_landmark3, left_eye_landmark4)
+        try:
+            right_eye_ratio = right_eye_horizontal_distance / right_eye_verticle_distance
+        except:
+            right_eye_ratio = 0 
+        try:
+            left_eye_ratio = left_eye_horizontal_distance / left_eye_verticle_distance
+        except:
+            left_eye_ratio=0
+        # eyes_ratio = (right_eye_ratio + left_eye_ratio) / 2
+
+        return [right_eye_ratio, left_eye_ratio]
+    
+    def BlinkCounter(self,
+                     eyes_ratio):
+        
+        if eyes_ratio[0] > 4 or eyes_ratio[1] > 4:
+            if self.COUNTER == 0:
+                self.blink_start_time = time.time()
+            self.COUNTER += 1
+        else:
+            if self.COUNTER > 4:
+                self.TOTAL_BLINKS += 1
+                blink_duration = time.time() - self.blink_start_time
+                self.blink_durations.append(blink_duration)
+                self.COUNTER = 0
+        
+        return [self.TOTAL_BLINKS, self.blink_durations]
+    
+
+def InitialiseVariables():
+    return BlinkDetector()
+
+class Facetrack(BlinkDetector):
+    def __init__(self):
+        super().__init__()
+
+        # create object for mediapipe face_mesh
+        self.mediapipe_face_mesh = self.FaceMeshInitialiser(max_num_faces=1,
+                                                            min_detection_confidence=0.6,
+                                                            min_tracking_confidence=0.7)
+        self.frame = None
+        self.avg_blink_duration=0
+        self.list_blinks=[]
+
+    def predict(self,img):
+        self.rgb_frame = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
+        self.results = self.mediapipe_face_mesh.process(self.rgb_frame)
+        if self.results.multi_face_landmarks:
+            self.mesh_coordinates = self.LandmarksDetector(img, self.results, draw=True)
+            self.eyes_ratio = self.BlinkRatioCalculator(self.mesh_coordinates)
+            self.list_blinks = self.BlinkCounter(self.eyes_ratio)
+            if self.list_blinks[1]:
+                try:
+                    self.avg_blink_duration = sum(self.list_blinks[1]) / len(self.list_blinks[1])
+                except:
+                    self.avg_blink_duration = sum(self.list_blinks[1])
+
+                self.blink_durations = self.list_blinks[1]
+        if len(self.list_blinks)>0 :
+
+            self.TOTAL_BLINKS = self.list_blinks[0]
+        else:
+            self.TOTAL_BLINKS = 0
+            
+
+def eye_track_predict(fc,frames,fps):
+    preds=[]
+    for frame in tqdm(frames):
+        if frame is not None:
+            frame=np.copy(frame)
+            fc.predict(frame)
+            data=fc.TOTAL_BLINKS
+        else:
+            data='frame error'
+        preds.append(data)
+    return preds,fc.blink_durations,fc.TOTAL_BLINKS

functions/fer.py

@@ -0,0 +1,170 @@
+import cv2
+import torch
+import torchvision.transforms as transforms
+from PIL import Image
+import numpy as np
+import timm
+from tqdm import tqdm
+import torch.nn as nn
+import os
+import matplotlib.pyplot as plt
+
+import torch.nn.functional as F
+import dlib
+import pandas as pd
+
+
+# dnn_net = cv2.dnn.readNetFromCaffe("models/deploy.prototxt", "models/res10_300x300_ssd_iter_140000.caffemodel")
+
+# # Initialize dlib's facial landmark predictor
+# predictor = dlib.shape_predictor("models/shape_predictor_68_face_landmarks.dat")
+
+
+def extract_face(image, net, predictor):
+    # Prepare the image for DNN face detection
+    (h, w) = image.shape[:2]
+    blob = cv2.dnn.blobFromImage(cv2.resize(image, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))
+    net.setInput(blob)
+    detections = net.forward()
+
+    # Loop over the detections
+    for i in range(0, detections.shape[2]):
+        confidence = detections[0, 0, i, 2]
+
+        # Filter out weak detections
+        if confidence > 0.5:
+            box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
+            (startX, startY, endX, endY) = box.astype("int")
+
+            # Convert bounding box to dlib rectangle format
+            dlib_rect = dlib.rectangle(int(startX), int(startY), int(endX), int(endY))
+            gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+            landmarks = predictor(gray, dlib_rect)
+            landmarks_np = np.array([[p.x, p.y] for p in landmarks.parts()])
+            x, y, w, h = cv2.boundingRect(landmarks_np)
+            x -= 25
+            y -= 25
+            w += 50
+            h += 50
+
+            x = max(0, x)
+            y = max(0, y)
+            w = min(w, image.shape[1] - x)
+            h = min(h, image.shape[0] - y)
+            # Crop and resize the face
+            try:
+                face_crop = cv2.resize(face_crop, (224, 224))
+            except:
+                face_crop = cv2.resize(image, (224, 224))
+            return face_crop
+    return None
+
+
+
+
+class Model:
+    def __init__(self,fps,fer_model):
+        self.device="cuda" if torch.cuda.is_available() else "cpu"
+        self.transform = transforms.Compose([transforms.Resize((224, 224)),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])]
+        )
+        self.fermodel= timm.create_model("tf_efficientnet_b0_ns", pretrained=False)
+        self.fermodel.classifier = torch.nn.Identity()
+        self.fermodel.classifier=nn.Sequential(
+        nn.Linear(in_features=1280, out_features=7)
+        )
+        self.fermodel = torch.load(
+        fer_model,
+        map_location=self.device)
+        self.fermodel.to(self.device)
+
+        self.class_labels = ["angry", "disgust", "fear", "happy", "neutral", "sad", "surprised"]
+        self.emotion_reorder = {
+        0: 6,
+        1: 5,
+        2: 4,
+        3: 1,
+        4: 0,
+        5: 2,
+        6: 3,
+        }
+        self.label_dict = {
+                            0: "angry",
+                            1: "disgust",
+                            2: "fear",
+                            3: "happy",
+                            4: "neutral",
+                            5: "sad",
+                            6: "surprised",
+                        }
+        self.class_wise_frame_count=None
+        self.emotion_count = [0] * 7
+        self.frame_count=0
+        self.fps=fps
+        self.df=None
+        self.faces_=0
+    def predict(self,frames):
+        emotion_list=[]
+        emt=[]
+        for frame in tqdm(frames):
+            if frame is not None:
+                frame=np.copy(frame)
+                face_pil = Image.fromarray(
+                                cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+                            )
+                face_tensor = self.transform(face_pil).unsqueeze(0).to(self.device)
+                with torch.no_grad():
+                    output = self.fermodel(face_tensor)
+                    _, predicted = torch.max(output, 1)
+                    emotion = self.emotion_reorder[predicted.item()]
+                    if isinstance(emotion, np.ndarray):
+                        emotion = (
+                            emotion.astype(float).item()
+                            if emotion.size == 1
+                            else emotion.tolist()
+                                    )
+                    emotion = torch.tensor(
+                                    [emotion], dtype=torch.float32
+                                )  # Ensures it's a tensor
+                    emotion.to(self.device)
+                    emt.append(emotion)
+                self.emotion_count[predicted.item()] += 1
+                label = f"{self.label_dict[predicted.item()]}"
+                emotion_list.append(label)
+            else:
+                emt.append('frame error')
+                emotion_list.append('frame error')
+        return emotion_list,emt
+        
+    def get_data(self,emotion_list,emt):
+        self.class_wise_frame_count = dict(zip(self.class_labels, self.emotion_count))
+        return emotion_list,self.class_wise_frame_count,emt
+
+def fer_predict(video_frames,fps,model):
+    emotion_list,emt=model.predict(video_frames)
+    return model.get_data(emotion_list,emt)
+
+def filter(list1,list2):
+    filtered_list1 = [x for i, x in enumerate(list1) if list2[i]!='fnf']
+    filtered_list2 = [x for x in list2 if x!='fnf']
+    return [filtered_list1,filtered_list2]
+
+def plot_graph(x,y,var,path):
+    y = [value if isinstance(value, (int, float)) else np.nan for value in y]
+    print(len(y))
+    plt.plot(range(len(x)), y, linestyle='-')
+    plt.xlabel('Frame')
+    plt.ylabel(var)
+    plt.title(f'{var} Values vs Frame')
+    plt.savefig(path)
+    plt.clf()
+
+
+
+# def save_frames(frames,folder_path):
+#     for i in tqdm(range(len(frames))):
+#         frame_filename = os.path.join(folder_path, f'frame_{i+1:04d}.jpg')
+#         # Save the frame as a .jpg file
+#         frame=cv2.cvtColor(frames[i],cv2.COLOR_BGR2RGB)
+#         cv2.imwrite(frame_filename, frame)

functions/models.py

@@ -0,0 +1,61 @@
+import os 
+import cv2
+import speech_recognition as sr
+import dlib
+import sys 
+os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'
+from functions.eye_track import Facetrack
+from functions.fer import Model
+from functions.valence_arousal import load_models
+from tensorflow.keras.models import load_model # type: ignore
+def load_values(file_path):
+    values_dict = {}
+    with open(file_path, 'r') as file:
+        for line in file:
+            word, value = line.strip().split('\t')
+            values_dict[word.lower()] = float(value)
+    return values_dict
+
+models_folder='models'
+
+fer_model_path=os.path.join(models_folder,'22.6_AffectNet_10K_part2.pt')
+arousal_dict_path=os.path.join(models_folder,'arousal-NRC-VAD-Lexicon.txt')
+dominance_dict_path=os.path.join(models_folder,'dominance-NRC-VAD-Lexicon.txt')
+valence_arousal_model=os.path.join(models_folder,'emotion_model.pt')
+val_ar_feat_path=os.path.join(models_folder,'resnet_features.pt')
+speech_model=os.path.join(models_folder,'speech.keras')
+valence_dict_path=os.path.join(models_folder,'valence-NRC-VAD-Lexicon.txt')
+
+
+print("Loading models ")
+
+#Face detection models
+dnn_net = cv2.dnn.readNetFromCaffe(os.path.join(models_folder,"deploy.prototxt"), os.path.join(models_folder,"res10_300x300_ssd_iter_140000.caffemodel"))
+predictor = dlib.shape_predictor(os.path.join(models_folder,"shape_predictor_68_face_landmarks.dat"))
+# print("face models loaded ")
+
+#FER model
+fer_model=Model(fps=30,fer_model=fer_model_path)
+# print("fer model loaded ")
+
+#Speech model 
+model_s = load_model(speech_model)
+# print("speech model loaded ")
+
+#Load valence,arousal,dominance_dicts
+valence_dict = load_values(valence_dict_path)
+arousal_dict = load_values(arousal_dict_path)
+dominance_dict = load_values(dominance_dict_path)
+
+
+#Loading valence_arousal_models
+resnet,emotion_model=load_models(valence_arousal_model,val_ar_feat_path)
+
+models_dict={
+    'face':(dnn_net,predictor),
+    'speech':model_s,
+    'fer':fer_model,
+    'vad':(valence_dict,arousal_dict,dominance_dict),
+    "valence_fer":(resnet,emotion_model)
+    }   
+print("models loaded")

functions/speech.py

@@ -0,0 +1,203 @@
+import speech_recognition as sr
+import librosa
+import os
+import nltk
+import matplotlib.pyplot as plt
+os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'
+import tensorflow
+from tensorflow.keras.models import load_model # type: ignore
+import numpy as np
+import pandas as pd
+import soundfile as sf
+import statistics
+from pyAudioAnalysis import audioSegmentation as aS
+
+import nltk
+nltk.download('punkt_tab')
+nltk.download('averaged_perceptron_tagger_eng')
+
+label_mapping = {
+    0: 'angry',
+    1: 'disgust',
+    2: 'fear',
+    3: 'happy',
+    4: 'neutral',
+    5: 'sad',
+    6: 'surprise',
+}
+def features_extractor(file_name):
+    audio, sample_rate = librosa.load(file_name, res_type='kaiser_best')
+
+    mfccs_features = librosa.feature.mfcc(y=audio, sr=sample_rate, n_mfcc=40)
+    mfccs_scaled_features = np.mean(mfccs_features.T, axis=0)
+
+    # Extract Chroma Features
+    chroma = librosa.feature.chroma_stft(y=audio, sr=sample_rate,n_chroma=12)
+    chroma_scaled_features = np.mean(chroma.T, axis=0)
+
+    # Extract Mel Spectrogram Features
+    mel = librosa.feature.melspectrogram(y=audio, sr=sample_rate)
+    mel_scaled_features = np.mean(mel.T, axis=0)
+
+    # Concatenate all features into a single array
+    features = np.hstack((mfccs_scaled_features,  chroma_scaled_features, mel_scaled_features))
+
+    return features
+
+def predict_emotions(audio_path, interval,model_s):
+    audio_data, samplerate = sf.read(audio_path)
+    duration = len(audio_data) / samplerate
+    emotions = []
+
+    for start in np.arange(0, duration, interval):
+        end = start + interval
+        if end > duration:
+            end = duration
+        segment = audio_data[int(start*samplerate):int(end*samplerate)]
+        segment_path = 'segment.wav'
+        sf.write(segment_path, segment, samplerate)
+        # Extract features
+        feat = features_extractor(segment_path)
+        if feat is not None:
+            feat = feat.reshape(1, -1)
+            predictions = np.argmax(model_s.predict(feat),axis=1)
+            emotions.append(label_mapping[predictions[0]])
+    return emotions
+
+def recognize_speech_from_file(audio_file_path):
+    recognizer = sr.Recognizer()  # Create a recognizer instance
+    audio_file = sr.AudioFile(audio_file_path)  # Load the audio file
+    with audio_file as source:  # Use the audio file as the source
+        audio = recognizer.record(source)  # Record the audio
+    try:
+        # Recognize the speech using Google's Web Speech API
+        transcript = recognizer.recognize_google(audio)
+        return transcript  # Return the transcript
+    except sr.UnknownValueError:  # If the speech is unintelligible
+        return None
+    except sr.RequestError as e:  # If there's an error with the API request
+        print(f"Could not request results from Google Speech Recognition service; {e}")
+        return None
+
+def count_words(text):
+    words = text.split()  # Split the text into words
+    return len(words)  # Return the number of words
+
+def estimate_syllables(text):
+    syllable_count = 0  # Initialize syllable count
+    words = text.split()  # Split the text into words
+    for word in words:  # Iterate through each word
+        # Count the vowels in the word to estimate syllables
+        syllable_count += len([c for c in word if c.lower() in 'aeiou'])
+    return syllable_count  # Return the syllable count
+
+def get_speaking_rate(file_path, transcript):
+    y, sr = librosa.load(file_path, sr=None)  # Load the audio file
+    total_duration = len(y) / sr  # Calculate the total duration of the audio
+    num_syllables = estimate_syllables(transcript)  # Estimate the number of syllables
+    speaking_rate = num_syllables / total_duration if total_duration > 0 else 0  # Calculate the speaking rate
+    return speaking_rate  # Return the speaking rate
+
+def calculate_pause_metrics(file_path):
+    y, sr = librosa.load(file_path, sr=None)  # Load the audio file
+    # Remove silence and get the segments
+    segments = aS.silence_removal(y, sr, 0.020, 0.020, smooth_window=1.0, weight=0.3, plot=False)
+    total_duration = len(y) / sr  # Calculate the total duration
+    speech_duration = sum([end - start for start, end in segments])  # Calculate the speech duration
+    pause_duration = total_duration - speech_duration  # Calculate the pause duration
+    num_pauses = len(segments) - 1 if len(segments) > 0 else 0  # Calculate the number of pauses
+    average_pause_length = pause_duration / num_pauses if num_pauses > 0 else 0  # Calculate the average pause length
+    return average_pause_length  # Return the average pause length and number of pauses
+
+def calculate_articulation_rate(file_path, transcript):
+    y, sr = librosa.load(file_path, sr=None)  # Load the audio file
+    # Remove silence and get the segments
+    segments = aS.silence_removal(y, sr, 0.020, 0.020, smooth_window=1.0, weight=0.3, plot=False)
+    speech_duration = sum([end - start for start, end in segments])  # Calculate the speech duration
+    num_syllables = estimate_syllables(transcript)  # Estimate the number of syllables
+    articulation_rate = num_syllables / speech_duration if speech_duration > 0 else 0  # Calculate the articulation rate
+    return articulation_rate  # Return the articulation rate
+
+
+def pos_tag_and_filter(transcript):
+    words = nltk.word_tokenize(transcript)
+    pos_tags = nltk.pos_tag(words)
+    
+    # Define important POS tags
+    important_tags = {'NN', 'NNS', 'NNP', 'NNPS', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ', 'JJ', 'JJR', 'JJS', 'RB', 'RBR', 'RBS'}  
+    filtered_words = []
+    for word, tag in pos_tags:
+        if tag in important_tags:
+            filtered_words.append((word, tag))
+    return filtered_words
+
+def load_values(file_path):
+    values_dict = {}
+    with open(file_path, 'r') as file:
+        for line in file:
+            word, value = line.strip().split('\t')
+            values_dict[word.lower()] = float(value)
+    return values_dict
+
+# Map values to filtered words
+def map_values_to_filtered_words(filtered_words, valence_dict, arousal_dict, dominance_dict):
+    mapped_values = []
+    word_weights = {}
+    for word in filtered_words:
+        valence = valence_dict.get(word.lower())
+        arousal = arousal_dict.get(word.lower())
+        dominance = dominance_dict.get(word.lower())
+        if valence is not None and arousal is not None and dominance is not None:
+            valence=(valence+1)/2
+            arousal=(arousal+1)/2
+            mapped_values.append((word, valence, arousal,dominance,1))
+            # Calculate importance weight (sum of valence, arousal, and dominance)
+            word_weights[word] = valence + arousal + dominance
+        else:
+            mapped_values.append((word, 'not found', 'not found','not found',0))
+            word_weights[word] = 0
+    return mapped_values,word_weights
+def generate_word_cloud(word_weights):
+    if len(word_weights)>0:
+        return word_weights
+def analyze_audio(file_path,valence_dict,arousal_dict,dominance_dict):
+    # Get the transcript of the audio
+
+    # transcript = "I want you to act like he's coming back, both of you. Don't think I haven't noticed you since he in..."
+    transcript = recognize_speech_from_file(file_path)
+    print(transcript)
+    if not transcript:  # If transcript is not available
+        transcript = "I want you to act like he's coming back, both of you. Don't think I haven't noticed you since he in..."
+
+    filtered_words_with_tags = pos_tag_and_filter(transcript)
+    filtered_words = [word for word, tag in filtered_words_with_tags]
+
+
+    mapped_values,word_weights = map_values_to_filtered_words(filtered_words, valence_dict, arousal_dict, dominance_dict)
+    # Calculate various metrics
+
+    word_weights=generate_word_cloud(word_weights)
+    word_count = count_words(transcript)  # Count the number of words
+    speaking_rate = get_speaking_rate(file_path, transcript) # Calculate the speaking rate
+    average_pause_length = calculate_pause_metrics(file_path)  # Calculate pause metrics
+    articulation_rate = calculate_articulation_rate(file_path, transcript)  # Calculate the articulation rate
+    
+    word={} 
+    word['word_count']=word_count
+    word['word_weights']=word_weights
+    word['speaking_rate']=speaking_rate
+    word['average_pause_length']=average_pause_length
+    word['articulation_rate']=articulation_rate
+    word['mapped_values']=mapped_values
+    return word
+
+def speech_predict(audio_path,model_s,valence_dict,arousal_dict,dominance_dict):
+    
+    interval = 3.0  # Set the interval for emotion detection segments
+    emotions = predict_emotions(audio_path, interval,model_s)
+    
+    # Save emotions to a log file
+    # Extrapolate major emotions
+    major_emotion = statistics.mode(emotions)
+    word = analyze_audio(audio_path,valence_dict,arousal_dict,dominance_dict)
+    return emotions,major_emotion,word

functions/valence_arousal.py

@@ -0,0 +1,93 @@
+from torchvision import models
+import torch.nn as nn
+import torch
+import torch.nn.functional as F
+import torchvision.transforms as transforms
+from PIL import Image
+import cv2
+
+def create_emotion_model(num_ftrs, num_emotions):
+    return nn.Sequential(
+        nn.Linear(num_ftrs + num_emotions, 128),
+        nn.ReLU(),
+        nn.Linear(128, 64),
+        nn.ReLU(),
+        nn.Linear(64, 2),
+    )
+def load_models(val_model_path,val_featmodel_path):
+    transform = transforms.Compose(
+        [
+            transforms.Resize((224, 224)),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ]
+    )
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    resnet = models.resnet18(pretrained=False)
+    num_ftrs = resnet.fc.in_features
+    resnet.fc = nn.Identity()
+    resnet.load_state_dict(
+        torch.load(
+            val_featmodel_path,
+            map_location=device
+        )
+    )
+    resnet = resnet.to(device)
+
+    # num_ftrs = resnet.fc.in_features
+    num_emotions = 1
+    emotion_model = create_emotion_model(num_ftrs, num_emotions).to(device)
+    emotion_model.load_state_dict(
+        torch.load(
+            val_model_path,
+            map_location=device
+        )
+    )
+    return resnet,emotion_model
+
+
+
+def va_predict(emotion_model,resnet,faces,emotions):
+    transform = transforms.Compose(
+        [
+            transforms.Resize((224, 224)),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ]
+    )
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    def model_forward(images, emotions):
+        resnet_features = resnet(images)
+        batch_size = resnet_features.size(0)
+        emotions = emotions.view(batch_size, -1)
+        x = torch.cat((resnet_features, emotions), dim=1)
+        output = emotion_model(x)
+        return output
+
+    arousal_list = []
+    valence_list = []
+    stress_list = []
+    from tqdm import tqdm
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    for face, emotion in tqdm(zip(faces, emotions)):
+        if face is not None:
+            face_pil = Image.fromarray(cv2.cvtColor(face, cv2.COLOR_BGR2RGB))
+            face_tensor = transform(face_pil).unsqueeze(0).to(device)
+            # print(emotion)
+            # print(emotion)
+            emotion = emotion.to(device)
+            output_va = model_forward(face_tensor, emotion)
+            arousal = output_va[0][0].item()
+            norm_arousal = float(output_va[0][0].item()) / 2 + 0.5
+            valence = output_va[0][1].item()
+            norm_valence = float(output_va[0][1].item()) / 2 + 0.5
+            stress = (1 - norm_valence) * norm_arousal
+            arousal_list.append(arousal)
+            valence_list.append(valence)
+            stress_list.append(stress)
+        else:
+            arousal_list.append('frame error')
+            valence_list.append('frame error')
+            stress_list.append('frame error')
+    return valence_list, arousal_list, stress_list

models/22.6_AffectNet_10K_part2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92ef53adb843700faa3c54ae6f3e0f4105e04e099f9190dd66aafc360afdb2bf
+size 16425358

models/deploy.prototxt

@@ -0,0 +1,1789 @@
+input: "data"
+input_shape {
+  dim: 1
+  dim: 3
+  dim: 300
+  dim: 300
+}
+
+layer {
+  name: "data_bn"
+  type: "BatchNorm"
+  bottom: "data"
+  top: "data_bn"
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+}
+layer {
+  name: "data_scale"
+  type: "Scale"
+  bottom: "data_bn"
+  top: "data_bn"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  param {
+    lr_mult: 2.0
+    decay_mult: 1.0
+  }
+  scale_param {
+    bias_term: true
+  }
+}
+layer {
+  name: "conv1_h"
+  type: "Convolution"
+  bottom: "data_bn"
+  top: "conv1_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  param {
+    lr_mult: 2.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 32
+    pad: 3
+    kernel_size: 7
+    stride: 2
+    weight_filler {
+      type: "msra"
+      variance_norm: FAN_OUT
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "conv1_bn_h"
+  type: "BatchNorm"
+  bottom: "conv1_h"
+  top: "conv1_h"
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+}
+layer {
+  name: "conv1_scale_h"
+  type: "Scale"
+  bottom: "conv1_h"
+  top: "conv1_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  param {
+    lr_mult: 2.0
+    decay_mult: 1.0
+  }
+  scale_param {
+    bias_term: true
+  }
+}
+layer {
+  name: "conv1_relu"
+  type: "ReLU"
+  bottom: "conv1_h"
+  top: "conv1_h"
+}
+layer {
+  name: "conv1_pool"
+  type: "Pooling"
+  bottom: "conv1_h"
+  top: "conv1_pool"
+  pooling_param {
+    kernel_size: 3
+    stride: 2
+  }
+}
+layer {
+  name: "layer_64_1_conv1_h"
+  type: "Convolution"
+  bottom: "conv1_pool"
+  top: "layer_64_1_conv1_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 32
+    bias_term: false
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "msra"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "layer_64_1_bn2_h"
+  type: "BatchNorm"
+  bottom: "layer_64_1_conv1_h"
+  top: "layer_64_1_conv1_h"
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+}
+layer {
+  name: "layer_64_1_scale2_h"
+  type: "Scale"
+  bottom: "layer_64_1_conv1_h"
+  top: "layer_64_1_conv1_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  param {
+    lr_mult: 2.0
+    decay_mult: 1.0
+  }
+  scale_param {
+    bias_term: true
+  }
+}
+layer {
+  name: "layer_64_1_relu2"
+  type: "ReLU"
+  bottom: "layer_64_1_conv1_h"
+  top: "layer_64_1_conv1_h"
+}
+layer {
+  name: "layer_64_1_conv2_h"
+  type: "Convolution"
+  bottom: "layer_64_1_conv1_h"
+  top: "layer_64_1_conv2_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 32
+    bias_term: false
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "msra"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "layer_64_1_sum"
+  type: "Eltwise"
+  bottom: "layer_64_1_conv2_h"
+  bottom: "conv1_pool"
+  top: "layer_64_1_sum"
+}
+layer {
+  name: "layer_128_1_bn1_h"
+  type: "BatchNorm"
+  bottom: "layer_64_1_sum"
+  top: "layer_128_1_bn1_h"
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+}
+layer {
+  name: "layer_128_1_scale1_h"
+  type: "Scale"
+  bottom: "layer_128_1_bn1_h"
+  top: "layer_128_1_bn1_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  param {
+    lr_mult: 2.0
+    decay_mult: 1.0
+  }
+  scale_param {
+    bias_term: true
+  }
+}
+layer {
+  name: "layer_128_1_relu1"
+  type: "ReLU"
+  bottom: "layer_128_1_bn1_h"
+  top: "layer_128_1_bn1_h"
+}
+layer {
+  name: "layer_128_1_conv1_h"
+  type: "Convolution"
+  bottom: "layer_128_1_bn1_h"
+  top: "layer_128_1_conv1_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 128
+    bias_term: false
+    pad: 1
+    kernel_size: 3
+    stride: 2
+    weight_filler {
+      type: "msra"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "layer_128_1_bn2"
+  type: "BatchNorm"
+  bottom: "layer_128_1_conv1_h"
+  top: "layer_128_1_conv1_h"
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+}
+layer {
+  name: "layer_128_1_scale2"
+  type: "Scale"
+  bottom: "layer_128_1_conv1_h"
+  top: "layer_128_1_conv1_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  param {
+    lr_mult: 2.0
+    decay_mult: 1.0
+  }
+  scale_param {
+    bias_term: true
+  }
+}
+layer {
+  name: "layer_128_1_relu2"
+  type: "ReLU"
+  bottom: "layer_128_1_conv1_h"
+  top: "layer_128_1_conv1_h"
+}
+layer {
+  name: "layer_128_1_conv2"
+  type: "Convolution"
+  bottom: "layer_128_1_conv1_h"
+  top: "layer_128_1_conv2"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 128
+    bias_term: false
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "msra"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "layer_128_1_conv_expand_h"
+  type: "Convolution"
+  bottom: "layer_128_1_bn1_h"
+  top: "layer_128_1_conv_expand_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 128
+    bias_term: false
+    pad: 0
+    kernel_size: 1
+    stride: 2
+    weight_filler {
+      type: "msra"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "layer_128_1_sum"
+  type: "Eltwise"
+  bottom: "layer_128_1_conv2"
+  bottom: "layer_128_1_conv_expand_h"
+  top: "layer_128_1_sum"
+}
+layer {
+  name: "layer_256_1_bn1"
+  type: "BatchNorm"
+  bottom: "layer_128_1_sum"
+  top: "layer_256_1_bn1"
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+}
+layer {
+  name: "layer_256_1_scale1"
+  type: "Scale"
+  bottom: "layer_256_1_bn1"
+  top: "layer_256_1_bn1"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  param {
+    lr_mult: 2.0
+    decay_mult: 1.0
+  }
+  scale_param {
+    bias_term: true
+  }
+}
+layer {
+  name: "layer_256_1_relu1"
+  type: "ReLU"
+  bottom: "layer_256_1_bn1"
+  top: "layer_256_1_bn1"
+}
+layer {
+  name: "layer_256_1_conv1"
+  type: "Convolution"
+  bottom: "layer_256_1_bn1"
+  top: "layer_256_1_conv1"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 256
+    bias_term: false
+    pad: 1
+    kernel_size: 3
+    stride: 2
+    weight_filler {
+      type: "msra"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "layer_256_1_bn2"
+  type: "BatchNorm"
+  bottom: "layer_256_1_conv1"
+  top: "layer_256_1_conv1"
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+}
+layer {
+  name: "layer_256_1_scale2"
+  type: "Scale"
+  bottom: "layer_256_1_conv1"
+  top: "layer_256_1_conv1"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  param {
+    lr_mult: 2.0
+    decay_mult: 1.0
+  }
+  scale_param {
+    bias_term: true
+  }
+}
+layer {
+  name: "layer_256_1_relu2"
+  type: "ReLU"
+  bottom: "layer_256_1_conv1"
+  top: "layer_256_1_conv1"
+}
+layer {
+  name: "layer_256_1_conv2"
+  type: "Convolution"
+  bottom: "layer_256_1_conv1"
+  top: "layer_256_1_conv2"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 256
+    bias_term: false
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "msra"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "layer_256_1_conv_expand"
+  type: "Convolution"
+  bottom: "layer_256_1_bn1"
+  top: "layer_256_1_conv_expand"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 256
+    bias_term: false
+    pad: 0
+    kernel_size: 1
+    stride: 2
+    weight_filler {
+      type: "msra"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "layer_256_1_sum"
+  type: "Eltwise"
+  bottom: "layer_256_1_conv2"
+  bottom: "layer_256_1_conv_expand"
+  top: "layer_256_1_sum"
+}
+layer {
+  name: "layer_512_1_bn1"
+  type: "BatchNorm"
+  bottom: "layer_256_1_sum"
+  top: "layer_512_1_bn1"
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+}
+layer {
+  name: "layer_512_1_scale1"
+  type: "Scale"
+  bottom: "layer_512_1_bn1"
+  top: "layer_512_1_bn1"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  param {
+    lr_mult: 2.0
+    decay_mult: 1.0
+  }
+  scale_param {
+    bias_term: true
+  }
+}
+layer {
+  name: "layer_512_1_relu1"
+  type: "ReLU"
+  bottom: "layer_512_1_bn1"
+  top: "layer_512_1_bn1"
+}
+layer {
+  name: "layer_512_1_conv1_h"
+  type: "Convolution"
+  bottom: "layer_512_1_bn1"
+  top: "layer_512_1_conv1_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 128
+    bias_term: false
+    pad: 1
+    kernel_size: 3
+    stride: 1 # 2
+    weight_filler {
+      type: "msra"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "layer_512_1_bn2_h"
+  type: "BatchNorm"
+  bottom: "layer_512_1_conv1_h"
+  top: "layer_512_1_conv1_h"
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+}
+layer {
+  name: "layer_512_1_scale2_h"
+  type: "Scale"
+  bottom: "layer_512_1_conv1_h"
+  top: "layer_512_1_conv1_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  param {
+    lr_mult: 2.0
+    decay_mult: 1.0
+  }
+  scale_param {
+    bias_term: true
+  }
+}
+layer {
+  name: "layer_512_1_relu2"
+  type: "ReLU"
+  bottom: "layer_512_1_conv1_h"
+  top: "layer_512_1_conv1_h"
+}
+layer {
+  name: "layer_512_1_conv2_h"
+  type: "Convolution"
+  bottom: "layer_512_1_conv1_h"
+  top: "layer_512_1_conv2_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 256
+    bias_term: false
+    pad: 2 # 1
+    kernel_size: 3
+    stride: 1
+    dilation: 2
+    weight_filler {
+      type: "msra"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "layer_512_1_conv_expand_h"
+  type: "Convolution"
+  bottom: "layer_512_1_bn1"
+  top: "layer_512_1_conv_expand_h"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  convolution_param {
+    num_output: 256
+    bias_term: false
+    pad: 0
+    kernel_size: 1
+    stride: 1 # 2
+    weight_filler {
+      type: "msra"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0.0
+    }
+  }
+}
+layer {
+  name: "layer_512_1_sum"
+  type: "Eltwise"
+  bottom: "layer_512_1_conv2_h"
+  bottom: "layer_512_1_conv_expand_h"
+  top: "layer_512_1_sum"
+}
+layer {
+  name: "last_bn_h"
+  type: "BatchNorm"
+  bottom: "layer_512_1_sum"
+  top: "layer_512_1_sum"
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+  param {
+    lr_mult: 0.0
+  }
+}
+layer {
+  name: "last_scale_h"
+  type: "Scale"
+  bottom: "layer_512_1_sum"
+  top: "layer_512_1_sum"
+  param {
+    lr_mult: 1.0
+    decay_mult: 1.0
+  }
+  param {
+    lr_mult: 2.0
+    decay_mult: 1.0
+  }
+  scale_param {
+    bias_term: true
+  }
+}
+layer {
+  name: "last_relu"
+  type: "ReLU"
+  bottom: "layer_512_1_sum"
+  top: "fc7"
+}
+
+layer {
+  name: "conv6_1_h"
+  type: "Convolution"
+  bottom: "fc7"
+  top: "conv6_1_h"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 128
+    pad: 0
+    kernel_size: 1
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv6_1_relu"
+  type: "ReLU"
+  bottom: "conv6_1_h"
+  top: "conv6_1_h"
+}
+layer {
+  name: "conv6_2_h"
+  type: "Convolution"
+  bottom: "conv6_1_h"
+  top: "conv6_2_h"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 256
+    pad: 1
+    kernel_size: 3
+    stride: 2
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv6_2_relu"
+  type: "ReLU"
+  bottom: "conv6_2_h"
+  top: "conv6_2_h"
+}
+layer {
+  name: "conv7_1_h"
+  type: "Convolution"
+  bottom: "conv6_2_h"
+  top: "conv7_1_h"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 64
+    pad: 0
+    kernel_size: 1
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv7_1_relu"
+  type: "ReLU"
+  bottom: "conv7_1_h"
+  top: "conv7_1_h"
+}
+layer {
+  name: "conv7_2_h"
+  type: "Convolution"
+  bottom: "conv7_1_h"
+  top: "conv7_2_h"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 128
+    pad: 1
+    kernel_size: 3
+    stride: 2
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv7_2_relu"
+  type: "ReLU"
+  bottom: "conv7_2_h"
+  top: "conv7_2_h"
+}
+layer {
+  name: "conv8_1_h"
+  type: "Convolution"
+  bottom: "conv7_2_h"
+  top: "conv8_1_h"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 64
+    pad: 0
+    kernel_size: 1
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv8_1_relu"
+  type: "ReLU"
+  bottom: "conv8_1_h"
+  top: "conv8_1_h"
+}
+layer {
+  name: "conv8_2_h"
+  type: "Convolution"
+  bottom: "conv8_1_h"
+  top: "conv8_2_h"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 128
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv8_2_relu"
+  type: "ReLU"
+  bottom: "conv8_2_h"
+  top: "conv8_2_h"
+}
+layer {
+  name: "conv9_1_h"
+  type: "Convolution"
+  bottom: "conv8_2_h"
+  top: "conv9_1_h"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 64
+    pad: 0
+    kernel_size: 1
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv9_1_relu"
+  type: "ReLU"
+  bottom: "conv9_1_h"
+  top: "conv9_1_h"
+}
+layer {
+  name: "conv9_2_h"
+  type: "Convolution"
+  bottom: "conv9_1_h"
+  top: "conv9_2_h"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 128
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv9_2_relu"
+  type: "ReLU"
+  bottom: "conv9_2_h"
+  top: "conv9_2_h"
+}
+layer {
+  name: "conv4_3_norm"
+  type: "Normalize"
+  bottom: "layer_256_1_bn1"
+  top: "conv4_3_norm"
+  norm_param {
+    across_spatial: false
+    scale_filler {
+      type: "constant"
+      value: 20
+    }
+    channel_shared: false
+  }
+}
+layer {
+  name: "conv4_3_norm_mbox_loc"
+  type: "Convolution"
+  bottom: "conv4_3_norm"
+  top: "conv4_3_norm_mbox_loc"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 16
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv4_3_norm_mbox_loc_perm"
+  type: "Permute"
+  bottom: "conv4_3_norm_mbox_loc"
+  top: "conv4_3_norm_mbox_loc_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "conv4_3_norm_mbox_loc_flat"
+  type: "Flatten"
+  bottom: "conv4_3_norm_mbox_loc_perm"
+  top: "conv4_3_norm_mbox_loc_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "conv4_3_norm_mbox_conf"
+  type: "Convolution"
+  bottom: "conv4_3_norm"
+  top: "conv4_3_norm_mbox_conf"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 8 # 84
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv4_3_norm_mbox_conf_perm"
+  type: "Permute"
+  bottom: "conv4_3_norm_mbox_conf"
+  top: "conv4_3_norm_mbox_conf_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "conv4_3_norm_mbox_conf_flat"
+  type: "Flatten"
+  bottom: "conv4_3_norm_mbox_conf_perm"
+  top: "conv4_3_norm_mbox_conf_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "conv4_3_norm_mbox_priorbox"
+  type: "PriorBox"
+  bottom: "conv4_3_norm"
+  bottom: "data"
+  top: "conv4_3_norm_mbox_priorbox"
+  prior_box_param {
+    min_size: 30.0
+    max_size: 60.0
+    aspect_ratio: 2
+    flip: true
+    clip: false
+    variance: 0.1
+    variance: 0.1
+    variance: 0.2
+    variance: 0.2
+    step: 8
+    offset: 0.5
+  }
+}
+layer {
+  name: "fc7_mbox_loc"
+  type: "Convolution"
+  bottom: "fc7"
+  top: "fc7_mbox_loc"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 24
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "fc7_mbox_loc_perm"
+  type: "Permute"
+  bottom: "fc7_mbox_loc"
+  top: "fc7_mbox_loc_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "fc7_mbox_loc_flat"
+  type: "Flatten"
+  bottom: "fc7_mbox_loc_perm"
+  top: "fc7_mbox_loc_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "fc7_mbox_conf"
+  type: "Convolution"
+  bottom: "fc7"
+  top: "fc7_mbox_conf"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 12 # 126
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "fc7_mbox_conf_perm"
+  type: "Permute"
+  bottom: "fc7_mbox_conf"
+  top: "fc7_mbox_conf_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "fc7_mbox_conf_flat"
+  type: "Flatten"
+  bottom: "fc7_mbox_conf_perm"
+  top: "fc7_mbox_conf_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "fc7_mbox_priorbox"
+  type: "PriorBox"
+  bottom: "fc7"
+  bottom: "data"
+  top: "fc7_mbox_priorbox"
+  prior_box_param {
+    min_size: 60.0
+    max_size: 111.0
+    aspect_ratio: 2
+    aspect_ratio: 3
+    flip: true
+    clip: false
+    variance: 0.1
+    variance: 0.1
+    variance: 0.2
+    variance: 0.2
+    step: 16
+    offset: 0.5
+  }
+}
+layer {
+  name: "conv6_2_mbox_loc"
+  type: "Convolution"
+  bottom: "conv6_2_h"
+  top: "conv6_2_mbox_loc"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 24
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv6_2_mbox_loc_perm"
+  type: "Permute"
+  bottom: "conv6_2_mbox_loc"
+  top: "conv6_2_mbox_loc_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "conv6_2_mbox_loc_flat"
+  type: "Flatten"
+  bottom: "conv6_2_mbox_loc_perm"
+  top: "conv6_2_mbox_loc_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "conv6_2_mbox_conf"
+  type: "Convolution"
+  bottom: "conv6_2_h"
+  top: "conv6_2_mbox_conf"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 12 # 126
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv6_2_mbox_conf_perm"
+  type: "Permute"
+  bottom: "conv6_2_mbox_conf"
+  top: "conv6_2_mbox_conf_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "conv6_2_mbox_conf_flat"
+  type: "Flatten"
+  bottom: "conv6_2_mbox_conf_perm"
+  top: "conv6_2_mbox_conf_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "conv6_2_mbox_priorbox"
+  type: "PriorBox"
+  bottom: "conv6_2_h"
+  bottom: "data"
+  top: "conv6_2_mbox_priorbox"
+  prior_box_param {
+    min_size: 111.0
+    max_size: 162.0
+    aspect_ratio: 2
+    aspect_ratio: 3
+    flip: true
+    clip: false
+    variance: 0.1
+    variance: 0.1
+    variance: 0.2
+    variance: 0.2
+    step: 32
+    offset: 0.5
+  }
+}
+layer {
+  name: "conv7_2_mbox_loc"
+  type: "Convolution"
+  bottom: "conv7_2_h"
+  top: "conv7_2_mbox_loc"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 24
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv7_2_mbox_loc_perm"
+  type: "Permute"
+  bottom: "conv7_2_mbox_loc"
+  top: "conv7_2_mbox_loc_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "conv7_2_mbox_loc_flat"
+  type: "Flatten"
+  bottom: "conv7_2_mbox_loc_perm"
+  top: "conv7_2_mbox_loc_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "conv7_2_mbox_conf"
+  type: "Convolution"
+  bottom: "conv7_2_h"
+  top: "conv7_2_mbox_conf"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 12 # 126
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv7_2_mbox_conf_perm"
+  type: "Permute"
+  bottom: "conv7_2_mbox_conf"
+  top: "conv7_2_mbox_conf_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "conv7_2_mbox_conf_flat"
+  type: "Flatten"
+  bottom: "conv7_2_mbox_conf_perm"
+  top: "conv7_2_mbox_conf_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "conv7_2_mbox_priorbox"
+  type: "PriorBox"
+  bottom: "conv7_2_h"
+  bottom: "data"
+  top: "conv7_2_mbox_priorbox"
+  prior_box_param {
+    min_size: 162.0
+    max_size: 213.0
+    aspect_ratio: 2
+    aspect_ratio: 3
+    flip: true
+    clip: false
+    variance: 0.1
+    variance: 0.1
+    variance: 0.2
+    variance: 0.2
+    step: 64
+    offset: 0.5
+  }
+}
+layer {
+  name: "conv8_2_mbox_loc"
+  type: "Convolution"
+  bottom: "conv8_2_h"
+  top: "conv8_2_mbox_loc"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 16
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv8_2_mbox_loc_perm"
+  type: "Permute"
+  bottom: "conv8_2_mbox_loc"
+  top: "conv8_2_mbox_loc_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "conv8_2_mbox_loc_flat"
+  type: "Flatten"
+  bottom: "conv8_2_mbox_loc_perm"
+  top: "conv8_2_mbox_loc_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "conv8_2_mbox_conf"
+  type: "Convolution"
+  bottom: "conv8_2_h"
+  top: "conv8_2_mbox_conf"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 8 # 84
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv8_2_mbox_conf_perm"
+  type: "Permute"
+  bottom: "conv8_2_mbox_conf"
+  top: "conv8_2_mbox_conf_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "conv8_2_mbox_conf_flat"
+  type: "Flatten"
+  bottom: "conv8_2_mbox_conf_perm"
+  top: "conv8_2_mbox_conf_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "conv8_2_mbox_priorbox"
+  type: "PriorBox"
+  bottom: "conv8_2_h"
+  bottom: "data"
+  top: "conv8_2_mbox_priorbox"
+  prior_box_param {
+    min_size: 213.0
+    max_size: 264.0
+    aspect_ratio: 2
+    flip: true
+    clip: false
+    variance: 0.1
+    variance: 0.1
+    variance: 0.2
+    variance: 0.2
+    step: 100
+    offset: 0.5
+  }
+}
+layer {
+  name: "conv9_2_mbox_loc"
+  type: "Convolution"
+  bottom: "conv9_2_h"
+  top: "conv9_2_mbox_loc"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 16
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv9_2_mbox_loc_perm"
+  type: "Permute"
+  bottom: "conv9_2_mbox_loc"
+  top: "conv9_2_mbox_loc_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "conv9_2_mbox_loc_flat"
+  type: "Flatten"
+  bottom: "conv9_2_mbox_loc_perm"
+  top: "conv9_2_mbox_loc_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "conv9_2_mbox_conf"
+  type: "Convolution"
+  bottom: "conv9_2_h"
+  top: "conv9_2_mbox_conf"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 8 # 84
+    pad: 1
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "conv9_2_mbox_conf_perm"
+  type: "Permute"
+  bottom: "conv9_2_mbox_conf"
+  top: "conv9_2_mbox_conf_perm"
+  permute_param {
+    order: 0
+    order: 2
+    order: 3
+    order: 1
+  }
+}
+layer {
+  name: "conv9_2_mbox_conf_flat"
+  type: "Flatten"
+  bottom: "conv9_2_mbox_conf_perm"
+  top: "conv9_2_mbox_conf_flat"
+  flatten_param {
+    axis: 1
+  }
+}
+layer {
+  name: "conv9_2_mbox_priorbox"
+  type: "PriorBox"
+  bottom: "conv9_2_h"
+  bottom: "data"
+  top: "conv9_2_mbox_priorbox"
+  prior_box_param {
+    min_size: 264.0
+    max_size: 315.0
+    aspect_ratio: 2
+    flip: true
+    clip: false
+    variance: 0.1
+    variance: 0.1
+    variance: 0.2
+    variance: 0.2
+    step: 300
+    offset: 0.5
+  }
+}
+layer {
+  name: "mbox_loc"
+  type: "Concat"
+  bottom: "conv4_3_norm_mbox_loc_flat"
+  bottom: "fc7_mbox_loc_flat"
+  bottom: "conv6_2_mbox_loc_flat"
+  bottom: "conv7_2_mbox_loc_flat"
+  bottom: "conv8_2_mbox_loc_flat"
+  bottom: "conv9_2_mbox_loc_flat"
+  top: "mbox_loc"
+  concat_param {
+    axis: 1
+  }
+}
+layer {
+  name: "mbox_conf"
+  type: "Concat"
+  bottom: "conv4_3_norm_mbox_conf_flat"
+  bottom: "fc7_mbox_conf_flat"
+  bottom: "conv6_2_mbox_conf_flat"
+  bottom: "conv7_2_mbox_conf_flat"
+  bottom: "conv8_2_mbox_conf_flat"
+  bottom: "conv9_2_mbox_conf_flat"
+  top: "mbox_conf"
+  concat_param {
+    axis: 1
+  }
+}
+layer {
+  name: "mbox_priorbox"
+  type: "Concat"
+  bottom: "conv4_3_norm_mbox_priorbox"
+  bottom: "fc7_mbox_priorbox"
+  bottom: "conv6_2_mbox_priorbox"
+  bottom: "conv7_2_mbox_priorbox"
+  bottom: "conv8_2_mbox_priorbox"
+  bottom: "conv9_2_mbox_priorbox"
+  top: "mbox_priorbox"
+  concat_param {
+    axis: 2
+  }
+}
+
+layer {
+  name: "mbox_conf_reshape"
+  type: "Reshape"
+  bottom: "mbox_conf"
+  top: "mbox_conf_reshape"
+  reshape_param {
+    shape {
+      dim: 0
+      dim: -1
+      dim: 2
+    }
+  }
+}
+layer {
+  name: "mbox_conf_softmax"
+  type: "Softmax"
+  bottom: "mbox_conf_reshape"
+  top: "mbox_conf_softmax"
+  softmax_param {
+    axis: 2
+  }
+}
+layer {
+  name: "mbox_conf_flatten"
+  type: "Flatten"
+  bottom: "mbox_conf_softmax"
+  top: "mbox_conf_flatten"
+  flatten_param {
+    axis: 1
+  }
+}
+
+layer {
+  name: "detection_out"
+  type: "DetectionOutput"
+  bottom: "mbox_loc"
+  bottom: "mbox_conf_flatten"
+  bottom: "mbox_priorbox"
+  top: "detection_out"
+  include {
+    phase: TEST
+  }
+  detection_output_param {
+    num_classes: 2
+    share_location: true
+    background_label_id: 0
+    nms_param {
+      nms_threshold: 0.45
+      top_k: 400
+    }
+    code_type: CENTER_SIZE
+    keep_top_k: 200
+    confidence_threshold: 0.01
+  }
+}

models/emotion_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19c707063376d618b3e56df7537b6e103c7426e759c63ce5a4c33df414ce3612
+size 299634

models/res10_300x300_ssd_iter_140000.caffemodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a56a11a57a4a295956b0660b4a3d76bbdca2206c4961cea8efe7d95c7cb2f2d
+size 10666211

models/resnet_features.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58abd9efad0ad01490f42835b4dd082b346e0bfa3b6b9a6041295de905cf688f
+size 44786742

models/shape_predictor_68_face_landmarks.dat

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbdc2cb80eb9aa7a758672cbfdda32ba6300efe9b6e6c7a299ff7e736b11b92f
+size 99693937

models/speech.keras

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b57a6a602c56250c004da5c8b83d3a46b5d6fb20999b1457f7a6294457d9008c
+size 12300234