import warnings warnings.filterwarnings('ignore', category=UserWarning, module='tensorflow') import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0' import logging logging.getLogger('absl').setLevel(logging.ERROR) from functions.models import models_dict from functions.helper import extract_faces_from_frames,make_pdf from functions.video import eyebrow,detect_blinks,detect_yawns,detect_smiles from functions.valence_arousal import va_predict from functions.fer import fer_predict,plot_graph from functions.helper import plot_facial_expression_graphs from moviepy.editor import VideoFileClip import json # from trash import detect_eyes_in_faces import pandas as pd from typing import Callable from functions.audio import extract_audio_features asrmodel=models_dict['asrmodel'] asrproc=models_dict['asrproc'] sentipipe=models_dict['sentipipe'] valence_arousal_model=models_dict['valence_fer'][1] val_ar_feat_model=models_dict['valence_fer'][0] fer_model=models_dict['fer'] smile_cascade=models_dict['smile_cascade'] dnn_net=models_dict['face'][0] predictor=models_dict['face'][1] fps=30 session_data={} def analyze_live_video(video_path: str, uid: str, user_id: str, count: int, final: bool, log: Callable[[str], None]): try: global session_data if uid not in session_data: session_data[uid]={ "vcount":[], "duration":[], "audio":[], "blinks":[], "yawn":[], "smile":[], "eyebrow":[], "fer": [], "valence":[], "arousal":[], "stress":[], } print(f"UID: {uid}, User ID: {user_id}, Count: {count}, Final: {final}, Video: {video_path}") print(f"analysing video for question - {count}") output_dir = os.path.join('output', uid) os.makedirs(output_dir,exist_ok=True) folder_path=os.path.join(output_dir,f'{count}') os.makedirs(folder_path,exist_ok=True) meta_data_path=os.path.join(folder_path,'metadata.json') valence_plot=os.path.join(folder_path,"vas.png") word_cloud=os.path.join(folder_path,'wordcloud.jpg') df_path=os.path.join(folder_path,'data.csv') pdf_filename = os.path.join(folder_path,"formatted_output_with_plots.pdf") video_clip=VideoFileClip(video_path) video_clip=video_clip.set_fps(fps) duration=video_clip.duration print(duration) audio=video_clip.audio audio_path = os.path.join(folder_path,'extracted_audio.wav') print(audio_path) audio.write_audiofile(audio_path) video_frames=[frame for frame in video_clip.iter_frames()] faces, landmarks, sizes=extract_faces_from_frames(video_frames,dnn_net,predictor) # faces=[extract_face(frame) for frame in tqdm(video_frames)] af,pitches=extract_audio_features(audio_path,asrmodel,asrproc,sentipipe,duration,word_cloud) pitches=[float(pitch) for pitch in pitches] fer_emotions,class_wise_frame_count,em_tensors=fer_predict(faces,fps,fer_model) valence_list,arousal_list,stress_list=va_predict(valence_arousal_model,val_ar_feat_model,faces,list(em_tensors)) timestamps=[j/fps for j in range(len(valence_list))] eyebrow_dist=eyebrow(landmarks,sizes) print('eyebrow done') blink_count, ear_ratios=detect_blinks(landmarks,sizes,fps) ear_ratios=[float(pitch) for pitch in ear_ratios] print('blinks done',blink_count) smiles, smile_ratios, total_smiles, smile_durations,smile_threshold=detect_smiles(landmarks,sizes) smile_ratios=[float(smile) for smile in smile_ratios] print('smiles done',total_smiles) yawns, yawn_ratios, total_yawns, yawn_durations=detect_yawns(landmarks,sizes) print('ywan done') thresholds=[smile_threshold,0.225,0.22] buffer = plot_facial_expression_graphs(smile_ratios, ear_ratios, yawn_ratios, thresholds, 'path_to_save_plot.pdf') # print("detect_eyes : ",detect_eyes_in_faces(faces)) y_vals = [valence_list, arousal_list, stress_list,eyebrow_dist,pitches] labels = ['Valence', 'Arousal', 'Stress',"EyeBrowDistance","Pitch"] buf=plot_graph(timestamps, y_vals, labels, valence_plot) print('graph_plotted') meta_data={} meta_data['duration']=duration meta_data['facial_emotion_recognition'] = { "class_wise_frame_count": class_wise_frame_count, } meta_data['audio']=af make_pdf(pdf_filename,meta_data,buf,buffer) with open(meta_data_path, 'w') as json_file: json.dump(meta_data, json_file, indent=4) df=pd.DataFrame( { 'timestamps':timestamps, 'fer': fer_emotions, 'valence': valence_list, 'arousal': arousal_list, 'stress': stress_list, 'eyebrow':eyebrow_dist, } ) df.to_csv(df_path,index=False) except Exception as e: print("Error analyzing video...: ", e) # analyze_live_video('s1.mp4','1',1,1,True,print)