Update commonsenseConstraint.py

commonsenseConstraint.py

@@ -545,191 +545,3 @@ def boolean_evaluation(query_data, tested_data):
             print(return_info[key][1])
             return False
     return True
-
-# if __name__ == '__main__':
-#     number_list = extract_numbers_from_filenames('/home/xj/toolAugEnv/code/toolConstraint/data/annotation/lrz')
-#     # json_data = json.load(open('/home/xj/toolAugEnv/code/toolConstraint/data/annotation/x/annotation_4.json'))
-#     query_data = load_line_json_data('/home/xj/toolAugEnv/code/toolConstraint/data/query/lrz.jsonl')
-#     for idx in number_list:
-#         json_data = json.load(open(f'/home/xj/toolAugEnv/code/toolConstraint/data/annotation/lrz/annotation_{idx}.json'))
-#         print(str(idx), evaluation(query_data[idx-1], json_data))
-#     # json_data = json.load(open(f'/home/xj/toolAugEnv/code/toolConstraint/results/turbo16k-turbo16k/plan_{idx}.json'))
-#     # query_data = load_line_json_data('/home/xj/toolAugEnv/code/toolConstraint/data/query/test.jsonl')[idx-1]
-#     # help me write all function name in this file, just the name
-#     # 
-#     # list all function name in this file
-#     # ['is_reasonalbe_visiting_city', 'is_valiable_restaurants', 'is_valiable_attractions', 'is_valiable_transportation', 'is_valid_information_in_current_city', 'is_valid_information_in_sandbox']
-#     # print(is_valiable_restaurants(query_data, json_data))
-
-# if __name__ == "__main__":
-#     user = 'zk'
-#     query_data_list = load_line_json_data(f'/home/xj/toolAugEnv/code/toolConstraint/data/query/{user}.jsonl')
-#     idx_number_list = extract_numbers_from_filenames(f'/home/xj/toolAugEnv/code/toolConstraint/data/annotation/{user}')
-#     commonsense_statistic= {level:{day:[] for day in [3,5,7]} for level in ['easy','medium','hard']} 
-#     for idx in idx_number_list:
-#         print(idx)
-#         query_data = query_data_list[idx-1]
-#         generated_plan = json.load(open(f'/home/xj/toolAugEnv/code/toolConstraint/results/turbo16k-turbo16k/{user}/plan_{idx}.json'))
-#         # generated_plan = generated_plan[:-1]
-#         if generated_plan[-1]['gpt-3.5-turbo-16k-result'] != 'Plan Fail':
-#             info_box = evaluation(query_data, generated_plan[-1]['gpt-3.5-turbo-16k-result'])
-#             generated_plan[-1]['toolAug-commonsense'] = info_box
-#         else:
-#             generated_plan[-1]['toolAug-commonsense'] = None
-#             info_box = None
-#         commonsense_statistic[query_data['level']][query_data['days']].append(info_box)
-#         with open(f'/home/xj/toolAugEnv/code/toolConstraint/results/turbo16k-turbo16k/{user}/plan_{idx}.json','w') as f:
-#             json.dump(generated_plan,f)
-
-#     with open(f'/home/xj/toolAugEnv/code/toolConstraint/results/turbo16k-turbo16k/{user}/commonsense_statistic.json','w') as f:
-#         json.dump(commonsense_statistic,f)
-
-# if __name__ == "__main__":
-#     user = 'all'
-#     model_type = ['chatgpt','gpt4','greedy_search'][2]
-#     query_data_list = load_line_json_data(f'/home/xj/toolAugEnv/code/toolConstraint/data/query/{user}.jsonl')
-#     # idx_number_list = extract_numbers_from_filenames(f'/home/xj/toolAugEnv/code/toolConstraint/data/annotation/{user}')
-#     idx_number_list = [i for i in range(1,501)]
-#     commonsense_statistic= {level:{day:[] for day in [3,5,7]} for level in ['easy','medium','hard']} 
-    
-#     for idx in idx_number_list:
-#         print(idx)
-#         query_data = query_data_list[idx-1]
-#         generated_plan = json.load(open(f'/home/xj/toolAugEnv/code/toolConstraint/results/pre2/{user}/plan_{idx}.json'))
-#         # generated_plan = generated_plan[:-1]
-#         if model_type == 'greedy_search':
-#             info_box = evaluation(query_data, generated_plan[-1][f'greedy_search_plan'])
-#         else:
-#             info_box = evaluation(query_data, generated_plan[-1][f'{model_type}_human_collected_info_results_parsed'])
-#         generated_plan[-1][f'{model_type}_with_human_collected_commonsense'] = info_box
-#         commonsense_statistic[query_data['level']][query_data['days']].append(info_box)
-
-#         with open(f'/home/xj/toolAugEnv/code/toolConstraint/results/pre2/{user}/plan_{idx}.json','w') as f:
-#             json.dump(generated_plan,f)
-
-#     with open(f'/home/xj/toolAugEnv/code/toolConstraint/results/pre2/{user}/{model_type}_with_human_collected_commonsense_statistic.json','w') as f:
-#         json.dump(commonsense_statistic,f)
-
-
-# if __name__ == "__main__":
-#     user = 'all'
-#     query_data_list = load_line_json_data(f'/home/xj/toolAugEnv/code/toolConstraint/data/query/{user}.jsonl')
-#     idx_number_list = extract_numbers_from_filenames(f'/home/xj/toolAugEnv/code/toolConstraint/data/annotation/{user}')
-#     hardConstraint_statistic= {level:{day:[] for day in [3,5,7]} for level in ['easy','medium','hard']} 
-#     not_satified = []
-#     for idx in tqdm(idx_number_list):
-#         # print(idx)
-#         query_data = query_data_list[idx-1]
-#         generated_plan = json.load(open(f'/home/xj/toolAugEnv/code/toolConstraint/data/annotation/{user}/annotation_{idx}.json'))
-
-#         if not boolean_evaluation(query_data, generated_plan):
-#             not_satified.append(idx)
-#             print(idx)
-#         generated_plan = generated_plan[:-1]
-#     print(not_satified)
-
-if __name__ == "__main__":
-    set_type = ["train",'dev','test'][0]
-    query_data_list = load_line_json_data(f'/home/xj/toolAugEnv/code/toolConstraint/data/final_data/{set_type}/query/query.jsonl')
-    # idx_number_list = extract_numbers_from_filenames(f'/home/xj/toolAugEnv/code/toolConstraint/data/final_data/{set_type}/plan')
-    commonsense_statistic= {level:{day:[] for day in [3,5,7]} for level in ['easy','medium','hard']} 
-    not_satified = []
-    # print( idx_number_list)
-    for idx in tqdm(range(1,len(query_data_list)+1)):
-        # print(idx)
-        query_data = query_data_list[idx-1]
-        generated_plan = json.load(open(f'/home/xj/toolAugEnv/code/toolConstraint/data/final_data/{set_type}/plan/plan_{idx}.json'))
-        try:
-            store_plan = json.load(open(f'/home/xj/toolAugEnv/code/toolConstraint/results/{set_type}/plan_{idx}.json'))
-        except FileNotFoundError:
-            store_plan = [{}]
-        info_box = evaluation(query_data,generated_plan[1])
-        # if not boolean_evaluation(query_data, generated_plan[1]):
-        #     not_satified.append(idx)
-        #     print(idx)
-        # print(store_plan[-1])
-        store_plan[-1][f'human_anno_commonsense_constraint'] = info_box
-        with open(f'/home/xj/toolAugEnv/code/toolConstraint/results/{set_type}/plan_{idx}.json','w') as f:
-             json.dump(store_plan,f)
-        commonsense_statistic[query_data['level']][query_data['days']].append(info_box)
-    print(not_satified)
-    with open(f'/home/xj/toolAugEnv/code/toolConstraint/results/{set_type}/human_anno_commonsense_constraint.json','w') as f:
-        json.dump(commonsense_statistic,f)
-
-# if __name__ == "__main__":
-#     user = 'all'
-#     model_type = ['chatgpt','gpt4'][1]
-#     query_data_list = load_line_json_data(f'/home/xj/toolAugEnv/code/toolConstraint/data/query/{user}.jsonl')
-#     # idx_number_list = extract_numbers_from_filenames(f'/home/xj/toolAugEnv/code/toolConstraint/data/annotation/{user}')
-#     idx_number_list = [i for i in range(1,501)]
-#     commonsense_statistic= {level:{day:[] for day in [3,5,7]} for level in ['easy','medium','hard']} 
-#     cnt = 0
-#     for idx in idx_number_list:
-#         # print(idx)
-#         query_data = query_data_list[idx-1]
-#         generated_plan = json.load(open(f'/home/xj/toolAugEnv/code/toolConstraint/results/pre/{user}/plan_{idx}.json'))[-1]['gpt4_human_collected_info_results_parsed']
-#         # generated_plan = generated_plan[:-1]
-
-#         if not boolean_evaluation(query_data, generated_plan):
-#             cnt += 1
-#             print(idx)
-#     print(cnt)
-
-# if __name__ == "__main__":
-#     parser = argparse.ArgumentParser(description="")
-#     # model_type = ['gpt-3.5-turbo-1106','gpt-4-1106-preview','greedy_search','mistral-7B-32K','gemini2','mixtral','gpt-3.5-turbo-11062'][-1]
-#     # method = ['direct','cot','react','reflexion','tool-use'][-1]
-#     # set_type = ['dev','test'][0]
-#     parser.add_argument("--model_type", type=str, default="gpt-3.5-turbo-1106")
-#     parser.add_argument("--method", type=str, default="direct")
-#     parser.add_argument("--set_type", type=str, default="dev")
-#     args = parser.parse_args()
-#     directory = f'/home/xj/toolAugEnv/code/toolConstraint/data/final_data/{args.set_type}'
-#     query_data_list = load_line_json_data(os.path.join(directory, 'query/query.jsonl'))
-#     # idx_number_list = extract_numbers_from_filenames(f'/home/xj/toolAugEnv/code/toolConstraint/data/annotation/{user}')
-#     idx_number_list = [i for i in range(1,len(query_data_list)+1)]
-#     commonsense_statistic= {level:{day:[] for day in [3,5,7]} for level in ['easy','medium','hard']}
-#     deliver_cnt = 0 
-#     if args.method == 'tool-use':
-#         suffix = ''
-#     else:
-#         suffix = '_with_human_info'
-#     for idx in tqdm(idx_number_list):
-#         # print(idx)
-#         query_data = query_data_list[idx-1]
-#         generated_plan = json.load(open(f'/home/xj/toolAugEnv/code/toolConstraint/results/{args.set_type}/plan_{idx}.json'))
-#         # generated_plan = generated_plan[:-1]
-#         if args.model_type == 'greedy_search':
-#             info_box = evaluation(query_data, generated_plan[-1][f'greedy_search_plan'])
-#         else:
-#             if args.method == 'tool-use':
-#                 suffix2 = ''
-#             else:
-#                 suffix2 = '_collected'
-#             if generated_plan[-1][f'{args.model_type}_{args.method}{suffix2}_info_results'] and generated_plan[-1][f'{args.model_type}_{args.method}{suffix2}_info_results']!='Max Token Length Exceeded.':
-#                 try:
-#                     info_box = evaluation(query_data, generated_plan[-1][f'{args.model_type}_{args.method}{suffix}_results_parsed'])
-#                 except KeyError:
-#                     info_box = None
-#                     generated_plan[-1][f'{args.model_type}_{args.method}{suffix2}_info_results'] = ""
-#                 except IndexError:
-#                     info_box = None
-#                     generated_plan[-1][f'{args.model_type}_{args.method}{suffix2}_info_results'] = ""
-#             else:
-#                 info_box = None
-#         if info_box:
-#             deliver_cnt += 1
-#         generated_plan[-1][f'{args.model_type}_{args.method}{suffix}_commonsense_constraint'] = info_box
-#         commonsense_statistic[query_data['level']][query_data['days']].append(info_box)
-
-#         with open(f'/home/xj/toolAugEnv/code/toolConstraint/results/{args.set_type}/plan_{idx}.json','w') as f:
-#             json.dump(generated_plan,f)
-
-#     with open(f'/home/xj/toolAugEnv/code/toolConstraint/results/{args.set_type}/{args.model_type}_{args.method}{suffix}_commonsense_constraint.json','w') as f:
-#         json.dump(commonsense_statistic,f)
-    
-#     if args.set_type == 'dev':
-#         print(f"Model:{args.model_type} Method:{args.method} Set: {args.set_type} \nDeliver Rate: {deliver_cnt/180}" )
-#     elif args.set_type == 'test':
-#         print(f"Model:{args.model_type} Method:{args.method} Set: {args.set_type} \nDeliver Rate: {deliver_cnt/1000}" )
-