diff --git "a/Infer/phi35_general_coco_lre4_e1_general_refine_final.json" "b/Infer/phi35_general_coco_lre4_e1_general_refine_final.json" --- "a/Infer/phi35_general_coco_lre4_e1_general_refine_final.json" +++ "b/Infer/phi35_general_coco_lre4_e1_general_refine_final.json" @@ -1,4002 +1,4002 @@ [ { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.4976]\", \"lift_point\": \"[0.5565, 0.4976]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5788, 0.2193, lift_point: 0.5788, 0.2193" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5788, 0.2193]\", \"lift_point\": \"[0.5788, 0.2193]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Korean restaurants\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Search for good Korean restaurants\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good Korean restaurants\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for my laptop\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"nice wallpapers for laptop\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for laptop\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.7128]\", \"lift_point\": \"[0.6314, 0.7128]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.436]\", \"lift_point\": \"[0.123, 0.436]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1344, 0.2843, lift_point: 0.1344, 0.2843" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1344, 0.2843]\", \"lift_point\": \"[0.1344, 0.2843]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6262, 0.4764, lift_point: 0.6262, 0.4764" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6262, 0.4764]\", \"lift_point\": \"[0.6262, 0.4764]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.5129]\", \"lift_point\": \"[0.891, 0.5129]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9018, 0.494, lift_point: 0.9018, 0.494" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.494]\", \"lift_point\": \"[0.9018, 0.494]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology? \"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0466, 0.5782, lift_point: 0.0466, 0.5782" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0466, 0.5782]\", \"lift_point\": \"[0.0466, 0.5782]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6869]\", \"lift_point\": \"[0.7886, 0.6869]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7909, 0.6879, lift_point: 0.7909, 0.6879" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3115, 0.2876]\", \"lift_point\": \"[0.3115, 0.2876]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.5463]\", \"lift_point\": \"[0.0652, 0.5463]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.5492]\", \"lift_point\": \"[0.0639, 0.5492]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0518, 0.4301, lift_point: 0.0518, 0.4301" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4301]\", \"lift_point\": \"[0.0518, 0.4301]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Houston?\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0741, 0.9326, lift_point: 0.0741, 0.9326" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0741, 0.9326]\", \"lift_point\": \"[0.0741, 0.9326]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8067, 0.617]\", \"lift_point\": \"[0.8067, 0.617]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6348, 0.8499, lift_point: 0.6348, 0.8499" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Denver\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \" good restaurant in Denver?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant in Denver?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1898, 0.6857]\", \"lift_point\": \"[0.1898, 0.6857]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"How much does the HisenseTV cost?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach blanket?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"nice beach blanket?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach blanket?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1256, 0.4299]\", \"lift_point\": \"[0.1256, 0.4299]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1217, 0.3766]\", \"lift_point\": \"[0.1217, 0.3766]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.4865]\", \"lift_point\": \"[0.0638, 0.4865]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0831, 0.5901, lift_point: 0.0831, 0.5901" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0831, 0.5901]\", \"lift_point\": \"[0.0831, 0.5901]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.4459]\", \"lift_point\": \"[0.0575, 0.4459]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0505, 0.418, lift_point: 0.0505, 0.418" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.418]\", \"lift_point\": \"[0.0505, 0.418]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3757, 0.5036]\", \"lift_point\": \"[0.3757, 0.5036]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3726, 0.4941, lift_point: 0.3726, 0.4941" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3726, 0.4941]\", \"lift_point\": \"[0.3726, 0.4941]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.125, 0.4441]\", \"lift_point\": \"[0.125, 0.4441]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats a good restaurant in Chicago?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8358, 0.6103]\", \"lift_point\": \"[0.8358, 0.6103]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8732, 0.5281, lift_point: 0.8732, 0.5281" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8732, 0.5281]\", \"lift_point\": \"[0.8732, 0.5281]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5474, 0.1745]\", \"lift_point\": \"[0.5474, 0.1745]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5412, 0.1542, lift_point: 0.5412, 0.1542" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5412, 0.1542]\", \"lift_point\": \"[0.5412, 0.1542]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.4462]\", \"lift_point\": \"[0.063, 0.4462]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0613, 0.9393, lift_point: 0.0613, 0.9393" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.9393]\", \"lift_point\": \"[0.0613, 0.9393]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.6427]\", \"lift_point\": \"[0.0649, 0.6427]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0724, 0.5384, lift_point: 0.0724, 0.5384" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.5384]\", \"lift_point\": \"[0.0724, 0.5384]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3066, 0.5383]\", \"lift_point\": \"[0.3066, 0.5383]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2712, 0.3339, lift_point: 0.2712, 0.3339" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2712, 0.3339]\", \"lift_point\": \"[0.2712, 0.3339]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7989, 0.7124, lift_point: 0.7989, 0.7124" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7124]\", \"lift_point\": \"[0.7989, 0.7124]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2454, 0.408]\", \"lift_point\": \"[0.2454, 0.408]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Puerto Rico?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Puerto Rico news\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Puerto Rico news\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3321, 0.2254]\", \"lift_point\": \"[0.3321, 0.2254]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3651, 0.4097, lift_point: 0.3651, 0.4097" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.4097]\", \"lift_point\": \"[0.3651, 0.4097]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.8177]\", \"lift_point\": \"[0.5469, 0.8177]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5443, 0.8308, lift_point: 0.5443, 0.8308" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5443, 0.8308]\", \"lift_point\": \"[0.5443, 0.8308]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7609, 0.4401, lift_point: 0.7609, 0.4401" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7609, 0.4401]\", \"lift_point\": \"[0.7609, 0.4401]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from IGN?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"ign lates video\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign lates video\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3258, 0.4925]\", \"lift_point\": \"[0.3258, 0.4925]\", \"typed_text\": \"\"", - "gt": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7393, 0.3145]\", \"lift_point\": \"[0.7393, 0.3145]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7529, 0.3197, lift_point: 0.7529, 0.3197" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.3197]\", \"lift_point\": \"[0.7529, 0.3197]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the prime minister of Canada?\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6864, 0.7784, lift_point: 0.6864, 0.7784" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6864, 0.7784]\", \"lift_point\": \"[0.6864, 0.7784]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1548, 0.4823]\", \"lift_point\": \"[0.1548, 0.4823]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6132, 0.2649, lift_point: 0.6132, 0.2649" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6132, 0.2649]\", \"lift_point\": \"[0.6132, 0.2649]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.8076]\", \"lift_point\": \"[0.0582, 0.8076]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0619, 0.7812, lift_point: 0.0619, 0.7812" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.7812]\", \"lift_point\": \"[0.0619, 0.7812]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1089, 0.4993]\", \"lift_point\": \"[0.1089, 0.4993]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1045, 0.5368, lift_point: 0.1045, 0.5368" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1045, 0.5368]\", \"lift_point\": \"[0.1045, 0.5368]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4936, 0.9286, lift_point: 0.4936, 0.9286" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.285]\", \"lift_point\": \"[0.1315, 0.285]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.136, 0.3202, lift_point: 0.136, 0.3202" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.3202]\", \"lift_point\": \"[0.136, 0.3202]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.8472]\", \"lift_point\": \"[0.0729, 0.8472]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7191, 0.5092, lift_point: 0.7191, 0.5092" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7191, 0.5092]\", \"lift_point\": \"[0.7191, 0.5092]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.057, 0.5478]\", \"lift_point\": \"[0.057, 0.5478]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0484, 0.4007, lift_point: 0.0484, 0.4007" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0484, 0.4007]\", \"lift_point\": \"[0.0484, 0.4007]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0949, 0.5281]\", \"lift_point\": \"[0.0949, 0.5281]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1028, 0.4722, lift_point: 0.1028, 0.4722" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4722]\", \"lift_point\": \"[0.1028, 0.4722]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.4665]\", \"lift_point\": \"[0.0679, 0.4665]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0769, 0.9198, lift_point: 0.0769, 0.9198" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0769, 0.9198]\", \"lift_point\": \"[0.0769, 0.9198]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.459]\", \"lift_point\": \"[0.7521, 0.459]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7521, 0.4897, lift_point: 0.7521, 0.4897" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.4897]\", \"lift_point\": \"[0.7521, 0.4897]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.149, 0.8472, lift_point: 0.149, 0.8472" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.149, 0.8472]\", \"lift_point\": \"[0.149, 0.8472]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7427, 0.615]\", \"lift_point\": \"[0.7427, 0.615]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7122, 0.6291, lift_point: 0.7122, 0.6291" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7122, 0.6291]\", \"lift_point\": \"[0.7122, 0.6291]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5565, 0.9406, lift_point: 0.5565, 0.9406" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1337, 0.6943]\", \"lift_point\": \"[0.1337, 0.6943]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1432, 0.4758, lift_point: 0.1432, 0.4758" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.4758]\", \"lift_point\": \"[0.1432, 0.4758]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7444, 0.5085]\", \"lift_point\": \"[0.7444, 0.5085]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2387, 0.6137, lift_point: 0.2387, 0.6137" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.6137]\", \"lift_point\": \"[0.2387, 0.6137]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5144, 0.9441, lift_point: 0.5144, 0.9441" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.508]\", \"lift_point\": \"[0.0645, 0.508]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1525, 0.0525, lift_point: 0.1525, 0.0525" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1525, 0.0525]\", \"lift_point\": \"[0.1525, 0.0525]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1772, 0.578, lift_point: 0.1772, 0.578" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1772, 0.578]\", \"lift_point\": \"[0.1772, 0.578]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6389, 0.3453]\", \"lift_point\": \"[0.6389, 0.3453]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5905, 0.3914, lift_point: 0.5905, 0.3914" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5905, 0.3914]\", \"lift_point\": \"[0.5905, 0.3914]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4924, 0.688]\", \"lift_point\": \"[0.4924, 0.688]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4997, 0.6738, lift_point: 0.4997, 0.6738" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4997, 0.6738]\", \"lift_point\": \"[0.4997, 0.6738]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Vietnam\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"news in Vietnam?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Vietnam?\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"What are the best selling refrigerators at Home Depot\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2009, 0.2577, lift_point: 0.2009, 0.2577" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2009, 0.2577]\", \"lift_point\": \"[0.2009, 0.2577]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2344, 0.508]\", \"lift_point\": \"[0.2344, 0.508]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in cosmology\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"latest news in cosmology\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in cosmology\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5144, 0.9441, lift_point: 0.5144, 0.9441" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3375, 0.4266]\", \"lift_point\": \"[0.3375, 0.4266]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8072, 0.4337, lift_point: 0.8072, 0.4337" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8072, 0.4337]\", \"lift_point\": \"[0.8072, 0.4337]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2158, 0.1338, lift_point: 0.2158, 0.1338" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2158, 0.1338]\", \"lift_point\": \"[0.2158, 0.1338]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.129, 0.9333]\", \"lift_point\": \"[0.129, 0.9333]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0908, 0.5762, lift_point: 0.0908, 0.5762" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.5762]\", \"lift_point\": \"[0.0908, 0.5762]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Calendar app\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Calendar\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Calendar\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1258, 0.7713]\", \"lift_point\": \"[0.1258, 0.7713]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1787, 0.6526, lift_point: 0.1787, 0.6526" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.6526]\", \"lift_point\": \"[0.1787, 0.6526]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.418, 0.719, lift_point: 0.418, 0.719" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.418, 0.719]\", \"lift_point\": \"[0.418, 0.719]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2776, 0.3345]\", \"lift_point\": \"[0.2776, 0.3345]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2711, 0.4538, lift_point: 0.2711, 0.4538" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2711, 0.4538]\", \"lift_point\": \"[0.2711, 0.4538]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8868, 0.5256]\", \"lift_point\": \"[0.8868, 0.5256]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8207, 0.56, lift_point: 0.8207, 0.56" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8207, 0.56]\", \"lift_point\": \"[0.8207, 0.56]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1896, 0.3994]\", \"lift_point\": \"[0.1896, 0.3994]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2733, 0.4317, lift_point: 0.2733, 0.4317" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2733, 0.4317]\", \"lift_point\": \"[0.2733, 0.4317]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6323, 0.3296]\", \"lift_point\": \"[0.6323, 0.3296]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6357, 0.3296, lift_point: 0.6357, 0.3296" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6357, 0.3296]\", \"lift_point\": \"[0.6357, 0.3296]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.4649]\", \"lift_point\": \"[0.0644, 0.4649]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0606, 0.5044, lift_point: 0.0606, 0.5044" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.5044]\", \"lift_point\": \"[0.0606, 0.5044]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Mexico?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"What is the capital of Mexico\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Mexico\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2868, 0.5126]\", \"lift_point\": \"[0.2868, 0.5126]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2741, 0.5065, lift_point: 0.2741, 0.5065" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2741, 0.5065]\", \"lift_point\": \"[0.2741, 0.5065]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.4743]\", \"lift_point\": \"[0.061, 0.4743]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats the weather like in Mumbai?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mumbai?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.0644]\", \"lift_point\": \"[0.0649, 0.0644]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1904, 0.5908, lift_point: 0.1904, 0.5908" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1904, 0.5908]\", \"lift_point\": \"[0.1904, 0.5908]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.6167]\", \"lift_point\": \"[0.8002, 0.6167]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7981, 0.6113, lift_point: 0.7981, 0.6113" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3464, 0.7445]\", \"lift_point\": \"[0.3464, 0.7445]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5192, 0.696, lift_point: 0.5192, 0.696" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5192, 0.696]\", \"lift_point\": \"[0.5192, 0.696]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7787, 0.6859]\", \"lift_point\": \"[0.7787, 0.6859]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7849, 0.6884, lift_point: 0.7849, 0.6884" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6884]\", \"lift_point\": \"[0.7849, 0.6884]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.4453]\", \"lift_point\": \"[0.1318, 0.4453]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.5317]\", \"lift_point\": \"[0.0612, 0.5317]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.048, 0.8925, lift_point: 0.048, 0.8925" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.8925]\", \"lift_point\": \"[0.048, 0.8925]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.052]\", \"lift_point\": \"[0.0549, 0.052]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0515, 0.0547, lift_point: 0.0515, 0.0547" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0547]\", \"lift_point\": \"[0.0515, 0.0547]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5687, 0.5023, lift_point: 0.5687, 0.5023" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5687, 0.5023]\", \"lift_point\": \"[0.5687, 0.5023]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1274, 0.8517]\", \"lift_point\": \"[0.1274, 0.8517]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.5047]\", \"lift_point\": \"[0.0628, 0.5047]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.035, 0.6353, lift_point: 0.035, 0.6353" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.035, 0.6353]\", \"lift_point\": \"[0.035, 0.6353]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8956, 0.3634]\", \"lift_point\": \"[0.8956, 0.3634]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6372, 0.1618]\", \"lift_point\": \"[0.6372, 0.1618]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9114, 0.3703, lift_point: 0.9114, 0.3703" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9114, 0.3703]\", \"lift_point\": \"[0.9114, 0.3703]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.882, 0.6251, lift_point: 0.882, 0.6251" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.6251]\", \"lift_point\": \"[0.882, 0.6251]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2293, 0.4964]\", \"lift_point\": \"[0.2293, 0.4964]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2253, 0.6068, lift_point: 0.2253, 0.6068" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2253, 0.6068]\", \"lift_point\": \"[0.2253, 0.6068]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1436, 0.3267, lift_point: 0.1436, 0.3267" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.3267]\", \"lift_point\": \"[0.1436, 0.3267]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0681, 0.4349]\", \"lift_point\": \"[0.0681, 0.4349]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0529, 0.3639, lift_point: 0.0529, 0.3639" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.3639]\", \"lift_point\": \"[0.0529, 0.3639]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3087, 0.2179]\", \"lift_point\": \"[0.3087, 0.2179]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of canada\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"What is the capital of Canada\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Canada\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2865, 0.5043]\", \"lift_point\": \"[0.2865, 0.5043]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2245, 0.6318, lift_point: 0.2245, 0.6318" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2245, 0.6318]\", \"lift_point\": \"[0.2245, 0.6318]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"hotels in Philadelphia\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0767, 0.4565]\", \"lift_point\": \"[0.0767, 0.4565]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0754, 0.5816, lift_point: 0.0754, 0.5816" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0754, 0.5816]\", \"lift_point\": \"[0.0754, 0.5816]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8715, 0.6588]\", \"lift_point\": \"[0.8715, 0.6588]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4388, 0.4604, lift_point: 0.4388, 0.4604" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4388, 0.4604]\", \"lift_point\": \"[0.4388, 0.4604]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1262, 0.3662]\", \"lift_point\": \"[0.1262, 0.3662]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1215, 0.2697, lift_point: 0.1215, 0.2697" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.2697]\", \"lift_point\": \"[0.1215, 0.2697]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"capital of Paraguay\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7369, 0.622]\", \"lift_point\": \"[0.7369, 0.622]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7345, 0.6311, lift_point: 0.7345, 0.6311" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.6311]\", \"lift_point\": \"[0.7345, 0.6311]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1059, 0.5084]\", \"lift_point\": \"[0.1059, 0.5084]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2742, 0.3428, lift_point: 0.2742, 0.3428" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2742, 0.3428]\", \"lift_point\": \"[0.2742, 0.3428]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1544, 0.0272, lift_point: 0.1544, 0.0272" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1544, 0.0272]\", \"lift_point\": \"[0.1544, 0.0272]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0629, 0.7847]\", \"lift_point\": \"[0.0629, 0.7847]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0712, 0.6123, lift_point: 0.0712, 0.6123" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.6123]\", \"lift_point\": \"[0.0712, 0.6123]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.5082]\", \"lift_point\": \"[0.8864, 0.5082]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8897, 0.2232, lift_point: 0.8897, 0.2232" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8897, 0.2232]\", \"lift_point\": \"[0.8897, 0.2232]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in new york\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"What time is it in New York?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1286, 0.4124]\", \"lift_point\": \"[0.1286, 0.4124]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1288, 0.7106, lift_point: 0.1288, 0.7106" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.7106]\", \"lift_point\": \"[0.1288, 0.7106]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats the news in India?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.429, 0.922, lift_point: 0.429, 0.922" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.4139]\", \"lift_point\": \"[0.1306, 0.4139]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1841, 0.5421, lift_point: 0.1841, 0.5421" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1841, 0.5421]\", \"lift_point\": \"[0.1841, 0.5421]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3356, 0.4963]\", \"lift_point\": \"[0.3356, 0.4963]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3217, 0.4698, lift_point: 0.3217, 0.4698" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3217, 0.4698]\", \"lift_point\": \"[0.3217, 0.4698]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6923]\", \"lift_point\": \"[0.7944, 0.6923]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7904, 0.6969]\", \"lift_point\": \"[0.7904, 0.6969]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3964, 0.5443, lift_point: 0.3964, 0.5443" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3964, 0.5443]\", \"lift_point\": \"[0.3964, 0.5443]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5565, 0.9406, lift_point: 0.5565, 0.9406" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0666, 0.363, lift_point: 0.0666, 0.363" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0666, 0.363]\", \"lift_point\": \"[0.0666, 0.363]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.0749]\", \"lift_point\": \"[0.0655, 0.0749]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0567, 0.7902, lift_point: 0.0567, 0.7902" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.7902]\", \"lift_point\": \"[0.0567, 0.7902]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0658, 0.4786]\", \"lift_point\": \"[0.0658, 0.4786]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.062, 0.3378, lift_point: 0.062, 0.3378" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3378]\", \"lift_point\": \"[0.062, 0.3378]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.078, 0.8768, lift_point: 0.078, 0.8768" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1397, 0.3945]\", \"lift_point\": \"[0.1397, 0.3945]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1365, 0.4779]\", \"lift_point\": \"[0.1365, 0.4779]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2061, 0.4438, lift_point: 0.2061, 0.4438" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2061, 0.4438]\", \"lift_point\": \"[0.2061, 0.4438]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2779, 0.2959]\", \"lift_point\": \"[0.2779, 0.2959]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3067, 0.2148, lift_point: 0.3067, 0.2148" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3067, 0.2148]\", \"lift_point\": \"[0.3067, 0.2148]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1386, 0.5999]\", \"lift_point\": \"[0.1386, 0.5999]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1419, 0.6285, lift_point: 0.1419, 0.6285" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.6285]\", \"lift_point\": \"[0.1419, 0.6285]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0852, 0.3729, lift_point: 0.0852, 0.3729" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0852, 0.3729]\", \"lift_point\": \"[0.0852, 0.3729]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.387]\", \"lift_point\": \"[0.117, 0.387]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1159, 0.3367, lift_point: 0.1159, 0.3367" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.3367]\", \"lift_point\": \"[0.1159, 0.3367]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0526, 0.4695]\", \"lift_point\": \"[0.0526, 0.4695]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0661, 0.5783, lift_point: 0.0661, 0.5783" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0661, 0.5783]\", \"lift_point\": \"[0.0661, 0.5783]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0611, 0.439, lift_point: 0.0611, 0.439" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0611, 0.439]\", \"lift_point\": \"[0.0611, 0.439]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in delhi\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"weather like in Delhi\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Delhi\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats the news in Barbados?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6461, 0.9417, lift_point: 0.6461, 0.9417" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.9417]\", \"lift_point\": \"[0.6461, 0.9417]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"flights from NYC to Buenos aires\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7954, 0.6954]\", \"lift_point\": \"[0.7954, 0.6954]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7998, 0.6996, lift_point: 0.7998, 0.6996" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6996]\", \"lift_point\": \"[0.7998, 0.6996]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8897, 0.5134]\", \"lift_point\": \"[0.8897, 0.5134]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9078, 0.4758, lift_point: 0.9078, 0.4758" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9078, 0.4758]\", \"lift_point\": \"[0.9078, 0.4758]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest walmart\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"nearest walmart\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest walmart\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7977, 0.6952]\", \"lift_point\": \"[0.7977, 0.6952]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7834, 0.698, lift_point: 0.7834, 0.698" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7834, 0.698]\", \"lift_point\": \"[0.7834, 0.698]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2424, 0.2047]\", \"lift_point\": \"[0.2424, 0.2047]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2755, 0.2146, lift_point: 0.2755, 0.2146" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2755, 0.2146]\", \"lift_point\": \"[0.2755, 0.2146]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6167]\", \"lift_point\": \"[0.7859, 0.6167]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7798, 0.5928, lift_point: 0.7798, 0.5928" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.5928]\", \"lift_point\": \"[0.7798, 0.5928]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3498, 0.3592, lift_point: 0.3498, 0.3592" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.3592]\", \"lift_point\": \"[0.3498, 0.3592]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3252, 0.4976]\", \"lift_point\": \"[0.3252, 0.4976]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5058, 0.6866, lift_point: 0.5058, 0.6866" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.6866]\", \"lift_point\": \"[0.5058, 0.6866]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1176, 0.2434]\", \"lift_point\": \"[0.1176, 0.2434]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1283, 0.6011, lift_point: 0.1283, 0.6011" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.6011]\", \"lift_point\": \"[0.1283, 0.6011]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.5268]\", \"lift_point\": \"[0.5744, 0.5268]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1069, 0.3495]\", \"lift_point\": \"[0.1069, 0.3495]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1115, 0.2639, lift_point: 0.1115, 0.2639" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.2639]\", \"lift_point\": \"[0.1115, 0.2639]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7981, 0.6113, lift_point: 0.7981, 0.6113" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3298, 0.4224]\", \"lift_point\": \"[0.3298, 0.4224]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3223, 0.42, lift_point: 0.3223, 0.42" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3223, 0.42]\", \"lift_point\": \"[0.3223, 0.42]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5047, 0.6838]\", \"lift_point\": \"[0.5047, 0.6838]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3652, 0.5103, lift_point: 0.3652, 0.5103" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3652, 0.5103]\", \"lift_point\": \"[0.3652, 0.5103]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1237, 0.3175]\", \"lift_point\": \"[0.1237, 0.3175]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7048, 0.6341, lift_point: 0.7048, 0.6341" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7048, 0.6341]\", \"lift_point\": \"[0.7048, 0.6341]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7891, 0.6835, lift_point: 0.7891, 0.6835" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for the best Mexican restaurants\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0472, 0.9329, lift_point: 0.0472, 0.9329" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.9329]\", \"lift_point\": \"[0.0472, 0.9329]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6861]\", \"lift_point\": \"[0.7768, 0.6861]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7751, 0.6872, lift_point: 0.7751, 0.6872" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7751, 0.6872]\", \"lift_point\": \"[0.7751, 0.6872]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4488, 0.5619, lift_point: 0.4488, 0.5619" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4488, 0.5619]\", \"lift_point\": \"[0.4488, 0.5619]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good pizza restaurants\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"good pizza restaurants\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good pizza restaurants\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7166, 0.9415]\", \"lift_point\": \"[0.7166, 0.9415]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6006, 0.4735, lift_point: 0.6006, 0.4735" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6006, 0.4735]\", \"lift_point\": \"[0.6006, 0.4735]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1328, 0.8076]\", \"lift_point\": \"[0.1328, 0.8076]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1975, 0.5157, lift_point: 0.1975, 0.5157" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1975, 0.5157]\", \"lift_point\": \"[0.1975, 0.5157]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7795, 0.6913]\", \"lift_point\": \"[0.7795, 0.6913]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.774, 0.6905, lift_point: 0.774, 0.6905" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6349, 0.1767]\", \"lift_point\": \"[0.6349, 0.1767]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4735, 0.8521, lift_point: 0.4735, 0.8521" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8521]\", \"lift_point\": \"[0.4735, 0.8521]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8895, 0.4724]\", \"lift_point\": \"[0.8895, 0.4724]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8872, 0.2415, lift_point: 0.8872, 0.2415" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.2415]\", \"lift_point\": \"[0.8872, 0.2415]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8171, 0.6154]\", \"lift_point\": \"[0.8171, 0.6154]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5185, 0.6138, lift_point: 0.5185, 0.6138" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5185, 0.6138]\", \"lift_point\": \"[0.5185, 0.6138]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.9348]\", \"lift_point\": \"[0.0594, 0.9348]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0583, 0.9216, lift_point: 0.0583, 0.9216" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.9216]\", \"lift_point\": \"[0.0583, 0.9216]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1354, 0.3584]\", \"lift_point\": \"[0.1354, 0.3584]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"lipstick\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lipstick\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"How big is a dinosaur?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1077, 0.5283]\", \"lift_point\": \"[0.1077, 0.5283]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0983, 0.4783, lift_point: 0.0983, 0.4783" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0983, 0.4783]\", \"lift_point\": \"[0.0983, 0.4783]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0669, 0.4072]\", \"lift_point\": \"[0.0669, 0.4072]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0648, 0.5254, lift_point: 0.0648, 0.5254" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0648, 0.5254]\", \"lift_point\": \"[0.0648, 0.5254]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1367, 0.295]\", \"lift_point\": \"[0.1367, 0.295]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1305, 0.3664, lift_point: 0.1305, 0.3664" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1305, 0.3664]\", \"lift_point\": \"[0.1305, 0.3664]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4966, 0.621]\", \"lift_point\": \"[0.4966, 0.621]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4836, 0.8681, lift_point: 0.4836, 0.8681" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4836, 0.8681]\", \"lift_point\": \"[0.4836, 0.8681]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.898, 0.5161]\", \"lift_point\": \"[0.898, 0.5161]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8698, 0.5392, lift_point: 0.8698, 0.5392" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8698, 0.5392]\", \"lift_point\": \"[0.8698, 0.5392]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3567, 0.5044]\", \"lift_point\": \"[0.3567, 0.5044]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3531, 0.4998, lift_point: 0.3531, 0.4998" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.4998]\", \"lift_point\": \"[0.3531, 0.4998]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.879, 0.497]\", \"lift_point\": \"[0.879, 0.497]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7828, 0.8909, lift_point: 0.7828, 0.8909" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.8909]\", \"lift_point\": \"[0.7828, 0.8909]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2763, 0.3775]\", \"lift_point\": \"[0.2763, 0.3775]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3034, 0.3769, lift_point: 0.3034, 0.3769" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.3769]\", \"lift_point\": \"[0.3034, 0.3769]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6348, 0.8499, lift_point: 0.6348, 0.8499" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.4994]\", \"lift_point\": \"[0.0563, 0.4994]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0488, 0.5993, lift_point: 0.0488, 0.5993" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0488, 0.5993]\", \"lift_point\": \"[0.0488, 0.5993]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1243, 0.3053]\", \"lift_point\": \"[0.1243, 0.3053]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1278, 0.3068, lift_point: 0.1278, 0.3068" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.3068]\", \"lift_point\": \"[0.1278, 0.3068]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4842]\", \"lift_point\": \"[0.0578, 0.4842]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2971, 0.4607, lift_point: 0.2971, 0.4607" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2971, 0.4607]\", \"lift_point\": \"[0.2971, 0.4607]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.3426]\", \"lift_point\": \"[0.1238, 0.3426]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1347, 0.4935, lift_point: 0.1347, 0.4935" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.4935]\", \"lift_point\": \"[0.1347, 0.4935]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of brazil\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"What is the capital of Brazil?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brazil?\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Seoul\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"flights from Helsinki to Seoul\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Seoul\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7771, 0.6929]\", \"lift_point\": \"[0.7771, 0.6929]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7789, 0.6906, lift_point: 0.7789, 0.6906" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6906]\", \"lift_point\": \"[0.7789, 0.6906]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5144, 0.9441, lift_point: 0.5144, 0.9441" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Seoul to Barcelona\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0881, 0.9045, lift_point: 0.0881, 0.9045" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0881, 0.9045]\", \"lift_point\": \"[0.0881, 0.9045]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.9039]\", \"lift_point\": \"[0.0697, 0.9039]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0591, 0.9085, lift_point: 0.0591, 0.9085" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9085]\", \"lift_point\": \"[0.0591, 0.9085]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.313, 0.3787]\", \"lift_point\": \"[0.313, 0.3787]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3837, 0.3738, lift_point: 0.3837, 0.3738" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3837, 0.3738]\", \"lift_point\": \"[0.3837, 0.3738]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.1001]\", \"lift_point\": \"[0.136, 0.1001]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1358, 0.1133, lift_point: 0.1358, 0.1133" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1358, 0.1133]\", \"lift_point\": \"[0.1358, 0.1133]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0629, 0.4787]\", \"lift_point\": \"[0.0629, 0.4787]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0534, 0.5295, lift_point: 0.0534, 0.5295" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.5295]\", \"lift_point\": \"[0.0534, 0.5295]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3683, 0.9673, lift_point: 0.3683, 0.9673" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2094, 0.8079]\", \"lift_point\": \"[0.2094, 0.8079]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0818, 0.0549, lift_point: 0.0818, 0.0549" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.0549]\", \"lift_point\": \"[0.0818, 0.0549]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5701, 0.7052, lift_point: 0.5701, 0.7052" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5701, 0.7052]\", \"lift_point\": \"[0.5701, 0.7052]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8982, 0.7229, lift_point: 0.8982, 0.7229" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8982, 0.7229]\", \"lift_point\": \"[0.8982, 0.7229]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.897, 0.501]\", \"lift_point\": \"[0.897, 0.501]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9056, 0.518, lift_point: 0.9056, 0.518" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.518]\", \"lift_point\": \"[0.9056, 0.518]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1311, 0.6471]\", \"lift_point\": \"[0.1311, 0.6471]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1398, 0.5773, lift_point: 0.1398, 0.5773" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.5773]\", \"lift_point\": \"[0.1398, 0.5773]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.5198]\", \"lift_point\": \"[0.0579, 0.5198]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0553, 0.4296, lift_point: 0.0553, 0.4296" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.4296]\", \"lift_point\": \"[0.0553, 0.4296]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in India\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats the news in India?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in San Francisco\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"2 bedroom apartment rent in san francisco\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent in san francisco\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8863, 0.4885]\", \"lift_point\": \"[0.8863, 0.4885]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.884, 0.53, lift_point: 0.884, 0.53" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.884, 0.53]\", \"lift_point\": \"[0.884, 0.53]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3657, 0.8723]\", \"lift_point\": \"[0.3657, 0.8723]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3585, 0.6635, lift_point: 0.3585, 0.6635" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3585, 0.6635]\", \"lift_point\": \"[0.3585, 0.6635]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.678]\", \"lift_point\": \"[0.3718, 0.678]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1492, 0.2963, lift_point: 0.1492, 0.2963" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1492, 0.2963]\", \"lift_point\": \"[0.1492, 0.2963]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from GameSpot Reviews\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0634, 0.3804, lift_point: 0.0634, 0.3804" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3804]\", \"lift_point\": \"[0.0634, 0.3804]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7981, 0.6113, lift_point: 0.7981, 0.6113" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.9345]\", \"lift_point\": \"[0.0557, 0.9345]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"capital of Paraguay\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3725, 0.5031]\", \"lift_point\": \"[0.3725, 0.5031]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3702, 0.4935, lift_point: 0.3702, 0.4935" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3702, 0.4935]\", \"lift_point\": \"[0.3702, 0.4935]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.5249]\", \"lift_point\": \"[0.0639, 0.5249]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0553, 0.498, lift_point: 0.0553, 0.498" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.498]\", \"lift_point\": \"[0.0553, 0.498]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7827, 0.6728]\", \"lift_point\": \"[0.7827, 0.6728]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.893, 0.3801, lift_point: 0.893, 0.3801" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.893, 0.3801]\", \"lift_point\": \"[0.893, 0.3801]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Chinese restaurants\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"good chinese restaurant \"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good chinese restaurant \"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.8444]\", \"lift_point\": \"[0.0576, 0.8444]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0688, 0.7975, lift_point: 0.0688, 0.7975" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.7975]\", \"lift_point\": \"[0.0688, 0.7975]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.9234]\", \"lift_point\": \"[0.0588, 0.9234]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0551, 0.9139, lift_point: 0.0551, 0.9139" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9139]\", \"lift_point\": \"[0.0551, 0.9139]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6248, 0.1787]\", \"lift_point\": \"[0.6248, 0.1787]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3498, 0.5828, lift_point: 0.3498, 0.5828" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.5828]\", \"lift_point\": \"[0.3498, 0.5828]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8316, 0.5064]\", \"lift_point\": \"[0.8316, 0.5064]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5938, 0.718, lift_point: 0.5938, 0.718" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5938, 0.718]\", \"lift_point\": \"[0.5938, 0.718]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5469, 0.3295, lift_point: 0.5469, 0.3295" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.3295]\", \"lift_point\": \"[0.5469, 0.3295]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar?\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0664, 0.2897, lift_point: 0.0664, 0.2897" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.2897]\", \"lift_point\": \"[0.0664, 0.2897]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.6765]\", \"lift_point\": \"[0.1189, 0.6765]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.126, 0.5796, lift_point: 0.126, 0.5796" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.126, 0.5796]\", \"lift_point\": \"[0.126, 0.5796]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.5046]\", \"lift_point\": \"[0.0618, 0.5046]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0645, 0.5103, lift_point: 0.0645, 0.5103" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.5103]\", \"lift_point\": \"[0.0645, 0.5103]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.0727]\", \"lift_point\": \"[0.0594, 0.0727]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0551, 0.0814, lift_point: 0.0551, 0.0814" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.0814]\", \"lift_point\": \"[0.0551, 0.0814]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Houston?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats a good restaurant in Houston?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Houston?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.4341]\", \"lift_point\": \"[0.0591, 0.4341]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8882, 0.509]\", \"lift_point\": \"[0.8882, 0.509]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0491, 0.9077, lift_point: 0.0491, 0.9077" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0491, 0.9077]\", \"lift_point\": \"[0.0491, 0.9077]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6292, 0.6956]\", \"lift_point\": \"[0.6292, 0.6956]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5841, 0.1064, lift_point: 0.5841, 0.1064" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.1064]\", \"lift_point\": \"[0.5841, 0.1064]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7915, 0.7148, lift_point: 0.7915, 0.7148" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"What is the speed of light\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.5423]\", \"lift_point\": \"[0.0581, 0.5423]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.064, 0.3033, lift_point: 0.064, 0.3033" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.3033]\", \"lift_point\": \"[0.064, 0.3033]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5845, 0.5072]\", \"lift_point\": \"[0.5845, 0.5072]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7261, 0.2077, lift_point: 0.7261, 0.2077" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7261, 0.2077]\", \"lift_point\": \"[0.7261, 0.2077]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.4748]\", \"lift_point\": \"[0.8925, 0.4748]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8012, 0.6957, lift_point: 0.8012, 0.6957" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8012, 0.6957]\", \"lift_point\": \"[0.8012, 0.6957]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1262, 0.4275]\", \"lift_point\": \"[0.1262, 0.4275]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1319, 0.4011, lift_point: 0.1319, 0.4011" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.4011]\", \"lift_point\": \"[0.1319, 0.4011]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.8475]\", \"lift_point\": \"[0.0651, 0.8475]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0679, 0.4665, lift_point: 0.0679, 0.4665" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.4665]\", \"lift_point\": \"[0.0679, 0.4665]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6203]\", \"lift_point\": \"[0.7948, 0.6203]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8252, 0.6178, lift_point: 0.8252, 0.6178" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.6178]\", \"lift_point\": \"[0.8252, 0.6178]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.5754]\", \"lift_point\": \"[0.0585, 0.5754]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0431, 0.5396, lift_point: 0.0431, 0.5396" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0431, 0.5396]\", \"lift_point\": \"[0.0431, 0.5396]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Zurich\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"hotels in Zurich\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Zurich\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7972, 0.6824]\", \"lift_point\": \"[0.7972, 0.6824]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7985, 0.6824, lift_point: 0.7985, 0.6824" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7985, 0.6824]\", \"lift_point\": \"[0.7985, 0.6824]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1331, 0.3845]\", \"lift_point\": \"[0.1331, 0.3845]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1377, 0.363]\", \"lift_point\": \"[0.1377, 0.363]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new lipstick on Sephora\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"sephora\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at McDonalds?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"menu at McDonalds?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8917, 0.4429]\", \"lift_point\": \"[0.8917, 0.4429]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7768, 0.7205, lift_point: 0.7768, 0.7205" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.7205]\", \"lift_point\": \"[0.7768, 0.7205]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5857, 0.8448]\", \"lift_point\": \"[0.5857, 0.8448]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5415, 0.6211, lift_point: 0.5415, 0.6211" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5415, 0.6211]\", \"lift_point\": \"[0.5415, 0.6211]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.3499]\", \"lift_point\": \"[0.1412, 0.3499]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1477, 0.5548, lift_point: 0.1477, 0.5548" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1477, 0.5548]\", \"lift_point\": \"[0.1477, 0.5548]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8933, 0.3738, lift_point: 0.8933, 0.3738" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8933, 0.3738]\", \"lift_point\": \"[0.8933, 0.3738]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Paris\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"flights from NYC to Paris\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Paris\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6277, 0.698]\", \"lift_point\": \"[0.6277, 0.698]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6352, 0.6476, lift_point: 0.6352, 0.6476" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6476]\", \"lift_point\": \"[0.6352, 0.6476]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8986, 0.5991]\", \"lift_point\": \"[0.8986, 0.5991]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7462, 0.5963, lift_point: 0.7462, 0.5963" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7462, 0.5963]\", \"lift_point\": \"[0.7462, 0.5963]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2091, 0.6976]\", \"lift_point\": \"[0.2091, 0.6976]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1698, 0.5617, lift_point: 0.1698, 0.5617" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1698, 0.5617]\", \"lift_point\": \"[0.1698, 0.5617]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5105, 0.493]\", \"lift_point\": \"[0.5105, 0.493]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5113, 0.4446, lift_point: 0.5113, 0.4446" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5113, 0.4446]\", \"lift_point\": \"[0.5113, 0.4446]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1522, 0.3142]\", \"lift_point\": \"[0.1522, 0.3142]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3545, 0.5048, lift_point: 0.3545, 0.5048" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3545, 0.5048]\", \"lift_point\": \"[0.3545, 0.5048]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"new iPad cost\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3447, 0.4696]\", \"lift_point\": \"[0.3447, 0.4696]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3899, 0.2991, lift_point: 0.3899, 0.2991" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3899, 0.2991]\", \"lift_point\": \"[0.3899, 0.2991]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter app\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"twitter\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3275, 0.6995]\", \"lift_point\": \"[0.3275, 0.6995]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1216, 0.8219]\", \"lift_point\": \"[0.1216, 0.8219]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0669, 0.0763]\", \"lift_point\": \"[0.0669, 0.0763]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8455, 0.5367]\", \"lift_point\": \"[0.8455, 0.5367]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8673, 0.497, lift_point: 0.8673, 0.497" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8673, 0.497]\", \"lift_point\": \"[0.8673, 0.497]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.8056]\", \"lift_point\": \"[0.0653, 0.8056]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6396, 0.4336, lift_point: 0.6396, 0.4336" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6396, 0.4336]\", \"lift_point\": \"[0.6396, 0.4336]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8145, 0.6085, lift_point: 0.8145, 0.6085" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3748, 0.9512, lift_point: 0.3748, 0.9512" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.5328]\", \"lift_point\": \"[0.1412, 0.5328]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1352, 0.5249, lift_point: 0.1352, 0.5249" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5249]\", \"lift_point\": \"[0.1352, 0.5249]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3358, 0.8682]\", \"lift_point\": \"[0.3358, 0.8682]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.332, 0.86, lift_point: 0.332, 0.86" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.86]\", \"lift_point\": \"[0.332, 0.86]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space technology\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"latest news in space technology?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space technology?\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at IHOP\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"menu on ihop\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu on ihop\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.4067]\", \"lift_point\": \"[0.0587, 0.4067]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1643, 0.4469]\", \"lift_point\": \"[0.1643, 0.4469]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9164]\", \"lift_point\": \"[0.0638, 0.9164]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.06, 0.9095, lift_point: 0.06, 0.9095" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.06, 0.9095]\", \"lift_point\": \"[0.06, 0.9095]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_BACK#" + "gt": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2067, 0.7565]\", \"lift_point\": \"[0.2067, 0.7565]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1823, 0.9005, lift_point: 0.1823, 0.9005" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.9005]\", \"lift_point\": \"[0.1823, 0.9005]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Bahamas?\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0436, 0.3128, lift_point: 0.0436, 0.3128" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0436, 0.3128]\", \"lift_point\": \"[0.0436, 0.3128]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Barbados\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"news in barbados\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in barbados\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0523, 0.4996]\", \"lift_point\": \"[0.0523, 0.4996]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1742, 0.6779, lift_point: 0.1742, 0.6779" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1742, 0.6779]\", \"lift_point\": \"[0.1742, 0.6779]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1387, 0.4999]\", \"lift_point\": \"[0.1387, 0.4999]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1415, 0.5113, lift_point: 0.1415, 0.5113" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1415, 0.5113]\", \"lift_point\": \"[0.1415, 0.5113]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0658, 0.9189]\", \"lift_point\": \"[0.0658, 0.9189]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0654, 0.9074, lift_point: 0.0654, 0.9074" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.9074]\", \"lift_point\": \"[0.0654, 0.9074]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.113, 0.6793, lift_point: 0.113, 0.6793" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.6793]\", \"lift_point\": \"[0.113, 0.6793]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6657, 0.1264]\", \"lift_point\": \"[0.6657, 0.1264]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7727, 0.8562, lift_point: 0.7727, 0.8562" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7727, 0.8562]\", \"lift_point\": \"[0.7727, 0.8562]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1188, 0.4086]\", \"lift_point\": \"[0.1188, 0.4086]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1161, 0.481, lift_point: 0.1161, 0.481" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.481]\", \"lift_point\": \"[0.1161, 0.481]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7831, 0.6797]\", \"lift_point\": \"[0.7831, 0.6797]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7663, 0.7143, lift_point: 0.7663, 0.7143" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7663, 0.7143]\", \"lift_point\": \"[0.7663, 0.7143]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3582, 0.2986]\", \"lift_point\": \"[0.3582, 0.2986]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.411, 0.1532, lift_point: 0.411, 0.1532" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.411, 0.1532]\", \"lift_point\": \"[0.411, 0.1532]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6626, 0.7481]\", \"lift_point\": \"[0.6626, 0.7481]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6771, 0.7448, lift_point: 0.6771, 0.7448" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6771, 0.7448]\", \"lift_point\": \"[0.6771, 0.7448]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.5574]\", \"lift_point\": \"[0.0645, 0.5574]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0471, 0.4793, lift_point: 0.0471, 0.4793" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0471, 0.4793]\", \"lift_point\": \"[0.0471, 0.4793]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2745, 0.4277]\", \"lift_point\": \"[0.2745, 0.4277]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3341, 0.3463, lift_point: 0.3341, 0.3463" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3341, 0.3463]\", \"lift_point\": \"[0.3341, 0.3463]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.325, 0.2933, lift_point: 0.325, 0.2933" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.325, 0.2933]\", \"lift_point\": \"[0.325, 0.2933]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2735, 0.9223, lift_point: 0.2735, 0.9223" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.9223]\", \"lift_point\": \"[0.2735, 0.9223]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami?\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0563, 0.4573, lift_point: 0.0563, 0.4573" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.4573]\", \"lift_point\": \"[0.0563, 0.4573]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2654, 0.3563]\", \"lift_point\": \"[0.2654, 0.3563]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0681, 0.1163, lift_point: 0.0681, 0.1163" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0681, 0.1163]\", \"lift_point\": \"[0.0681, 0.1163]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1797, 0.8461]\", \"lift_point\": \"[0.1797, 0.8461]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6048, 0.3794, lift_point: 0.6048, 0.3794" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6048, 0.3794]\", \"lift_point\": \"[0.6048, 0.3794]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.4961]\", \"lift_point\": \"[0.0614, 0.4961]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0574, 0.4573, lift_point: 0.0574, 0.4573" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.4573]\", \"lift_point\": \"[0.0574, 0.4573]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4335, 0.8971, lift_point: 0.4335, 0.8971" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4335, 0.8971]\", \"lift_point\": \"[0.4335, 0.8971]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3954, 0.4839, lift_point: 0.3954, 0.4839" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3954, 0.4839]\", \"lift_point\": \"[0.3954, 0.4839]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the president of the United States?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \" president of the United States\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" president of the United States\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.257, 0.3947]\", \"lift_point\": \"[0.257, 0.3947]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3276, 0.2949, lift_point: 0.3276, 0.2949" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3276, 0.2949]\", \"lift_point\": \"[0.3276, 0.2949]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1142, 0.5269]\", \"lift_point\": \"[0.1142, 0.5269]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1063, 0.5713, lift_point: 0.1063, 0.5713" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.5713]\", \"lift_point\": \"[0.1063, 0.5713]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8983, 0.4957]\", \"lift_point\": \"[0.8983, 0.4957]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8713, 0.5122, lift_point: 0.8713, 0.5122" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8713, 0.5122]\", \"lift_point\": \"[0.8713, 0.5122]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"the nearest grocery store \"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the nearest grocery store \"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.4667]\", \"lift_point\": \"[0.0615, 0.4667]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0662, 0.3991, lift_point: 0.0662, 0.3991" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.3991]\", \"lift_point\": \"[0.0662, 0.3991]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Italy\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1161, 0.279, lift_point: 0.1161, 0.279" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.279]\", \"lift_point\": \"[0.1161, 0.279]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1945, 0.5838]\", \"lift_point\": \"[0.1945, 0.5838]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6762]\", \"lift_point\": \"[0.7851, 0.6762]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7716, 0.67, lift_point: 0.7716, 0.67" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7716, 0.67]\", \"lift_point\": \"[0.7716, 0.67]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1285, 0.4895]\", \"lift_point\": \"[0.1285, 0.4895]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0586, 0.9156, lift_point: 0.0586, 0.9156" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.9156]\", \"lift_point\": \"[0.0586, 0.9156]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6849]\", \"lift_point\": \"[0.7839, 0.6849]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7913, 0.6927, lift_point: 0.7913, 0.6927" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.6927]\", \"lift_point\": \"[0.7913, 0.6927]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7727, 0.8562]\", \"lift_point\": \"[0.7727, 0.8562]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1093, 0.3357, lift_point: 0.1093, 0.3357" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.3357]\", \"lift_point\": \"[0.1093, 0.3357]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1464, 0.3747]\", \"lift_point\": \"[0.1464, 0.3747]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0508, 0.0719]\", \"lift_point\": \"[0.0508, 0.0719]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0437, 0.8153, lift_point: 0.0437, 0.8153" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.8153]\", \"lift_point\": \"[0.0437, 0.8153]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0578, 0.0742, lift_point: 0.0578, 0.0742" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.0742]\", \"lift_point\": \"[0.0578, 0.0742]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1464, 0.4379]\", \"lift_point\": \"[0.1464, 0.4379]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1936, 0.6753, lift_point: 0.1936, 0.6753" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1936, 0.6753]\", \"lift_point\": \"[0.1936, 0.6753]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7761, 0.6931]\", \"lift_point\": \"[0.7761, 0.6931]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7789, 0.6894, lift_point: 0.7789, 0.6894" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6894]\", \"lift_point\": \"[0.7789, 0.6894]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.208, 0.5184, lift_point: 0.208, 0.5184" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.208, 0.5184]\", \"lift_point\": \"[0.208, 0.5184]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.71, 0.3772]\", \"lift_point\": \"[0.71, 0.3772]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6367, 0.3181, lift_point: 0.6367, 0.3181" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6367, 0.3181]\", \"lift_point\": \"[0.6367, 0.3181]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3412, 0.3256]\", \"lift_point\": \"[0.3412, 0.3256]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3553, 0.5071, lift_point: 0.3553, 0.5071" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3553, 0.5071]\", \"lift_point\": \"[0.3553, 0.5071]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8944, 0.4964]\", \"lift_point\": \"[0.8944, 0.4964]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8789, 0.4552, lift_point: 0.8789, 0.4552" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.4552]\", \"lift_point\": \"[0.8789, 0.4552]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6377, 0.4206, lift_point: 0.6377, 0.4206" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6377, 0.4206]\", \"lift_point\": \"[0.6377, 0.4206]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1263, 0.4059]\", \"lift_point\": \"[0.1263, 0.4059]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1303, 0.3898, lift_point: 0.1303, 0.3898" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1303, 0.3898]\", \"lift_point\": \"[0.1303, 0.3898]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Hotels in Miami\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Hotels in Miami\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.4986]\", \"lift_point\": \"[0.0596, 0.4986]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0609, 0.4861, lift_point: 0.0609, 0.4861" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.4861]\", \"lift_point\": \"[0.0609, 0.4861]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2355, 0.5046]\", \"lift_point\": \"[0.2355, 0.5046]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1213, 0.8467, lift_point: 0.1213, 0.8467" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1213, 0.8467]\", \"lift_point\": \"[0.1213, 0.8467]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5256, 0.4326, lift_point: 0.5256, 0.4326" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5256, 0.4326]\", \"lift_point\": \"[0.5256, 0.4326]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new foundation (skincare) product\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"new foundation (skincare) product\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new foundation (skincare) product\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8869, 0.4468]\", \"lift_point\": \"[0.8869, 0.4468]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8891, 0.1918, lift_point: 0.8891, 0.1918" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8891, 0.1918]\", \"lift_point\": \"[0.8891, 0.1918]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4748, 0.6991, lift_point: 0.4748, 0.6991" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4748, 0.6991]\", \"lift_point\": \"[0.4748, 0.6991]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4887, 0.6881]\", \"lift_point\": \"[0.4887, 0.6881]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5056, 0.673, lift_point: 0.5056, 0.673" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5056, 0.673]\", \"lift_point\": \"[0.5056, 0.673]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a blue whale?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"How big is a blue whale?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a blue whale?\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_BACK#" + "gt": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3346, 0.1938]\", \"lift_point\": \"[0.3346, 0.1938]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2896, 0.1464, lift_point: 0.2896, 0.1464" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2896, 0.1464]\", \"lift_point\": \"[0.2896, 0.1464]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.8026]\", \"lift_point\": \"[0.0586, 0.8026]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0615, 0.7935, lift_point: 0.0615, 0.7935" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7935]\", \"lift_point\": \"[0.0615, 0.7935]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2569, 0.1959]\", \"lift_point\": \"[0.2569, 0.1959]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2512, 0.2042, lift_point: 0.2512, 0.2042" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2512, 0.2042]\", \"lift_point\": \"[0.2512, 0.2042]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.4739]\", \"lift_point\": \"[0.0582, 0.4739]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0619, 0.3492, lift_point: 0.0619, 0.3492" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.3492]\", \"lift_point\": \"[0.0619, 0.3492]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.5165]\", \"lift_point\": \"[0.0664, 0.5165]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2126, 0.6494, lift_point: 0.2126, 0.6494" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2126, 0.6494]\", \"lift_point\": \"[0.2126, 0.6494]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8843, 0.5112]\", \"lift_point\": \"[0.8843, 0.5112]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2485, 0.2867]\", \"lift_point\": \"[0.2485, 0.2867]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2588, 0.1962, lift_point: 0.2588, 0.1962" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.1962]\", \"lift_point\": \"[0.2588, 0.1962]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6954]\", \"lift_point\": \"[0.7887, 0.6954]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.9223]\", \"lift_point\": \"[0.0646, 0.9223]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.067, 0.912, lift_point: 0.067, 0.912" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.912]\", \"lift_point\": \"[0.067, 0.912]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Philadelphia\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"2 bedroom apartment rent for in Philadelphia\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Philadelphia\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.4831]\", \"lift_point\": \"[0.8974, 0.4831]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.909, 0.527, lift_point: 0.909, 0.527" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.909, 0.527]\", \"lift_point\": \"[0.909, 0.527]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2144, 0.4624]\", \"lift_point\": \"[0.2144, 0.4624]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2909, 0.6381, lift_point: 0.2909, 0.6381" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2909, 0.6381]\", \"lift_point\": \"[0.2909, 0.6381]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0595, 0.8357]\", \"lift_point\": \"[0.0595, 0.8357]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.23]\", \"lift_point\": \"[0.1233, 0.23]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.117, 0.3334, lift_point: 0.117, 0.3334" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.3334]\", \"lift_point\": \"[0.117, 0.3334]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1231, 0.4159]\", \"lift_point\": \"[0.1231, 0.4159]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1097, 0.3402, lift_point: 0.1097, 0.3402" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1097, 0.3402]\", \"lift_point\": \"[0.1097, 0.3402]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5378, 0.1475]\", \"lift_point\": \"[0.5378, 0.1475]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5619, 0.1167, lift_point: 0.5619, 0.1167" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5619, 0.1167]\", \"lift_point\": \"[0.5619, 0.1167]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.4652]\", \"lift_point\": \"[0.0582, 0.4652]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0506, 0.8524, lift_point: 0.0506, 0.8524" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0506, 0.8524]\", \"lift_point\": \"[0.0506, 0.8524]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1162, 0.5879]\", \"lift_point\": \"[0.1162, 0.5879]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1315, 0.4787, lift_point: 0.1315, 0.4787" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.4787]\", \"lift_point\": \"[0.1315, 0.4787]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8747, 0.8995]\", \"lift_point\": \"[0.8747, 0.8995]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8361, 0.2475, lift_point: 0.8361, 0.2475" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2475]\", \"lift_point\": \"[0.8361, 0.2475]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.6835]\", \"lift_point\": \"[0.7912, 0.6835]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7898, 0.6787, lift_point: 0.7898, 0.6787" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9303, 0.4941, lift_point: 0.9303, 0.4941" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1285, 0.4849]\", \"lift_point\": \"[0.1285, 0.4849]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1788, 0.6302, lift_point: 0.1788, 0.6302" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1788, 0.6302]\", \"lift_point\": \"[0.1788, 0.6302]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Barcelona to Mexico city\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Search for flights from Barcelona to Mexico city\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Mexico city\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.5377]\", \"lift_point\": \"[0.0634, 0.5377]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.5149]\", \"lift_point\": \"[0.9018, 0.5149]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9472, 0.4803, lift_point: 0.9472, 0.4803" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9472, 0.4803]\", \"lift_point\": \"[0.9472, 0.4803]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.1024]\", \"lift_point\": \"[0.0598, 0.1024]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3054, 0.3688]\", \"lift_point\": \"[0.3054, 0.3688]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1958, 0.8918, lift_point: 0.1958, 0.8918" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1958, 0.8918]\", \"lift_point\": \"[0.1958, 0.8918]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2148, 0.6118]\", \"lift_point\": \"[0.2148, 0.6118]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.223, 0.9017]\", \"lift_point\": \"[0.223, 0.9017]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4877, 0.9055, lift_point: 0.4877, 0.9055" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2877, 0.4986]\", \"lift_point\": \"[0.2877, 0.4986]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2559, 0.3466, lift_point: 0.2559, 0.3466" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2559, 0.3466]\", \"lift_point\": \"[0.2559, 0.3466]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.777, 0.4037, lift_point: 0.777, 0.4037" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2676, 0.6951]\", \"lift_point\": \"[0.2676, 0.6951]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2509, 0.5219, lift_point: 0.2509, 0.5219" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2509, 0.5219]\", \"lift_point\": \"[0.2509, 0.5219]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.5218]\", \"lift_point\": \"[0.0594, 0.5218]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1115, 0.3638, lift_point: 0.1115, 0.3638" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.3638]\", \"lift_point\": \"[0.1115, 0.3638]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.4847]\", \"lift_point\": \"[0.8887, 0.4847]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3666, 0.506]\", \"lift_point\": \"[0.3666, 0.506]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3669, 0.4972, lift_point: 0.3669, 0.4972" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3669, 0.4972]\", \"lift_point\": \"[0.3669, 0.4972]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8888, 0.4811, lift_point: 0.8888, 0.4811" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8888, 0.4811]\", \"lift_point\": \"[0.8888, 0.4811]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.184, 0.6846]\", \"lift_point\": \"[0.184, 0.6846]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2187, 0.6753, lift_point: 0.2187, 0.6753" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2187, 0.6753]\", \"lift_point\": \"[0.2187, 0.6753]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7857, 0.6916]\", \"lift_point\": \"[0.7857, 0.6916]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8814, 0.3866, lift_point: 0.8814, 0.3866" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3866]\", \"lift_point\": \"[0.8814, 0.3866]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2484, 0.4769]\", \"lift_point\": \"[0.2484, 0.4769]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2193, 0.3718, lift_point: 0.2193, 0.3718" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.3718]\", \"lift_point\": \"[0.2193, 0.3718]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4216, 0.5167]\", \"lift_point\": \"[0.4216, 0.5167]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5648, 0.6812, lift_point: 0.5648, 0.6812" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5648, 0.6812]\", \"lift_point\": \"[0.5648, 0.6812]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5609, 0.9437, lift_point: 0.5609, 0.9437" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5609, 0.9437]\", \"lift_point\": \"[0.5609, 0.9437]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8896, 0.8898]\", \"lift_point\": \"[0.8896, 0.8898]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2926, 0.9444, lift_point: 0.2926, 0.9444" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2926, 0.9444]\", \"lift_point\": \"[0.2926, 0.9444]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.9291]\", \"lift_point\": \"[0.0597, 0.9291]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1702, 0.5937, lift_point: 0.1702, 0.5937" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1702, 0.5937]\", \"lift_point\": \"[0.1702, 0.5937]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Huffington Post\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0548, 0.3905, lift_point: 0.0548, 0.3905" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.3905]\", \"lift_point\": \"[0.0548, 0.3905]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest T-Mobile Store\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"How do I get to the nearest T-Mobile Store?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest T-Mobile Store?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.7326]\", \"lift_point\": \"[0.5737, 0.7326]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6219, 0.6098, lift_point: 0.6219, 0.6098" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6219, 0.6098]\", \"lift_point\": \"[0.6219, 0.6098]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Canada?\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0637, 0.3928, lift_point: 0.0637, 0.3928" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.3928]\", \"lift_point\": \"[0.0637, 0.3928]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6354, 0.5054]\", \"lift_point\": \"[0.6354, 0.5054]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2245, 0.2275]\", \"lift_point\": \"[0.2245, 0.2275]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0515, 0.0421, lift_point: 0.0515, 0.0421" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6465, 0.1766]\", \"lift_point\": \"[0.6465, 0.1766]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6505, 0.6362, lift_point: 0.6505, 0.6362" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.6362]\", \"lift_point\": \"[0.6505, 0.6362]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1289, 0.4648]\", \"lift_point\": \"[0.1289, 0.4648]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2091, 0.3261, lift_point: 0.2091, 0.3261" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2091, 0.3261]\", \"lift_point\": \"[0.2091, 0.3261]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.8247]\", \"lift_point\": \"[0.0588, 0.8247]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0464, 0.825, lift_point: 0.0464, 0.825" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.825]\", \"lift_point\": \"[0.0464, 0.825]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1359, 0.4376]\", \"lift_point\": \"[0.1359, 0.4376]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1461, 0.4406, lift_point: 0.1461, 0.4406" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1461, 0.4406]\", \"lift_point\": \"[0.1461, 0.4406]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8137, 0.6754]\", \"lift_point\": \"[0.8137, 0.6754]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8466, 0.6103, lift_point: 0.8466, 0.6103" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8466, 0.6103]\", \"lift_point\": \"[0.8466, 0.6103]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1436, 0.6246, lift_point: 0.1436, 0.6246" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.6246]\", \"lift_point\": \"[0.1436, 0.6246]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.429, 0.922, lift_point: 0.429, 0.922" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.6127]\", \"lift_point\": \"[0.7885, 0.6127]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7873, 0.6244, lift_point: 0.7873, 0.6244" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7873, 0.6244]\", \"lift_point\": \"[0.7873, 0.6244]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1785, 0.5049]\", \"lift_point\": \"[0.1785, 0.5049]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.173, 0.2474, lift_point: 0.173, 0.2474" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2474]\", \"lift_point\": \"[0.173, 0.2474]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.348, 0.3778]\", \"lift_point\": \"[0.348, 0.3778]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_BACK#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0699, 0.4292]\", \"lift_point\": \"[0.0699, 0.4292]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0711, 0.9155, lift_point: 0.0711, 0.9155" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0711, 0.9155]\", \"lift_point\": \"[0.0711, 0.9155]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7949, 0.6997]\", \"lift_point\": \"[0.7949, 0.6997]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.783, 0.7229, lift_point: 0.783, 0.7229" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.7229]\", \"lift_point\": \"[0.783, 0.7229]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1296, 0.3987]\", \"lift_point\": \"[0.1296, 0.3987]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6825]\", \"lift_point\": \"[0.7859, 0.6825]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8015, 0.6568, lift_point: 0.8015, 0.6568" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8015, 0.6568]\", \"lift_point\": \"[0.8015, 0.6568]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3355, 0.4965]\", \"lift_point\": \"[0.3355, 0.4965]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5565, 0.9406, lift_point: 0.5565, 0.9406" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1435, 0.3255]\", \"lift_point\": \"[0.1435, 0.3255]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1423, 0.3399, lift_point: 0.1423, 0.3399" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.3399]\", \"lift_point\": \"[0.1423, 0.3399]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3203, 0.3245]\", \"lift_point\": \"[0.3203, 0.3245]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5601, 0.371, lift_point: 0.5601, 0.371" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5601, 0.371]\", \"lift_point\": \"[0.5601, 0.371]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1445, 0.5074]\", \"lift_point\": \"[0.1445, 0.5074]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1482, 0.6046, lift_point: 0.1482, 0.6046" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1482, 0.6046]\", \"lift_point\": \"[0.1482, 0.6046]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7944, 0.6887, lift_point: 0.7944, 0.6887" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.078, 0.8768, lift_point: 0.078, 0.8768" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3196, 0.5087]\", \"lift_point\": \"[0.3196, 0.5087]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.332, 0.4957, lift_point: 0.332, 0.4957" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.4957]\", \"lift_point\": \"[0.332, 0.4957]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1786, 0.4172]\", \"lift_point\": \"[0.1786, 0.4172]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3812, 0.4281, lift_point: 0.3812, 0.4281" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3812, 0.4281]\", \"lift_point\": \"[0.3812, 0.4281]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7462, 0.3048]\", \"lift_point\": \"[0.7462, 0.3048]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7589, 0.3169, lift_point: 0.7589, 0.3169" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7589, 0.3169]\", \"lift_point\": \"[0.7589, 0.3169]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.6914]\", \"lift_point\": \"[0.4877, 0.6914]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3444, 0.5381, lift_point: 0.3444, 0.5381" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3444, 0.5381]\", \"lift_point\": \"[0.3444, 0.5381]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.888, 0.5123]\", \"lift_point\": \"[0.888, 0.5123]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8352, 0.9414, lift_point: 0.8352, 0.9414" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.9414]\", \"lift_point\": \"[0.8352, 0.9414]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5363, 0.6639]\", \"lift_point\": \"[0.5363, 0.6639]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6403, 0.4089, lift_point: 0.6403, 0.4089" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6403, 0.4089]\", \"lift_point\": \"[0.6403, 0.4089]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5449, 0.8768]\", \"lift_point\": \"[0.5449, 0.8768]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5654, 0.8708, lift_point: 0.5654, 0.8708" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5654, 0.8708]\", \"lift_point\": \"[0.5654, 0.8708]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3199, 0.4774]\", \"lift_point\": \"[0.3199, 0.4774]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3159, 0.2876]\", \"lift_point\": \"[0.3159, 0.2876]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2525, 0.3556]\", \"lift_point\": \"[0.2525, 0.3556]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2598, 0.2621, lift_point: 0.2598, 0.2621" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2598, 0.2621]\", \"lift_point\": \"[0.2598, 0.2621]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.4421]\", \"lift_point\": \"[0.0647, 0.4421]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0721, 0.4079, lift_point: 0.0721, 0.4079" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4079]\", \"lift_point\": \"[0.0721, 0.4079]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3722, 0.6794]\", \"lift_point\": \"[0.3722, 0.6794]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1954, 0.309, lift_point: 0.1954, 0.309" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1954, 0.309]\", \"lift_point\": \"[0.1954, 0.309]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.558, 0.9406, lift_point: 0.558, 0.9406" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6931]\", \"lift_point\": \"[0.7859, 0.6931]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7686, 0.7016, lift_point: 0.7686, 0.7016" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7686, 0.7016]\", \"lift_point\": \"[0.7686, 0.7016]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.473, 0.3296]\", \"lift_point\": \"[0.473, 0.3296]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7301, 0.9206, lift_point: 0.7301, 0.9206" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7301, 0.9206]\", \"lift_point\": \"[0.7301, 0.9206]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1394, 0.9532, lift_point: 0.1394, 0.9532" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1394, 0.9532]\", \"lift_point\": \"[0.1394, 0.9532]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7956, 0.9421]\", \"lift_point\": \"[0.7956, 0.9421]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1774, 0.2732, lift_point: 0.1774, 0.2732" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1774, 0.2732]\", \"lift_point\": \"[0.1774, 0.2732]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4075, 0.2072, lift_point: 0.4075, 0.2072" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4075, 0.2072]\", \"lift_point\": \"[0.4075, 0.2072]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5554, 0.4946]\", \"lift_point\": \"[0.5554, 0.4946]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.713, 0.7036, lift_point: 0.713, 0.7036" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.713, 0.7036]\", \"lift_point\": \"[0.713, 0.7036]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6877]\", \"lift_point\": \"[0.7952, 0.6877]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6506, 0.6833, lift_point: 0.6506, 0.6833" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6506, 0.6833]\", \"lift_point\": \"[0.6506, 0.6833]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5483, 0.1974]\", \"lift_point\": \"[0.5483, 0.1974]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5418, 0.1546, lift_point: 0.5418, 0.1546" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5418, 0.1546]\", \"lift_point\": \"[0.5418, 0.1546]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0767, 0.8795]\", \"lift_point\": \"[0.0767, 0.8795]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2993, 0.5542]\", \"lift_point\": \"[0.2993, 0.5542]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.305, 0.6055, lift_point: 0.305, 0.6055" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.305, 0.6055]\", \"lift_point\": \"[0.305, 0.6055]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Houston\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"2 bed room apartment rent in houston\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bed room apartment rent in houston\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Denver?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats a good restaurant in Denver\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Denver\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.4769]\", \"lift_point\": \"[0.0582, 0.4769]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0619, 0.546, lift_point: 0.0619, 0.546" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.546]\", \"lift_point\": \"[0.0619, 0.546]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1448, 0.3648]\", \"lift_point\": \"[0.1448, 0.3648]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2667, 0.3762, lift_point: 0.2667, 0.3762" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2667, 0.3762]\", \"lift_point\": \"[0.2667, 0.3762]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0629, 0.5364]\", \"lift_point\": \"[0.0629, 0.5364]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0537, 0.3913, lift_point: 0.0537, 0.3913" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.3913]\", \"lift_point\": \"[0.0537, 0.3913]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8139, 0.8516, lift_point: 0.8139, 0.8516" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8139, 0.8516]\", \"lift_point\": \"[0.8139, 0.8516]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0626, 0.8066]\", \"lift_point\": \"[0.0626, 0.8066]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0556, 0.7814, lift_point: 0.0556, 0.7814" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.7814]\", \"lift_point\": \"[0.0556, 0.7814]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Sweden\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"capital of sweden?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of sweden?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8771, 0.4957]\", \"lift_point\": \"[0.8771, 0.4957]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7746, 0.6711, lift_point: 0.7746, 0.6711" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7746, 0.6711]\", \"lift_point\": \"[0.7746, 0.6711]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1289, 0.2978]\", \"lift_point\": \"[0.1289, 0.2978]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1361, 0.2898, lift_point: 0.1361, 0.2898" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1361, 0.2898]\", \"lift_point\": \"[0.1361, 0.2898]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6135]\", \"lift_point\": \"[0.8049, 0.6135]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7966, 0.5681, lift_point: 0.7966, 0.5681" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7966, 0.5681]\", \"lift_point\": \"[0.7966, 0.5681]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7857, 0.6036]\", \"lift_point\": \"[0.7857, 0.6036]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6333, 0.6176, lift_point: 0.6333, 0.6176" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6333, 0.6176]\", \"lift_point\": \"[0.6333, 0.6176]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1256, 0.4002, lift_point: 0.1256, 0.4002" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1256, 0.4002]\", \"lift_point\": \"[0.1256, 0.4002]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7844, 0.6948]\", \"lift_point\": \"[0.7844, 0.6948]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.885, 0.1574]\", \"lift_point\": \"[0.885, 0.1574]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.876, 0.5326, lift_point: 0.876, 0.5326" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.876, 0.5326]\", \"lift_point\": \"[0.876, 0.5326]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"news in Pakistan\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Pakistan\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4996, 0.1605]\", \"lift_point\": \"[0.4996, 0.1605]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.491, 0.2181, lift_point: 0.491, 0.2181" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.491, 0.2181]\", \"lift_point\": \"[0.491, 0.2181]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5611, 0.6412, lift_point: 0.5611, 0.6412" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5611, 0.6412]\", \"lift_point\": \"[0.5611, 0.6412]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0629, 0.8993]\", \"lift_point\": \"[0.0629, 0.8993]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.173, 0.892, lift_point: 0.173, 0.892" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.892]\", \"lift_point\": \"[0.173, 0.892]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.5349]\", \"lift_point\": \"[0.1388, 0.5349]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1316, 0.703, lift_point: 0.1316, 0.703" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1316, 0.703]\", \"lift_point\": \"[0.1316, 0.703]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4744, 0.5237]\", \"lift_point\": \"[0.4744, 0.5237]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6927, 0.5764, lift_point: 0.6927, 0.5764" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6927, 0.5764]\", \"lift_point\": \"[0.6927, 0.5764]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5326, 0.5293]\", \"lift_point\": \"[0.5326, 0.5293]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5045, 0.5531, lift_point: 0.5045, 0.5531" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5045, 0.5531]\", \"lift_point\": \"[0.5045, 0.5531]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1406, 0.9582, lift_point: 0.1406, 0.9582" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.9582]\", \"lift_point\": \"[0.1406, 0.9582]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5565, 0.9406, lift_point: 0.5565, 0.9406" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow on Sephora\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \" a new eyeshadow on Sephora\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" a new eyeshadow on Sephora\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2935, 0.4897]\", \"lift_point\": \"[0.2935, 0.4897]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"latest news in space exploration\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3266, 0.511]\", \"lift_point\": \"[0.3266, 0.511]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3462, 0.1159, lift_point: 0.3462, 0.1159" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3462, 0.1159]\", \"lift_point\": \"[0.3462, 0.1159]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2999, 0.2785, lift_point: 0.2999, 0.2785" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2999, 0.2785]\", \"lift_point\": \"[0.2999, 0.2785]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Bolivia\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"news in Bolivia\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Bolivia\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1221, 0.4484]\", \"lift_point\": \"[0.1221, 0.4484]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1341, 0.2067, lift_point: 0.1341, 0.2067" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.2067]\", \"lift_point\": \"[0.1341, 0.2067]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"hotels in Philadelphia\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3845, 0.5099]\", \"lift_point\": \"[0.3845, 0.5099]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4944]\", \"lift_point\": \"[0.0646, 0.4944]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0621, 0.536, lift_point: 0.0621, 0.536" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.536]\", \"lift_point\": \"[0.0621, 0.536]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Suriname\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"news in suriname?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in suriname?\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Bolivia?\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0817, 0.4715, lift_point: 0.0817, 0.4715" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.4715]\", \"lift_point\": \"[0.0817, 0.4715]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.5005]\", \"lift_point\": \"[0.0618, 0.5005]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0627, 0.7127, lift_point: 0.0627, 0.7127" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.7127]\", \"lift_point\": \"[0.0627, 0.7127]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7184, 0.9297]\", \"lift_point\": \"[0.7184, 0.9297]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7977, 0.8485, lift_point: 0.7977, 0.8485" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7977, 0.8485]\", \"lift_point\": \"[0.7977, 0.8485]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7771, 0.3033]\", \"lift_point\": \"[0.7771, 0.3033]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8085, 0.5081, lift_point: 0.8085, 0.5081" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8085, 0.5081]\", \"lift_point\": \"[0.8085, 0.5081]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4936, 0.9286, lift_point: 0.4936, 0.9286" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1149, 0.4278]\", \"lift_point\": \"[0.1149, 0.4278]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9402]\", \"lift_point\": \"[0.066, 0.9402]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1186, 0.6529, lift_point: 0.1186, 0.6529" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1186, 0.6529]\", \"lift_point\": \"[0.1186, 0.6529]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Malaysia?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"malaysia\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"malaysia\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2488, 0.3989]\", \"lift_point\": \"[0.2488, 0.3989]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1219, 0.266, lift_point: 0.1219, 0.266" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1219, 0.266]\", \"lift_point\": \"[0.1219, 0.266]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calendar and show me this week's events? \"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Open the calendar and show me this weeks events? \"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calendar and show me this weeks events? \"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8425, 0.8956]\", \"lift_point\": \"[0.8425, 0.8956]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7761, 0.7667, lift_point: 0.7761, 0.7667" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7761, 0.7667]\", \"lift_point\": \"[0.7761, 0.7667]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2825, 0.4568]\", \"lift_point\": \"[0.2825, 0.4568]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7979, 0.5803, lift_point: 0.7979, 0.5803" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.5803]\", \"lift_point\": \"[0.7979, 0.5803]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.5064]\", \"lift_point\": \"[0.1298, 0.5064]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1143, 0.3895]\", \"lift_point\": \"[0.1143, 0.3895]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4782, 0.8456, lift_point: 0.4782, 0.8456" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"samsung tv cost\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"How much does the Samsung TV cost?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the Samsung TV cost?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2502, 0.316]\", \"lift_point\": \"[0.2502, 0.316]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2995, 0.2829, lift_point: 0.2995, 0.2829" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2995, 0.2829]\", \"lift_point\": \"[0.2995, 0.2829]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3321, 0.1299, lift_point: 0.3321, 0.1299" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3321, 0.1299]\", \"lift_point\": \"[0.3321, 0.1299]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8285, 0.587]\", \"lift_point\": \"[0.8285, 0.587]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8188, 0.69, lift_point: 0.8188, 0.69" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8188, 0.69]\", \"lift_point\": \"[0.8188, 0.69]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8048, 0.614]\", \"lift_point\": \"[0.8048, 0.614]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8421, 0.6318, lift_point: 0.8421, 0.6318" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8421, 0.6318]\", \"lift_point\": \"[0.8421, 0.6318]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7264, 0.1487, lift_point: 0.7264, 0.1487" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.1487]\", \"lift_point\": \"[0.7264, 0.1487]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2495, 0.0452, lift_point: 0.2495, 0.0452" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2495, 0.0452]\", \"lift_point\": \"[0.2495, 0.0452]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8877, 0.5062]\", \"lift_point\": \"[0.8877, 0.5062]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7756, 0.6926, lift_point: 0.7756, 0.6926" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6926]\", \"lift_point\": \"[0.7756, 0.6926]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1167, 0.4031, lift_point: 0.1167, 0.4031" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1167, 0.4031]\", \"lift_point\": \"[0.1167, 0.4031]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0669, 0.4945]\", \"lift_point\": \"[0.0669, 0.4945]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0509, 0.1702, lift_point: 0.0509, 0.1702" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.1702]\", \"lift_point\": \"[0.0509, 0.1702]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7797, 0.6878]\", \"lift_point\": \"[0.7797, 0.6878]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7842, 0.5074, lift_point: 0.7842, 0.5074" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.5074]\", \"lift_point\": \"[0.7842, 0.5074]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8948, 0.536]\", \"lift_point\": \"[0.8948, 0.536]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9173, 0.8478, lift_point: 0.9173, 0.8478" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9173, 0.8478]\", \"lift_point\": \"[0.9173, 0.8478]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7805, 0.6881]\", \"lift_point\": \"[0.7805, 0.6881]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7698, 0.6702, lift_point: 0.7698, 0.6702" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7698, 0.6702]\", \"lift_point\": \"[0.7698, 0.6702]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0426, 0.819, lift_point: 0.0426, 0.819" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0426, 0.819]\", \"lift_point\": \"[0.0426, 0.819]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.4976]\", \"lift_point\": \"[0.0564, 0.4976]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0473, 0.3652, lift_point: 0.0473, 0.3652" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.3652]\", \"lift_point\": \"[0.0473, 0.3652]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"hotels in Washington DC\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.6132]\", \"lift_point\": \"[0.1159, 0.6132]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0596, 0.9275, lift_point: 0.0596, 0.9275" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5875, 0.5166]\", \"lift_point\": \"[0.5875, 0.5166]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5567, 0.4671, lift_point: 0.5567, 0.4671" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5567, 0.4671]\", \"lift_point\": \"[0.5567, 0.4671]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8976, 0.5154]\", \"lift_point\": \"[0.8976, 0.5154]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1164, 0.1471, lift_point: 0.1164, 0.1471" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.1471]\", \"lift_point\": \"[0.1164, 0.1471]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of sweden\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"capital of sweden ?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of sweden ?\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5445, 0.8611, lift_point: 0.5445, 0.8611" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5445, 0.8611]\", \"lift_point\": \"[0.5445, 0.8611]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3542, 0.4973]\", \"lift_point\": \"[0.3542, 0.4973]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8904, 0.5199]\", \"lift_point\": \"[0.8904, 0.5199]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9174, 0.4332, lift_point: 0.9174, 0.4332" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9174, 0.4332]\", \"lift_point\": \"[0.9174, 0.4332]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1407, 0.4285]\", \"lift_point\": \"[0.1407, 0.4285]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1406, 0.4874, lift_point: 0.1406, 0.4874" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.4874]\", \"lift_point\": \"[0.1406, 0.4874]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0478, 0.2489]\", \"lift_point\": \"[0.0478, 0.2489]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Pakistan\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Pakistan\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1345, 0.3419]\", \"lift_point\": \"[0.1345, 0.3419]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2757, 0.5502, lift_point: 0.2757, 0.5502" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5502]\", \"lift_point\": \"[0.2757, 0.5502]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Guyana?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats the news in Guyana?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Guyana?\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3187, 0.2868, lift_point: 0.3187, 0.2868" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3187, 0.2868]\", \"lift_point\": \"[0.3187, 0.2868]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2568, 0.2237]\", \"lift_point\": \"[0.2568, 0.2237]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7796, 0.2768, lift_point: 0.7796, 0.2768" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7796, 0.2768]\", \"lift_point\": \"[0.7796, 0.2768]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1185, 0.5459]\", \"lift_point\": \"[0.1185, 0.5459]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1174, 0.4815, lift_point: 0.1174, 0.4815" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.4815]\", \"lift_point\": \"[0.1174, 0.4815]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0809, 0.8737, lift_point: 0.0809, 0.8737" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.897, 0.5098]\", \"lift_point\": \"[0.897, 0.5098]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7786, 0.6886]\", \"lift_point\": \"[0.7786, 0.6886]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Where can I buy a nice beach towel?\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0687, 0.918]\", \"lift_point\": \"[0.0687, 0.918]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.073, 0.8872, lift_point: 0.073, 0.8872" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.8872]\", \"lift_point\": \"[0.073, 0.8872]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of the United Kingdom?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"What is the capital of the United Kingdom\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of the United Kingdom\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5463, 0.4976]\", \"lift_point\": \"[0.5463, 0.4976]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1028, 0.4618, lift_point: 0.1028, 0.4618" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4618]\", \"lift_point\": \"[0.1028, 0.4618]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9116, 0.5564]\", \"lift_point\": \"[0.9116, 0.5564]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9183, 0.3832, lift_point: 0.9183, 0.3832" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9183, 0.3832]\", \"lift_point\": \"[0.9183, 0.3832]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Paris\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"flights from nyc to paris\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to paris\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1221, 0.2835]\", \"lift_point\": \"[0.1221, 0.2835]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.0609]\", \"lift_point\": \"[0.0578, 0.0609]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0598, 0.1024, lift_point: 0.0598, 0.1024" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.1024]\", \"lift_point\": \"[0.0598, 0.1024]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5144, 0.9441, lift_point: 0.5144, 0.9441" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats the time in New York?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3324, 0.2437]\", \"lift_point\": \"[0.3324, 0.2437]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3443, 0.3518]\", \"lift_point\": \"[0.3443, 0.3518]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5399, 0.5883, lift_point: 0.5399, 0.5883" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5399, 0.5883]\", \"lift_point\": \"[0.5399, 0.5883]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6365, 0.188]\", \"lift_point\": \"[0.6365, 0.188]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5651, 0.1852, lift_point: 0.5651, 0.1852" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5651, 0.1852]\", \"lift_point\": \"[0.5651, 0.1852]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8846, 0.502]\", \"lift_point\": \"[0.8846, 0.502]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8814, 0.5825, lift_point: 0.8814, 0.5825" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.5825]\", \"lift_point\": \"[0.8814, 0.5825]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8962, 0.497]\", \"lift_point\": \"[0.8962, 0.497]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6539, 0.8814, lift_point: 0.6539, 0.8814" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6539, 0.8814]\", \"lift_point\": \"[0.6539, 0.8814]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9077, 0.5068]\", \"lift_point\": \"[0.9077, 0.5068]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9338, 0.5386, lift_point: 0.9338, 0.5386" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9338, 0.5386]\", \"lift_point\": \"[0.9338, 0.5386]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.4984]\", \"lift_point\": \"[0.8925, 0.4984]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.777, 0.6024, lift_point: 0.777, 0.6024" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.6024]\", \"lift_point\": \"[0.777, 0.6024]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5126, 0.1778]\", \"lift_point\": \"[0.5126, 0.1778]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.607, 0.1523, lift_point: 0.607, 0.1523" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.607, 0.1523]\", \"lift_point\": \"[0.607, 0.1523]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.508, 0.6983]\", \"lift_point\": \"[0.508, 0.6983]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5051, 0.6922, lift_point: 0.5051, 0.6922" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5051, 0.6922]\", \"lift_point\": \"[0.5051, 0.6922]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.475, 0.4932, lift_point: 0.475, 0.4932" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.475, 0.4932]\", \"lift_point\": \"[0.475, 0.4932]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1529, 0.4357]\", \"lift_point\": \"[0.1529, 0.4357]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.128, 0.3125, lift_point: 0.128, 0.3125" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.128, 0.3125]\", \"lift_point\": \"[0.128, 0.3125]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest T-Mobile Store\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"how do i get nearest t mobile store\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get nearest t mobile store\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"twittwer\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twittwer\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.689]\", \"lift_point\": \"[0.7895, 0.689]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3141, 0.9451]\", \"lift_point\": \"[0.3141, 0.9451]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6975]\", \"lift_point\": \"[0.7894, 0.6975]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Mexico?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"What is the capital of Mexico?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Mexico?\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1991, 0.4725]\", \"lift_point\": \"[0.1991, 0.4725]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2007, 0.6839, lift_point: 0.2007, 0.6839" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2007, 0.6839]\", \"lift_point\": \"[0.2007, 0.6839]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_BACK#" + "gt": "Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1062, 0.3767]\", \"lift_point\": \"[0.1062, 0.3767]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1079, 0.3517, lift_point: 0.1079, 0.3517" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1079, 0.3517]\", \"lift_point\": \"[0.1079, 0.3517]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3246, 0.3199, lift_point: 0.3246, 0.3199" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3246, 0.3199]\", \"lift_point\": \"[0.3246, 0.3199]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8784, 0.9095]\", \"lift_point\": \"[0.8784, 0.9095]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.842, 0.554, lift_point: 0.842, 0.554" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.842, 0.554]\", \"lift_point\": \"[0.842, 0.554]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1323, 0.3975]\", \"lift_point\": \"[0.1323, 0.3975]\", \"typed_text\": \"\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.39, 0.2406]\", \"lift_point\": \"[0.39, 0.2406]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8977, 0.5023]\", \"lift_point\": \"[0.8977, 0.5023]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8796, 0.5176, lift_point: 0.8796, 0.5176" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8796, 0.5176]\", \"lift_point\": \"[0.8796, 0.5176]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Lowes? \"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"What is the price of a 12 ladder at Lowes? \"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Lowes? \"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"HisenseTV cost\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"HisenseTV cost\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.9167]\", \"lift_point\": \"[0.0682, 0.9167]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0555, 0.9183, lift_point: 0.0555, 0.9183" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.9183]\", \"lift_point\": \"[0.0555, 0.9183]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1763, 0.3982]\", \"lift_point\": \"[0.1763, 0.3982]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.369, 0.4993, lift_point: 0.369, 0.4993" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.369, 0.4993]\", \"lift_point\": \"[0.369, 0.4993]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0683, 0.927]\", \"lift_point\": \"[0.0683, 0.927]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.148, 0.9353, lift_point: 0.148, 0.9353" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.148, 0.9353]\", \"lift_point\": \"[0.148, 0.9353]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1307, 0.2776]\", \"lift_point\": \"[0.1307, 0.2776]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1421, 0.4265, lift_point: 0.1421, 0.4265" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1421, 0.4265]\", \"lift_point\": \"[0.1421, 0.4265]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Buenos aires\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Hotel in Buenos aires\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Hotel in Buenos aires\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8796, 0.4947]\", \"lift_point\": \"[0.8796, 0.4947]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8672, 0.3458, lift_point: 0.8672, 0.3458" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8672, 0.3458]\", \"lift_point\": \"[0.8672, 0.3458]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in San Diego\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats a good restaurant in San Diego?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Diego?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3223, 0.2248]\", \"lift_point\": \"[0.3223, 0.2248]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5144, 0.9441, lift_point: 0.5144, 0.9441" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6909]\", \"lift_point\": \"[0.7947, 0.6909]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7898, 0.6984, lift_point: 0.7898, 0.6984" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6984]\", \"lift_point\": \"[0.7898, 0.6984]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3364, 0.5066]\", \"lift_point\": \"[0.3364, 0.5066]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4966, 0.621, lift_point: 0.4966, 0.621" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4966, 0.621]\", \"lift_point\": \"[0.4966, 0.621]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2329, 0.8893]\", \"lift_point\": \"[0.2329, 0.8893]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7689, 0.4649]\", \"lift_point\": \"[0.7689, 0.4649]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1515, 0.9091, lift_point: 0.1515, 0.9091" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1515, 0.9091]\", \"lift_point\": \"[0.1515, 0.9091]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4201, 0.9891, lift_point: 0.4201, 0.9891" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4201, 0.9891]\", \"lift_point\": \"[0.4201, 0.9891]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1364, 0.8484]\", \"lift_point\": \"[0.1364, 0.8484]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2094, 0.8261, lift_point: 0.2094, 0.8261" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2094, 0.8261]\", \"lift_point\": \"[0.2094, 0.8261]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7793, 0.1463]\", \"lift_point\": \"[0.7793, 0.1463]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1252, 0.3535]\", \"lift_point\": \"[0.1252, 0.3535]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5797, 0.5502, lift_point: 0.5797, 0.5502" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5797, 0.5502]\", \"lift_point\": \"[0.5797, 0.5502]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology?\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0585, 0.442, lift_point: 0.0585, 0.442" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.442]\", \"lift_point\": \"[0.0585, 0.442]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4861, 0.1534, lift_point: 0.4861, 0.1534" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4861, 0.1534]\", \"lift_point\": \"[0.4861, 0.1534]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2674, 0.5932]\", \"lift_point\": \"[0.2674, 0.5932]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6355, 0.1884]\", \"lift_point\": \"[0.6355, 0.1884]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6075, 0.6952, lift_point: 0.6075, 0.6952" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6075, 0.6952]\", \"lift_point\": \"[0.6075, 0.6952]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0504, 0.2105, lift_point: 0.0504, 0.2105" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.2105]\", \"lift_point\": \"[0.0504, 0.2105]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami? \"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami? \"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1248, 0.4038]\", \"lift_point\": \"[0.1248, 0.4038]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1419, 0.6766, lift_point: 0.1419, 0.6766" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.6766]\", \"lift_point\": \"[0.1419, 0.6766]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0775, 0.9231, lift_point: 0.0775, 0.9231" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0775, 0.9231]\", \"lift_point\": \"[0.0775, 0.9231]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.43, 0.3015, lift_point: 0.43, 0.3015" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.43, 0.3015]\", \"lift_point\": \"[0.43, 0.3015]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0951, 0.5065]\", \"lift_point\": \"[0.0951, 0.5065]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0862, 0.4742, lift_point: 0.0862, 0.4742" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0862, 0.4742]\", \"lift_point\": \"[0.0862, 0.4742]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.127, 0.3781]\", \"lift_point\": \"[0.127, 0.3781]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1372, 0.4749, lift_point: 0.1372, 0.4749" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1372, 0.4749]\", \"lift_point\": \"[0.1372, 0.4749]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8648, 0.5908]\", \"lift_point\": \"[0.8648, 0.5908]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5689, 0.5117, lift_point: 0.5689, 0.5117" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5689, 0.5117]\", \"lift_point\": \"[0.5689, 0.5117]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6749, 0.6251]\", \"lift_point\": \"[0.6749, 0.6251]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.657, 0.6143, lift_point: 0.657, 0.6143" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.657, 0.6143]\", \"lift_point\": \"[0.657, 0.6143]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4288, 0.9313, lift_point: 0.4288, 0.9313" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7306, 0.6166]\", \"lift_point\": \"[0.7306, 0.6166]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7162, 0.636, lift_point: 0.7162, 0.636" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7162, 0.636]\", \"lift_point\": \"[0.7162, 0.636]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4989, 0.5141]\", \"lift_point\": \"[0.4989, 0.5141]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3199, 0.6084, lift_point: 0.3199, 0.6084" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3199, 0.6084]\", \"lift_point\": \"[0.3199, 0.6084]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated burger restaurants\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \" top rated burger restaurants\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" top rated burger restaurants\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.4924]\", \"lift_point\": \"[0.0677, 0.4924]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8791, 0.4472]\", \"lift_point\": \"[0.8791, 0.4472]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8899, 0.5686, lift_point: 0.8899, 0.5686" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5686]\", \"lift_point\": \"[0.8899, 0.5686]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1088, 0.4164]\", \"lift_point\": \"[0.1088, 0.4164]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1172, 0.2172, lift_point: 0.1172, 0.2172" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.2172]\", \"lift_point\": \"[0.1172, 0.2172]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3647, 0.8777]\", \"lift_point\": \"[0.3647, 0.8777]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3412, 0.8923, lift_point: 0.3412, 0.8923" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3412, 0.8923]\", \"lift_point\": \"[0.3412, 0.8923]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.6877]\", \"lift_point\": \"[0.7923, 0.6877]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7886, 0.6901, lift_point: 0.7886, 0.6901" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7854, 0.6865, lift_point: 0.7854, 0.6865" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6865]\", \"lift_point\": \"[0.7854, 0.6865]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.6793]\", \"lift_point\": \"[0.768, 0.6793]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7656, 0.6793, lift_point: 0.7656, 0.6793" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7656, 0.6793]\", \"lift_point\": \"[0.7656, 0.6793]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3078, 0.1589]\", \"lift_point\": \"[0.3078, 0.1589]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4812, 0.2229, lift_point: 0.4812, 0.2229" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4812, 0.2229]\", \"lift_point\": \"[0.4812, 0.2229]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7927, 0.6885]\", \"lift_point\": \"[0.7927, 0.6885]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7912, 0.7041, lift_point: 0.7912, 0.7041" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.7041]\", \"lift_point\": \"[0.7912, 0.7041]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4396, 0.1947, lift_point: 0.4396, 0.1947" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4396, 0.1947]\", \"lift_point\": \"[0.4396, 0.1947]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1216, 0.2006]\", \"lift_point\": \"[0.1216, 0.2006]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1057, 0.6469, lift_point: 0.1057, 0.6469" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1057, 0.6469]\", \"lift_point\": \"[0.1057, 0.6469]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7795, 0.6826]\", \"lift_point\": \"[0.7795, 0.6826]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8714, 0.3907, lift_point: 0.8714, 0.3907" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8714, 0.3907]\", \"lift_point\": \"[0.8714, 0.3907]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.12, 0.3894, lift_point: 0.12, 0.3894" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.3894]\", \"lift_point\": \"[0.12, 0.3894]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7876, 0.173, lift_point: 0.7876, 0.173" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7876, 0.173]\", \"lift_point\": \"[0.7876, 0.173]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brazil?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"capital of Brazil\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Brazil\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Lyft app\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"lyft app\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lyft app\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7925, 0.614]\", \"lift_point\": \"[0.7925, 0.614]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7783, 0.6389, lift_point: 0.7783, 0.6389" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9026, 0.4977]\", \"lift_point\": \"[0.9026, 0.4977]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.885, 0.4626, lift_point: 0.885, 0.4626" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.885, 0.4626]\", \"lift_point\": \"[0.885, 0.4626]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1435, 0.7849]\", \"lift_point\": \"[0.1435, 0.7849]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.151, 0.8536, lift_point: 0.151, 0.8536" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.151, 0.8536]\", \"lift_point\": \"[0.151, 0.8536]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1377, 0.7861]\", \"lift_point\": \"[0.1377, 0.7861]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.6835]\", \"lift_point\": \"[0.7946, 0.6835]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1247, 0.4082]\", \"lift_point\": \"[0.1247, 0.4082]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1417, 0.3636, lift_point: 0.1417, 0.3636" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1417, 0.3636]\", \"lift_point\": \"[0.1417, 0.3636]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1779, 0.3465]\", \"lift_point\": \"[0.1779, 0.3465]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8068, 0.611]\", \"lift_point\": \"[0.8068, 0.611]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8873, 0.5619]\", \"lift_point\": \"[0.8873, 0.5619]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9061, 0.8273, lift_point: 0.9061, 0.8273" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9061, 0.8273]\", \"lift_point\": \"[0.9061, 0.8273]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2708, 0.4597, lift_point: 0.2708, 0.4597" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2708, 0.4597]\", \"lift_point\": \"[0.2708, 0.4597]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2659, 0.3664]\", \"lift_point\": \"[0.2659, 0.3664]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3263, 0.4572, lift_point: 0.3263, 0.4572" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3263, 0.4572]\", \"lift_point\": \"[0.3263, 0.4572]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.07, 0.9181, lift_point: 0.07, 0.9181" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2467, 0.5052]\", \"lift_point\": \"[0.2467, 0.5052]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3094, 0.5558, lift_point: 0.3094, 0.5558" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3094, 0.5558]\", \"lift_point\": \"[0.3094, 0.5558]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1491, 0.9393, lift_point: 0.1491, 0.9393" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.9393]\", \"lift_point\": \"[0.1491, 0.9393]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Venezuela?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"the news in Venezuela\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Venezuela\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.323, 0.1973]\", \"lift_point\": \"[0.323, 0.1973]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3658, 0.1658, lift_point: 0.3658, 0.1658" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3658, 0.1658]\", \"lift_point\": \"[0.3658, 0.1658]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1218, 0.4095]\", \"lift_point\": \"[0.1218, 0.4095]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1146, 0.4143, lift_point: 0.1146, 0.4143" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1146, 0.4143]\", \"lift_point\": \"[0.1146, 0.4143]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8821, 0.4968]\", \"lift_point\": \"[0.8821, 0.4968]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7807, 0.642, lift_point: 0.7807, 0.642" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.642]\", \"lift_point\": \"[0.7807, 0.642]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.4998]\", \"lift_point\": \"[0.1787, 0.4998]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3041, 0.3748, lift_point: 0.3041, 0.3748" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3041, 0.3748]\", \"lift_point\": \"[0.3041, 0.3748]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6477, 0.4744]\", \"lift_point\": \"[0.6477, 0.4744]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5543, 0.7882, lift_point: 0.5543, 0.7882" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5543, 0.7882]\", \"lift_point\": \"[0.5543, 0.7882]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2774, 0.1749]\", \"lift_point\": \"[0.2774, 0.1749]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3179, 0.4504, lift_point: 0.3179, 0.4504" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3179, 0.4504]\", \"lift_point\": \"[0.3179, 0.4504]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9026, 0.522]\", \"lift_point\": \"[0.9026, 0.522]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9121, 0.5982, lift_point: 0.9121, 0.5982" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9121, 0.5982]\", \"lift_point\": \"[0.9121, 0.5982]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.3638]\", \"lift_point\": \"[0.1115, 0.3638]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1095, 0.535, lift_point: 0.1095, 0.535" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1095, 0.535]\", \"lift_point\": \"[0.1095, 0.535]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7945, 0.6839]\", \"lift_point\": \"[0.7945, 0.6839]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7906, 0.6852, lift_point: 0.7906, 0.6852" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3255, 0.4885]\", \"lift_point\": \"[0.3255, 0.4885]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3583, 0.6785, lift_point: 0.3583, 0.6785" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3583, 0.6785]\", \"lift_point\": \"[0.3583, 0.6785]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7949, 0.6254]\", \"lift_point\": \"[0.7949, 0.6254]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8259, 0.57, lift_point: 0.8259, 0.57" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8259, 0.57]\", \"lift_point\": \"[0.8259, 0.57]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new perfume\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"new perfume\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new perfume\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.5193]\", \"lift_point\": \"[0.8857, 0.5193]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.9036, 0.4394, lift_point: 0.9036, 0.4394" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9036, 0.4394]\", \"lift_point\": \"[0.9036, 0.4394]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7936, 0.6979, lift_point: 0.7936, 0.6979" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats the news in Vietnam?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.6178]\", \"lift_point\": \"[0.8252, 0.6178]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8132, 0.6673, lift_point: 0.8132, 0.6673" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8132, 0.6673]\", \"lift_point\": \"[0.8132, 0.6673]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1276, 0.4477]\", \"lift_point\": \"[0.1276, 0.4477]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1236, 0.4752, lift_point: 0.1236, 0.4752" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1236, 0.4752]\", \"lift_point\": \"[0.1236, 0.4752]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1563, 0.5084]\", \"lift_point\": \"[0.1563, 0.5084]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3141, 0.9451, lift_point: 0.3141, 0.9451" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3141, 0.9451]\", \"lift_point\": \"[0.3141, 0.9451]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1357, 0.4491]\", \"lift_point\": \"[0.1357, 0.4491]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1411, 0.7654, lift_point: 0.1411, 0.7654" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.7654]\", \"lift_point\": \"[0.1411, 0.7654]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4, 0.914, lift_point: 0.4, 0.914" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4, 0.914]\", \"lift_point\": \"[0.4, 0.914]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_ENTER#" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2892, 0.4823, lift_point: 0.2892, 0.4823" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2892, 0.4823]\", \"lift_point\": \"[0.2892, 0.4823]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in india\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats the news in India?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7091, 0.4199]\", \"lift_point\": \"[0.7091, 0.4199]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7042, 0.4983, lift_point: 0.7042, 0.4983" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7042, 0.4983]\", \"lift_point\": \"[0.7042, 0.4983]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Portland\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats a good restaurant in Portland?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1385, 0.4273]\", \"lift_point\": \"[0.1385, 0.4273]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1428, 0.4671, lift_point: 0.1428, 0.4671" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1428, 0.4671]\", \"lift_point\": \"[0.1428, 0.4671]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8881, 0.5096]\", \"lift_point\": \"[0.8881, 0.5096]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6359, 0.164]\", \"lift_point\": \"[0.6359, 0.164]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5025, 0.8773, lift_point: 0.5025, 0.8773" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5025, 0.8773]\", \"lift_point\": \"[0.5025, 0.8773]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5038, 0.4206, lift_point: 0.5038, 0.4206" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5038, 0.4206]\", \"lift_point\": \"[0.5038, 0.4206]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3216, 0.223]\", \"lift_point\": \"[0.3216, 0.223]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.2264, 0.1648, lift_point: 0.2264, 0.1648" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2264, 0.1648]\", \"lift_point\": \"[0.2264, 0.1648]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Drake video\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Play the new Drake \"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the new Drake \"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3079, 0.428, lift_point: 0.3079, 0.428" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3079, 0.428]\", \"lift_point\": \"[0.3079, 0.428]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4457, 0.1982, lift_point: 0.4457, 0.1982" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4457, 0.1982]\", \"lift_point\": \"[0.4457, 0.1982]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in San Francisco?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Whats the time in San Francisco\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in San Francisco\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.5038]\", \"lift_point\": \"[0.0615, 0.5038]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0497, 0.5548, lift_point: 0.0497, 0.5548" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0497, 0.5548]\", \"lift_point\": \"[0.0497, 0.5548]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6461, 0.3885, lift_point: 0.6461, 0.3885" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.3885]\", \"lift_point\": \"[0.6461, 0.3885]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.4957]\", \"lift_point\": \"[0.0614, 0.4957]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0565, 0.3216, lift_point: 0.0565, 0.3216" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.3216]\", \"lift_point\": \"[0.0565, 0.3216]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7965, 0.0738, lift_point: 0.7965, 0.0738" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7965, 0.0738]\", \"lift_point\": \"[0.7965, 0.0738]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1269, 0.4594]\", \"lift_point\": \"[0.1269, 0.4594]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1111, 0.206, lift_point: 0.1111, 0.206" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1111, 0.206]\", \"lift_point\": \"[0.1111, 0.206]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in San Francisco?\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0561, 0.479, lift_point: 0.0561, 0.479" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.479]\", \"lift_point\": \"[0.0561, 0.479]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.267, 0.3025]\", \"lift_point\": \"[0.267, 0.3025]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3741, 0.3787, lift_point: 0.3741, 0.3787" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3741, 0.3787]\", \"lift_point\": \"[0.3741, 0.3787]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in seattle\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"hotels in seattle\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in seattle\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8015, 0.622]\", \"lift_point\": \"[0.8015, 0.622]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8086, 0.6235, lift_point: 0.8086, 0.6235" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8086, 0.6235]\", \"lift_point\": \"[0.8086, 0.6235]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8828, 0.6863]\", \"lift_point\": \"[0.8828, 0.6863]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8609, 0.9214, lift_point: 0.8609, 0.9214" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8609, 0.9214]\", \"lift_point\": \"[0.8609, 0.9214]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3776, 0.5482]\", \"lift_point\": \"[0.3776, 0.5482]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6163, 0.5461, lift_point: 0.6163, 0.5461" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6163, 0.5461]\", \"lift_point\": \"[0.6163, 0.5461]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8838, 0.4552]\", \"lift_point\": \"[0.8838, 0.4552]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8867, 0.412, lift_point: 0.8867, 0.412" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.412]\", \"lift_point\": \"[0.8867, 0.412]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.8377]\", \"lift_point\": \"[0.0574, 0.8377]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Up#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.809]\", \"lift_point\": \"[0.0587, 0.809]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0662, 0.8267, lift_point: 0.0662, 0.8267" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.8267]\", \"lift_point\": \"[0.0662, 0.8267]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6668, 0.4068, lift_point: 0.6668, 0.4068" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6668, 0.4068]\", \"lift_point\": \"[0.6668, 0.4068]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.6477, 0.07, lift_point: 0.6477, 0.07" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6477, 0.07]\", \"lift_point\": \"[0.6477, 0.07]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.3924]\", \"lift_point\": \"[0.1347, 0.3924]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1408, 0.5501, lift_point: 0.1408, 0.5501" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1408, 0.5501]\", \"lift_point\": \"[0.1408, 0.5501]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.5298]\", \"lift_point\": \"[0.1196, 0.5298]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.119, 0.822, lift_point: 0.119, 0.822" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.119, 0.822]\", \"lift_point\": \"[0.119, 0.822]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7909, 0.6879, lift_point: 0.7909, 0.6879" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3773, 0.4787]\", \"lift_point\": \"[0.3773, 0.4787]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5953, 0.6958, lift_point: 0.5953, 0.6958" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5953, 0.6958]\", \"lift_point\": \"[0.5953, 0.6958]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.4231]\", \"lift_point\": \"[0.1326, 0.4231]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1215, 0.6063, lift_point: 0.1215, 0.6063" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.6063]\", \"lift_point\": \"[0.1215, 0.6063]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0651, 0.3924, lift_point: 0.0651, 0.3924" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.3924]\", \"lift_point\": \"[0.0651, 0.3924]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5609, 0.9437]\", \"lift_point\": \"[0.5609, 0.9437]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5624, 0.9437, lift_point: 0.5624, 0.9437" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.3325, 0.5901, lift_point: 0.3325, 0.5901" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3325, 0.5901]\", \"lift_point\": \"[0.3325, 0.5901]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8082, 0.4998]\", \"lift_point\": \"[0.8082, 0.4998]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1883, 0.8366, lift_point: 0.1883, 0.8366" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1883, 0.8366]\", \"lift_point\": \"[0.1883, 0.8366]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.4535, 0.855, lift_point: 0.4535, 0.855" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4535, 0.855]\", \"lift_point\": \"[0.4535, 0.855]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2442, 0.4882]\", \"lift_point\": \"[0.2442, 0.4882]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1405, 0.2047, lift_point: 0.1405, 0.2047" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1405, 0.2047]\", \"lift_point\": \"[0.1405, 0.2047]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.0814]\", \"lift_point\": \"[0.0551, 0.0814]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.052, 0.5992, lift_point: 0.052, 0.5992" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5992]\", \"lift_point\": \"[0.052, 0.5992]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in Berlin\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"time in berlin\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in berlin\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Bruno Mars video\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"Bruno Mars\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Bruno Mars\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6776]\", \"lift_point\": \"[0.7938, 0.6776]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7647, 0.6666, lift_point: 0.7647, 0.6666" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6666]\", \"lift_point\": \"[0.7647, 0.6666]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #STATUS_TASK_COMPLETE#" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.781, 0.6938]\", \"lift_point\": \"[0.781, 0.6938]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5761, 0.9414, lift_point: 0.5761, 0.9414" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5761, 0.9414]\", \"lift_point\": \"[0.5761, 0.9414]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1385, 0.6125]\", \"lift_point\": \"[0.1385, 0.6125]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1491, 0.5146, lift_point: 0.1491, 0.5146" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.5146]\", \"lift_point\": \"[0.1491, 0.5146]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8013, 0.1648]\", \"lift_point\": \"[0.8013, 0.1648]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8013, 0.1648, lift_point: 0.8013, 0.1648" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8013, 0.1648]\", \"lift_point\": \"[0.8013, 0.1648]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"starbucks\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \" Starbucks app\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Starbucks app\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.4744]\", \"lift_point\": \"[0.8887, 0.4744]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8785, 0.458, lift_point: 0.8785, 0.458" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.458]\", \"lift_point\": \"[0.8785, 0.458]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2866, 0.4464]\", \"lift_point\": \"[0.2866, 0.4464]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.284, 0.4831, lift_point: 0.284, 0.4831" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.284, 0.4831]\", \"lift_point\": \"[0.284, 0.4831]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8356, 0.6015]\", \"lift_point\": \"[0.8356, 0.6015]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.81, 0.502, lift_point: 0.81, 0.502" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.502]\", \"lift_point\": \"[0.81, 0.502]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.5274]\", \"lift_point\": \"[0.8975, 0.5274]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8862, 0.5515, lift_point: 0.8862, 0.5515" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.5515]\", \"lift_point\": \"[0.8862, 0.5515]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.061, 0.053, lift_point: 0.061, 0.053" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8058, 0.6208]\", \"lift_point\": \"[0.8058, 0.6208]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.8079, 0.6236, lift_point: 0.8079, 0.6236" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6236]\", \"lift_point\": \"[0.8079, 0.6236]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1286, 0.4499]\", \"lift_point\": \"[0.1286, 0.4499]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1306, 0.2331, lift_point: 0.1306, 0.2331" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.2331]\", \"lift_point\": \"[0.1306, 0.2331]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.439]\", \"lift_point\": \"[0.0551, 0.439]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0448, 0.4535, lift_point: 0.0448, 0.4535" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0448, 0.4535]\", \"lift_point\": \"[0.0448, 0.4535]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "Action Plan: [PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" + "gt": "Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3088, 0.2159]\", \"lift_point\": \"[0.3088, 0.2159]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1874, 0.6457]\", \"lift_point\": \"[0.1874, 0.6457]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.18, 0.7055, lift_point: 0.18, 0.7055" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.18, 0.7055]\", \"lift_point\": \"[0.18, 0.7055]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a rocket?\"", - "gt": "The action is #TYPE#; And the content is : \"typed_text\": \"What is the speed of a rocket?\"" + "gt": "Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a rocket?\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5856, 0.9453]\", \"lift_point\": \"[0.5856, 0.9453]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.5279, 0.7919, lift_point: 0.5279, 0.7919" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5279, 0.7919]\", \"lift_point\": \"[0.5279, 0.7919]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.6917]\", \"lift_point\": \"[0.7722, 0.6917]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7736, 0.7043, lift_point: 0.7736, 0.7043" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.7043]\", \"lift_point\": \"[0.7736, 0.7043]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1056, 0.2084]\", \"lift_point\": \"[0.1056, 0.2084]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.1056, 0.2133, lift_point: 0.1056, 0.2133" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1056, 0.2133]\", \"lift_point\": \"[0.1056, 0.2133]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7359, 0.9374, lift_point: 0.7359, 0.9374" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0613, 0.3336, lift_point: 0.0613, 0.3336" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.3336]\", \"lift_point\": \"[0.0613, 0.3336]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0671, 0.904]\", \"lift_point\": \"[0.0671, 0.904]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.0735, 0.9084, lift_point: 0.0735, 0.9084" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0735, 0.9084]\", \"lift_point\": \"[0.0735, 0.9084]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6871]\", \"lift_point\": \"[0.7849, 0.6871]\", \"typed_text\": \"\"", - "gt": "The action is #DUAL_POINT#; touch_point: 0.7808, 0.6931, lift_point: 0.7808, 0.6931" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6931]\", \"lift_point\": \"[0.7808, 0.6931]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", - "gt": "The action is #PRESS_HOME#" + "gt": "Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { - "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" }, { "pred": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3256, 0.1964]\", \"lift_point\": \"[0.3256, 0.1964]\", \"typed_text\": \"\"", - "gt": "The action is #Scrolling Down#" + "gt": "Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"" } ] \ No newline at end of file