Upload test_max_token.py with huggingface_hub
Browse files- test_max_token.py +3 -1
test_max_token.py
CHANGED
|
@@ -154,7 +154,9 @@ def main():
|
|
| 154 |
return_tensors="pt",
|
| 155 |
return_dict=True,
|
| 156 |
).to(model.device)
|
| 157 |
-
|
|
|
|
|
|
|
| 158 |
now_token_nums = inputs['input_ids'].shape
|
| 159 |
max_len_val.append(now_token_nums)
|
| 160 |
with open('max_token_nums.json','w') as f:
|
|
|
|
| 154 |
return_tensors="pt",
|
| 155 |
return_dict=True,
|
| 156 |
).to(model.device)
|
| 157 |
+
for attri in inputs:
|
| 158 |
+
print(attri,inputs[attri].shape)
|
| 159 |
+
break
|
| 160 |
now_token_nums = inputs['input_ids'].shape
|
| 161 |
max_len_val.append(now_token_nums)
|
| 162 |
with open('max_token_nums.json','w') as f:
|