Update utils.py
Browse files
utils.py
CHANGED
|
@@ -12,7 +12,7 @@ from dotenv import load_dotenv
|
|
| 12 |
load_dotenv()
|
| 13 |
HF_READ=os.environ["HF_READ"]
|
| 14 |
HF_WRITE=os.environ["HF_WRITE"]
|
| 15 |
-
|
| 16 |
model_base_url={}
|
| 17 |
LANGUAGE="MOROCCAN Arabic"
|
| 18 |
HF_DATASET="abdeljalilELmajjodi/Mohadata"
|
|
@@ -101,7 +101,7 @@ def load_upload_ds_hf(df):
|
|
| 101 |
print("[INFO] dataset loaded successfully")
|
| 102 |
new_ds=Dataset.from_pandas(df,preserve_index=False)
|
| 103 |
updated_ds=concatenate_datasets([dataset_stream,new_ds])
|
| 104 |
-
updated_ds.push_to_hub("
|
| 105 |
print("[INFO] dataset uploaded successfully")
|
| 106 |
|
| 107 |
async def load_upload_ds_hf_async(df):
|
|
|
|
| 12 |
load_dotenv()
|
| 13 |
HF_READ=os.environ["HF_READ"]
|
| 14 |
HF_WRITE=os.environ["HF_WRITE"]
|
| 15 |
+
print(HF_READ,HF_WRITE)
|
| 16 |
model_base_url={}
|
| 17 |
LANGUAGE="MOROCCAN Arabic"
|
| 18 |
HF_DATASET="abdeljalilELmajjodi/Mohadata"
|
|
|
|
| 101 |
print("[INFO] dataset loaded successfully")
|
| 102 |
new_ds=Dataset.from_pandas(df,preserve_index=False)
|
| 103 |
updated_ds=concatenate_datasets([dataset_stream,new_ds])
|
| 104 |
+
updated_ds.push_to_hub("atlasia/Mohadata_Dataset",token=HF_WRITE)
|
| 105 |
print("[INFO] dataset uploaded successfully")
|
| 106 |
|
| 107 |
async def load_upload_ds_hf_async(df):
|