Update app.py
Browse files
app.py
CHANGED
@@ -1,103 +1,638 @@
|
|
1 |
-
import
|
2 |
-
import pandas as pd
|
3 |
import os
|
4 |
-
import
|
5 |
-
import
|
6 |
-
|
7 |
-
from
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
|
71 |
-
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
|
79 |
-
|
80 |
-
|
81 |
-
|
82 |
-
data
|
83 |
-
|
84 |
-
|
85 |
-
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
90 |
else:
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
|
102 |
-
|
103 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import argparse
|
|
|
2 |
import os
|
3 |
+
import random
|
4 |
+
import re
|
5 |
+
import sys
|
6 |
+
from datetime import datetime
|
7 |
+
from typing import Dict, List, Tuple, Any
|
8 |
+
|
9 |
+
import pandas as pd
|
10 |
+
from huggingface_hub import login
|
11 |
+
from transformers import pipeline
|
12 |
+
from datasets import load_dataset
|
13 |
+
|
14 |
+
import gradio as gr
|
15 |
+
|
16 |
+
from huggingface_hub import HfApi, HfFolder
|
17 |
+
|
18 |
+
|
19 |
+
def extract_label(input_string: str) -> Tuple[str, str]:
|
20 |
+
"""
|
21 |
+
Extracts the label and its description from a given input string.
|
22 |
+
Args:
|
23 |
+
input_string (str): The input string containing a label and its description, separated by a colon.
|
24 |
+
Returns:
|
25 |
+
Tuple[str, str]
|
26 |
+
Raises:
|
27 |
+
ValueError: If the input string does not contain a colon.
|
28 |
+
"""
|
29 |
+
if ":" not in input_string:
|
30 |
+
raise ValueError(
|
31 |
+
"Input string must contain a ':' separating the label and description."
|
32 |
+
)
|
33 |
+
parts = input_string.split(":", 1)
|
34 |
+
return parts[0].strip(), parts[1].strip()
|
35 |
+
|
36 |
+
|
37 |
+
def parse_string(input_string: str) -> Tuple[str, str]:
|
38 |
+
"""
|
39 |
+
Parses a string containing `OUTPUT:` and `REASONING:` sections and extracts their values.
|
40 |
+
Args:
|
41 |
+
input_string (str): The input string containing `OUTPUT:` and `REASONING:` labels.
|
42 |
+
Returns:
|
43 |
+
Tuple[str, str]: A tuple containing two strings:
|
44 |
+
- The content following `OUTPUT:`.
|
45 |
+
- The content following `REASONING:`.
|
46 |
+
Raises:
|
47 |
+
ValueError: If the input string does not match the expected format with both `OUTPUT:` and `REASONING:` sections.
|
48 |
+
Note:
|
49 |
+
- The function is case-sensitive and assumes `OUTPUT:` and `REASONING:` are correctly capitalized.
|
50 |
+
"""
|
51 |
+
# Use regular expressions to extract OUTPUT and REASONING
|
52 |
+
match = re.search(r"OUTPUT:\s*(.+?)\s*REASONING:\s*(.+)", input_string, re.DOTALL)
|
53 |
+
|
54 |
+
if not match:
|
55 |
+
raise ValueError(
|
56 |
+
"The generated response is not in the expected 'OUTPUT:... REASONING:...' format."
|
57 |
+
)
|
58 |
+
|
59 |
+
output = match.group(1).strip()
|
60 |
+
reasoning = match.group(2).strip()
|
61 |
+
|
62 |
+
return output, reasoning
|
63 |
+
|
64 |
+
|
65 |
+
def sdg(
|
66 |
+
sample_size: int,
|
67 |
+
labels: List[str],
|
68 |
+
label_descriptions: str,
|
69 |
+
categories_types: Dict[str, List[str]],
|
70 |
+
use_case: str,
|
71 |
+
prompt_examples: str,
|
72 |
+
model: str,
|
73 |
+
max_new_tokens: int,
|
74 |
+
batch_size: int,
|
75 |
+
output_dir: str,
|
76 |
+
save_reasoning: bool,
|
77 |
+
) -> Tuple[str, str, str]:
|
78 |
+
"""
|
79 |
+
Generates synthetic data based on specified categories and labels.
|
80 |
+
Args:
|
81 |
+
sample_size (int): The number of synthetic data samples to generate.
|
82 |
+
labels (List[str]): The labels used to classify the synthetic data.
|
83 |
+
label_descriptions (str): A description of the meaning of each label.
|
84 |
+
categories_types (Dict[str, List[str]]): The categories and their types for data generation and diversification.
|
85 |
+
use_case (str): The use case of the synthetic data to provide context for the language model.
|
86 |
+
prompt_examples (str): The examples used in the Few-Shot or Chain-of-Thought prompting.
|
87 |
+
model (str): The large language model used for generating the synthetic data.
|
88 |
+
max_new_tokens (int): The maximum number of new tokens to generate for each sample.
|
89 |
+
batch_size (int): The number of samples per batch to append to the output file.
|
90 |
+
output_dir (str): The directory path where the output file will be saved.
|
91 |
+
save_reasoning (bool): Whether to save the reasoning or explanation behind the generated data.
|
92 |
+
Returns:
|
93 |
+
Tuple[str, str, str]: A tuple containing:
|
94 |
+
- A status message indicating the save location of the synthetic data.
|
95 |
+
- The path to the output CSV file.
|
96 |
+
- The timestamp used in the filename.
|
97 |
+
"""
|
98 |
+
categories = list(categories_types.keys())
|
99 |
+
|
100 |
+
# Generate filename with current date and time
|
101 |
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
102 |
+
output_path = os.path.join(output_dir, f"{timestamp}.csv")
|
103 |
+
|
104 |
+
num_batches = (sample_size + batch_size - 1) // batch_size
|
105 |
+
|
106 |
+
for batch in range(num_batches):
|
107 |
+
start = batch * batch_size
|
108 |
+
end = min(start + batch_size, sample_size)
|
109 |
+
|
110 |
+
batch_data = []
|
111 |
+
|
112 |
+
batch_random_labels = random.choices(labels, k=batch_size)
|
113 |
+
batch_random_categories = random.choices(categories, k=batch_size)
|
114 |
+
|
115 |
+
for i in range(start, end):
|
116 |
+
random_type = random.choices(
|
117 |
+
categories_types[batch_random_categories[i - start]]
|
118 |
+
)
|
119 |
+
prompt = f"""You should create synthetic data for specified labels and categories.
|
120 |
+
This is especially useful for {use_case}.
|
121 |
+
*Label Descriptions*
|
122 |
+
{label_descriptions}
|
123 |
+
*Examples*
|
124 |
+
{prompt_examples}
|
125 |
+
####################
|
126 |
+
Generate one output for the classification below.
|
127 |
+
You may use the examples I have provided as a guide, but you cannot simply modify or rewrite them.
|
128 |
+
Only return the OUTPUT and REASONING. The first token in your response must be OUTPUT.
|
129 |
+
Do not return the LABEL, CATEGORY, or TYPE.
|
130 |
+
LABEL: {batch_random_labels[i - start]}
|
131 |
+
CATEGORY: {batch_random_categories[i - start]}
|
132 |
+
TYPE: {random_type}
|
133 |
+
OUTPUT:
|
134 |
+
REASONING:
|
135 |
+
"""
|
136 |
+
messages = [
|
137 |
+
{
|
138 |
+
"role": "system",
|
139 |
+
"content": f"You are a helpful assistant designed to generate synthetic data for {use_case} with labels {labels} in categories {categories}. The first token in your generated text must be OUTPUT: This must be followed by the token REASONING: as in the prompt examples.",
|
140 |
+
},
|
141 |
+
{"role": "user", "content": prompt},
|
142 |
+
]
|
143 |
+
generator = pipeline("text-generation", model=model)
|
144 |
+
result = generator(messages, max_new_tokens=max_new_tokens)[0][
|
145 |
+
"generated_text"
|
146 |
+
][-1]["content"]
|
147 |
+
|
148 |
+
text, reasoning = parse_string(result)
|
149 |
+
|
150 |
+
entry = {
|
151 |
+
"text": text,
|
152 |
+
"label": batch_random_labels[i - start],
|
153 |
+
"source": model,
|
154 |
+
}
|
155 |
+
|
156 |
+
if save_reasoning:
|
157 |
+
entry["reasoning"] = reasoning
|
158 |
+
|
159 |
+
batch_data.append(entry)
|
160 |
+
|
161 |
+
batch_df = pd.DataFrame(batch_data)
|
162 |
+
|
163 |
+
if batch == 0:
|
164 |
+
batch_df.to_csv(output_path, mode="w", index=False)
|
165 |
else:
|
166 |
+
batch_df.to_csv(output_path, mode="a", header=False, index=False)
|
167 |
+
|
168 |
+
return f"Synthetic data saved to {output_path}", output_path, timestamp
|
169 |
+
|
170 |
+
|
171 |
+
def main() -> None:
|
172 |
+
"""
|
173 |
+
Main entry point for running the synthetic data generator.
|
174 |
+
"""
|
175 |
+
|
176 |
+
def run_sdg(
|
177 |
+
sample_size: int,
|
178 |
+
model: str,
|
179 |
+
max_new_tokens: int,
|
180 |
+
save_reasoning: bool,
|
181 |
+
token: str,
|
182 |
+
state: Dict,
|
183 |
+
label_boxes: List[Dict[str, str]],
|
184 |
+
use_case: str,
|
185 |
+
prompt_examples: str,
|
186 |
+
category_boxes: List[Dict[str, str]],
|
187 |
+
) -> Tuple[str, Dict]:
|
188 |
+
"""
|
189 |
+
Runs the synthetic data generation process and updates the application state.
|
190 |
+
Args:
|
191 |
+
sample_size (int): The total number of synthetic data samples to generate.
|
192 |
+
model (str): The large language model used for generating the synthetic data.
|
193 |
+
max_new_tokens (int): The maximum number of new tokens to generate for each sample.
|
194 |
+
save_reasoning (bool): Whether to save the reasoning or explanation behind the generated data.
|
195 |
+
token (str): The Hugging Face token for authentication.
|
196 |
+
state (Dict): The application state to store the output path and timestamp.
|
197 |
+
label_boxes (List[Dict[str, str]]): A list of label description dictionaries.
|
198 |
+
use_case (str): A string for the use case description.
|
199 |
+
prompt_examples (str): A string for prompt examples.
|
200 |
+
category_boxes (List[Dict[str, str]]): A list of category and type dictionaries .
|
201 |
+
Returns:
|
202 |
+
Tuple[str, Dict]: A tuple containing:
|
203 |
+
- A status message indicating the result of the generation process.
|
204 |
+
- The updated application state with the output path and timestamp.
|
205 |
+
"""
|
206 |
+
try:
|
207 |
+
login(token)
|
208 |
+
except Exception as e:
|
209 |
+
return f"Error logging in with token: {e}", None, None
|
210 |
+
|
211 |
+
label_descriptions = ""
|
212 |
+
labels = []
|
213 |
+
for box in label_boxes:
|
214 |
+
label_descriptions += box["content"] + "\n"
|
215 |
+
label, _ = extract_label(box["content"])
|
216 |
+
labels.append(label)
|
217 |
+
|
218 |
+
categories_types = {}
|
219 |
+
for box in category_boxes:
|
220 |
+
category, types = extract_label(box["content"])
|
221 |
+
categories_types[category.strip()] = [t.strip() for t in types.split(",")]
|
222 |
+
|
223 |
+
status, output_path, timestamp = sdg(
|
224 |
+
sample_size=sample_size,
|
225 |
+
labels=labels,
|
226 |
+
label_descriptions=label_descriptions,
|
227 |
+
categories_types=categories_types,
|
228 |
+
use_case=use_case,
|
229 |
+
prompt_examples=prompt_examples,
|
230 |
+
model=model,
|
231 |
+
max_new_tokens=max_new_tokens,
|
232 |
+
batch_size=20,
|
233 |
+
output_dir="./",
|
234 |
+
save_reasoning=save_reasoning,
|
235 |
+
)
|
236 |
+
state["output_path"] = output_path
|
237 |
+
state["timestamp"] = timestamp
|
238 |
+
return status, state
|
239 |
+
|
240 |
+
with gr.Blocks(css_paths="styles.css") as demo:
|
241 |
+
gr.Markdown(
|
242 |
+
"# Synthetic Data Generator",
|
243 |
+
elem_id="header",
|
244 |
+
elem_classes="text-center",
|
245 |
+
)
|
246 |
+
gr.Markdown(
|
247 |
+
"**Use Language Models to Create Datasets for Specified Labels and Categories**",
|
248 |
+
elem_classes="text-center",
|
249 |
+
)
|
250 |
+
with gr.Tab("Data Generator"):
|
251 |
+
with gr.Row(): # A row for two columns
|
252 |
+
with gr.Column(): # First column
|
253 |
+
gr.Markdown(
|
254 |
+
"## Setup & Configure",
|
255 |
+
elem_classes="text-center",
|
256 |
+
)
|
257 |
+
gr.Markdown("### Use Case")
|
258 |
+
use_case = gr.Textbox(
|
259 |
+
show_label=False,
|
260 |
+
info="Example. customer service",
|
261 |
+
autofocus=True,
|
262 |
+
)
|
263 |
+
label_boxes = gr.State([])
|
264 |
+
gr.Markdown("### Labels")
|
265 |
+
with gr.Row():
|
266 |
+
new_label = gr.Textbox(
|
267 |
+
show_label=False,
|
268 |
+
info="Example. polite: Text is considerate and shows respect and good manners, often including courteous phrases and a friendly tone.",
|
269 |
+
placeholder="Use a colon to separate each label and its description.",
|
270 |
+
)
|
271 |
+
add_label_button = gr.Button("Add", elem_classes="btn", scale=0)
|
272 |
+
|
273 |
+
def add_item(
|
274 |
+
label_boxes: List[Dict[str, str]], new_content: str
|
275 |
+
) -> Tuple[List[Dict[str, str]], str]:
|
276 |
+
"""
|
277 |
+
Adds a new label or category to the list if the input is not empty.
|
278 |
+
Args:
|
279 |
+
label_boxes (List[Dict[str, str]]): A list containing dictionaries representing the current labels or categories.
|
280 |
+
new_content (str): The new label or category content to add.
|
281 |
+
Returns:
|
282 |
+
Tuple[List[Dict[str, str]], str]: A tuple containing the updated list of labels or categories and an empty string to clear the input field.
|
283 |
+
"""
|
284 |
+
if new_content.strip():
|
285 |
+
return (
|
286 |
+
label_boxes + [{"content": new_content.strip()}],
|
287 |
+
"",
|
288 |
+
)
|
289 |
+
return label_boxes, ""
|
290 |
+
|
291 |
+
add_label_button.click(
|
292 |
+
add_item, [label_boxes, new_label], [label_boxes, new_label]
|
293 |
+
)
|
294 |
+
|
295 |
+
@gr.render(inputs=label_boxes)
|
296 |
+
def render_boxes(box_list: List[Dict[str, str]]) -> None:
|
297 |
+
"""
|
298 |
+
Renders a list of labels in a Gradio interface.
|
299 |
+
Args:
|
300 |
+
box_list (List[Dict[str, str]]): A list containing dictionaries representing the categories to render.
|
301 |
+
"""
|
302 |
+
with gr.Accordion(
|
303 |
+
f"Number of Entered Labels ({len(box_list)})"
|
304 |
+
):
|
305 |
+
for box in box_list:
|
306 |
+
with gr.Row():
|
307 |
+
gr.Textbox(
|
308 |
+
box["content"],
|
309 |
+
show_label=False,
|
310 |
+
container=False,
|
311 |
+
)
|
312 |
+
delete_button = gr.Button(
|
313 |
+
"Delete", scale=0, variant="stop"
|
314 |
+
)
|
315 |
+
|
316 |
+
def delete(
|
317 |
+
box: Dict[str, str] = box,
|
318 |
+
) -> List[Dict[str, str]]:
|
319 |
+
"""
|
320 |
+
Deletes a specific box from the list of labels.
|
321 |
+
Args:
|
322 |
+
box (Dict[str, str]): The box to be removed from the list.
|
323 |
+
Returns:
|
324 |
+
List[Dict[str, str]]: The updated list of labels after the box is removed.
|
325 |
+
"""
|
326 |
+
box_list.remove(box)
|
327 |
+
return box_list
|
328 |
+
|
329 |
+
delete_button.click(delete, None, [label_boxes])
|
330 |
+
|
331 |
+
category_boxes = gr.State([])
|
332 |
+
gr.Markdown("### Categories")
|
333 |
+
with gr.Row():
|
334 |
+
new_category = gr.Textbox(
|
335 |
+
show_label=False,
|
336 |
+
info="Example. travel: hotel, airline, train",
|
337 |
+
placeholder="Use a colon to separate each category and its types.",
|
338 |
+
)
|
339 |
+
add_category_button = gr.Button(
|
340 |
+
"Add", elem_classes="btn", scale=0
|
341 |
+
)
|
342 |
+
add_category_button.click(
|
343 |
+
add_item,
|
344 |
+
[category_boxes, new_category],
|
345 |
+
[category_boxes, new_category],
|
346 |
+
)
|
347 |
+
|
348 |
+
@gr.render(inputs=category_boxes)
|
349 |
+
def render_boxes(box_list: List[Dict[str, str]]) -> None:
|
350 |
+
"""
|
351 |
+
Renders a list of categories in a Gradio interface.
|
352 |
+
Args:
|
353 |
+
box_list (List[Dict[str, str]]): A list containing dictionaries representing the categories to render.
|
354 |
+
"""
|
355 |
+
with gr.Accordion(
|
356 |
+
f"Number of Entered Categories ({len(box_list)})"
|
357 |
+
):
|
358 |
+
for box in box_list:
|
359 |
+
with gr.Row():
|
360 |
+
gr.Textbox(
|
361 |
+
box["content"],
|
362 |
+
show_label=False,
|
363 |
+
container=False,
|
364 |
+
)
|
365 |
+
delete_button = gr.Button(
|
366 |
+
"Delete", scale=0, variant="stop"
|
367 |
+
)
|
368 |
+
|
369 |
+
def delete(
|
370 |
+
box: Dict[str, str] = box,
|
371 |
+
) -> List[Dict[str, str]]:
|
372 |
+
"""
|
373 |
+
Deletes a specific box from the list of categories.
|
374 |
+
Args:
|
375 |
+
box (Dict[str, str]): The box to be removed from the list.
|
376 |
+
Returns:
|
377 |
+
List[Dict[str, str]]: The updated list of categories after the box is removed.
|
378 |
+
"""
|
379 |
+
box_list.remove(box)
|
380 |
+
return box_list
|
381 |
+
|
382 |
+
delete_button.click(delete, None, [category_boxes])
|
383 |
+
|
384 |
+
gr.Markdown(
|
385 |
+
"### Guiding Examples",
|
386 |
+
)
|
387 |
+
prompt_examples = gr.Textbox(
|
388 |
+
show_label=False,
|
389 |
+
info="""Example.
|
390 |
+
LABEL: polite
|
391 |
+
CATEGORY: food and drink
|
392 |
+
TYPE: cafe
|
393 |
+
OUTPUT: Thank you for visiting! While we prepare your coffee, feel free to relax or browse our selection of pastries. Let us know if we can make your day even better!
|
394 |
+
REASONING: This text is polite because it expresses gratitude and encourages the customer to feel at ease with a welcoming tone. Phrases like "Let us know if we can make your day even better" show warmth and consideration, enhancing the customer experience.""",
|
395 |
+
placeholder="Include all examples in this box. Use the format\n'LABEL: label_name\nCATEGORY: category_name\nTYPE: type_name\nOUTPUT: generated_output\nREASONING: reasoning'",
|
396 |
+
lines=6,
|
397 |
+
)
|
398 |
+
gr.Markdown("### Language Model")
|
399 |
+
model = gr.Dropdown(
|
400 |
+
label="Model",
|
401 |
+
choices=[
|
402 |
+
"google/gemma-3-1b-it",
|
403 |
+
"HuggingFaceTB/SmolLM2-1.7B-Instruct",
|
404 |
+
"meta-llama/Llama-3.2-3B-Instruct",
|
405 |
+
],
|
406 |
+
value="google/gemma-3-1b-it",
|
407 |
+
)
|
408 |
+
max_new_tokens = gr.Number(
|
409 |
+
label="Max New Tokens", value=256, minimum=64
|
410 |
+
)
|
411 |
+
token = gr.Textbox(
|
412 |
+
label="Hugging Face Token",
|
413 |
+
type="password",
|
414 |
+
info="Enter a 'Read' Hugging Face token to access gated language models, or a 'Write' token to push the generated data to Hugging Face.",
|
415 |
+
)
|
416 |
+
with gr.Column(): # Second column
|
417 |
+
gr.Markdown(
|
418 |
+
"## Generate & Export",
|
419 |
+
elem_classes="text-center",
|
420 |
+
)
|
421 |
+
gr.Markdown("### Status")
|
422 |
+
status = gr.Textbox(label="Status")
|
423 |
+
gr.Markdown("### Actions")
|
424 |
+
sample_size = gr.Number(label="Sample Size", value=1, minimum=1)
|
425 |
+
save_reasoning = gr.Checkbox(label="Save Reasoning", value=True)
|
426 |
+
generate_button = gr.Button(
|
427 |
+
"Generate!", interactive=False, elem_classes="btn"
|
428 |
+
)
|
429 |
+
download_button = gr.Button(
|
430 |
+
"Download CSV", interactive=False, elem_classes="btn"
|
431 |
+
)
|
432 |
+
file_output = gr.File(label="Download!", visible=False)
|
433 |
+
repo_id = gr.Textbox(
|
434 |
+
label="Hugging Face Repo ID",
|
435 |
+
placeholder="your-username/your-repo-name",
|
436 |
+
)
|
437 |
+
is_public_repo = gr.Checkbox(
|
438 |
+
label="Make Repository Public", value=False
|
439 |
+
)
|
440 |
+
push_button = gr.Button(
|
441 |
+
"Push to Hugging Face", interactive=False, elem_classes="btn"
|
442 |
+
)
|
443 |
+
gr.Markdown(
|
444 |
+
"### Sample Output",
|
445 |
+
)
|
446 |
+
dataset = load_dataset("intel/polite-guard", data_files={"validation": "data/validation/val_cot.csv"})
|
447 |
+
dataset = dataset["validation"].to_pandas().sample(n=5)
|
448 |
+
|
449 |
+
dataframe = gr.Dataframe(value=dataset, show_label=False)
|
450 |
+
|
451 |
+
state = gr.State({"output_path": None, "timestamp": None})
|
452 |
+
|
453 |
+
def toggle_button(
|
454 |
+
token_value: str,
|
455 |
+
label_value: List[Dict[str, str]],
|
456 |
+
category_value: List[Dict[str, str]],
|
457 |
+
use_case_value: str,
|
458 |
+
example_value: str,
|
459 |
+
) -> Dict[str, Any]:
|
460 |
+
"""
|
461 |
+
Toggles the interactivity of the generate button based on input values.
|
462 |
+
Args:
|
463 |
+
token_value (str): The Hugging Face token value.
|
464 |
+
label_value (List[Dict[str, str]]): A list of label description dictionaries.
|
465 |
+
category_value (List[Dict[str, str]]): A list of category and type dictionaries.
|
466 |
+
use_case_value (str): A string for the use case description.
|
467 |
+
example_value (str): A string for prompt examples.
|
468 |
+
Returns:
|
469 |
+
Dict[str, Any]: A dictionary containing the updated interactivity state of the generate button.
|
470 |
+
"""
|
471 |
+
return gr.update(
|
472 |
+
interactive=all(
|
473 |
+
[
|
474 |
+
token_value,
|
475 |
+
label_value,
|
476 |
+
category_value,
|
477 |
+
use_case_value,
|
478 |
+
example_value,
|
479 |
+
]
|
480 |
+
)
|
481 |
+
)
|
482 |
+
|
483 |
+
token.change(
|
484 |
+
toggle_button,
|
485 |
+
inputs=[token, label_boxes, category_boxes, use_case, prompt_examples],
|
486 |
+
outputs=generate_button,
|
487 |
+
)
|
488 |
+
label_boxes.change(
|
489 |
+
toggle_button,
|
490 |
+
inputs=[token, label_boxes, category_boxes, use_case, prompt_examples],
|
491 |
+
outputs=generate_button,
|
492 |
+
)
|
493 |
+
category_boxes.change(
|
494 |
+
toggle_button,
|
495 |
+
inputs=[token, label_boxes, category_boxes, use_case, prompt_examples],
|
496 |
+
outputs=generate_button,
|
497 |
+
)
|
498 |
+
use_case.change(
|
499 |
+
toggle_button,
|
500 |
+
inputs=[token, label_boxes, category_boxes, use_case, prompt_examples],
|
501 |
+
outputs=generate_button,
|
502 |
+
)
|
503 |
+
prompt_examples.change(
|
504 |
+
toggle_button,
|
505 |
+
inputs=[token, label_boxes, category_boxes, use_case, prompt_examples],
|
506 |
+
outputs=generate_button,
|
507 |
+
)
|
508 |
+
|
509 |
+
def enable_buttons(state: Dict[str, Any]) -> List[Any]:
|
510 |
+
"""
|
511 |
+
Enables the interactivity of the download and push buttons and loads a preview of the generated data.
|
512 |
+
Args:
|
513 |
+
state (Dict[str, Any]): The application state containing the output file path.
|
514 |
+
Returns:
|
515 |
+
List[Any]: A list containing:
|
516 |
+
- An update to make the download button interactive.
|
517 |
+
- An update to make the push button interactive.
|
518 |
+
- A DataFrame preview of the generated data.
|
519 |
+
"""
|
520 |
+
df = pd.read_csv(state["output_path"]).head()
|
521 |
+
return [gr.update(interactive=True), gr.update(interactive=True), df]
|
522 |
+
|
523 |
+
generate_button.click(
|
524 |
+
run_sdg,
|
525 |
+
inputs=[
|
526 |
+
sample_size,
|
527 |
+
model,
|
528 |
+
max_new_tokens,
|
529 |
+
save_reasoning,
|
530 |
+
token,
|
531 |
+
state,
|
532 |
+
label_boxes,
|
533 |
+
use_case,
|
534 |
+
prompt_examples,
|
535 |
+
category_boxes,
|
536 |
+
],
|
537 |
+
outputs=[status, state],
|
538 |
+
).success(
|
539 |
+
enable_buttons,
|
540 |
+
inputs=[state],
|
541 |
+
outputs=[download_button, push_button, dataframe],
|
542 |
+
)
|
543 |
+
|
544 |
+
def download_csv(state: Dict) -> str:
|
545 |
+
"""
|
546 |
+
Generate the file path for downloading a CSV file.
|
547 |
+
Args:
|
548 |
+
state (Dict): The application state.
|
549 |
+
Returns:
|
550 |
+
str: The file path to the CSV file for download.
|
551 |
+
"""
|
552 |
+
return state[
|
553 |
+
"output_path"
|
554 |
+
] # Return the file path to trigger the download
|
555 |
+
|
556 |
+
def push_to_huggingface(
|
557 |
+
repo_id: str,
|
558 |
+
token_value: str,
|
559 |
+
is_public: bool,
|
560 |
+
state: Dict,
|
561 |
+
) -> str:
|
562 |
+
"""
|
563 |
+
Pushes the generated synthetic data file to the Hugging Face Hub.
|
564 |
+
Args:
|
565 |
+
repo_id (str): The ID of the Hugging Face repository (e.g., "username/repo-name").
|
566 |
+
token_value (str): The Hugging Face token for authentication.
|
567 |
+
is_public (bool): Whether to make the repository public.
|
568 |
+
state (Dict): The application state containing the output file path and timestamp.
|
569 |
+
Returns:
|
570 |
+
str: A message indicating the result of the upload process.
|
571 |
+
"""
|
572 |
+
try:
|
573 |
+
api = HfApi(token=token_value)
|
574 |
+
except Exception as e:
|
575 |
+
return f"Invalid token for writing to Hugging Face: {e}"
|
576 |
+
|
577 |
+
try:
|
578 |
+
# Ensure the repository exists, creating it if it doesn't
|
579 |
+
api.create_repo(
|
580 |
+
repo_id=repo_id,
|
581 |
+
repo_type="dataset",
|
582 |
+
exist_ok=True,
|
583 |
+
private=not is_public,
|
584 |
+
)
|
585 |
+
|
586 |
+
api.upload_file(
|
587 |
+
path_or_fileobj=state["output_path"],
|
588 |
+
path_in_repo=f"{state['timestamp']}.csv",
|
589 |
+
repo_id=repo_id,
|
590 |
+
repo_type="dataset",
|
591 |
+
)
|
592 |
+
except Exception as e:
|
593 |
+
return f"Error uploading file to Hugging Face: {e}"
|
594 |
+
visibility = "public" if is_public else "private"
|
595 |
+
return f"File pushed to {visibility} Hugging Face Hub at {repo_id}/{state['timestamp']}.csv"
|
596 |
+
|
597 |
+
download_button.click(download_csv, inputs=state, outputs=file_output).then(
|
598 |
+
lambda: gr.update(visible=True), outputs=file_output
|
599 |
+
)
|
600 |
+
|
601 |
+
push_button.click(
|
602 |
+
push_to_huggingface,
|
603 |
+
inputs=[repo_id, token, is_public_repo, state],
|
604 |
+
outputs=status,
|
605 |
+
)
|
606 |
+
|
607 |
+
with gr.Tab("About"):
|
608 |
+
with gr.Row():
|
609 |
+
with gr.Column(scale=1):
|
610 |
+
gr.Image(
|
611 |
+
"polite-guard.png",
|
612 |
+
show_download_button=False,
|
613 |
+
show_fullscreen_button=False,
|
614 |
+
show_label=False,
|
615 |
+
show_share_button=False,
|
616 |
+
container=False,
|
617 |
+
)
|
618 |
+
with gr.Column(scale=3):
|
619 |
+
gr.Markdown(
|
620 |
+
"""
|
621 |
+
This synthetic data generator, part of Intel's [Polite Guard](https://huggingface.co/Intel/polite-guard) project, utilizes a specified language model to generate synthetic data for a given use case.
|
622 |
+
If you find this project valuable, please consider giving it a ❤️ on Hugging Face and sharing it with your network.
|
623 |
+
Visit
|
624 |
+
- [Polite Guard GitHub repository](https://github.com/intel/polite-guard) for the source code that you can run through the command line,
|
625 |
+
- [Synthetic Data Generation with Language Models: A Practical Guide](https://medium.com/p/0ff98eb226a1) to learn more about the implementation of this data generator, and
|
626 |
+
- [Polite Guard Dataset](https://huggingface.co/datasets/Intel/polite-guard) for an example of a dataset generated using this data generator.
|
627 |
+
## Privacy Notice
|
628 |
+
Please note that this data generator uses AI technology and you are interacting with a chat model.
|
629 |
+
Prompts that are being used during the demo and your personal information will not be stored.
|
630 |
+
For information regarding the handling of personal data collected refer to the Global Privacy Notice (https://www.intel.com/content/www/us/en/privacy/intelprivacy-notice.html), which encompass our privacy practices.
|
631 |
+
"""
|
632 |
+
)
|
633 |
+
|
634 |
+
demo.launch()
|
635 |
+
|
636 |
+
|
637 |
+
if __name__ == "__main__":
|
638 |
+
main()
|