updated README

README.md

@@ -29,11 +29,46 @@ It is defined as the exponentiated average negative log-likelihood of a sequence
 For more information, see https://huggingface.co/docs/transformers/perplexity
 
 ## How to Use
-At minimum, this metric requires the model and text as inputs.
+At minimum, this metric requires the model and data as inputs.
 ```python
->>> perplexity = evaluate.load("d-matrix/perplexity", module_type="metric")
+>>> import evaluate
+>>> perplexity = evaluate.load("perplexity", module_type="metric")
 >>> input_texts = ["lorem ipsum", "Happy Birthday!", "Bienvenue"]
->>> results = perplexity.compute(model='distilgpt2',text=input_texts)
+>>> results = perplexity.compute(model='distilgpt2',data=input_texts)
 >>> print(results)
 {'accuracy': 1.0}
-```
+```
+
+### Inputs
+- **model** (`Union`[`str`,`AutoModelForCausalLM`]): model used for calculating Perplexity
+- **data** (`list` of `str`): input text, each separate text snippet is one list entry.
+- **device** (`str`): device to run on, defaults to 'cuda' when available.
+- **max_length** (`int`): maximum sequence length, defaults to 2048.
+
+### Output Values
+- **loss** (`float`): the loss of the model predictions compared to the reference
+- **perplexity**(`float`): measures the uncertainty of a model predicting text. Model performance is better when perplexity is lower.
+
+Output Example(s):
+```python
+{'accuracy': 1.0}
+```
+This metric outputs a dictionary, containing the loss and perplexity score.
+
+### Examples
+```python
+>>> import evaluate
+>>> from datasets import load_dataset
+>>> perplexity = evaluate.load("d-matrix/perplexity", module_type="metric")
+>>> input_texts = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")["text"][:10] 
+>>> results = perplexity.compute(model='distilgpt2',data=input_texts)
+>>> print(list(results.keys()))
+['loss', 'perplexity']
+>>> print(results['loss']) 
+3.8299286365509033
+>>> print(results['perplexity']) 
+46.05925369262695
+```
+
+## Citation(s)
+https://huggingface.co/docs/transformers/perplexity

perplexity.py

@@ -20,7 +20,7 @@ Args:
                     causal versions of t5, and more (the full list can be found
                     in the AutoModelForCausalLM documentation here:
                     https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForCausalLM )
-    text (list of str): input text, each separate text snippet is one list entry.
+    data (list of str): input text, each separate text snippet is one list entry.
     device (str): device to run on, defaults to 'cuda' when available.
     max_length (int): maximum sequence length, defaults to 2048.
 Returns:
@@ -31,7 +31,7 @@ Examples:
         >>> perplexity = evaluate.load("dmx_perplexity", module_type="metric")
         >>> input_texts = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")["text"][:10] # doctest: +SKIP
         >>> results = perplexity.compute(model='distilgpt2',
-        ...                              text=input_texts)
+        ...                              data=input_texts)
         >>> print(list(results.keys()))
         ['loss', 'perplexity']
         >>> print(results['loss']) # doctest: +SKIP
@@ -40,8 +40,8 @@ Examples:
         46.05925369262695
 """
 
-
-class DmxPerplexity(evaluate.Metric):
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class Perplexity(evaluate.Metric):
     def _info(self):
         return evaluate.MetricInfo(
             module_type="metric",
@@ -58,7 +58,7 @@ class DmxPerplexity(evaluate.Metric):
 
     def _compute(
         self,
-        text,
+        data,
         model: Union[str, AutoModelForCausalLM],
         device=None,
         max_length=None,
@@ -91,7 +91,7 @@ class DmxPerplexity(evaluate.Metric):
             max_seq_len = 2048
 
         model = model.to(device)
-        encodings = tokenizer("\n\n".join(text), return_tensors="pt")
+        encodings = tokenizer("\n\n".join(data), return_tensors="pt")
 
         stride = max_seq_len
         seq_len = encodings.input_ids.size(1)

requirements.txt

@@ -1,3 +1,4 @@
+git+https://github.com/huggingface/evaluate@main
 evaluate
 transformers
 torch