Update README.md
Browse files
README.md
CHANGED
|
@@ -55,7 +55,7 @@ tokenizer = AutoTokenizer.from_pretrained("inclusionAI/Ring-mini-linear-2.0-GPTQ
|
|
| 55 |
sampling_params = SamplingParams(temperature=0.6, top_p=1.0, max_tokens=16384)
|
| 56 |
|
| 57 |
|
| 58 |
-
llm = LLM(model="inclusionAI/Ring-
|
| 59 |
|
| 60 |
|
| 61 |
prompt = "Give me a short introduction to large language models."
|
|
|
|
| 55 |
sampling_params = SamplingParams(temperature=0.6, top_p=1.0, max_tokens=16384)
|
| 56 |
|
| 57 |
|
| 58 |
+
llm = LLM(model="inclusionAI/Ring-flash-linear-2.0-GPTQ-int4", dtype='auto', enable_prefix_caching=False, max_num_seqs=128)
|
| 59 |
|
| 60 |
|
| 61 |
prompt = "Give me a short introduction to large language models."
|