Initial commit

README.md

@@ -0,0 +1,2 @@
+# n‑gram LM (demo)
+A constant‑time n‑gram language model implemented in PyTorch. Designed for tiny‑footprint inference with vLLM.

config.json

@@ -0,0 +1,12 @@
+{
+  "architectures": [
+    "NGramLM"
+  ],
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "hidden_size": 128,
+  "model_type": "ngram",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "vocab_size": 50257
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ee89fa1a7910052da774d3041fad6bbf34a4e10d20dd8f03258afb33e7d41a4
+size 25731800

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}