Upload inference.py

inference.py

@@ -0,0 +1,18 @@
+# inference.py
+from tokenizers import Tokenizer
+from typing import List, Dict
+
+# 加载 tokenizer
+tokenizer = Tokenizer.from_pretrained(".")  # 从当前目录加载
+
+def tokenize(text: str) -> Dict[str, List[str]]:
+    """接收文本，返回分词结果"""
+    encoded = tokenizer.encode(text)
+    return {
+        "tokens": encoded.tokens,
+        "ids": encoded.ids
+    }
+
+# 测试示例（可选）
+if __name__ == "__main__":
+    print(tokenize("Hello, this is a test."))