Rename handler.py to tokenizer.py

handler.py

@@ -1,12 +0,0 @@
-class EndpointHandler:
-    def __init__(self):
-        from transformers import AutoModelForSequenceClassification, AutoTokenizer
-        self.tokenizer = AutoTokenizer.from_pretrained(".")
-        self.model = AutoModelForSequenceClassification.from_pretrained(".")
-
-    def __call__(self, inputs: dict) -> dict:
-        texts = inputs.get("inputs")
-        tokens = self.tokenizer(texts, return_tensors="pt", padding=True)
-        outputs = self.model(**tokens)
-        # convert outputs to JSON-serializable form
-        return {"logits": outputs.logits.tolist()}

tokenizer.py

@@ -0,0 +1,14 @@
+import numpy as np
+import torch
+import tiktoken
+from torch import nn
+tokenizer = tiktoken.get_encoding("cl100k_base")
+
+def text_to_token_ids(text, tokenizer):
+    encoded = tokenizer.encode(text, allowed_special={'<|endoftext|>'})
+    encoded_tensor = torch.tensor(encoded).unsqueeze(0)
+    return encoded_tensor
+
+def token_ids_to_text(token_ids, tokenizer):
+    flat = token_ids.squeeze(0)
+    return tokenizer.decode(flat.tolist())