add custom handler

handler.py

@@ -0,0 +1,80 @@
+from typing import Dict, List, Any
+from unsloth import FastLanguageModel
+class EndpointHandler():
+    def __init__(self, path="prashanthbsp/DeepSeek-R1-Distill-Llama-8B-unsloth-bnb-4bit-reasoning-cpg-entity-v1"):
+        # Preload all the elements you are going to need at inference.
+        # pseudo:
+        # self.model= load_model(path)
+        max_seq_length = 2048
+        dtype = None
+        load_in_4bit = True
+        model, tokenizer = FastLanguageModel.from_pretrained(
+            model_name = path,
+            max_seq_length = max_seq_length,
+            dtype = dtype,
+            load_in_4bit = load_in_4bit,
+        )
+        self.model = model
+        self.tokenizer = tokenizer
+
+    def __call__(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """
+       data args:
+            inputs (:obj: `str` | `PIL.Image` | `np.array`)
+            kwargs
+      Return:
+            A :obj:`list` | `dict`: will be serialized and returned
+        """
+
+        # pseudo
+        # self.model(input)
+        inputs = data.pop("inputs", data)
+        context = inputs.pop("context", inputs)
+        prompt_style = """Below is an instruction that describes a task, paired with an input that provides further context.
+            Write a response that appropriately completes the request.
+            Before answering, think carefully about the task to ensure a logical and accurate response.
+
+            ### Instruction
+            You are a helpful assistant analyzing social media posts. Your task is to extract ANY food, beverage, or supplement entities mentioned in the post and determine whether each entity is used as an ingredient or consumed as a product.
+
+            Guidelines:
+            - Extract ONLY food, beverage, or supplement entities mentioned in the post
+            - An entity is considered an ingredient if it's used as part of a recipe or combined with other foods
+            - An entity is considered a product if it's a food, beverage, or supplement consumed as is
+            - Focus on specific items rather than general categories when possible
+
+            Main thing to note - we ONLY want to extract food, beverage, or supplement entities, nothing else
+
+            Output in JSON format only:
+            {{
+            "entities": [
+                {{
+                "entity": "name of first entity",
+                "type": "ingredient or product"
+                }},
+                {{
+                "entity": "name of second entity",
+                "type": "ingredient or product"
+                }}
+            ]
+            }}
+
+            If no entities are found, output:
+            {{
+            "entities": []
+            }}
+
+            ### Social Media Post:
+            {0}
+            ### Response:
+            <think>{1}"""
+        FastLanguageModel.for_inference(model)
+        inputs = tokenizer([prompt_style.format(context, "")], return_tensors="pt").to("cuda")
+        outputs = model.generate(
+            input_ids=inputs.input_ids,
+            attention_mask=inputs.attention_mask,
+            max_new_tokens=1200,
+            use_cache=True,
+        )
+        response = tokenizer.batch_decode(outputs)
+        return response[0].split("### Response:")[1]