a intent parser using llama.cpp and mistral-7b

llama_intents.py

import json
from typing import List

from llama_cpp import ChatCompletionMessage, Llama, LlamaGrammar

TRAINING_INTENTS = [
    {
        "name": "get_current_weather",
        "description": "Get the current weather in a given location",
        "parameters": {
            "type": "object",
            "properties": {
                "location": {
                    "type": "string",
                    "description": "The city and state, e.g. San Francisco, CA",
                },
                "unit": {"type": "string", "enum": ["metric", "uscs"]},
            },
            "required": ["location"],
        },
    },
    {
        "name": "binary_arithmetic",
        "description": "Perform binary arithmetic operations on two operands.",
        "parameters": {
            "type": "object",
            "properties": {
                "left_op": {
                    "type": ["integer", "number"],
                    "description": "The left operand.",
                },
                "right_op": {
                    "type": ["integer", "number"],
                    "description": "The right operand.",
                },
                "operator": {
                    "type": "string",
                    "description": "The arithmetic operator. Supported operators are '+', '-', '*', '/', '%'.",
                    "enum": ["+", "-", "*", "/", "%"],
                },
            },
            "required": ["left_op", "right_op", "operator"],
        },
    },
]


class IntentLLama:
    def __init__(self, model, grammar="json.gbnf"):
        self.system_prompt = ChatCompletionMessage(
            role="system",
            content="""My name is Vincent and I am a helpful assistant. 
            I can make function calls to retrieve information such as the current weather in a given location.
            \n{ "function_call": { "name": "binary_arithmetic", "arguments": { "left_op": 10, "right_op": 5, "operator": "+" } } }""",
        )
        self.llm = Llama(model_path=model, verbose=False)
        self.llama_grammar = LlamaGrammar.from_file(grammar)

    def generate_chat_sequence(self,
                               user_query: str,
                               function_list: list,
                               ) -> List[ChatCompletionMessage]:
        function_messages = [
            ChatCompletionMessage(role="function", content=json.dumps(func_def))
            for func_def in function_list
        ]
        user_message = ChatCompletionMessage(role="user", content=user_query)
        messages = [self.system_prompt] + function_messages + [user_message]
        return messages

    def answer_with_function(self, query):
        messages = self.generate_chat_sequence(query, TRAINING_INTENTS)
        response = self.llm.create_chat_completion(messages=messages,
                                                   grammar=self.llama_grammar,
                                                   temperature=0)
        assistant_content = response["choices"][0]["message"]["content"]
        return json.loads(assistant_content)

    def get_intent(self, query):
        func = self.answer_with_function(query)
        return {
            "intent_type": func["function_call"]["name"],
            "entities": func["function_call"].get("arguments", {}),
        }


llm = IntentLLama("mistral-7b-openorca.Q5_K_S.gguf")
ans = llm.get_intent("What is 10 times 12345?")
# {'intent_type': 'binary_arithmetic', 'entities': {'left_op': 10, 'right_op': 12345, 'operator': '*'}}
print(ans)
ans = llm.get_intent("What is the weather like in Boston?")
# {'intent_type': 'weather', 'entities': {'location': 'Boston'}}
print(ans)
ans = llm.get_intent("What time is it")
# {'intent_type': 'time', 'entities': {}}
print(ans)
ans = llm.get_intent("Tell me a joke")
print(ans)
# {'intent_type': 'tell_joke', 'entities': {}}

get the json grammar here: https://github.com/ggerganov/llama.cpp/tree/master/grammars
get models from TheBloke: https://huggingface.co/TheBloke?search_models=gguf