OPENAI_KEY=
NGC_API_KEY=YOUR_NGC_API_KEY

Then, we import all the libraries, load the API Keys, and define the models we want to use from the NVIDIA catalog.

import os

import matplotlib.pyplot as plt
import pandas as pd
import scikit_posthocs as sp
import seaborn as sns
import utils
from datasets import load_dataset
from dotenv import load_dotenv
from generator import Generator
load_dotenv('env/var.env')
# models
llama8b = Generator(model='meta/llama3-8b-instruct', ngc_key=os.getenv("NGC_API_KEY"))
mistral7b = Generator(model="mistralai/mistral-7b-instruct-v0.3", ngc_key=os.getenv("NGC_API_KEY"))
llama70b = Generator(model="meta/llama3-70b-instruct", ngc_key=os.getenv("NGC_API_KEY"))
mixtral = Generator(model="mistralai/mixtral-8x22b-instruct-v0.1", ngc_key=os.getenv("NGC_API_KEY"))

The Generator class is responsible for loading the model and creating the Prompt Template powered by LangChain. It formats the query and the context before passing it to the LLM to get a response.

from langchain import PromptTemplate
from langchain_openai import ChatOpenAI
from langchain.chains import LLMChain

class Generator:
    """Generator, aka LLM, to provide an answer based on some question and context"""
    def __init__(self, model: str, ngc_key: str) -> None:
        # template
        self.template = """
            Use the following pieces of context to give a succinct and clear answer to the question at the end:
            {context}
            Question: {question}
            Answer:
        """
        # llm
        self.llm = ChatOpenAI(
          base_url="",
            api_key=ngc_key,
            model=model,
            temperature=0.1)
        # create prompt template
        self.prompt = PromptTemplate(
            template=self.template, input_variables=["context", "question"]
        )
    def generate_answer(self, context: str, question: str) -> str:
        """
        Get the answer from llm based on context and user's question
        Args:
            context (str): most similar document retrieved
            question (str): user's question
        Returns:
            str: llm answer
        """
        query_llm = LLMChain(
            llm=self.llm,
            prompt=self.prompt,
            llm_kwargs={"max_tokens": 2000},
        )
        answer = query_llm.invoke(
            {"context": context, "question": question}
        )
        
        return answer['text']

With the LLMs loaded, we fetch the SQuAD dataset from HuggingFace and shuffle it to ensure enough variety in the question themes.

squad = load_dataset("squad", split="train")
squad = squad.shuffle()

Now, we can apply the RAQ loop over 100 questions and contexts and record the above metrics.

for i in range(100):
    context = squad[i]['context']
    query = squad[i]['question']
    answer = squad[i]['answers']['text'][0]

    # llama 8b
    answer_llama, words_per_second, words = utils.get_llm_response(llama8b, context, query)
    llama8b_metrics["words_per_second"].append(words_per_second)
    llama8b_metrics["words"].append(words)
    # mistral 7b 
    answer_mistral, words_per_second, words = utils.get_llm_response(mistral7b, context, query)
    mistral7b_metrics["words_per_second"].append(words_per_second)
    mistral7b_metrics["words"].append(words)
    # llama 70b
    answer_llama70b, words_per_second, words = utils.get_llm_response(llama70b, context, query)
    llama70b_metrics["words_per_second"].append(words_per_second)
    llama70b_metrics["words"].append(words)
    # mixtral
    answer_mixtral, words_per_second, words = utils.get_llm_response(mixtral, context, query)
    mixtral_metrics["words_per_second"].append(words_per_second)
    mixtral_metrics["words"].append(words)
    # GPT-3.5 rank
    llm_answers_dict = {'llama8b': answer_llama, 'mistral7b': answer_mistral, 'llama70b': answer_llama70b, 'mixtral': answer_mixtral}
    rank = utils.get_gpt_rank(answer, llm_answers_dict, os.getenv("OPENAI_API_KEY"))
    llama8b_metrics["rank"].append(rank.index('1')+1)
    mistral7b_metrics["rank"].append(rank.index('2')+1)
    llama70b_metrics["rank"].append(rank.index('3')+1)
    mixtral_metrics["rank"].append(rank.index('4')+1)

The function get_llm_response receives the loaded LLM, the context, and the question and returns the LLM answer as well as the quantitative metrics.

def get_llm_response(model: Generator, context: str, query: str) -> Tuple[str, int, int]:
    """
    Generates an answer from a given LLM based on context and query
    returns the answer and the number of words per second and the total number of words
    Args:
        model (Generator): LLM
        context (str): context data
        query (str): question
    Returns:
        Tuple[str, int, int]: answer, words_per_second, words
    """

    init_time = time.time()
    answer_llm = model.get_answer(context, query)
    total_time = time.time()-init_time
    words_per_second = len(re.sub("[^a-zA-Z']+", ' ', answer_llm).split())/total_time
    words = len(re.sub("[^a-zA-Z']+", ' ', answer_llm).split())
    return answer_llm, words_per_second, words

On the other hand, the function get_gpt_rank implements the RAQ core logic. It is responsible for receiving the ground truth answer and each of the LLM answers and sending a request to GPT-3.5 to rank them based on correctness.

def get_gpt_rank(true_answer: str, llm_answers: dict, openai_key: str) -> list:
    """
    Implements RAQ core: based on the true answer, it uses GPT-3.5 to rank the answers of the LLMs
    Args:
        true_answer (str): correct answer
        llm_answers (dict): LLM answers
        openai_key (str): open ai key
    Returns:
        list: rank of LLM IDs
    """
    
    # get a formatted output from OpenAI
    functions = define_open_ai_function()
    gpt_query = f"""Based on the correct answer: {true_answer}, rank the IDs of the following four answers from the most to the least correct one:
        ID: 1 Answer: {re.sub("[^a-zA-Z0-9']+", ' ', llm_answers['llama8b'])}
        ID: 2 Answer: {re.sub("[^a-zA-Z0-9']+", ' ', llm_answers['mistral7b'])}
        ID: 3 Answer: {re.sub("[^a-zA-Z0-9']+", ' ', llm_answers['llama70b'])}
        ID: 4 Answer: {re.sub("[^a-zA-Z0-9']+", ' ', llm_answers['mixtral'])}"""
    completion = OpenAI(api_key=openai_key).chat.completions.create(
        model="gpt-3.5-turbo",
        messages=[{"role": "user", "content": gpt_query}],
        functions=functions,
        function_call={"name": "return_rank"},
    )
    response_message = completion.choices[0].message.function_call.arguments
    rank = ast.literal_eval(response_message)["rank"].split(",")
    if len(rank) == 1:
        rank = list(rank[0])
    return rank

From Figure 3, it is evident that LLaMA 3 8B is the fastest LLM, producing an average of approximately 43 words per second. In terms of answer length, Mistral 7B produces longer answers, with an average answer length of 24 words, significantly more than LLaMA 70B, which produces only 8 words. Finally, based on the independent LLM rankings, Mistral 7B had the best average rank of approximately 2.25, while LLaMA 3 8B was the worst-performing LLM with an average rank of approximately 2.8.

Table 2 displays the results of the Dunn post-hoc test, comparing the performance of different language models. Each cell indicates whether the difference in performance between the respective models is statistically significant at a 5% significance level. “Significant” denotes a statistically significant difference (p-value ≤ 0.05), while “Not Significant” indicates no statistically significant difference (p-value > 0.05).

For the selected significance level, the Dunn test result shows that Mistral 7 B’s performance is significantly different from LLaMA 3 8 B’s but not significantly different from the other LLMs. One strategy to increase the likelihood of detecting significant differences in the remaining comparisons is to increase the test’s sample size (the number of examples used to rank the models). With a larger sample size, we may obtain smaller p-values if there are indeed differences between the ranks.

p_values = sp.posthoc_dunn([mistral7b_metrics['rank'], llama8b_metrics['rank'], llama70b_metrics['rank'], mixtral_metrics['rank']], p_adjust='holm')
p_values > 0.05

FAs stated earlier, the benefit of RAQ is that it can be used to assess any set of LLMs on any domain/subject/use case rather than relying on traditional benchmarks. This means that, depending on the data set used, different models will show up as leaders during this evaluation.

Conclusion

The fast progress in developing more powerful LLMs and making them easily accessible for everyone brings new challenges on the adoption side. Companies are looking for new ways of integrating them into internal and external tools. Nevertheless, there is a big trade-off between control, privacy, flexibility, and security with the current approaches to select and deploy these models.

In this article, we introduce RAQ, a novel framework to assess and compare the quality of the answers of a set of LLMs. It makes the selection of an LLM for a new use case to be objective scalable, and flexible. The flexibility comes from the fact that it can be applied when an organization has previous private examples with which they want to test the set of LLMs. But it can work even when this dataset is unavailable.

Having selected the LLM, we explored NIM as a solution for deploying the model at scale. It ensures privacy, security, and scalability out of the box, regardless of whether the model is deployed in physical hardware, a private cloud, or as a hosting service.

We applied RAQ and used NIM to compare the performance of the Mistral and Meta models (small and large versions). We used a reading comprehension dataset to illustrate how to use RAQ. Mistral 7B was the best model in our setup, generating significantly better answers than LLaMA 3 8B. The latter was actually the fastest model, producing more words per second. This shows that selecting a model usually involves a trade-off between quality and speed.

About me

Serial entrepreneur and leader in the AI space. I develop AI products for businesses and invest in AI-focused startups. Founder @ ZAAI | LinkedIn | X/Twitter

References

[1] https://developer.nvidia.com/blog/nvidia-nim-offers-optimized-inference-microservices-for-deploying-ai-models-at-scale/

[2] Dunn, O. J. (1964) Multiple comparisons using rank sums. Technometrics. 6, 241–252. doi:10.1080/00401706.1964.10490181.