a subjective evaluation of a few open LLMs

import os
import pytest

from llm_lib import classify_sentiment

@pytest.mark.parametrize("headline_input,expected", [
    (
            {
                'headline_text': ('Asure Partners with Key Benefit Administrators',
                                  'to Offer Proactive Health Management Plan (PHMP) to Clients')},
            {'possible_sentiments': ['bullish', 'neutral', 'slightly bullish', 'very bullish']}
    ),
    (
            {
                'headline_text': ('Everbridge Cancels Fourth Quarter',
                                  'and Full Year 2023 Financial Results Conference Call')},
            {'possible_sentiments': ['bearish', 'neutral', 'slightly bearish', 'uncertain', 'very bearish']}
    ),
    (
            {
                'headline_text': ("This Analyst With 87% Accuracy Rate Sees Around 12% Upside In Masco -",
                                  "Here Are 5 Stock Picks For Last Week From Wall Street's Most Accurate Analysts "
                                  "- Masco (NYSE:MAS)")},
            {'possible_sentiments': ['bullish', 'slightly bullish', 'very bullish']}
    ),
    (
            {'headline_text': 'Tesla leads 11% annual drop in EV prices as demand slowdown continues'},
            {'possible_sentiments': ['bearish', 'slightly bearish', 'very bearish']}
    ),
    (
            {'headline_text': "Elon Musk Dispatches Tesla's 'Fireman' to China Amid Slowing Sales"},
            {'possible_sentiments': ['bearish', 'slightly bearish']}
    ),
    (
            {'headline_text': "OpenAI co-founder Ilya Sutskever says he will leave the startup"},
            {'possible_sentiments': ['bearish', 'neutral', 'slightly bearish', 'uncertain']}
    ),
    (
            {'headline_text': "Hedge funds cut stakes in Magnificent Seven to invest in broader AI boom"},
            {'possible_sentiments': ['bearish', 'bullish', 'neutral', 'slightly bearish', 'slightly bullish']} # the "broader AI boom" part can be seen as bullish
    )
])
def test_classify_sentiment(headline_input, expected):
    assert classify_sentiment(**headline_input) in expected['possible_sentiments']

def classify_sentiment(headline_text: str, model_name: str = current_model):
    template = """You are a stocks market professional. Your job is to label a headline with a sentiment IN ENGLISH.

Headlines that mention upside should be considered bullish. 
Any headline that mentions a sales decline, a drop in stock prices, a factory glut, an economic slowdown, increased selling pressure, or other negative economic indicators should be considered bearish instead of neutral. 
Only label a headline as neutral if it does not have any clear positive or negative sentiment or business implication.

You'll prefix a bullish or bearish sentiment with "very" if the headline is particularly positive or negative in its implications.
On the other hand, you'll prefix a slightly bullish or slightly bearish sentiment with "slightly" if the headline is only slightly positive or negative in its implications.

Here is the headline text you need to label, delimited by dashes:

--------------------------------------------------
{headline_text}
--------------------------------------------------

Here is the list of the possible sentiments, delimited by commas:

,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
very bullish
bullish
slightly bullish
neutral
slightly bearish
bearish
very bearish
uncertain
volatile
,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

You are to output ONLY ONE SENTIMENT WITH THE EXACT WORDING, from the provided list of sentiments.
DO NOT add additional content, punctuation, explanation, characters, or any formatting in your output."""
    sentiment_prompt = PromptTemplate.from_template(template)
    chain = sentiment_prompt | get_model(model_name)
    output = chain.invoke({"headline_text": headline_text})
    return output.content.strip().lower()

| Model              | Status | Time (s) |
|--------------------|--------|----------|
| llama3             | OK     | 17.68    |
| phi3               | OK     | 17.84    |
| aya                | OK     | 21.68    |
| mistral            | OK     | 21.76    |
| mistral-openorca   | OK     | 22.20    |
| gemma2             | OK     | 23.14    |
| phi3:medium-128k   | OK     | 45.87    |
| phi3:14b           | OK     | 47.36    |
| aya:35b            | OK     | 77.99    |
| llama3:70b         | OK     | 144.62   |
| qwen2:72b          | OK     | 148.25   |
| command-r-plus     | OK     | 239.20   |
| qwen2              | OKKO   | 16.11    |