From 3900e0adebd98b43d601b8a02243ee7019efc405 Mon Sep 17 00:00:00 2001
From: Cole Medin <cole@dynamous.ai>
Date: Wed, 13 Nov 2024 09:08:59 -0600
Subject: [PATCH] Updating the LLM eval agent to work with Ollama

---
 llm-agent-evaluation-framework/.env.example   |   7 ++++-
 .../requirements.txt                          | Bin 441 -> 9274 bytes
 llm-agent-evaluation-framework/runnable.py    |  27 ++++++++++++++----
 3 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/llm-agent-evaluation-framework/.env.example b/llm-agent-evaluation-framework/.env.example
index 55e9601..7fbcdf4 100644
--- a/llm-agent-evaluation-framework/.env.example
+++ b/llm-agent-evaluation-framework/.env.example
@@ -28,7 +28,12 @@ HUGGINGFACEHUB_API_TOKEN=
 # And all Anthropic models you can use here -
 # https://docs.anthropic.com/en/docs/about-claude/models
 # A good default to go with here is gpt-4o-mini, claude-3-5-sonnet-20240620, or llama3-groq-70b-8192-tool-use-preview
-LLM_MODEL=gpt-4o-mini
+LLM_MODEL=
+
+# The provider you want to use for your LLMs
+# If you don't specify this, the script will try to determine your provider from your model, but this isn't guaranteed to work!
+# Possible providers: openai, anthropic, groq, ollama
+LLM_PROVIDER=
 
 # Get your personal Asana access token through the developer console in Asana. 
 # Feel free to follow these instructions -
diff --git a/llm-agent-evaluation-framework/requirements.txt b/llm-agent-evaluation-framework/requirements.txt
index 3e9e42c480833e989ccde9eb3ffe55f4f74bfc20..1e1590a472f6be37e30e8ed5bca2126cb7012dfc 100644
GIT binary patch
literal 9274
zcmb`NOK)4r5rywMKz<4Xlaw{_B8vfn1Q^T!PG*s0B}f!0$(Bf&q(-)We3J9^xrcT6
zkY@}G3|X?ey1MGrsYf^czyDr#cipPH?Jm1V{rS?(+t-!;eXl>8?l1Z?>HgH6>1*2k
zvHOEoykpyKwS)Dk*2dkN{_m36Nb;szHeJY#n+{}$GM{wLopjBcF0@RVt~b5Zz1BE$
z-PYe~9b2MfDgCP^frjgDq@TaYu6b`e`o7awv@NxMD_twe+;y|vnbYpwfz-XE&ZG}i
z*t60}Q~i70J?W>dc0cqsV-LH7&r&<U2%lLy*B2N*>USIMjO`ow1v<##mu)pR8zG0!
z^KQ}B!T8Vm%3jc4@8ZKLoM11B-mLqm(-!@1*7)-7E3K{@O53LUaQ9sMpPGjYeTM}r
zeSefM@o3z%+25-czopKbOKaj>XkKaWon)?eYq4e@k6E+&R$8_?Z`IBXGDZhvX6+Qv
zU+9Z<*fME2gU__bVAg!M1iTwJ7TEt;<NI0`aaPVK{Ib4j>!Zf7wW3W<+;uO_0(KD-
zuruoHf>+3m^z%}<J+~cK?Tj$-yzzP{echD=o{kl1`^tGllG7Z0*z7eBTIs~gb|Mm-
zV7~23kQd0_^n4lb*cLqCHmE;IAACn!vNhg?G3epcNBL;UvxWqxn-Mepex?({2Ug4&
zpLea@<TNsuvI(3w!t8_o#*+9#-q>3r1s9RL?mo-o`<)coZM}ch>Z5F%OEV`R8&Ays
zvxeQZP$dtER@RHJ*9`^slf%m$9(eJt?Z=kIfIY$8R?KDHO1^}hSXV58aTmP@Fc_4u
z!R!l{=G`yy;9iocW*6d!vzn|~>-Ryac)Mt5f-JQy>tmhostGG{CVU99T)#ov9z-T*
z2czLhL~bjao*EWJI=up4(KykmKGE#j&uXOJHe7Nh(IB^$^4syrxt0A3owAX4M3~je
zefJ-8mCCtQe4JIr9^ZV@-`nbYaWqvOO~!!yF^6wt4NNCKqaM+bP6hjl*rbmeexBYj
z>>ljLH+*#b!pcJSPz!bVw)YiWiBi1Wx6qPQ9(QI=x{`f(2Sb1DjFEhE-M?*C8a=9j
ztvvP-Z>Sk`-CuMX{XY2%o>Z)3Tg^`L9W)<Wwk-88GM7$Hk4)sx`%2}rLEijG-A>dT
zS?bcN`?T|zh?fOawpp{Vv{&t5Z`Mb$fGpa!igQZ(wy`P54)YuIsZMVXFtmk<&5%d7
zktqQ$BXJzPieF^*Gm03)4%fKIeBPlkX&rFVpYkNWk|#_3P2Hfc!kwo+Ys?t*)ebt_
zQqpVfBMx+x^a|u1>pT5EQ;p-3-#-Xp*DiZ^A*6`prFMX}Cm~o<8PAk;xC<V5CRsPT
z%@_O5UT}nok4>wy2m3&pI>Gu6&G!$IaE63QMvv-E|L{D+-kKplNV8|2$`ERdaY_&K
z*>w5vXWCbmeTmz~{QX+A&xn#=HJi&WY%bg*w-F{xQACTgZJTBB{FEGwcUARV<^Jhx
zPF>HGl}R66A7#<AzKF)XYCsUf1I~5#{@7T>+ENnOZ=KUVKf@e)3Vg(VCjOcPT!D=6
zu@v?BwYi?1j_Ij=!zrw?Ik7yw!!?UL030#@iGpo(p1-y;y#skO&TKs;e^|||AJk7t
zlexzk{o8yZcIXUa;*B%q)M?}v6@Vz^4D53aJGFzTe->V58u?F`s<|O&?#I{@I^Ev=
zGMV8yEa%ib8e5#{`%)YE$Ek<);H=MFw0#ke>7!&X6HWM>c!fzdfl{d|w|;8-KB}Ir
zlo#eNtIh-PT`Om@I%@X#hKc3YlWd4CZeBlY=R(#DrHqtyZsZj|5l2pr+@xn2tYIC`
zkl5;Prh~{a{YIbX-<q$%GHVYyXWDkTLT6%R;`Ke8@3~jA(~O{o`}|=&jHeI6oLfGf
zgT0P5_1y8c_1x;eh(G2L?9062YGT>wgqv$2K_$sM1`^|jh-W;i5uJV{nT>QWbv88*
zRBU@`qi#KGjrh5z60NHHOs(7grp5}0x)v<u9sZ|Nf@byUQ!^L)$tI&tc5>sWy8>)H
z>p68QciC?bsa12;1^PRY@mm7Ak#T<W`(^Kw5vpF7wc|GrB!}-bhZTW!&-by!6=S9@
zVa5-qZ*a^6&)n!di%zinEEK|4>xLDPppt-+=Lv4A#5A`c+v@xCukC_enK#IOred<?
zv00r7b$@mZUncz>{r5T}GwHYLr*ry#3-ibnPqQ_hn?dhX0nE{_;rMlJ&K)L6to%Lo
zp1#eVDYJfy)peWrR`fQiLVj`i7T(Y+sBmCS4KqWQZ6@;X$R$^*Z$ZiLaQD)eiT*Us
z>)l+(I@z5$EfMx~#_Ud4wIyVFWjkGuH-Sr44BrgU<+nB0A|#mmsh3`JrE-oxYGomL
zW_EPAUxA`u-&}7jMK7lQuC+=XcTM$up1F{lwO_$3Pv3&gS8Y!)@D0z^f^5GKN^^a8
zodf4#jY5J~N^3+1Rhrrp%c!x{L5<*W-M1GyieFDMA5c+)H`z}GAp?0ezG+Aiks;Dl
z67HKIj?M81O|hn`jlHiv$J@%WoL^(m#Hy=NBC~084gRsd-&?7#bW6_j?7#~o`s)lN
zHj3DIXQ_4A+gjyLw-OKN=wX@p4nmQp8}YS&gSGF}|B^eL#E^j-k${WIIXq*%rWrgN
zN?f#4Glg3hkqLv4X4-WAV|VrO_tJGE?_5ud|G_F2B}ecOi_HHzAs$tRVS#-w9b{4>
ziH2~3DTuYZmJPnU!cgA5BZZ~!8`pgUH2y@?6EUc`K7MH4@*)XSSMqzvvFJk{%($mB
zH<Gv0W*wHpwel`qebjKLFQ<YfI@nS18_I*2>wqzPT@Jm}R&?T>r@F$ZwEAv(E_>fK
z>BASpA(v{G>sRJj<A*h#<H2Jno9We&t(RF_c52T`em)3m*9PP)oA{S@K9$=JP(f~p
z-phekwZ^@CDZlf+@utr!*zuq4`|hXi$F`R|F0KDml&O=~-QN_s7ySWa;ykQxuz~vs
z-#s;RZ%;I)jT6pxB<KXg`tE6u82fgB%qN}ZX}mHE+cU`?%A9_!bQXoT$9qz5=pxMP
zSnTYoJw!dX6#RCNATqoVUMp&KV>51*pWGSi)y3JyZ@|i&<)tKV<?IFvzX9-WLx19q
zI%?IEn(R9yC%C3#hb3WQ?Dq?#5oZ#iB04g>>hVbjRdORcIGq|68}JZ2YA>&zaBlb+
z+;{DC%i02Z7QNC{j9b>j&YG$S)!Ff2-1?5Ql-oF2=1-8>F|>j^B%iSiZ)4`5Pxt4M
zXy=oXdXG1se(wGuZhtT9+*==dq{0qf(XEHMc-^q}YeX1))>z?O@+3|5<6W2N1oh!-
zJom@_Tb^r@adkB4_3U@tsDt(!TgZ6$N{wZD;9kc~xNbc|S$IN(m}csg%st!Mn87L{
z#x}NOUp~up#v$Wr$uk4`ho7@K|BVT2yuCV4z!mFCzt;{w6Q~o0lHAmJ5dl@-i%Zid
zt=J3l<a{teeT~z|o;6*u*8d}*UUP^2!k==P$GpgpCw>d9N=^+hZ->$sLZ3G+VzB=f
z;R**XvEW6X)EjqoA+PhAm>U~W=X2P6dg<7KjO)Z}vd5n;8Sk1pU*Cz>!+KuxD4cS%
zUfa$8YoH%ez5QYa&kH-}tvQ8XWB_+1W?B&OSw^hRA}jqy<w?U>rW-sd>#5VQu=wwO
z;W#}v-7pkxxi@i!F(F4vB5y0hJg}_g=Iz!&pJTE$Q%bg&|14jwghB2G)bYbl{HTMK
zr^h?2cle3YvwWmZR^1uaQQk%PpA2MA>YL;BZ>?;^&wuSOH~)!;CGpOazpIREG<>m!
P7x~Kkfo|_Y!ub9lZ8vMt

literal 441
zcmY*V%Z|e!47~F%pg^|!IPfK_9$e}I5-|ynmHPK}n@D9ZNFEO}<ADkU)}GX)zQlTO
zA(Oo3hV*Ccr1Y#Nk$}7l5b22+rOg|WKrNKKM)zVXGwyuJ+YX<GA1*ITr`Ya?n*O&d
zxWndqfb)t3@o*UONXOm_r{|z*bPnD7Nd5-yB{!*AR3tvip=<@H-f~&6RBQP@rA4RO
zN_VXny#7K;+<>cR!63qtVzXAO`SYm47g)pYsec5-TsNldxhLlMoP9zGYm`TyE=6{?
jmb?2f1EaSfGw_?@<!Qlt@t$X%jVQQvtgQ}2mSB+wcD;{0

diff --git a/llm-agent-evaluation-framework/runnable.py b/llm-agent-evaluation-framework/runnable.py
index ec1df9c..a4b3633 100644
--- a/llm-agent-evaluation-framework/runnable.py
+++ b/llm-agent-evaluation-framework/runnable.py
@@ -10,6 +10,7 @@ import json
 import os
 
 from langchain_groq import ChatGroq
+from langchain_ollama import ChatOllama
 from langchain_openai import ChatOpenAI
 from langchain_anthropic import ChatAnthropic
 from langchain_core.messages import ToolMessage, AIMessage
@@ -21,6 +22,14 @@ from tools.vector_db_tools import available_vector_db_functions
 
 load_dotenv()
 model = os.getenv('LLM_MODEL', 'gpt-4o')
+provider = os.getenv('LLM_PROVIDER', 'auto')
+
+provider_mapping = {
+    "openai": ChatOpenAI,
+    "anthropic": ChatAnthropic,
+    "ollama": ChatOllama,
+    "llama": ChatGroq
+}
 
 model_mapping = {
     "gpt": ChatOpenAI,
@@ -53,10 +62,16 @@ def get_local_model():
 available_functions = available_asana_functions | available_drive_functions | available_vector_db_functions
 tools = [tool for _, tool in available_functions.items()]
 
-for key, chatbot_class in model_mapping.items():
-    if key in model.lower():
-        chatbot = chatbot_class(model=model) if key != "huggingface" else chatbot_class(llm=get_local_model())
-        break
+if provider == "auto":
+    for key, chatbot_class in model_mapping.items():
+        if key in model.lower():
+            chatbot = chatbot_class(model=model) if key != "huggingface" else chatbot_class(llm=get_local_model())
+            break
+else:
+    for key, chatbot_class in provider_mapping.items():
+        if key in provider.lower():
+            chatbot = chatbot_class(model=model) if key != "huggingface" else chatbot_class(llm=get_local_model())
+            break
 
 chatbot_with_tools = chatbot.bind_tools(tools)
 
@@ -70,7 +85,7 @@ class GraphState(TypedDict):
     """
     messages: Annotated[list[AnyMessage], add_messages]
 
-async def call_model(state: GraphState, config: RunnableConfig) -> Dict[str, AnyMessage]:
+def call_model(state: GraphState, config: RunnableConfig) -> Dict[str, AnyMessage]:
     """
     Function that calls the model to generate a response.
 
@@ -88,7 +103,7 @@ async def call_model(state: GraphState, config: RunnableConfig) -> Dict[str, Any
     ))
 
     # Invoke the chatbot with the binded tools
-    response = await chatbot_with_tools.ainvoke(messages, config)
+    response = chatbot_with_tools.invoke(messages, config)
     # print("Response from model:", response)
 
     # We return an object because this will get added to the existing list