Jeffrey MorganBuild Bigger With Small AI: Running Small Models Locally을 보고 정리했다. 항상 큰 모델에 대한 얘기만 강조되다 보니 작은 모델로는 무엇이 가능한가 싶었었는데 이 발표가 이해에 많은 도움이 되었다.

RAG 예시 코드

from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Settings
from llama_index.core.node_parser import TokenTextSplitter
from llama_index.vector_stores.duckdb import DuckDBVectorStore
from llama_index.core import StorageContext
from llama_index.llms.ollama import Ollama
from llama_index.embeddings.ollama import OllamaEmbedding

# models
Settings.embed_model = OllamaEmbedding(model_name="all-minilm")
Settings.llm = Olama (model="gemma2:2b", temperature=0, request_timeout=360.0)

# load documents into a vector store (DuckDB)
documents = SimpleDirectoryReader(input_files=["facts.txt"]).\
              load_data(show_progress=True)
splitter = TokenTextSplitter(separator="\n", chunk_size=64, chunk_overlap=0)
vector_store = DuckDBVectorStore()
storage_context = StorageContext.from_defaults(vector_store=vector_store)
index = VectorStoreIndex(splitter.get_nodes_from_documents(documents), \
  storage_context=storage_context, show_progress=True)
query_engine = index.as_query_engine()

try:
    while True:
        user_query = input (">>> ")
        response = query_ engine. query (user_query)
        print (response)
except KeyboardInterrupt:
    exit()

Tool Calling 예시 코드

import duckdb
from langchain_core.tools import tool
from langchain_ollama import ChatOllama
from langchain_core.messages import HumanMessage, ToolMessage

con = duckdb.connect(database="ducks.duckdb")
schema = con.execute(f"DESCRIBE ducks"). fetchdf()
schema_str = schema.to_string(index=False)

@tool
def query(query: str) -> str:
    """Queries the database for information and returns the result.
    Args:
    query: The query to run against the database.
    """
    return str(con.execute(query).fetchone()[0])

llm = ChatOllama (model="qwen2.5-coder") .bind_tools([query])
try:
    while True:
        user_query = input(">>> ")
        messages = [HumanMessage(f"You are provided You are given a DuckDB   \\
         schema for table 'ducks': \n\n{schema_strschema_str}\n\n.\n\nAnswer \\
          the user query: '{user_query}' in a single sentence.")]
        ai_msg = llm.invoke(messages)
        messages.append(ai_msg)

        for tool_call in ai_msg.tool_calls:
            print('>>> tool_call:', tool_call)
            selected_tool = {"query": query}[tool_call["name"].lower()]
            tool_output = selected_tool.invoke(tool_call["args"])
            print('>>> tool_output:', tool_output)
            messages.append(ToolMessage(tool_output, tool_call_id=tool_call["id"]))

        response = llm.invoke(messages)
        print(response.content)
except KeyboardInterrupt:
    exit()

색상을 바꿔요

눈에 편한 색상을 골라보세요 :)

Darkreader 플러그인으로 선택한 색상이 제대로 표시되지 않을 수 있습니다.