Refine config and runtime flow
This commit is contained in:
parent
85e082dd46
commit
b3be444625
16 changed files with 642 additions and 137 deletions
|
|
@ -5,12 +5,9 @@ import logging
|
|||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, cast
|
||||
|
||||
from llama_cpp import Llama, llama_cpp as llama_cpp_lib # type: ignore[import-not-found]
|
||||
|
||||
from constants import LLM_LANGUAGE, MODEL_DIR, MODEL_NAME, MODEL_PATH, MODEL_URL
|
||||
from constants import MODEL_DIR, MODEL_NAME, MODEL_PATH, MODEL_URL
|
||||
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
|
|
@ -36,7 +33,8 @@ SYSTEM_PROMPT = (
|
|||
|
||||
|
||||
class LlamaProcessor:
|
||||
def __init__(self, verbose=False):
|
||||
def __init__(self, verbose: bool = False):
|
||||
Llama, llama_cpp_lib = _load_llama_bindings()
|
||||
ensure_model()
|
||||
if not verbose:
|
||||
os.environ.setdefault("LLAMA_CPP_LOG_LEVEL", "ERROR")
|
||||
|
|
@ -100,6 +98,16 @@ def ensure_model():
|
|||
raise
|
||||
|
||||
|
||||
def _load_llama_bindings():
|
||||
try:
|
||||
from llama_cpp import Llama, llama_cpp as llama_cpp_lib # type: ignore[import-not-found]
|
||||
except ModuleNotFoundError as exc:
|
||||
raise RuntimeError(
|
||||
"llama-cpp-python is not installed; install dependencies with `uv sync`"
|
||||
) from exc
|
||||
return Llama, llama_cpp_lib
|
||||
|
||||
|
||||
def _extract_chat_text(payload: Any) -> str:
|
||||
if "choices" in payload and payload["choices"]:
|
||||
choice = payload["choices"][0]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue