Add benchmark-driven model promotion workflow and pipeline stages
Some checks failed
ci / test-and-build (push) Has been cancelled
Some checks failed
ci / test-and-build (push) Has been cancelled
This commit is contained in:
parent
98b13d1069
commit
8c1f7c1e13
38 changed files with 5300 additions and 503 deletions
147
src/config_ui.py
147
src/config_ui.py
|
|
@ -10,13 +10,6 @@ import gi
|
|||
|
||||
from config import (
|
||||
Config,
|
||||
DEFAULT_EXTERNAL_API_BASE_URL,
|
||||
DEFAULT_EXTERNAL_API_KEY_ENV_VAR,
|
||||
DEFAULT_EXTERNAL_API_MAX_RETRIES,
|
||||
DEFAULT_EXTERNAL_API_MODEL,
|
||||
DEFAULT_EXTERNAL_API_PROVIDER,
|
||||
DEFAULT_EXTERNAL_API_TIMEOUT_MS,
|
||||
DEFAULT_LLM_PROVIDER,
|
||||
DEFAULT_STT_PROVIDER,
|
||||
)
|
||||
from constants import DEFAULT_CONFIG_PATH
|
||||
|
|
@ -42,28 +35,16 @@ class ConfigUiResult:
|
|||
def infer_runtime_mode(cfg: Config) -> str:
|
||||
is_canonical = (
|
||||
cfg.stt.provider.strip().lower() == DEFAULT_STT_PROVIDER
|
||||
and cfg.llm.provider.strip().lower() == DEFAULT_LLM_PROVIDER
|
||||
and not bool(cfg.external_api.enabled)
|
||||
and not bool(cfg.models.allow_custom_models)
|
||||
and not cfg.models.whisper_model_path.strip()
|
||||
and not cfg.models.llm_model_path.strip()
|
||||
)
|
||||
return RUNTIME_MODE_MANAGED if is_canonical else RUNTIME_MODE_EXPERT
|
||||
|
||||
|
||||
def apply_canonical_runtime_defaults(cfg: Config) -> None:
|
||||
cfg.stt.provider = DEFAULT_STT_PROVIDER
|
||||
cfg.llm.provider = DEFAULT_LLM_PROVIDER
|
||||
cfg.external_api.enabled = False
|
||||
cfg.external_api.provider = DEFAULT_EXTERNAL_API_PROVIDER
|
||||
cfg.external_api.base_url = DEFAULT_EXTERNAL_API_BASE_URL
|
||||
cfg.external_api.model = DEFAULT_EXTERNAL_API_MODEL
|
||||
cfg.external_api.timeout_ms = DEFAULT_EXTERNAL_API_TIMEOUT_MS
|
||||
cfg.external_api.max_retries = DEFAULT_EXTERNAL_API_MAX_RETRIES
|
||||
cfg.external_api.api_key_env_var = DEFAULT_EXTERNAL_API_KEY_ENV_VAR
|
||||
cfg.models.allow_custom_models = False
|
||||
cfg.models.whisper_model_path = ""
|
||||
cfg.models.llm_model_path = ""
|
||||
|
||||
|
||||
class ConfigWindow:
|
||||
|
|
@ -280,6 +261,22 @@ class ConfigWindow:
|
|||
self._strict_startup_check = Gtk.CheckButton(label="Fail fast on startup validation errors")
|
||||
box.pack_start(self._strict_startup_check, False, False, 0)
|
||||
|
||||
safety_title = Gtk.Label()
|
||||
safety_title.set_markup("<span weight='bold'>Output safety</span>")
|
||||
safety_title.set_xalign(0.0)
|
||||
box.pack_start(safety_title, False, False, 0)
|
||||
|
||||
self._safety_enabled_check = Gtk.CheckButton(
|
||||
label="Enable fact-preservation guard (recommended)"
|
||||
)
|
||||
self._safety_enabled_check.connect("toggled", lambda *_: self._on_safety_guard_toggled())
|
||||
box.pack_start(self._safety_enabled_check, False, False, 0)
|
||||
|
||||
self._safety_strict_check = Gtk.CheckButton(
|
||||
label="Strict mode: reject output when facts are changed"
|
||||
)
|
||||
box.pack_start(self._safety_strict_check, False, False, 0)
|
||||
|
||||
runtime_title = Gtk.Label()
|
||||
runtime_title.set_markup("<span weight='bold'>Runtime management</span>")
|
||||
runtime_title.set_xalign(0.0)
|
||||
|
|
@ -287,8 +284,8 @@ class ConfigWindow:
|
|||
|
||||
runtime_copy = Gtk.Label(
|
||||
label=(
|
||||
"Aman-managed mode handles model downloads, updates, and safe defaults for you. "
|
||||
"Expert mode keeps Aman open-source friendly by exposing custom providers and models."
|
||||
"Aman-managed mode handles the canonical editor model lifecycle for you. "
|
||||
"Expert mode keeps Aman open-source friendly by letting you use custom Whisper paths."
|
||||
)
|
||||
)
|
||||
runtime_copy.set_xalign(0.0)
|
||||
|
|
@ -301,7 +298,7 @@ class ConfigWindow:
|
|||
|
||||
self._runtime_mode_combo = Gtk.ComboBoxText()
|
||||
self._runtime_mode_combo.append(RUNTIME_MODE_MANAGED, "Aman-managed (recommended)")
|
||||
self._runtime_mode_combo.append(RUNTIME_MODE_EXPERT, "Expert mode (custom models/providers)")
|
||||
self._runtime_mode_combo.append(RUNTIME_MODE_EXPERT, "Expert mode (custom Whisper path)")
|
||||
self._runtime_mode_combo.connect("changed", lambda *_: self._on_runtime_mode_changed(user_initiated=True))
|
||||
box.pack_start(self._runtime_mode_combo, False, False, 0)
|
||||
|
||||
|
|
@ -335,41 +332,6 @@ class ConfigWindow:
|
|||
expert_warning.get_content_area().pack_start(warning_label, True, True, 0)
|
||||
expert_box.pack_start(expert_warning, False, False, 0)
|
||||
|
||||
llm_provider_label = Gtk.Label(label="LLM provider")
|
||||
llm_provider_label.set_xalign(0.0)
|
||||
expert_box.pack_start(llm_provider_label, False, False, 0)
|
||||
|
||||
self._llm_provider_combo = Gtk.ComboBoxText()
|
||||
self._llm_provider_combo.append("local_llama", "Local llama.cpp")
|
||||
self._llm_provider_combo.append("external_api", "External API")
|
||||
self._llm_provider_combo.connect("changed", lambda *_: self._on_runtime_widgets_changed())
|
||||
expert_box.pack_start(self._llm_provider_combo, False, False, 0)
|
||||
|
||||
self._external_api_enabled_check = Gtk.CheckButton(label="Enable external API provider")
|
||||
self._external_api_enabled_check.connect("toggled", lambda *_: self._on_runtime_widgets_changed())
|
||||
expert_box.pack_start(self._external_api_enabled_check, False, False, 0)
|
||||
|
||||
external_model_label = Gtk.Label(label="External API model")
|
||||
external_model_label.set_xalign(0.0)
|
||||
expert_box.pack_start(external_model_label, False, False, 0)
|
||||
self._external_model_entry = Gtk.Entry()
|
||||
self._external_model_entry.connect("changed", lambda *_: self._on_runtime_widgets_changed())
|
||||
expert_box.pack_start(self._external_model_entry, False, False, 0)
|
||||
|
||||
external_base_url_label = Gtk.Label(label="External API base URL")
|
||||
external_base_url_label.set_xalign(0.0)
|
||||
expert_box.pack_start(external_base_url_label, False, False, 0)
|
||||
self._external_base_url_entry = Gtk.Entry()
|
||||
self._external_base_url_entry.connect("changed", lambda *_: self._on_runtime_widgets_changed())
|
||||
expert_box.pack_start(self._external_base_url_entry, False, False, 0)
|
||||
|
||||
external_key_env_label = Gtk.Label(label="External API key env var")
|
||||
external_key_env_label.set_xalign(0.0)
|
||||
expert_box.pack_start(external_key_env_label, False, False, 0)
|
||||
self._external_key_env_entry = Gtk.Entry()
|
||||
self._external_key_env_entry.connect("changed", lambda *_: self._on_runtime_widgets_changed())
|
||||
expert_box.pack_start(self._external_key_env_entry, False, False, 0)
|
||||
|
||||
self._allow_custom_models_check = Gtk.CheckButton(
|
||||
label="Allow custom local model paths"
|
||||
)
|
||||
|
|
@ -383,13 +345,6 @@ class ConfigWindow:
|
|||
self._whisper_model_path_entry.connect("changed", lambda *_: self._on_runtime_widgets_changed())
|
||||
expert_box.pack_start(self._whisper_model_path_entry, False, False, 0)
|
||||
|
||||
llm_model_path_label = Gtk.Label(label="Custom LLM model path")
|
||||
llm_model_path_label.set_xalign(0.0)
|
||||
expert_box.pack_start(llm_model_path_label, False, False, 0)
|
||||
self._llm_model_path_entry = Gtk.Entry()
|
||||
self._llm_model_path_entry.connect("changed", lambda *_: self._on_runtime_widgets_changed())
|
||||
expert_box.pack_start(self._llm_model_path_entry, False, False, 0)
|
||||
|
||||
self._runtime_error = Gtk.Label(label="")
|
||||
self._runtime_error.set_xalign(0.0)
|
||||
self._runtime_error.set_line_wrap(True)
|
||||
|
|
@ -429,7 +384,10 @@ class ConfigWindow:
|
|||
"- Press Esc while recording to cancel.\n\n"
|
||||
"Model/runtime tips:\n"
|
||||
"- Aman-managed mode (recommended) handles model lifecycle for you.\n"
|
||||
"- Expert mode lets you bring your own models/providers.\n\n"
|
||||
"- Expert mode lets you set custom Whisper model paths.\n\n"
|
||||
"Safety tips:\n"
|
||||
"- Keep fact guard enabled to prevent accidental name/number changes.\n"
|
||||
"- Strict safety blocks output on fact violations.\n\n"
|
||||
"Use the tray menu for pause/resume, config reload, and diagnostics."
|
||||
)
|
||||
)
|
||||
|
|
@ -489,17 +447,11 @@ class ConfigWindow:
|
|||
self._profile_combo.set_active_id(profile)
|
||||
self._show_notifications_check.set_active(bool(self._config.ux.show_notifications))
|
||||
self._strict_startup_check.set_active(bool(self._config.advanced.strict_startup))
|
||||
llm_provider = self._config.llm.provider.strip().lower()
|
||||
if llm_provider not in {"local_llama", "external_api"}:
|
||||
llm_provider = "local_llama"
|
||||
self._llm_provider_combo.set_active_id(llm_provider)
|
||||
self._external_api_enabled_check.set_active(bool(self._config.external_api.enabled))
|
||||
self._external_model_entry.set_text(self._config.external_api.model)
|
||||
self._external_base_url_entry.set_text(self._config.external_api.base_url)
|
||||
self._external_key_env_entry.set_text(self._config.external_api.api_key_env_var)
|
||||
self._safety_enabled_check.set_active(bool(self._config.safety.enabled))
|
||||
self._safety_strict_check.set_active(bool(self._config.safety.strict))
|
||||
self._on_safety_guard_toggled()
|
||||
self._allow_custom_models_check.set_active(bool(self._config.models.allow_custom_models))
|
||||
self._whisper_model_path_entry.set_text(self._config.models.whisper_model_path)
|
||||
self._llm_model_path_entry.set_text(self._config.models.llm_model_path)
|
||||
self._runtime_mode_combo.set_active_id(self._runtime_mode)
|
||||
self._sync_runtime_mode_ui(user_initiated=False)
|
||||
self._validate_runtime_settings()
|
||||
|
|
@ -525,6 +477,9 @@ class ConfigWindow:
|
|||
self._sync_runtime_mode_ui(user_initiated=False)
|
||||
self._validate_runtime_settings()
|
||||
|
||||
def _on_safety_guard_toggled(self) -> None:
|
||||
self._safety_strict_check.set_sensitive(self._safety_enabled_check.get_active())
|
||||
|
||||
def _sync_runtime_mode_ui(self, *, user_initiated: bool) -> None:
|
||||
mode = self._current_runtime_mode()
|
||||
self._runtime_mode = mode
|
||||
|
|
@ -541,36 +496,22 @@ class ConfigWindow:
|
|||
return
|
||||
|
||||
self._runtime_status_label.set_text(
|
||||
"Expert mode is active. You are responsible for provider, model, and environment compatibility."
|
||||
"Expert mode is active. You are responsible for custom Whisper path compatibility."
|
||||
)
|
||||
self._expert_expander.set_visible(True)
|
||||
self._expert_expander.set_expanded(True)
|
||||
self._set_expert_controls_sensitive(True)
|
||||
|
||||
def _set_expert_controls_sensitive(self, enabled: bool) -> None:
|
||||
provider = (self._llm_provider_combo.get_active_id() or "local_llama").strip().lower()
|
||||
allow_custom = self._allow_custom_models_check.get_active()
|
||||
external_fields_enabled = enabled and provider == "external_api"
|
||||
custom_path_enabled = enabled and allow_custom
|
||||
|
||||
self._llm_provider_combo.set_sensitive(enabled)
|
||||
self._external_api_enabled_check.set_sensitive(enabled)
|
||||
self._external_model_entry.set_sensitive(external_fields_enabled)
|
||||
self._external_base_url_entry.set_sensitive(external_fields_enabled)
|
||||
self._external_key_env_entry.set_sensitive(external_fields_enabled)
|
||||
self._allow_custom_models_check.set_sensitive(enabled)
|
||||
self._whisper_model_path_entry.set_sensitive(custom_path_enabled)
|
||||
self._llm_model_path_entry.set_sensitive(custom_path_enabled)
|
||||
|
||||
def _apply_canonical_runtime_defaults_to_widgets(self) -> None:
|
||||
self._llm_provider_combo.set_active_id(DEFAULT_LLM_PROVIDER)
|
||||
self._external_api_enabled_check.set_active(False)
|
||||
self._external_model_entry.set_text(DEFAULT_EXTERNAL_API_MODEL)
|
||||
self._external_base_url_entry.set_text(DEFAULT_EXTERNAL_API_BASE_URL)
|
||||
self._external_key_env_entry.set_text(DEFAULT_EXTERNAL_API_KEY_ENV_VAR)
|
||||
self._allow_custom_models_check.set_active(False)
|
||||
self._whisper_model_path_entry.set_text("")
|
||||
self._llm_model_path_entry.set_text("")
|
||||
|
||||
def _validate_runtime_settings(self) -> bool:
|
||||
mode = self._current_runtime_mode()
|
||||
|
|
@ -578,21 +519,6 @@ class ConfigWindow:
|
|||
self._runtime_error.set_text("")
|
||||
return True
|
||||
|
||||
provider = (self._llm_provider_combo.get_active_id() or "local_llama").strip().lower()
|
||||
if provider == "external_api" and not self._external_api_enabled_check.get_active():
|
||||
self._runtime_error.set_text(
|
||||
"Expert mode: enable External API provider when LLM provider is set to External API."
|
||||
)
|
||||
return False
|
||||
if provider == "external_api" and not self._external_model_entry.get_text().strip():
|
||||
self._runtime_error.set_text("Expert mode: External API model is required.")
|
||||
return False
|
||||
if provider == "external_api" and not self._external_base_url_entry.get_text().strip():
|
||||
self._runtime_error.set_text("Expert mode: External API base URL is required.")
|
||||
return False
|
||||
if provider == "external_api" and not self._external_key_env_entry.get_text().strip():
|
||||
self._runtime_error.set_text("Expert mode: External API key env var is required.")
|
||||
return False
|
||||
self._runtime_error.set_text("")
|
||||
return True
|
||||
|
||||
|
|
@ -646,23 +572,18 @@ class ConfigWindow:
|
|||
cfg.ux.profile = self._profile_combo.get_active_id() or "default"
|
||||
cfg.ux.show_notifications = self._show_notifications_check.get_active()
|
||||
cfg.advanced.strict_startup = self._strict_startup_check.get_active()
|
||||
cfg.safety.enabled = self._safety_enabled_check.get_active()
|
||||
cfg.safety.strict = self._safety_strict_check.get_active() and cfg.safety.enabled
|
||||
if self._current_runtime_mode() == RUNTIME_MODE_MANAGED:
|
||||
apply_canonical_runtime_defaults(cfg)
|
||||
return cfg
|
||||
|
||||
cfg.stt.provider = DEFAULT_STT_PROVIDER
|
||||
cfg.llm.provider = self._llm_provider_combo.get_active_id() or DEFAULT_LLM_PROVIDER
|
||||
cfg.external_api.enabled = self._external_api_enabled_check.get_active()
|
||||
cfg.external_api.model = self._external_model_entry.get_text().strip()
|
||||
cfg.external_api.base_url = self._external_base_url_entry.get_text().strip()
|
||||
cfg.external_api.api_key_env_var = self._external_key_env_entry.get_text().strip()
|
||||
cfg.models.allow_custom_models = self._allow_custom_models_check.get_active()
|
||||
if cfg.models.allow_custom_models:
|
||||
cfg.models.whisper_model_path = self._whisper_model_path_entry.get_text().strip()
|
||||
cfg.models.llm_model_path = self._llm_model_path_entry.get_text().strip()
|
||||
else:
|
||||
cfg.models.whisper_model_path = ""
|
||||
cfg.models.llm_model_path = ""
|
||||
return cfg
|
||||
|
||||
|
||||
|
|
@ -702,8 +623,8 @@ def show_help_dialog() -> None:
|
|||
dialog.set_title("Aman Help")
|
||||
dialog.format_secondary_text(
|
||||
"Press your hotkey to record, press it again to process, and press Esc while recording to "
|
||||
"cancel. Aman-managed mode is the canonical supported path; expert mode exposes custom "
|
||||
"providers/models for advanced users."
|
||||
"cancel. Keep fact guard enabled to prevent accidental fact changes. Aman-managed mode is "
|
||||
"the canonical supported path; expert mode exposes custom Whisper model paths for advanced users."
|
||||
)
|
||||
dialog.run()
|
||||
dialog.destroy()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue