fix: model cost/info matches first substring key (gpt-4o-mini billed as gpt-4o) (#1439)

* fix: match model name to the longest known key, not the first substring

* test: model key matching prefers the longest specific key
This commit is contained in:
Afonso Coutinho
2026-06-04 03:05:37 +01:00
committed by GitHub
parent 2efebcc278
commit eac354629a
3 changed files with 74 additions and 12 deletions

View File

@@ -0,0 +1,48 @@
"""Pin matchModelKey (static/js/model/matchKey.js).
Driven through `node --input-type=module` (same approach as test_compare_js.py);
skips when `node` is not installed.
Regression: model name -> info/pricing lookups returned the FIRST substring
match, so "gpt-4o-mini" matched the shorter "gpt-4o" key and was billed at
gpt-4o rates (~16x) with the wrong context window.
"""
import json
import shutil
import subprocess
from pathlib import Path
import pytest
_REPO = Path(__file__).resolve().parent.parent
_HELPER = _REPO / "static" / "js" / "model" / "matchKey.js"
_HAS_NODE = shutil.which("node") is not None
_KEYS = ["gpt-4o", "gpt-4o-mini", "gpt-4", "o1", "o1-mini", "o1-pro", "o3", "o3-mini"]
def _match(name):
js = (
f"import {{ matchModelKey }} from '{_HELPER.as_posix()}';"
f"console.log(JSON.stringify(matchModelKey({json.dumps(name)}, {json.dumps(_KEYS)})));"
)
proc = subprocess.run(
["node", "--input-type=module"],
input=js, capture_output=True, text=True, cwd=str(_REPO), timeout=30,
)
assert proc.returncode == 0, proc.stderr
return json.loads(proc.stdout.strip())
@pytest.mark.skipif(not _HAS_NODE, reason="node binary not on PATH")
def test_prefers_longest_specific_key():
assert _match("gpt-4o-mini") == "gpt-4o-mini"
assert _match("o1-mini") == "o1-mini"
assert _match("o1-pro") == "o1-pro"
assert _match("o3-mini") == "o3-mini"
@pytest.mark.skipif(not _HAS_NODE, reason="node binary not on PATH")
def test_base_model_and_unknown():
assert _match("gpt-4o-2024-08-06") == "gpt-4o"
assert _match("some-unknown-model") is None