Add single-call measurement for prepared investigation prompts

This commit is contained in:
Nico committed 2026-10-17 12:46:36 +02:00
1 parent 3989a07e05
commit 7c168182c1
4 files changed
+140 -2

No files matched your search

+36
View File
@@ -0,0 +1,36 @@
import copy
import pytest
from viano_bug_analysis.prompt_probe import prepare
from viano_bug_analysis.minimal_prompt_test import measure
from viano_bug_analysis.llm import LLMError
from test_minimal_prompt_test import provider
def request():return {'prompt_version':'fixture','messages':[{'role':'system','content':'Test instructions'},
{'role':'user','content':'Test context'}], 'output_schema':{'type':'object','required':['query'],
'properties':{'query':{'const':'database.schema'}},'additionalProperties':False}}
def test_native_schema_sent_once_without_wrapper_or_tools():
r=request();before=copy.deepcopy(r);case=prepare(provider(),r)
payload=case['payload']
assert set(payload)=={'systemInstruction','contents','generationConfig'}
assert payload['contents'][0]['parts'][0]['text']=='Test context'
assert payload['generationConfig']['responseJsonSchema']==r['output_schema']
assert payload['generationConfig']['thinkingConfig']=={'thinkingLevel':'low'}
assert r==before
def test_validates_the_prepared_schema_not_the_minimal_test_schema():
case=prepare(provider(),request())
def post(*args):return {'candidates':[{'finishReason':'STOP','content':{'parts':[{'text':'{"query":"database.schema"}'}]}}],
'usageMetadata':{'promptTokenCount':200,'candidatesTokenCount':8,'thoughtsTokenCount':30}}
result=measure(provider(),case,10000,post)
assert result['status']=='SUCCESS' and result['schema_valid']
assert result['usage']['input_tokens']==200
@pytest.mark.parametrize('field,value',[('messages',[]),('messages',[{'role':'user','content':'x'}])])
def test_rejects_invalid_envelopes(field,value):
r=request();r[field]=value
with pytest.raises(LLMError):prepare(provider(),r)