Add single-call measurement for prepared investigation prompts
This commit is contained in:
1 parent
3989a07e05
commit
7c168182c1
4 files changed
+140
-2
No files matched your search
@@ -0,0 +1,36 @@
|
||||
import copy
|
||||
import pytest
|
||||
from viano_bug_analysis.prompt_probe import prepare
|
||||
from viano_bug_analysis.minimal_prompt_test import measure
|
||||
from viano_bug_analysis.llm import LLMError
|
||||
from test_minimal_prompt_test import provider
|
||||
|
||||
|
||||
def request():return {'prompt_version':'fixture','messages':[{'role':'system','content':'Test instructions'},
|
||||
{'role':'user','content':'Test context'}], 'output_schema':{'type':'object','required':['query'],
|
||||
'properties':{'query':{'const':'database.schema'}},'additionalProperties':False}}
|
||||
|
||||
|
||||
def test_native_schema_sent_once_without_wrapper_or_tools():
|
||||
r=request();before=copy.deepcopy(r);case=prepare(provider(),r)
|
||||
payload=case['payload']
|
||||
assert set(payload)=={'systemInstruction','contents','generationConfig'}
|
||||
assert payload['contents'][0]['parts'][0]['text']=='Test context'
|
||||
assert payload['generationConfig']['responseJsonSchema']==r['output_schema']
|
||||
assert payload['generationConfig']['thinkingConfig']=={'thinkingLevel':'low'}
|
||||
assert r==before
|
||||
|
||||
|
||||
def test_validates_the_prepared_schema_not_the_minimal_test_schema():
|
||||
case=prepare(provider(),request())
|
||||
def post(*args):return {'candidates':[{'finishReason':'STOP','content':{'parts':[{'text':'{"query":"database.schema"}'}]}}],
|
||||
'usageMetadata':{'promptTokenCount':200,'candidatesTokenCount':8,'thoughtsTokenCount':30}}
|
||||
result=measure(provider(),case,10000,post)
|
||||
assert result['status']=='SUCCESS' and result['schema_valid']
|
||||
assert result['usage']['input_tokens']==200
|
||||
|
||||
|
||||
@pytest.mark.parametrize('field,value',[('messages',[]),('messages',[{'role':'user','content':'x'}])])
|
||||
def test_rejects_invalid_envelopes(field,value):
|
||||
r=request();r[field]=value
|
||||
with pytest.raises(LLMError):prepare(provider(),r)
|
||||
Reference in new issue
Block a user