37 lines
1.7 KiB
Python
37 lines
1.7 KiB
Python
import copy
|
|
import pytest
|
|
from viano_bug_analysis.prompt_probe import prepare
|
|
from viano_bug_analysis.minimal_prompt_test import measure
|
|
from viano_bug_analysis.llm import LLMError
|
|
from test_minimal_prompt_test import provider
|
|
|
|
|
|
def request():return {'prompt_version':'fixture','messages':[{'role':'system','content':'Test instructions'},
|
|
{'role':'user','content':'Test context'}], 'output_schema':{'type':'object','required':['query'],
|
|
'properties':{'query':{'const':'database.schema'}},'additionalProperties':False}}
|
|
|
|
|
|
def test_native_schema_sent_once_without_wrapper_or_tools():
|
|
r=request();before=copy.deepcopy(r);case=prepare(provider(),r)
|
|
payload=case['payload']
|
|
assert set(payload)=={'systemInstruction','contents','generationConfig'}
|
|
assert payload['contents'][0]['parts'][0]['text']=='Test context'
|
|
assert payload['generationConfig']['responseJsonSchema']==r['output_schema']
|
|
assert payload['generationConfig']['thinkingConfig']=={'thinkingLevel':'low'}
|
|
assert r==before
|
|
|
|
|
|
def test_validates_the_prepared_schema_not_the_minimal_test_schema():
|
|
case=prepare(provider(),request())
|
|
def post(*args):return {'candidates':[{'finishReason':'STOP','content':{'parts':[{'text':'{"query":"database.schema"}'}]}}],
|
|
'usageMetadata':{'promptTokenCount':200,'candidatesTokenCount':8,'thoughtsTokenCount':30}}
|
|
result=measure(provider(),case,10000,post)
|
|
assert result['status']=='SUCCESS' and result['schema_valid']
|
|
assert result['usage']['input_tokens']==200
|
|
|
|
|
|
@pytest.mark.parametrize('field,value',[('messages',[]),('messages',[{'role':'user','content':'x'}])])
|
|
def test_rejects_invalid_envelopes(field,value):
|
|
r=request();r[field]=value
|
|
with pytest.raises(LLMError):prepare(provider(),r)
|