62 lines
3.1 KiB
Python
62 lines
3.1 KiB
Python
import copy
|
|
import pytest
|
|
from viano_bug_analysis.prompt_probe import prepare
|
|
from viano_bug_analysis.minimal_prompt_test import measure
|
|
from viano_bug_analysis.llm import LLMError
|
|
from test_minimal_prompt_test import provider
|
|
|
|
|
|
def request():return {'prompt_version':'fixture','messages':[{'role':'system','content':'Test instructions'},
|
|
{'role':'user','content':'Test context'}], 'output_schema':{'type':'object','required':['query'],
|
|
'properties':{'query':{'const':'database.schema'}},'additionalProperties':False}}
|
|
|
|
|
|
def test_native_schema_sent_once_without_wrapper_or_tools():
|
|
r=request();before=copy.deepcopy(r);case=prepare(provider(),r)
|
|
payload=case['payload']
|
|
assert set(payload)=={'systemInstruction','contents','generationConfig'}
|
|
assert payload['contents'][0]['parts'][0]['text']=='Test context'
|
|
assert payload['generationConfig']['responseJsonSchema']['properties']['query']=={'type':'string','enum':['database.schema']}
|
|
assert case['validation_schema']==r['output_schema']
|
|
assert payload['generationConfig']['thinkingConfig']=={'thinkingLevel':'low'}
|
|
assert r==before
|
|
|
|
|
|
def test_validates_the_prepared_schema_not_the_minimal_test_schema():
|
|
case=prepare(provider(),request())
|
|
def post(*args):return {'candidates':[{'finishReason':'STOP','content':{'parts':[{'text':'{"query":"database.schema"}'}]}}],
|
|
'usageMetadata':{'promptTokenCount':200,'candidatesTokenCount':8,'thoughtsTokenCount':30}}
|
|
result=measure(provider(),case,10000,post)
|
|
assert result['status']=='SUCCESS' and result['schema_valid']
|
|
assert result['usage']['input_tokens']==200
|
|
|
|
|
|
@pytest.mark.parametrize('field,value',[('messages',[]),('messages',[{'role':'user','content':'x'}])])
|
|
def test_rejects_invalid_envelopes(field,value):
|
|
r=request();r[field]=value
|
|
with pytest.raises(LLMError):prepare(provider(),r)
|
|
|
|
|
|
def test_gemini_projection_types_const_and_keeps_strict_local_validation():
|
|
from viano_bug_analysis.prompt_probe import gemini_schema
|
|
from jsonschema import Draft202012Validator
|
|
r=request()
|
|
r['output_schema']['properties']['query']={'const':'database.schema','minLength':5,'pattern':'^database'}
|
|
original=copy.deepcopy(r)
|
|
c=prepare(provider(),r)
|
|
wire=c['payload']['generationConfig']['responseJsonSchema']
|
|
Draft202012Validator.check_schema(wire)
|
|
assert wire['properties']['query']=={'type':'string','enum':['database.schema']}
|
|
assert r==original and c['validation_schema']==r['output_schema']
|
|
assert gemini_schema({'type':['string','number','boolean','null']})=={
|
|
'anyOf':[{'type':v} for v in ['string','number','boolean','null']]}
|
|
|
|
|
|
def test_local_constraints_not_silently_dropped_from_response_check():
|
|
r=request();r['output_schema']['properties']['query']={'type':'string','pattern':'^database'}
|
|
c=prepare(provider(),r)
|
|
def post(*args):return {'candidates':[{'finishReason':'STOP','content':{'parts':[{'text':'{"query":"invented"}'}]}}],
|
|
'usageMetadata':{'promptTokenCount':200}}
|
|
result=measure(provider(),c,10000,post)
|
|
assert result['status']=='INVALID_JSON' and result['schema_valid'] is False
|