111 lines
6.3 KiB
Python
111 lines
6.3 KiB
Python
import asyncio
|
|
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from test_case_builder import case_setup
|
|
from test_case_interpreter import gateway
|
|
from test_investigation_tools import tool_case
|
|
from test_parameter_and_bootstrap_guard import bootstrap
|
|
from test_lean_investigation_decision import reply
|
|
from viano_bug_analysis.llm import LLMError, ProviderResponse
|
|
|
|
|
|
@pytest.mark.parametrize('fault,path,keyword,fields', [
|
|
('type','/update/working_brief/findings','type',[]),
|
|
('required','/update/working_brief','required',['retired']),
|
|
('extra','/update/working_brief','additionalProperties',['confidence']),
|
|
])
|
|
def test_bootstrap_gateway_reports_safe_nested_schema_location(fault,path,keyword,fields):
|
|
document=reply();document['update']['next_step']=None
|
|
brief=document['update']['working_brief']
|
|
if fault=='type':brief['findings']='PRIVATE_RESPONSE_VALUE'
|
|
elif fault=='required':brief.pop('retired')
|
|
else:brief['confidence']='PRIVATE_RESPONSE_VALUE'
|
|
class Adapter:
|
|
async def generate(self,*args,**kwargs):
|
|
return ProviderResponse(document,'fixture',100,50)
|
|
with pytest.raises(LLMError) as caught:
|
|
asyncio.run(gateway(Adapter()).complete_round(bootstrap()))
|
|
assert caught.value.code=='LLM_OUTPUT_SCHEMA'
|
|
details=caught.value.diagnostics
|
|
assert details['validation_path']==path
|
|
assert details['validation_keyword']==keyword
|
|
assert details['validation_fields']==fields
|
|
assert 'PRIVATE_RESPONSE_VALUE' not in json.dumps(details)
|
|
assert caught.value.metadata['input_tokens']==100
|
|
|
|
|
|
@pytest.mark.parametrize('value,actual', [('PRIVATE_VALUE','string'),(['PRIVATE_VALUE'],'array'),(None,'null'),(True,'boolean'),(42,'integer'),(1.5,'number')])
|
|
def test_finding_item_types_are_preserved_without_response_values(value,actual):
|
|
from jsonschema import Draft202012Validator
|
|
from viano_bug_analysis.llm.diagnostics import schema_feedback,sanitize
|
|
from viano_bug_analysis.case.contracts import validate
|
|
request=bootstrap();document=reply();document['update']['next_step']=None
|
|
document['update']['working_brief']['findings']=[value]
|
|
error=next(Draft202012Validator(request['output_schema']).iter_errors(document))
|
|
details=sanitize(schema_feedback(error,include_keyword=True))
|
|
assert details['validation_path']=='/update/working_brief/findings/0'
|
|
assert details['expected_types']==['object'] and details['actual_type']==actual
|
|
assert 'PRIVATE_VALUE' not in json.dumps(details)
|
|
validate(details,'llm-failure-diagnostics')
|
|
|
|
|
|
def test_type_sanitizer_rejects_arbitrary_values_and_distinguishes_bool():
|
|
from viano_bug_analysis.llm.diagnostics import sanitize
|
|
assert sanitize({'actual_type':'PRIVATE','expected_types':['object','PRIVATE',{},True]})=={'expected_types':['object']}
|
|
|
|
|
|
def test_old_contexts_and_other_error_paths_get_no_new_prompt_text():
|
|
from viano_bug_analysis.analysis.tool_guidance import finding_repair_instructions
|
|
assert finding_repair_instructions({'bootstrap_json_framing':True,'previous_rejection_details':{
|
|
'validation_path':'/update/working_brief/findings/0'}})==''
|
|
assert finding_repair_instructions({'bootstrap_json_framing':True,'previous_rejection_details':{
|
|
'validation_path':'/next/arguments','expected_types':['object']}})==''
|
|
|
|
|
|
def test_bootstrap_schema_feedback_survives_package_resume_and_repairs_once(tool_case,tmp_path,caplog):
|
|
from test_investigation_hybrid import HybridAdapter
|
|
from test_working_brief import service_for
|
|
from test_investigator import CASE,state
|
|
from viano_bug_analysis.analysis import Investigator,InvestigationConfig
|
|
from viano_bug_analysis.budget import BudgetManager
|
|
from viano_bug_analysis.lifecycle import CheckpointStore,Workspace
|
|
from viano_bug_analysis.package import PackageReader,PackageValidator
|
|
from viano_bug_analysis.tools import InvestigationToolRouter
|
|
class Adapter(HybridAdapter):
|
|
def __init__(self):super().__init__();self.sent=[]
|
|
async def generate(self,config,request,**kwargs):
|
|
self.sent.append(copy.deepcopy(request))
|
|
assert request['prompt_version']=='investigation_bootstrap_v22'
|
|
response=await super().generate(config,dict(request,prompt_version='investigation_project_v18'),**kwargs)
|
|
if len(self.sent)==1:
|
|
response.document['update']['working_brief']['findings']=['PRIVATE_RESPONSE_VALUE']
|
|
return response
|
|
adapter=Adapter();router=InvestigationToolRouter(project_research=service_for(tool_case))
|
|
loop=Investigator(tool_case[0],gateway(adapter),InvestigationConfig(
|
|
lean_loop=True,lean_protocol=2,lean_checkpoint_steps=0,bootstrap_json_framing=True),router=router)
|
|
first=asyncio.run(loop.step(*CASE))
|
|
assert first.status=='REJECTED' and first.code=='LLM_OUTPUT_SCHEMA'
|
|
assert not BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions']
|
|
with PackageReader(first.package) as reader:
|
|
context=json.loads(reader.read_bytes(state(first)['active']['context_ref']))
|
|
call=json.loads(reader.read_bytes(state(first)['hypotheses']['H1']['notes'][-1]['llm_ref']))
|
|
feedback=context['previous_rejection_details']
|
|
assert feedback=={'validation_path':'/update/working_brief/findings/0','validation_fields':[],
|
|
'expected_types':['object'],'actual_type':'string',
|
|
'requirement':'match_expected_type_at_validation_path'}
|
|
assert call['outcome']['diagnostics']['validation_path']==feedback['validation_path']
|
|
assert 'PRIVATE_RESPONSE_VALUE' not in json.dumps(call['outcome']['diagnostics'])
|
|
assert 'PRIVATE_RESPONSE_VALUE' not in caplog.text
|
|
assert call['outcome']['rejected_document']['update']['working_brief']['findings']==['PRIVATE_RESPONSE_VALUE']
|
|
portable=CheckpointStore(Workspace(tmp_path/'portable'));portable.intake(first.package)
|
|
result=asyncio.run(Investigator(portable,gateway(adapter),router=router).step(*CASE))
|
|
assert result.status=='PROGRESSED' and len(adapter.sent)==2
|
|
assert 'match_expected_type_at_validation_path' in json.dumps(adapter.sent[1])
|
|
assert '/update/working_brief/findings/0' in json.dumps(adapter.sent[1])
|
|
assert 'Finding type repair' in adapter.sent[1]['messages'][0]['content']
|
|
assert 'No $record/$table encoding in output' in adapter.sent[1]['messages'][0]['content']
|
|
assert PackageValidator().validate(result.package).valid
|