78 lines
4.2 KiB
Python
78 lines
4.2 KiB
Python
import asyncio
|
|
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from test_case_builder import case_setup
|
|
from test_case_interpreter import gateway
|
|
from test_investigation_tools import tool_case
|
|
from test_parameter_and_bootstrap_guard import bootstrap
|
|
from test_lean_investigation_decision import reply
|
|
from viano_bug_analysis.llm import LLMError, ProviderResponse
|
|
|
|
|
|
@pytest.mark.parametrize('fault,path,keyword,fields', [
|
|
('type','/update/working_brief/findings','type',[]),
|
|
('required','/update/working_brief','required',['retired']),
|
|
('extra','/update/working_brief','additionalProperties',['confidence']),
|
|
])
|
|
def test_bootstrap_gateway_reports_safe_nested_schema_location(fault,path,keyword,fields):
|
|
document=reply();document['update']['next_step']=None
|
|
brief=document['update']['working_brief']
|
|
if fault=='type':brief['findings']='PRIVATE_RESPONSE_VALUE'
|
|
elif fault=='required':brief.pop('retired')
|
|
else:brief['confidence']='PRIVATE_RESPONSE_VALUE'
|
|
class Adapter:
|
|
async def generate(self,*args,**kwargs):
|
|
return ProviderResponse(document,'fixture',100,50)
|
|
with pytest.raises(LLMError) as caught:
|
|
asyncio.run(gateway(Adapter()).complete_round(bootstrap()))
|
|
assert caught.value.code=='LLM_OUTPUT_SCHEMA'
|
|
details=caught.value.diagnostics
|
|
assert details['validation_path']==path
|
|
assert details['validation_keyword']==keyword
|
|
assert details['validation_fields']==fields
|
|
assert 'PRIVATE_RESPONSE_VALUE' not in json.dumps(details)
|
|
assert caught.value.metadata['input_tokens']==100
|
|
|
|
|
|
def test_bootstrap_schema_feedback_survives_package_resume_and_repairs_once(tool_case,tmp_path):
|
|
from test_investigation_hybrid import HybridAdapter
|
|
from test_working_brief import service_for
|
|
from test_investigator import CASE,state
|
|
from viano_bug_analysis.analysis import Investigator,InvestigationConfig
|
|
from viano_bug_analysis.budget import BudgetManager
|
|
from viano_bug_analysis.lifecycle import CheckpointStore,Workspace
|
|
from viano_bug_analysis.package import PackageReader,PackageValidator
|
|
from viano_bug_analysis.tools import InvestigationToolRouter
|
|
class Adapter(HybridAdapter):
|
|
def __init__(self):super().__init__();self.sent=[]
|
|
async def generate(self,config,request,**kwargs):
|
|
self.sent.append(copy.deepcopy(request))
|
|
assert request['prompt_version']=='investigation_bootstrap_v21'
|
|
response=await super().generate(config,dict(request,prompt_version='investigation_project_v18'),**kwargs)
|
|
if len(self.sent)==1:
|
|
response.document['update']['working_brief']['findings']='PRIVATE_RESPONSE_VALUE'
|
|
return response
|
|
adapter=Adapter();router=InvestigationToolRouter(project_research=service_for(tool_case))
|
|
loop=Investigator(tool_case[0],gateway(adapter),InvestigationConfig(
|
|
lean_loop=True,lean_protocol=2,lean_checkpoint_steps=0,bootstrap_json_framing=True),router=router)
|
|
first=asyncio.run(loop.step(*CASE))
|
|
assert first.status=='REJECTED' and first.code=='LLM_OUTPUT_SCHEMA'
|
|
assert not BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions']
|
|
with PackageReader(first.package) as reader:
|
|
context=json.loads(reader.read_bytes(state(first)['active']['context_ref']))
|
|
call=json.loads(reader.read_bytes(state(first)['hypotheses']['H1']['notes'][-1]['llm_ref']))
|
|
feedback=context['previous_rejection_details']
|
|
assert feedback=={'validation_path':'/update/working_brief/findings','validation_fields':[],
|
|
'requirement':'match_expected_type_at_validation_path'}
|
|
assert call['outcome']['diagnostics']['validation_path']==feedback['validation_path']
|
|
assert 'PRIVATE_RESPONSE_VALUE' not in json.dumps(call)
|
|
portable=CheckpointStore(Workspace(tmp_path/'portable'));portable.intake(first.package)
|
|
result=asyncio.run(Investigator(portable,gateway(adapter),router=router).step(*CASE))
|
|
assert result.status=='PROGRESSED' and len(adapter.sent)==2
|
|
assert 'match_expected_type_at_validation_path' in json.dumps(adapter.sent[1])
|
|
assert '/update/working_brief/findings' in json.dumps(adapter.sent[1])
|
|
assert PackageValidator().validate(result.package).valid
|