197 lines
8.8 KiB
Python
197 lines
8.8 KiB
Python
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from test_investigation_lean import lean_context
|
|
from test_case_builder import case_setup
|
|
from test_investigation_startup import Adapter
|
|
from test_investigation_tools import tool_case
|
|
from test_investigator import CASE, run, state
|
|
from test_working_brief import service_for
|
|
from test_case_interpreter import gateway
|
|
from viano_bug_analysis.analysis import Investigator, InvestigationConfig, InvestigationError
|
|
from viano_bug_analysis.analysis.investigation_prompt import build_prompt
|
|
from viano_bug_analysis.analysis.investigation_lean_v20 import validate_reply
|
|
from viano_bug_analysis.analysis.investigation_window import evidence_page, pages, PAGE_CHARACTERS
|
|
from viano_bug_analysis.llm import LLMError, ProviderResponse
|
|
from viano_bug_analysis.llm.investigation_size import characters, check, check_wire, LIMIT
|
|
from viano_bug_analysis.llm.gemini import GeminiProvider
|
|
from viano_bug_analysis.tools import InvestigationToolRouter
|
|
from viano_bug_analysis.package import PackageReader, PackageValidator
|
|
|
|
|
|
def context():
|
|
c = lean_context()
|
|
c['lean_protocol'] = 2
|
|
c['policy']['prompt_window_version'] = 1
|
|
return c
|
|
|
|
|
|
def test_window_is_small_and_canonical_context_is_untouched():
|
|
c = context()
|
|
before = copy.deepcopy(c)
|
|
request = build_prompt(c)
|
|
assert characters(request) <= 10_000
|
|
body = json.loads(request['messages'][1]['content'])
|
|
assert 'response_contract' in body and 'tool_schema' not in body['input']
|
|
assert c == before
|
|
|
|
|
|
def test_evidence_pages_are_lossless_and_explicitly_partial():
|
|
entry = {'record': {'hypothesis_id': 'H1'}, 'content': {'items': [
|
|
{'text': '\u00e4\U0001f600' * 8000, 'coverage': 'bounded'}]}}
|
|
raw = pages(entry)
|
|
first = evidence_page(entry)
|
|
assert first['complete_payload'] is False
|
|
fragments = [evidence_page(entry,i) for i in range(first['pages'])]
|
|
assert ''.join(p['fragment'] for p in fragments) == raw
|
|
assert all(len(p['fragment']) <= PAGE_CHARACTERS for p in fragments)
|
|
assert len({p['sha256'] for p in fragments}) == 1
|
|
with pytest.raises(InvestigationError, match='INVESTIGATION_EVIDENCE_PAGE'):
|
|
evidence_page(entry, first['pages'])
|
|
|
|
|
|
def test_long_history_does_not_accumulate_raw_results():
|
|
c = context()
|
|
seed = copy.deepcopy(next(iter(c['evidence'].values())))
|
|
sizes = []
|
|
for count in (3, 20, 100, 138):
|
|
c['evidence'] = {f'E{i:06d}': copy.deepcopy(seed) for i in range(1,count+1)}
|
|
for eid, entry in c['evidence'].items():
|
|
entry['record']['id'] = eid
|
|
entry['content']['items'] = [{'text': eid + 'x' * 30_000}]
|
|
c['previous_result'] = c['evidence'][f'E{count:06d}']
|
|
c['lean_session']['turns'] = []
|
|
request = build_prompt(c)
|
|
sizes.append(characters(request))
|
|
body = json.loads(request['messages'][1]['content'])
|
|
assert body['input']['archive']['omitted_from_view'] > 0
|
|
assert f'E{count:06d}' in body['input']['evidence']
|
|
assert max(sizes) < 20_000
|
|
assert max(sizes) - min(sizes) < 1500
|
|
|
|
|
|
def test_strict_controller_still_rejects_unknown_actions():
|
|
c = context()
|
|
with pytest.raises(InvestigationError, match='INVESTIGATION_OUTPUT'):
|
|
validate_reply(c, {'assessment': None, 'observation': '',
|
|
'next': {'kind': 'execute_shell', 'command': 'anything'}})
|
|
|
|
|
|
def test_size_guard_counts_schema_and_cached_wire_prefix():
|
|
request = {'messages': [{'role': 'system', 'content': 'a' * 10_000}],
|
|
'output_schema': {'type': 'object', 'description': 'b' * 10_000}}
|
|
with pytest.raises(LLMError, match='INVESTIGATION_PROMPT_LIMIT'):
|
|
check(request)
|
|
with pytest.raises(LLMError, match='INVESTIGATION_PROMPT_LIMIT'):
|
|
check_wire({'parts': [{'text': 'a' * 10_000}]},
|
|
[{'parts': [{'text': 'b' * 10_001}]}])
|
|
|
|
|
|
@pytest.mark.parametrize('version', ['investigation_project_v20', 'investigation_startup_v1',
|
|
'investigation_startup_review_v1', 'investigation_format_repair_v1'])
|
|
def test_oversized_requests_never_reach_adapter(version):
|
|
import asyncio
|
|
adapter = Adapter()
|
|
request = {'prompt_version':version,'privacy':'configured',
|
|
'messages':[{'role':'user','content':'x' * 20_001}],
|
|
'output_schema':{'type':'object'}}
|
|
with pytest.raises(LLMError, match='INVESTIGATION_PROMPT_LIMIT'):
|
|
asyncio.run(gateway(adapter).complete_round(request))
|
|
assert adapter.calls == []
|
|
request['prompt_version'] = 'case_plan_v2'
|
|
assert gateway(adapter).validate_request(request)['prompt_version'] == 'case_plan_v2'
|
|
|
|
|
|
def test_exact_character_limit_and_diagnostics():
|
|
from viano_bug_analysis.llm.investigation_size import compact
|
|
schema = {'type':'object'}
|
|
request = {'messages':[{'role':'user','content':'x' * (LIMIT-len(compact(schema)))}],
|
|
'output_schema':schema}
|
|
assert check(request) == LIMIT
|
|
request['messages'][0]['content'] += 'x'
|
|
with pytest.raises(LLMError) as error:
|
|
check(request)
|
|
assert error.value.diagnostics['prompt_characters'] == LIMIT+1
|
|
|
|
|
|
def loop_for(tool_case, adapter):
|
|
return Investigator(tool_case[0], gateway(adapter), InvestigationConfig(
|
|
lean_loop=True, lean_protocol=2, startup_batch_review=True,
|
|
lean_checkpoint_steps=0, prompt_window_version=1),
|
|
router=InvestigationToolRouter(project_research=service_for(tool_case)))
|
|
|
|
|
|
def test_bounded_startup_review_and_import(tool_case, tmp_path):
|
|
adapter = Adapter()
|
|
loop = loop_for(tool_case, adapter)
|
|
result = run(loop, steps=4)
|
|
assert state(result)['startup']['done']
|
|
assert len(adapter.calls) == 4
|
|
assert all('tool_schema' not in body['input'] for body in adapter.calls)
|
|
assert PackageValidator().validate(result.package).valid
|
|
# Replaying the stored bounded requests after an import retains bindings.
|
|
from viano_bug_analysis.lifecycle import CheckpointStore, Workspace
|
|
recipient = CheckpointStore(Workspace(tmp_path/'recipient'))
|
|
recipient.intake(result.package)
|
|
restored = Investigator(recipient, gateway(adapter), router=loop.router)
|
|
with PackageReader(recipient.latest(*CASE)) as reader:
|
|
assert restored._load(reader)['startup']['done']
|
|
|
|
|
|
def test_bounded_completion_after_checkpoints(tool_case):
|
|
class Finish(Adapter):
|
|
async def generate(self, config, request, **kwargs):
|
|
body = json.loads(request['messages'][1]['content'])
|
|
assert characters(request) <= LIMIT
|
|
if body['task'] == 'investigate_one_step':
|
|
out = {'assessment': None, 'observation': 'Ready to consolidate.',
|
|
'next': {'kind': 'checkpoint', 'intent': 'completion'}}
|
|
elif body['task'] == 'consolidate_investigation_session':
|
|
out = {'assessment': None, 'confidence': 'plausible', 'findings': [],
|
|
'uncertainties': ['No proven cause.'], 'next_question': None,
|
|
'retire': [], 'next': {'kind': 'finish', 'reason': 'Bounded investigation complete.'}}
|
|
else:
|
|
return await super().generate(config, request, **kwargs)
|
|
return ProviderResponse(out, 'fixture', 100, 50)
|
|
result = run(loop_for(tool_case, Finish()), steps=20)
|
|
assert result.status == 'COMPLETED'
|
|
assert PackageValidator().validate(result.package).valid
|
|
|
|
|
|
def test_recall_page_is_persisted_and_replayed(tool_case):
|
|
class Recall(Adapter):
|
|
async def generate(self, config, request, **kwargs):
|
|
body = json.loads(request['messages'][1]['content'])
|
|
if body['task'] == 'investigate_one_step':
|
|
self.calls.append(body)
|
|
out = {'assessment': None, 'observation': 'Read archived evidence.',
|
|
'next': {'kind':'recall','evidence_refs':['E000001'],'page':0}}
|
|
return ProviderResponse(out,'fixture',100,50)
|
|
return await super().generate(config,request,**kwargs)
|
|
loop = loop_for(tool_case, Recall())
|
|
result = run(loop, steps=5)
|
|
assert state(result)['recalled_pages']['H1'] == {'E000001':0}
|
|
result = run(loop, steps=1)
|
|
assert result.status != 'BLOCKED'
|
|
|
|
|
|
def test_bounded_cache_partition_is_exact_and_wire_is_bounded():
|
|
from viano_bug_analysis.llm.stable_context import split_document, restore_document
|
|
request = build_prompt(context())
|
|
body = json.loads(request['messages'][1]['content'])
|
|
parts = split_document(body,request['prompt_version'])
|
|
assert parts and restore_document(*parts) == body
|
|
system,prefix,live = GeminiProvider._messages(request,stable_case_context=True)
|
|
assert check_wire(system,prefix+live) <= LIMIT
|
|
|
|
|
|
def test_early_review_keeps_the_low_reasoning_route():
|
|
from viano_bug_analysis.llm.gateway import is_investigation_review
|
|
c = context()
|
|
c['lean_session']['checkpoint'] = True
|
|
c['early_review'] = {'request':{'status':'preparing'}}
|
|
request = build_prompt(c)
|
|
assert is_investigation_review(request)
|