90 lines
4.2 KiB
Python
90 lines
4.2 KiB
Python
import copy
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from test_diagnosis_reasoning import example
|
|
from test_handoff_outcomes import minimal
|
|
from viano_bug_analysis.analysis.diagnosis_builder import build_prompt, validate_evaluation
|
|
from viano_bug_analysis.analysis.change_request_contracts import (
|
|
build_prompt as handoff_prompt, make_document, render_change_request,
|
|
validate_request, ChangeRequestError,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize('early', [False, True])
|
|
def test_qualified_review_uses_same_contract_and_input(early):
|
|
_, context = example()
|
|
context['investigation']['early_stop']['status'] = 'requested' if early else 'none'
|
|
context['case'] = {}; context['interpretation'] = {}
|
|
kwargs = dict(modern=True, compact=True, consolidated=True, selected=True,
|
|
lean=True, reduced=True)
|
|
old = build_prompt(context, **kwargs)
|
|
new = build_prompt(context, qualified=True, **kwargs)
|
|
assert new['prompt_version'] == ('diagnosis_early_v8' if early else 'diagnosis_v8')
|
|
assert new['output_schema'] == old['output_schema']
|
|
assert new['messages'][1] == old['messages'][1]
|
|
system = new['messages'][0]['content']
|
|
assert 'Separate observed findings from causal inference' in system
|
|
assert 'neither exhaustive absence proofs' in system
|
|
assert 'Do not both declare a causal premise established' in system
|
|
assert 'QuestAccept' not in system and 'BUG-000017' not in system
|
|
|
|
|
|
def qualified_context():
|
|
payload, context = minimal()
|
|
context.update(case_id='case', package_id='package', analysis_revision=1,
|
|
diagnosis_ref='diagnosis.json', diagnosis_sha256='0'*64,
|
|
interpretation={'observed_behavior':[], 'expected_behavior':[]}, policy={}, case={},
|
|
uncertainty_handoff_version=1)
|
|
context['evidence']['E000001']['record']['content_sha256'] = '1'*64
|
|
diagnosis = context['diagnosis']['diagnosis']
|
|
diagnosis['remaining_uncertainty'] = ['The live deployment may differ from the snapshot.']
|
|
diagnosis['decision_basis'] = {'scope_limits':['Only the captured records were examined.']}
|
|
return payload, context
|
|
|
|
|
|
@pytest.mark.parametrize('status', ['ready', 'template', 'none'])
|
|
def test_caveats_are_copied_without_relying_on_model_or_changing_assignment(status):
|
|
payload, context = qualified_context()
|
|
payload['status'] = status
|
|
before = copy.deepcopy(context)
|
|
doc = make_document(context, payload, 'context.json', 'call.json')
|
|
assert doc['request']['status'] == status
|
|
assert doc['assignment'] == ('none' if status == 'none' else 'assigned')
|
|
assert not doc['human_dev_assigned']
|
|
assert doc['diagnostic_caveats'] == {
|
|
'remaining_uncertainty':context['diagnosis']['diagnosis']['remaining_uncertainty'],
|
|
'scope_limits':context['diagnosis']['diagnosis']['decision_basis']['scope_limits']}
|
|
assert 'The live deployment may differ' in render_change_request(doc)
|
|
doc['diagnostic_caveats']['remaining_uncertainty'].clear()
|
|
assert context == before
|
|
context.pop('uncertainty_handoff_version')
|
|
assert 'diagnostic_caveats' not in make_document(context, payload, 'context.json', 'call.json')
|
|
|
|
|
|
def test_handoff_prompt_changes_instructions_without_expanding_model_contract():
|
|
_, context = qualified_context()
|
|
old = handoff_prompt(context, compact=True, version='change_request_v8')
|
|
new = handoff_prompt(context, compact=True, version='change_request_v9')
|
|
assert old['output_schema'] == new['output_schema']
|
|
assert old['messages'][1] == new['messages'][1]
|
|
system = new['messages'][0]['content']
|
|
assert 'support or challenge' in system
|
|
assert 'uncertainty alone does not require status=none' in system
|
|
assert 'A successful build alone does not validate' in system
|
|
|
|
|
|
def test_open_assumptions_and_negative_findings_still_allow_bounded_confirmation():
|
|
payload, context = example()
|
|
payload['remaining_uncertainty'].append('A runtime activation path remains untested.')
|
|
validate_evaluation(payload, context)
|
|
assert payload['status'] == 'confirmed'
|
|
|
|
|
|
def test_model_cannot_supply_controller_owned_caveats():
|
|
payload, context = qualified_context()
|
|
payload['diagnostic_caveats'] = {'remaining_uncertainty':[], 'scope_limits':[]}
|
|
with pytest.raises(ChangeRequestError, match='CHANGE_REQUEST_OUTPUT'):
|
|
validate_request(payload, context)
|