From 0307ebfac0c9a0143822bb188269ce9dba47ae79 Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Sat, 17 Oct 2026 23:17:32 +0200 Subject: [PATCH] Configure lower reasoning effort for narrow early investigation reviews --- config/llm.example.toml | 2 + config/llm.gemini-cache.example.toml | 2 + docs/investigation-review-v16.md | 15 +++ src/viano_bug_analysis/llm/config.py | 10 +- src/viano_bug_analysis/llm/gateway.py | 25 ++++ .../schemas/llm-call-v1.schema.json | 12 ++ tests/test_review_reasoning_effort.py | 125 ++++++++++++++++++ 7 files changed, 188 insertions(+), 3 deletions(-) create mode 100644 tests/test_review_reasoning_effort.py diff --git a/config/llm.example.toml b/config/llm.example.toml index 1557222..b9e5fc6 100644 --- a/config/llm.example.toml +++ b/config/llm.example.toml @@ -21,6 +21,8 @@ max_output_tokens = 4096 # reasoning_effort = "medium" # Optional override for entity resolution planning and assessment only: # entity_resolution_reasoning_effort = "low" +# Optional override only for settling pending work before early diagnosis: +# investigation_review_reasoning_effort = "low" [[llm.providers]] id = "own-server" diff --git a/config/llm.gemini-cache.example.toml b/config/llm.gemini-cache.example.toml index 245f797..3a048e3 100644 --- a/config/llm.gemini-cache.example.toml +++ b/config/llm.gemini-cache.example.toml @@ -18,6 +18,8 @@ response_format = "json_object" max_output_tokens = 16384 reasoning_effort = "medium" entity_resolution_reasoning_effort = "low" +# Only the narrow early-review assessment, not regular investigation or diagnosis: +investigation_review_reasoning_effort = "low" # Enabled by default on this adapter. Stable case context plus instructions/schema; # live evidence is always complete. Explicit false is an operational override. diff --git a/docs/investigation-review-v16.md b/docs/investigation-review-v16.md index 5553508..6a978b7 100644 --- a/docs/investigation-review-v16.md +++ b/docs/investigation-review-v16.md @@ -27,3 +27,18 @@ Cache-Anfragen berücksichtigen v16. Der engere Review-Systemprompt und sein Aus Offline-Projektion derselben gespeicherten Bug20-Kontexte, ohne API: die beiden Review-Anfragen sinken von 24547 auf 16220 Zeichen (-33,92 %) und von 24182 auf 16725 (-30,84 %). Reguläre Anfragen verändern sich zwischen +2,50 % (zusätzliche klare Referenzregeln) und -0,92 % (mehr konsolidierbare Herkunftsobjekte). Diese Maßnahme ist dort primär Fehlervermeidung, keine große neue Inputreduktion. Einzelwerte: [investigation-v16-offline-20261017.json](measurements/investigation-v16-offline-20261017.json). Die Messung beweist weder weniger Denktokens noch tatsächliche Kostenersparnis. Denkintensität und Ausgabelimit wurden bewusst nicht pauschal abgesenkt. Der weiche Zeichenrichtwert bleibt bestehen; frische Gegenbelege werden dafür nicht abgeschnitten. Ein späterer Live-Regressionstest muss Aufrufzahl, Fehler, Denken, Cache und Ergebnisqualität gemeinsam vergleichen. + +## Separate Denkintensität für die Frühabschluss-Bewertung + +Pro Provider kann `investigation_review_reasoning_effort` gesetzt werden: + +```toml +reasoning_effort = "medium" +investigation_review_reasoning_effort = "low" +``` + +Nur der enge v16-Aufruf mit Aufgabe `assess_pending_result_for_early_review`, Status `preparing` und dem auf defer/reject begrenzten Antwortvertrag verwendet diesen Override. Normale Untersuchungsturns, Interpretation, Hypothesengenerierung und finale Diagnose behalten ihre Einstellungen. Für jeden Failover-Provider gilt dessen eigener Wert. Omission übernimmt die bisherige allgemeine Denkintensität; ohne Override bleibt auch der Konfigurationssnapshot mit früheren Operationen kompatibel. + +Gemini erhält den angeforderten Wert als thinkingLevel, der kompatible Adapter als reasoning_effort. Die tatsächlich unterstützten Werte hängen weiterhin vom Modell ab. Keine automatische Abschaltung des Denkens, kein kleineres Ausgabelimit und kein verändertes Hypothesenbudget. Die Call-Metadaten speichern bei Anwendung reasoning_profile=investigation_review und den angeforderten reasoning_effort, auch bei kontrolliert fehlgeschlagenen Antworten. Das beweist die übermittelte Einstellung, nicht deren interne Umsetzung durch den Anbieter; dafür sind gemessene Denktokens maßgeblich. + +Für den nächsten frischen lokalen Vergleich ist `config/llm.early-review.local.toml` vorbereitet: Kopie des bisherigen kompakten Profils, ergänzt um den Override low. Das vorige Profil wurde nicht geändert. Ein Konfigurationswechsel mitten in einer bereits gespeicherten LLM-Operation kann kontrolliert als LLM_OPERATION_CONFLICT abgelehnt werden; neue Vergleiche deshalb mit frischem Testworkspace starten. Es wurden keine kostenpflichtigen Aufrufe zur Einrichtung ausgeführt. diff --git a/src/viano_bug_analysis/llm/config.py b/src/viano_bug_analysis/llm/config.py index 37d5e0f..a64976e 100644 --- a/src/viano_bug_analysis/llm/config.py +++ b/src/viano_bug_analysis/llm/config.py @@ -34,6 +34,7 @@ class ProviderConfig: cost_currency: str = "USD" reasoning_effort: str | None = None entity_resolution_reasoning_effort: str | None = None + investigation_review_reasoning_effort: str | None = None cached_input_cost_per_million: float | None = None cache_storage_cost_per_million_hour: float | None = None context_cache: bool | None = None @@ -76,7 +77,8 @@ class ProviderConfig: raise LLMError("LLM_CONFIG") if self.context_cache is not None and type(self.context_cache) is not bool: raise LLMError('LLM_CONFIG') - for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort): + for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort, + self.investigation_review_reasoning_effort): if effort is not None and (not isinstance(effort, str) or effort not in {"none", "minimal", "low", "medium", "high"}): raise LLMError("LLM_CONFIG") positive(self.timeout_seconds) @@ -94,7 +96,8 @@ class ProviderConfig: or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}', self.cache_required_prompt_prefix)): raise LLMError('LLM_CONFIG') if self.adapter == 'gemini' and (not re.fullmatch(r'[A-Za-z0-9._-]+', self.model) - or self.reasoning_effort == 'none' or self.entity_resolution_reasoning_effort == 'none'): + or self.reasoning_effort == 'none' or self.entity_resolution_reasoning_effort == 'none' + or self.investigation_review_reasoning_effort == 'none'): raise LLMError('LLM_CONFIG') if not isinstance(self.cost_currency, str) or not re.fullmatch(r"[A-Z]{3}", self.cost_currency): raise LLMError("LLM_CONFIG") @@ -164,7 +167,8 @@ class GatewayConfig: 'cache_min_prefix_chars': 16000}.items(): if provider[name] == default: del provider[name] - for name in ("reasoning_effort", "entity_resolution_reasoning_effort"): + for name in ("reasoning_effort", "entity_resolution_reasoning_effort", + "investigation_review_reasoning_effort"): if provider[name] is None: del provider[name] return result diff --git a/src/viano_bug_analysis/llm/gateway.py b/src/viano_bug_analysis/llm/gateway.py index f7551a5..7ca21d9 100644 --- a/src/viano_bug_analysis/llm/gateway.py +++ b/src/viano_bug_analysis/llm/gateway.py @@ -34,6 +34,27 @@ def _schema_check(schema): _schema_check(value) +def is_investigation_review(request): + """Route the controller's narrow v16 review, never arbitrary research text.""" + if request['prompt_version'] != 'investigation_project_v16': + return False + try: + message = next(m for m in reversed(request['messages']) if m['role'] == 'user') + document = parse_manifest(message['content'].encode('utf-8')) + return (document.get('task') == 'assess_pending_result_for_early_review' + and document['input']['early_review']['request']['status'] == 'preparing' + and set(request['output_schema']['properties']['next']['properties']['kind']['enum']) + == {'defer','reject'}) + except (ValueError, TypeError, KeyError, AttributeError, StopIteration, RecursionError): + return False + + +def review_metadata(provider, request): + if provider.investigation_review_reasoning_effort is not None and is_investigation_review(request): + return {'reasoning_profile':'investigation_review', 'reasoning_effort':provider.reasoning_effort} + return {} + + class LLMGateway: def __init__(self, config, adapters=None): self.config = config @@ -79,6 +100,8 @@ class LLMGateway: if (provider.entity_resolution_reasoning_effort is not None and re.fullmatch(r"entity_resolution_(plan|assessment)_v[1-9][0-9]*", request["prompt_version"])): provider = replace(provider, reasoning_effort=provider.entity_resolution_reasoning_effort) + if provider.investigation_review_reasoning_effort is not None and is_investigation_review(request): + provider = replace(provider, reasoning_effort=provider.investigation_review_reasoning_effort) started = time.monotonic() response = None try: @@ -106,6 +129,7 @@ class LLMGateway: "estimated_cost": cost, "elapsed_seconds": round(time.monotonic() - started, 6), "attempts": attempts} metadata["cost_currency"] = provider.cost_currency if cost is not None else None metadata.update(usage) + metadata.update(review_metadata(provider, request)) if any(usage[k] is not None for k in ('estimated_input_cost', 'estimated_cached_input_cost', 'estimated_output_cost', 'estimated_cache_storage_cost')): metadata['cost_currency'] = provider.cost_currency @@ -159,6 +183,7 @@ class LLMGateway: 'input_tokens':input_tokens,'output_tokens':output_tokens,'cost_currency':None, 'elapsed_seconds':round(time.monotonic()-started,6),'attempts':copy.deepcopy(attempts)} error.metadata.update(usage_metadata(provider, response, details)) + error.metadata.update(review_metadata(provider, request)) if any(error.metadata[k] is not None for k in ('estimated_cost','estimated_input_cost', 'estimated_cached_input_cost','estimated_output_cost','estimated_cache_storage_cost')): error.metadata['cost_currency'] = provider.cost_currency diff --git a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json index 361bf78..33a18f5 100644 --- a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json +++ b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json @@ -104,6 +104,18 @@ "items": { "$ref": "#/$defs/http_attempt" } + }, + "reasoning_profile": { + "const": "investigation_review" + }, + "reasoning_effort": { + "enum": [ + "none", + "minimal", + "low", + "medium", + "high" + ] } } }, diff --git a/tests/test_review_reasoning_effort.py b/tests/test_review_reasoning_effort.py new file mode 100644 index 0000000..a30ccac --- /dev/null +++ b/tests/test_review_reasoning_effort.py @@ -0,0 +1,125 @@ +import asyncio +import json +from dataclasses import replace + +import pytest + +from test_investigation_review import review_context +from test_lean_investigation_decision import reply +from test_llm_gateway import provider, http_endpoint, active_case +from viano_bug_analysis.analysis.investigation_prompt import build_prompt +from viano_bug_analysis.llm import (GatewayConfig, ProviderConfig, ProviderResponse, + ProviderUnavailable, LLMGateway, LLMError, CaseLLMRunner) +from viano_bug_analysis.package import PackageReader, PackageValidator + + +def review_request(): + return build_prompt(review_context()) + + +def review_reply(): + result=reply();result['next']={'kind':'defer','reason':'Pending evidence assessed'} + return result + + +class RecordingAdapter: + def __init__(self, fail_first=False, error=False): + self.calls=[];self.fail_first=fail_first;self.error=error + + async def generate(self, config, request, *, max_response_bytes): + self.calls.append((config.id,config.reasoning_effort)) + if self.error:raise LLMError('LLM_RESPONSE') + if self.fail_first and len(self.calls)==1:raise ProviderUnavailable('LLM_CONNECTION') + return ProviderResponse(review_reply(),'fixture-model',100,20) + + +@pytest.mark.parametrize('mode',['review','normal','diagnosis','old_version','wrong_task','wrong_actions']) +def test_effort_override_applies_only_to_narrow_controller_review(mode): + req=review_request();c=review_context() + if mode=='normal':c['early_review']['request']=None;req=build_prompt(c) + elif mode=='old_version':c['prompt_version']='investigation_project_v15';req=build_prompt(c) + elif mode=='diagnosis':req['prompt_version']='diagnosis_early_v6' + elif mode=='wrong_task': + doc=json.loads(req['messages'][1]['content']);doc['task']='investigate_hypothesis' + req['messages'][1]['content']=json.dumps(doc) + elif mode=='wrong_actions':req['output_schema']['properties']['next']['properties']['kind']['enum'].append('tool') + cfg=GatewayConfig((provider(reasoning_effort='medium',investigation_review_reasoning_effort='low'),)) + adapter=RecordingAdapter();out=asyncio.run(LLMGateway(cfg,{'fixture':adapter}).complete_round(req)) + assert adapter.calls==[('first','low' if mode=='review' else 'medium')] + assert cfg.providers[0].reasoning_effort=='medium' + assert out['metadata'].get('reasoning_profile')==('investigation_review' if mode=='review' else None) + + +def test_provider_failover_uses_each_providers_review_setting(): + cfg=GatewayConfig((provider(reasoning_effort='high',investigation_review_reasoning_effort='low'), + provider('second',reasoning_effort='medium',investigation_review_reasoning_effort='minimal'))) + adapter=RecordingAdapter(fail_first=True) + out=asyncio.run(LLMGateway(cfg,{'fixture':adapter}).complete_round(review_request())) + assert adapter.calls==[('first','low'),('second','minimal')] + assert out['metadata']['reasoning_effort']=='minimal' + + +@pytest.mark.parametrize('bad',[True,0,'unknown',[],{}]) +def test_invalid_override_is_rejected(bad): + with pytest.raises(LLMError,match='LLM_CONFIG'): + provider(investigation_review_reasoning_effort=bad) + + +def test_omitted_override_preserves_snapshot_and_inherits_general_effort(): + cfg=GatewayConfig((provider(reasoning_effort='medium'),)) + assert 'investigation_review_reasoning_effort' not in cfg.snapshot()['providers'][0] + adapter=RecordingAdapter();asyncio.run(LLMGateway(cfg,{'fixture':adapter}).complete_round(review_request())) + assert adapter.calls==[('first','medium')] + changed=GatewayConfig((replace(cfg.providers[0],investigation_review_reasoning_effort='low'),)) + assert changed.snapshot()['providers'][0]['investigation_review_reasoning_effort']=='low' + + +def test_toml_loads_override(tmp_path): + p=tmp_path/'llm.toml';p.write_text('''[llm] +[[llm.providers]] +id="fixture" +model="fixture" +base_url="http://localhost/v1" +reasoning_effort="medium" +investigation_review_reasoning_effort="low" +''') + assert GatewayConfig.load(p).providers[0].investigation_review_reasoning_effort=='low' + + +def test_native_gemini_receives_low_thinking_level(monkeypatch): + from test_llm_caching import config, native, scripted + from viano_bug_analysis.llm.gemini import GeminiProvider + adapter=GeminiProvider();envelope=native() + envelope['candidates'][0]['content']['parts'][0]['text']=json.dumps(review_reply()) + seen=scripted(adapter,monkeypatch,[envelope]) + cfg=GatewayConfig((replace(config(),context_cache=False,reasoning_effort='medium', + investigation_review_reasoning_effort='low'),)) + out=asyncio.run(LLMGateway(cfg,{'gemini':adapter}).complete_round(review_request())) + assert seen[0][1]['generationConfig']['thinkingConfig']=={'thinkingLevel':'low'} + assert out['metadata']['reasoning_effort']=='low' + with pytest.raises(LLMError,match='LLM_CONFIG'): + replace(cfg.providers[0],investigation_review_reasoning_effort='none') + + +def test_compatible_transport_receives_low_reasoning_effort(http_endpoint): + url,seen,response=http_endpoint + response['body']['choices'][0]['message']['content']=json.dumps(review_reply()) + cfg=GatewayConfig((ProviderConfig(id='test',model='configured',base_url=url, + reasoning_effort='medium',investigation_review_reasoning_effort='low'),)) + asyncio.run(LLMGateway(cfg).complete_round(review_request())) + assert seen[0]['body']['reasoning_effort']=='low' + + +@pytest.mark.parametrize('failed',[False,True]) +def test_effective_setting_survives_checkpoint_and_replay(active_case,failed): + store,_,_=active_case;adapter=RecordingAdapter(error=failed) + cfg=GatewayConfig((provider(reasoning_effort='medium',investigation_review_reasoning_effort='low'),)) + runner=CaseLLMRunner(store,LLMGateway(cfg,{'fixture':adapter})) + req=review_request() + result=asyncio.run(runner.run('case-A','stable-package',operation_id='review-effort',request=req)) + assert PackageValidator().validate(result.package).valid + with PackageReader(result.package) as reader: + call=json.loads(reader.read_bytes(result.result_ref)) + assert call['outcome']['metadata']['reasoning_effort']=='low' + assert asyncio.run(runner.run('case-A','stable-package',operation_id='review-effort',request=req))==result + assert len(adapter.calls)==1