Add persistent profile-routed analysis worker and shared workflow

tests/profiles/vcore/test_worker_intake.py

@@ -0,0 +1,30 @@
+import asyncio
+import json
+
+from viano_bug_analysis.api import APIConfig, Client, Project, IntakeService, IntakeLimits
+from viano_bug_analysis.package import PackageReader
+from viano_bug_analysis.worker import AnalysisWorker, WorkerConfig
+from viano_bug_analysis.worker.config import binding
+from test_case_interpreter import gateway, FixtureAdapter
+
+
+def test_api_submission_is_admitted_and_built_without_manual_trial(source_pair, tmp_path):
+    client = Client('capture', 'UNUSED_API_TOKEN', ('core',))
+    project = Project('core', 'wow', 'vcore')
+    config = APIConfig(tmp_path / 'service', (project,), (client,), limits=IntakeLimits(min_free_bytes=1))
+    service = IntakeService(config)
+    data = {'kind': 'new_case', 'project_id': 'core', 'profile_id': 'vcore', 'case_id': 'BUG-000020'}
+    receipt, _ = service.submit(client, 'capture1', json.dumps(data).encode(), [('wowdiag', source_pair[0])])
+    def load(p):
+        return gateway(FixtureAdapter()), {}, binding(p)
+    worker = AnalysisWorker(WorkerConfig(config, min_free_bytes=1), loader=load)
+    asyncio.run(worker.run(once=True))
+    assert not list(worker.workspace.inbox.glob('*.zip'))
+    latest = worker.store.latest('BUG-000020', receipt['package_id'])
+    with PackageReader(latest) as reader:
+        assert reader.manifest['state'] == 'BUILDING_CASE'
+        assert reader.manifest['artifacts']['case']
+    status = service.status(receipt['submission_id'], client)
+    assert status['status'] == 'BUILDING_CASE'
+    assert status['analysis_revision'] == 1
+    assert status['worker']['status'] == 'YIELDED'

tests/test_worker.py

@@ -0,0 +1,283 @@
+import asyncio
+import json
+from dataclasses import replace
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from viano_bug_analysis.api import APIConfig, Client, Project
+from viano_bug_analysis.lifecycle import State
+from viano_bug_analysis.lifecycle.locking import FileLock
+from viano_bug_analysis.lifecycle.errors import LockBusy
+from viano_bug_analysis.package import PackageReader
+from viano_bug_analysis.worker import AnalysisWorker, WorkerConfig, WorkerError
+from viano_bug_analysis.worker.config import binding
+from test_case_builder import case_setup
+from test_case_interpreter import gateway, FixtureAdapter
+
+CASE = ('case-A', 'stable-package')
+
+
+@pytest.fixture(autouse=True)
+def project_manifest(manifest):
+    manifest['project'] = {'id': 'test', 'type': 'wow', 'profile': 'vcore'}
+
+
+@pytest.fixture
+def setup(case_setup):
+    store = case_setup[0]
+    project = Project('test', 'wow', 'vcore', planning_mode='separate')
+    config = WorkerConfig(APIConfig(store.workspace.root, (project,),
+        (Client('capture', 'TEST_TOKEN', ('test',)),)), min_free_bytes=1)
+    def loader(p):
+        return gateway(FixtureAdapter()), {'planning_mode': 'separate'}, binding(p)
+    return config, loader, store
+
+
+def test_real_case_build_then_interpret_and_restart(setup):
+    config, loader, store = setup
+    worker = AnalysisWorker(config, loader=loader)
+    assert asyncio.run(worker.tick()) == 1
+    with PackageReader(store.latest(*CASE)) as reader:
+        assert reader.manifest['state'] == State.BUILDING_CASE
+        assert reader.manifest['artifacts']['case']
+    # Fresh worker object has no in-memory queue. It recovers from managed packages.
+    resumed = AnalysisWorker(config, loader=loader)
+    assert asyncio.run(resumed.tick()) == 1
+    with PackageReader(store.latest(*CASE)) as reader:
+        assert reader.manifest['state'] == State.INTERPRETING_CASE
+        assert reader.manifest['artifacts']['case_interpretation']
+
+
+def test_scheduler_retry_is_durable_and_not_busy_loop(setup):
+    config, loader, store = setup
+    now = datetime.now(timezone.utc)
+    calls = []
+    async def dispatch(*args, **kwargs):
+        calls.append(kwargs)
+        return {'status': 'RETRY_PENDING', 'code': 'LLM_CONNECTION',
+                'retry_not_before': (now + timedelta(minutes=5)).isoformat()}
+    worker = AnalysisWorker(config, loader=loader, dispatcher=dispatch, clock=lambda: now)
+    assert asyncio.run(worker.tick()) == 1
+    again = AnalysisWorker(config, loader=loader, dispatcher=dispatch, clock=lambda: now)
+    assert asyncio.run(again.tick()) == 0
+    assert len(calls) == 1 and calls[0]['attempt_id'] == 'default'
+    after = AnalysisWorker(config, loader=loader, dispatcher=dispatch,
+        clock=lambda: now + timedelta(minutes=6))
+    assert asyncio.run(after.tick()) == 1
+
+
+@pytest.mark.parametrize('reason,automatic', [
+    ('AWAITING_TEST_FEEDBACK', True), ('INVALID_LLM_RESPONSES', True),
+    ('LLM_UNAVAILABLE', True), ('RESEARCH_UNAVAILABLE', False)])
+def test_worker_never_silently_bypasses_block(setup, reason, automatic):
+    config, loader, store = setup
+    store.advance(*CASE, State.BUILDING_CASE, step_id='start')
+    blocked = store.advance(*CASE, State.BLOCKED, step_id='block', reason=reason,
+        retry_not_before='2020-01-01T00:00:00Z', details={'automatic_retry': automatic})
+    async def forbidden(*args, **kwargs):
+        raise AssertionError('blocked case was dispatched')
+    worker = AnalysisWorker(config, loader=loader, dispatcher=forbidden)
+    assert asyncio.run(worker.tick()) == 0
+    assert store.latest(*CASE) == blocked
+    if reason != 'RESEARCH_UNAVAILABLE':
+        with pytest.raises(WorkerError, match='WORKER_RETRY_STATE'):
+            worker.retry(*CASE)
+    else:
+        worker.retry(*CASE)
+        with PackageReader(store.latest(*CASE)) as reader:
+            assert reader.manifest['state'] == 'BUILDING_CASE'
+
+
+def test_explicitly_permitted_due_retry_resumes(setup):
+    config, loader, store = setup
+    store.advance(*CASE, State.BUILDING_CASE, step_id='start')
+    store.advance(*CASE, State.BLOCKED, step_id='block', reason='RESEARCH_UNAVAILABLE',
+        retry_not_before='2020-01-01T00:00:00Z', details={'automatic_retry': True})
+    seen = []
+    async def dispatch(store, gateway, *identity, **kwargs):
+        with PackageReader(store.latest(*identity)) as reader:
+            seen.append(reader.manifest['state'])
+        return {'status': 'YIELDED'}
+    assert asyncio.run(AnalysisWorker(config, loader=loader, dispatcher=dispatch).tick()) == 1
+    assert seen == ['BUILDING_CASE']
+
+
+def test_one_service_and_trial_lease_exclusion(setup):
+    config, loader, store = setup
+    worker = AnalysisWorker(config, loader=loader)
+    with FileLock(worker.workspace.runtime / 'locks/analysis-service.lock'):
+        with pytest.raises(LockBusy):
+            asyncio.run(worker.run(once=True))
+    from viano_bug_analysis.lifecycle.storage import key
+    with FileLock(worker.workspace.runtime / 'locks' / ('trial-' + key(CASE[0]) + '.lock')):
+        assert asyncio.run(worker.tick()) == 0
+
+
+def test_exception_backoff_and_explicit_scheduler_release(setup):
+    config, loader, store = setup
+    calls = []
+    async def broken(*args, **kwargs):
+        calls.append(1)
+        raise ValueError('secret data must not appear in runtime state')
+    worker = AnalysisWorker(config, loader=loader, dispatcher=broken)
+    asyncio.run(worker.tick())
+    asyncio.run(worker.tick())
+    assert len(calls) == 1
+    record = next(worker.records.glob('*.json')).read_text()
+    assert 'secret data' not in record
+    worker.retry(*CASE)
+    asyncio.run(worker.tick())
+    assert len(calls) == 2
+
+
+def test_stop_requested_during_dispatch_finishes_step(setup):
+    config, loader, store = setup
+    async def dispatch(*args, **kwargs):
+        worker.request_stop()
+        return {'status': 'YIELDED'}
+    worker = AnalysisWorker(config, loader=loader, dispatcher=dispatch)
+    asyncio.run(worker.run())
+    assert json.loads((worker.root / 'status.json').read_bytes())['status'] == 'STOPPED'
+    assert next(worker.records.glob('*.json')).exists()
+
+
+def test_config_binding_persisted_before_dispatch_and_detects_restart_change(setup):
+    config, loader, store = setup
+    async def dispatch(*args, **kwargs):
+        assert next(worker.records.glob('*.json')).exists()
+        return {'status': 'YIELDED'}
+    worker = AnalysisWorker(config, loader=loader, dispatcher=dispatch)
+    asyncio.run(worker.tick())
+    changed = replace(config.intake.projects[0], planning_mode='joint')
+    updated = replace(config, intake=replace(config.intake, projects=(changed,)))
+    other = AnalysisWorker(updated, loader=loader, dispatcher=dispatch)
+    assert asyncio.run(other.tick()) == 0
+    record = json.loads(next(other.records.glob('*.json')).read_bytes())
+    assert record['status'] == 'ATTENTION' and record['code'] == 'WORKER_CONFIG_CHANGED'
+
+
+def test_unknown_project_held_without_model_call(setup):
+    config, loader, store = setup
+    changed = replace(config.intake.projects[0], id='other')
+    updated = replace(config, intake=APIConfig(config.intake.workspace, (changed,),
+        (Client('capture', 'TEST_TOKEN', ('other',)),)))
+    def forbidden(project):
+        raise AssertionError('unknown project')
+    worker = AnalysisWorker(updated, loader=forbidden)
+    assert asyncio.run(worker.tick()) == 0
+    assert json.loads(next(worker.records.glob('*.json')).read_bytes())['code'] == 'WORKER_PROJECT_UNKNOWN'
+
+
+def test_terminal_result_not_republished_every_pass(setup):
+    config, loader, store = setup
+    calls = []
+    async def dispatch(*args, **kwargs):
+        calls.append(1)
+        return {'status': 'OUTPUT_READY'}
+    worker = AnalysisWorker(config, loader=loader, dispatcher=dispatch)
+    asyncio.run(worker.tick())
+    asyncio.run(worker.tick())
+    assert len(calls) == 1
+
+
+def test_reopened_revision_does_not_skip_new_interpretation(setup):
+    config, loader, store = setup
+    worker = AnalysisWorker(config, loader=loader)
+    asyncio.run(worker.tick())
+    asyncio.run(worker.tick())
+    for state in ('GENERATING_HYPOTHESES', 'INVESTIGATING_HYPOTHESES', 'BUILDING_DIAGNOSIS', 'OUTPUT_READY', 'HANDED_OFF'):
+        store.advance(*CASE, state, step_id='test:' + state)
+    store.advance(*CASE, 'REOPENED', step_id='test:reopen')
+    asyncio.run(worker.tick())
+    with PackageReader(store.latest(*CASE)) as reader:
+        assert reader.manifest['analysis_revision'] == 2
+        assert reader.manifest['state'] == 'BUILDING_CASE'
+        # Old interpretation is deliberately retained in the append-only package.
+        old = json.loads(reader.read_bytes(reader.manifest['artifacts']['case_interpretation']))
+        assert old['analysis_revision'] == 1
+    asyncio.run(worker.tick())
+    with PackageReader(store.latest(*CASE)) as reader:
+        doc = json.loads(reader.read_bytes(reader.manifest['artifacts']['case_interpretation']))
+        assert doc['analysis_revision'] == 2
+
+
+def test_multiple_projects_and_poison_case_do_not_starve_other_cases(setup, case_setup, tmp_path):
+    from viano_bug_analysis.package.factory import PackageFactory
+    config, loader, store = setup
+    with PackageReader(case_setup[2]) as reader:
+        manifest = reader.manifest
+        contents = {ref: reader.read_bytes(ref) for ref in reader.files if ref != 'manifest.json'}
+    manifest.update(case_id='case-B', package_id='other-package', project={'id': 'second', 'type': 'wow', 'profile': 'vcore'})
+    PackageFactory().create(store.workspace.inbox / 'other.zip', manifest, contents)
+    project2 = Project('second', 'wow', 'vcore', planning_mode='separate')
+    intake = replace(config.intake, projects=config.intake.projects + (project2,))
+    seen = []
+    def load(project):
+        return project.id, {'project_id': project.id}, binding(project)
+    async def dispatch(store, gateway, case_id, package_id, **kwargs):
+        seen.append((case_id, gateway, kwargs['policies']['project_id']))
+        if case_id == 'case-A':
+            raise ValueError('broken first case')
+        return {'status': 'YIELDED'}
+    worker = AnalysisWorker(replace(config, intake=intake), loader=load, dispatcher=dispatch)
+    asyncio.run(worker.tick())
+    assert seen == [('case-A', 'test', 'test'), ('case-B', 'second', 'second')]
+    asyncio.run(worker.tick())
+    assert [v[0] for v in seen].count('case-A') == 1
+    assert [v[0] for v in seen].count('case-B') == 2
+
+
+def test_cli_check_status_and_empty_once_do_not_need_api_token(tmp_path, monkeypatch, capsys):
+    from viano_bug_analysis.worker.__main__ import main
+    import viano_bug_analysis.worker.__main__ as cli
+    monkeypatch.delenv('ABSENT_API_TOKEN', raising=False)
+    intake = tmp_path / 'api.toml'
+    intake.write_text('''[api]
+workspace = "service"
+[[projects]]
+id = "test"
+type = "wow"
+profile = "vcore"
+[[clients]]
+id = "capture"
+token_env = "ABSENT_API_TOKEN"
+projects = ["test"]
+''')
+    config = tmp_path / 'worker.toml'
+    config.write_text('[worker]\nintake_config = "api.toml"\nmin_free_bytes = 1\n')
+    loaded = []
+    monkeypatch.setattr(cli, 'load_project', lambda p: loaded.append(p.id))
+    assert main(['check', '--config', str(config)]) == 0
+    assert loaded == ['test']
+    assert main(['run', '--once', '--config', str(config)]) == 0
+    assert main(['status', '--config', str(config)]) == 0
+    assert 'STOPPED' in capsys.readouterr().out
+
+
+def test_storage_floor_pauses_without_dispatch(setup, monkeypatch):
+    import viano_bug_analysis.worker.service as module
+    from collections import namedtuple
+    config, loader, store = setup
+    Usage = namedtuple('Usage', 'total used free')
+    worker = AnalysisWorker(config, loader=loader)
+    monkeypatch.setattr(module.shutil, 'disk_usage', lambda *_: Usage(100, 100, 0))
+    assert asyncio.run(worker.tick()) == 0
+    assert json.loads((worker.root / 'status.json').read_bytes())['status'] == 'STORAGE_LOW'
+
+
+def test_exception_after_checkpoint_holds_actual_latest_generation(setup):
+    config, loader, store = setup
+    calls = []
+    async def dispatch(store, gateway, *identity, **kwargs):
+        calls.append(1)
+        store.advance(*identity, 'BUILDING_CASE', step_id='commit-before-error')
+        raise ValueError('unexpected after checkpoint')
+    worker = AnalysisWorker(config, loader=loader, dispatcher=dispatch)
+    asyncio.run(worker.tick())
+    asyncio.run(worker.tick())
+    assert len(calls) == 1
+    record = json.loads(next(worker.records.glob('*.json')).read_bytes())
+    with PackageReader(store.latest(*CASE)) as reader:
+        assert record['package_generation'] == reader.manifest['package_generation']
+    assert record['status'] == 'ATTENTION'