Add scoped operator controls and durable worker release queue

tests/dashboard-smoke.mjs

@@ -3,7 +3,9 @@ import {spawn} from 'node:child_process';
 import {mkdtemp, mkdir, writeFile, rm} from 'node:fs/promises';
 import {tmpdir} from 'node:os';
 import path from 'node:path';
-const port=18765, debugPort=18766;
+import {createServer} from 'node:net';
+async function freePort(){const server=createServer();await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));const port=server.address().port;await new Promise(resolve=>server.close(resolve));return port;}
+const port=await freePort(), debugPort=await freePort();
 const pause=ms=>new Promise(r=>setTimeout(r,ms));
 const output=path.resolve('service-data/ui-preview');await mkdir(output,{recursive:true});
 const profile=await mkdtemp(path.join(tmpdir(),'vba-browser-'));
@@ -20,7 +22,7 @@ try {
   ws=new WebSocket(tabs.find(t=>t.type==='page').webSocketDebuggerUrl);
   await new Promise((resolve,reject)=>{ws.onopen=resolve;ws.onerror=reject;});
   let next=0;const pending=new Map();
-  ws.onmessage=e=>{const m=JSON.parse(e.data);if(m.id){const p=pending.get(m.id);pending.delete(m.id);m.error?p.reject(new Error(m.error.message)):p.resolve(m.result);}
+  ws.onmessage=e=>{const m=JSON.parse(e.data);if(m.id){const p=pending.get(m.id);pending.delete(m.id);if(!p)return;m.error?p.reject(new Error(m.error.message)):p.resolve(m.result);}
     if(m.method==='Runtime.exceptionThrown')errors.push(m.params.exceptionDetails.text);
     if(m.method==='Log.entryAdded'&&m.params.entry.level==='error')errors.push(m.params.entry.text);};
   const send=(method,params={})=>new Promise((resolve,reject)=>{const id=++next;pending.set(id,{resolve,reject});ws.send(JSON.stringify({id,method,params}));});
@@ -31,6 +33,12 @@ try {
   await until(()=>evaluate("document.readyState === 'complete' && typeof render === 'function'"));
   await evaluate("document.getElementById('token').value='ui_fixture_"+'x'.repeat(40)+"';document.getElementById('login-form').dispatchEvent(new Event('submit',{bubbles:true,cancelable:true}));");
   await until(()=>evaluate("document.getElementById('token-count').textContent==='35.300'"));
+  await until(()=>evaluate("document.querySelector('#operation-projects button')"));
+  await evaluate("document.querySelector('#operation-projects button').click()");
+  await until(()=>evaluate("document.getElementById('operation-projects').textContent.includes('Neue Fälle pausiert')"));
+  if(!await evaluate("document.getElementById('operation-events').textContent.includes('preview')"))throw new Error('Operator audit missing');
+  await evaluate("document.querySelector('#operation-projects button').click()");
+  await until(()=>evaluate("document.getElementById('operation-projects').textContent.includes('Neue Fälle freigegeben')"));
   const state=await evaluate("({workers:document.querySelectorAll('.worker').length,cases:document.querySelectorAll('#case-rows tr').length,active:document.getElementById('active-count').textContent,correction:document.getElementById('correction-count').textContent,overflow:document.documentElement.scrollWidth>innerWidth,stored:localStorage.length,progress:document.querySelector('.worker [role=progressbar]').getAttribute('aria-valuenow')})");
   if(state.workers!==1||state.cases!==1||state.active!=='1'||state.stored!==0||state.overflow||state.progress!=='57')throw new Error(JSON.stringify(state));
   const categories=await evaluate("Array.from(document.querySelectorAll('#legend li')).map(e=>e.textContent)");
@@ -58,13 +66,16 @@ try {
   await evaluate("document.getElementById('logout').click()");
   if(!await evaluate("document.getElementById('content').hidden&&!document.getElementById('login').hidden&&document.getElementById('token-count').textContent==='—'"))throw new Error('Logout retained visible case data');
   if(!await evaluate("document.getElementById('case-detail').childElementCount===0&&document.getElementById('result-rows').childElementCount===0"))throw new Error('Logout retained detail data');
+  if(!await evaluate("document.getElementById('operation-events').childElementCount===0"))throw new Error('Logout retained audit');
   if(errors.filter(e=>!e.includes('404')).length)throw new Error(errors.join('\n'));
   console.log(JSON.stringify({status:'PASS',...state,console_errors:errors.filter(e=>!e.includes('404'))}));
 } finally {
-  ws?.close();browser?.kill();server?.kill();await pause(400);
+  // Let Chromium close its database handles before removing its own profile.
+  if(ws?.readyState===WebSocket.OPEN)ws.send(JSON.stringify({id:999999,method:'Browser.close'}));
+  await pause(1500);ws?.close();browser?.kill();server?.kill();await pause(400);
   for(const [folder,prefix] of [[profile,'vba-browser-'],[fixtureRoot,'vba-ui-data-']]) {
     const target=path.resolve(folder),relative=path.relative(path.resolve(tmpdir()),target);
     if(!relative||relative.startsWith('..')||path.isAbsolute(relative)||!path.basename(target).startsWith(prefix))throw new Error('Unsafe temporary cleanup');
-    await rm(target,{recursive:true,force:true,maxRetries:5,retryDelay:200});
+    await rm(target,{recursive:true,force:true,maxRetries:15,retryDelay:300});
   }
 }

tests/test_dashboard.py

@@ -198,3 +198,41 @@ def test_historical_artifacts_are_not_replaced_by_latest_revision():
         ('oldcp',{'analysis_revision':1,'details':{'operation':'diagnosis-complete','diagnosis_ref':'old'}})]
     assert references(reader,chain,1)['diagnosis']=='old'
     assert references(reader,chain,2)['diagnosis']=='new'
+
+
+def test_operator_permissions_scope_and_audit(case_setup,monkeypatch):
+    store,builder,*_=case_setup
+    builder.build('case-A','stable-package')
+    monkeypatch.setenv('CAPTURE_TOKEN','capture_'+'a'*40)
+    monkeypatch.setenv('OPERATOR_TOKEN','operator_'+'b'*40)
+    config=APIConfig(store.workspace.root,(Project('p','wow','vcore'),Project('other','wow','vcore')),
+        (Client('capture','CAPTURE_TOKEN',('p',)),Client('operator','OPERATOR_TOKEN',('p',),operator=True)))
+    app=create_app(config);client=TestClient(app)
+    capture={'Authorization':'Bearer capture_'+'a'*40}
+    operator={'Authorization':'Bearer operator_'+'b'*40}
+    assert client.get('/v1/operations').status_code==401
+    assert client.post('/v1/operations',headers=capture,json={'action':'pause','project_id':'p'}).status_code==403
+    assert client.post('/v1/operations',headers=operator,json={'action':'pause','project_id':'other'}).status_code==403
+    assert client.post('/v1/operations',headers=operator,json={'action':'pause','project_id':'p'}).status_code==200
+    view=client.get('/v1/operations',headers=capture).json()
+    assert view['projects']==[{'project_id':'p','paused':True}] and view['events']==[]
+    view=client.get('/v1/operations',headers=operator).json()
+    assert view['events'][-1]['actor']=='operator' and view['worker'] is None
+    assert client.post('/v1/operations',headers=operator,json={'action':'resume','project_id':'p','reset_budget':True}).status_code==422
+    assert client.post('/v1/operations',headers=operator,content='x'*4097).status_code==415
+    assert client.post('/v1/operations',headers=operator|{'Content-Type':'application/json'},content='x'*4097).status_code==413
+    from viano_bug_analysis.api.cases import case_key
+    from viano_bug_analysis.package import PackageReader
+    with PackageReader(store.latest('case-A','stable-package')) as reader:
+        generation=reader.manifest['package_generation']
+    command={'action':'retry','key':case_key('case-A','stable-package'),'package_generation':generation}
+    first=client.post('/v1/operations',headers=operator,json=command)
+    assert first.status_code==200 and first.json()['status']=='QUEUED'
+    assert client.post('/v1/operations',headers=operator,json=command).json()==first.json()
+    assert client.get('/v1/operations',headers=operator).json()['pending']==1
+    assert client.post('/v1/operations',headers=operator,json=command|{'package_generation':0}).status_code==409
+    store.advance('case-A','stable-package','BLOCKED',step_id='control:invalid',reason='INVALID_LLM_RESPONSES',
+        retry_not_before='2020-01-01T00:00:00Z')
+    with PackageReader(store.latest('case-A','stable-package')) as reader:
+        generation=reader.manifest['package_generation']
+    assert client.post('/v1/operations',headers=operator,json=command|{'package_generation':generation}).status_code==409

tests/test_worker.py

@@ -281,3 +281,41 @@ def test_exception_after_checkpoint_holds_actual_latest_generation(setup):
     with PackageReader(store.latest(*CASE)) as reader:
         assert record['package_generation'] == reader.manifest['package_generation']
     assert record['status'] == 'ATTENTION'
+
+
+def test_operator_pause_survives_restart_and_drains_started_cases(setup):
+    config,loader,store=setup
+    worker=AnalysisWorker(config,loader=loader)
+    worker.control.pause('operator','test',True)
+    assert asyncio.run(worker.tick())==0
+    worker=AnalysisWorker(config,loader=loader)
+    assert asyncio.run(worker.tick())==0
+    worker.control.pause('operator','test',False)
+    assert asyncio.run(worker.tick())==1
+    worker.control.pause('operator','test',True)
+    assert asyncio.run(worker.tick())==1 # already started case continues
+
+
+def test_operator_commands_recheck_generation_and_forbid_validation_reset(setup):
+    config,loader,store=setup
+    worker=AnalysisWorker(config,loader=loader)
+    store.advance(*CASE,State.BUILDING_CASE,step_id='operator:start')
+    store.advance(*CASE,State.BLOCKED,step_id='operator:block',reason='RESEARCH_UNAVAILABLE',
+        retry_not_before='2020-01-01T00:00:00Z',details={'automatic_retry':False})
+    with PackageReader(store.latest(*CASE)) as reader:
+        manifest=reader.manifest
+    worker.control.enqueue('operator','test',manifest)
+    worker._commands()
+    assert not worker.control.read()['commands']
+    assert worker.control.read()['events'][-1]['status']=='APPLIED'
+    worker.control.enqueue('operator','test',manifest) # stale UI cannot release new head
+    worker._commands()
+    assert worker.control.read()['events'][-1]['status']=='REJECTED'
+    store.advance(*CASE,State.BLOCKED,step_id='operator:invalid',reason='INVALID_LLM_RESPONSES',
+        retry_not_before='2020-01-01T00:00:00Z')
+    path=store.latest(*CASE)
+    with PackageReader(path) as reader:
+        worker.control.enqueue('operator','test',reader.manifest)
+    worker._commands()
+    assert store.latest(*CASE)==path
+    assert worker.control.read()['events'][-1]['status']=='REJECTED'

tests/ui_dashboard_server.py

@@ -13,6 +13,7 @@ from viano_bug_analysis.lifecycle.storage import atomic_json, json_bytes
 from viano_bug_analysis.package.factory import PackageFactory
 from viano_bug_analysis.case import CaseBuilder
 import viano_bug_analysis.api.dashboard as dashboard
+import viano_bug_analysis.api.operations as operations
 
 TOKEN = 'ui_fixture_' + 'x' * 40
 
@@ -90,10 +91,11 @@ if __name__ == '__main__':
         identity=seed(root,'BUG-000020','Wolpertinger fehlen in Dun Morogh')
         seed(root,'BUG-000017','Leuchtbrille – Kontrolllauf',completed=True)
         config=APIConfig(root,(Project('viano-core','wow','vcore'),),
-            (Client('preview','VBA_UI_TEST_TOKEN',('viano-core',)),))
+            (Client('preview','VBA_UI_TEST_TOKEN',('viano-core',),operator=True),))
         os.environ['VBA_UI_TEST_TOKEN']=TOKEN
         dashboard.read_status=lambda _: {'observer':'LIVE','service_id':'fixture-worker'}
+        operations.read_status=dashboard.read_status
         folder=root/'runtime/worker';folder.mkdir(exist_ok=True)
-        atomic_json(folder/'status.json',{'status':'ANALYZING','service_id':'fixture-worker',
+        atomic_json(folder/'status.json',{'status':'ANALYZING','service_id':'fixture-worker','updated_at':timestamp(),
             'case':{'case_id':identity[0],'package_id':identity[1]}})
         uvicorn.run(create_app(config),host='127.0.0.1',port=int(os.environ.get('VBA_UI_TEST_PORT','8765')),access_log=False)