fix: harden task recovery and DR contract handling

This commit is contained in:
2026-09-20 13:36:48 +08:00
parent 492676344a
commit f9d8feb6f0
49 changed files with 2083 additions and 165 deletions
+53
View File
@@ -3,6 +3,9 @@ import json
import sys
import tempfile
import threading
import socket
from concurrent.futures import ThreadPoolExecutor
from unittest.mock import patch
import unittest
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'coordinator'))
@@ -35,4 +38,54 @@ class HttpTest(unittest.TestCase):
self.assertEqual(self.request('POST','/v1/tasks',payload)[0],400)
self.assertEqual(self.request('GET','/not-found')[0],404)
def test_listener_can_queue_ten_fresh_connections_before_accept(self):
# Model a burst while the accept loop is busy. This is a capacity check,
# not a throughput or latency assertion on the host running the suite.
listener=make_server(self.c,'127.0.0.1',0,'test-secret')
clients=[]
try:
for _ in range(10):
try:
clients.append(socket.create_connection(('127.0.0.1',listener.server_port),timeout=2))
except OSError as error:
self.fail('listener could not queue ten clients: '+str(error))
finally:
for client in clients:client.close()
listener.server_close()
def test_ten_concurrent_submissions_are_independently_accepted_and_deduplicated(self):
gate=threading.Barrier(10)
def submit(index):
body=json.dumps(dict(client_request_id='burst-'+str(index),robot_id='robot_01',instruction='move water',known_info={}))
gate.wait(timeout=10)
return self.request('POST','/v1/tasks',body)
with ThreadPoolExecutor(max_workers=10) as pool:
results=list(pool.map(submit,range(10)))
self.assertEqual([code for code,_ in results],[202]*10)
self.assertEqual(len({task['task_id'] for _,task in results}),10)
self.assertTrue(all(task['status']=='QUEUED' for _,task in results))
with ThreadPoolExecutor(max_workers=10) as pool:
repeats=list(pool.map(submit,range(10)))
self.assertEqual([task['task_id'] for _,task in repeats],[task['task_id'] for _,task in results])
self.assertTrue(all(code==202 and task['deduplicated'] for code,task in repeats))
self.assertEqual(len(self.c.store.all()),10)
def test_recovery_backend_failure_is_sanitized_service_unavailable(self):
with patch.object(self.c,'intervene',side_effect=RuntimeError('private-token-123 from ROS')):
code,body=self.request('POST','/v1/tasks/task/interventions','{}',token='operator-secret')
self.assertEqual(code,503);self.assertEqual(body['error_code'],'RECONCILIATION_UNAVAILABLE')
self.assertNotIn('private-token',json.dumps(body))
def test_recovery_timeout_returns_json_instead_of_dropping_connection(self):
with patch.object(self.c,'intervene',side_effect=TimeoutError('private-token-123 outcome unknown')):
try:code,body=self.request('POST','/v1/tasks/task/interventions','{}',token='operator-secret')
except http.client.RemoteDisconnected:self.fail('recovery timeout dropped the HTTP connection without a JSON result')
self.assertEqual(code,504);self.assertEqual(body['error_code'],'RECONCILIATION_TIMEOUT')
self.assertNotIn('private-token',json.dumps(body))
def test_recovery_backend_is_not_called_with_ordinary_api_token(self):
with patch.object(self.c,'intervene') as reconcile:
self.assertEqual(self.request('POST','/v1/tasks/task/interventions','{}')[0],401)
reconcile.assert_not_called()
if __name__=='__main__':unittest.main()