fix: harden task recovery and DR contract handling
This commit is contained in:
@@ -3,6 +3,9 @@ import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import socket
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from unittest.mock import patch
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'coordinator'))
|
||||
@@ -35,4 +38,54 @@ class HttpTest(unittest.TestCase):
|
||||
self.assertEqual(self.request('POST','/v1/tasks',payload)[0],400)
|
||||
self.assertEqual(self.request('GET','/not-found')[0],404)
|
||||
|
||||
def test_listener_can_queue_ten_fresh_connections_before_accept(self):
|
||||
# Model a burst while the accept loop is busy. This is a capacity check,
|
||||
# not a throughput or latency assertion on the host running the suite.
|
||||
listener=make_server(self.c,'127.0.0.1',0,'test-secret')
|
||||
clients=[]
|
||||
try:
|
||||
for _ in range(10):
|
||||
try:
|
||||
clients.append(socket.create_connection(('127.0.0.1',listener.server_port),timeout=2))
|
||||
except OSError as error:
|
||||
self.fail('listener could not queue ten clients: '+str(error))
|
||||
finally:
|
||||
for client in clients:client.close()
|
||||
listener.server_close()
|
||||
|
||||
def test_ten_concurrent_submissions_are_independently_accepted_and_deduplicated(self):
|
||||
gate=threading.Barrier(10)
|
||||
def submit(index):
|
||||
body=json.dumps(dict(client_request_id='burst-'+str(index),robot_id='robot_01',instruction='move water',known_info={}))
|
||||
gate.wait(timeout=10)
|
||||
return self.request('POST','/v1/tasks',body)
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
results=list(pool.map(submit,range(10)))
|
||||
self.assertEqual([code for code,_ in results],[202]*10)
|
||||
self.assertEqual(len({task['task_id'] for _,task in results}),10)
|
||||
self.assertTrue(all(task['status']=='QUEUED' for _,task in results))
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
repeats=list(pool.map(submit,range(10)))
|
||||
self.assertEqual([task['task_id'] for _,task in repeats],[task['task_id'] for _,task in results])
|
||||
self.assertTrue(all(code==202 and task['deduplicated'] for code,task in repeats))
|
||||
self.assertEqual(len(self.c.store.all()),10)
|
||||
|
||||
def test_recovery_backend_failure_is_sanitized_service_unavailable(self):
|
||||
with patch.object(self.c,'intervene',side_effect=RuntimeError('private-token-123 from ROS')):
|
||||
code,body=self.request('POST','/v1/tasks/task/interventions','{}',token='operator-secret')
|
||||
self.assertEqual(code,503);self.assertEqual(body['error_code'],'RECONCILIATION_UNAVAILABLE')
|
||||
self.assertNotIn('private-token',json.dumps(body))
|
||||
|
||||
def test_recovery_timeout_returns_json_instead_of_dropping_connection(self):
|
||||
with patch.object(self.c,'intervene',side_effect=TimeoutError('private-token-123 outcome unknown')):
|
||||
try:code,body=self.request('POST','/v1/tasks/task/interventions','{}',token='operator-secret')
|
||||
except http.client.RemoteDisconnected:self.fail('recovery timeout dropped the HTTP connection without a JSON result')
|
||||
self.assertEqual(code,504);self.assertEqual(body['error_code'],'RECONCILIATION_TIMEOUT')
|
||||
self.assertNotIn('private-token',json.dumps(body))
|
||||
|
||||
def test_recovery_backend_is_not_called_with_ordinary_api_token(self):
|
||||
with patch.object(self.c,'intervene') as reconcile:
|
||||
self.assertEqual(self.request('POST','/v1/tasks/task/interventions','{}')[0],401)
|
||||
reconcile.assert_not_called()
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user