fix: harden task recovery and DR contract handling
This commit is contained in:
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ROS2 Coordinator -> PlanTask -> native BT -> receipt recovery regression.
|
||||
|
||||
Run in a built Humble overlay. Every endpoint is under /sim/robot_01 and all
|
||||
physical skills are fixtures; this is orchestration evidence, not robot/model
|
||||
acceptance. The second run drops only the already-decoded execution result at
|
||||
the coordinator event boundary, then reopens the same durable coordinator DB.
|
||||
"""
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
from native_workflow_regression import eventually, journal_records
|
||||
|
||||
|
||||
def run(root, binary, output):
|
||||
import rclpy
|
||||
from rclpy.executors import MultiThreadedExecutor
|
||||
from rclpy.node import Node
|
||||
from robot_bt_coordinator.plan_v2 import make_plan
|
||||
from robot_bt_coordinator.ros_backend import RosBackend
|
||||
from robot_bt_coordinator.service import Coordinator
|
||||
from bt_mock_servers.mock_skills import MockSkills, ACTION_TYPES
|
||||
|
||||
site_path = root/'config/sim_site_object_table.json'
|
||||
site = json.loads(site_path.read_text())
|
||||
instruction = '把货架A的一瓶水放到周转箱A'
|
||||
slots = {'items':[{'target_name':'water','quantity':1,'source_location':'shelf_A'}], 'destination':'tote_A'}
|
||||
plan = make_plan(instruction, slots, 'OBJECT_TABLE')
|
||||
scenarios = {
|
||||
'plan_task':[{'kind':'normal', 'plan':plan}],
|
||||
'verify_state':[{'kind':'passed'}],
|
||||
'navigate_semantic':[{'kind':'normal','final_pose':site['locations'][name]}
|
||||
for name in ['shelf_A_stop','tote_A_stop']*2],
|
||||
}
|
||||
actions = [name for name in ACTION_TYPES if name != 'execute_task']
|
||||
rclpy.init(args=['--ros-args','-p','initial_holding_state:=EMPTY',
|
||||
'-p','scenarios_json:='+json.dumps(json.dumps(scenarios)),
|
||||
'-p','enabled_actions:='+json.dumps(actions),
|
||||
'-p','enabled_topics:='+json.dumps(['robot_state','safety_state','visual_observation','dense_progress'])])
|
||||
server = MockSkills()
|
||||
bridge = Node('native_coordinator_regression',namespace='/sim/robot_01')
|
||||
executor = MultiThreadedExecutor(num_threads=8)
|
||||
executor.add_node(server);executor.add_node(bridge)
|
||||
thread = threading.Thread(target=executor.spin,daemon=True);thread.start()
|
||||
token = 'native-coordinator-test-recovery-token'
|
||||
operator = 'native-coordinator-regression'
|
||||
config = {'planning_context':site, 'planning_timeout':10, 'execution_timeout':30,
|
||||
'recovery_token':token, 'recovery_operator_id':operator, 'recovery_timeout':10}
|
||||
report = {'scope':'native ROS2 coordinator/executor with simulated physical skills','route':'OBJECT_TABLE'}
|
||||
coordinator = None
|
||||
process = None
|
||||
log = (output/'native-coordinator.log').open('w')
|
||||
|
||||
class DropFinal(RosBackend):
|
||||
"""Fault injection after genuine ROS deserialization, before DB commit."""
|
||||
def __init__(self):
|
||||
super().__init__(node=bridge,config=config)
|
||||
self.drop_task_id = None
|
||||
self.dropped = []
|
||||
def poll(self):
|
||||
events = super().poll()
|
||||
kept = []
|
||||
for event in events:
|
||||
if event.get('type') == 'execution_result' and event.get('task_id') == self.drop_task_id:
|
||||
self.dropped.append(copy.deepcopy(event))
|
||||
else:
|
||||
kept.append(event)
|
||||
return kept
|
||||
|
||||
def counts():
|
||||
with server.lock:
|
||||
return {name:server.counts[name] for name in ('plan_task','navigate_semantic','execute_manipulation','execute_posture')}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix='native-coordinator-regression-') as state:
|
||||
state = Path(state)
|
||||
journal = state/'executor';journal.mkdir()
|
||||
database = state/'coordinator.sqlite'
|
||||
try:
|
||||
args = [str(binary),'--ros-args','-r','__ns:=/sim/robot_01',
|
||||
'-p','robot_id:=robot_01','-p','allowed_robots:=[robot_01]',
|
||||
'-p','site_config_file:='+str(site_path),'-p','journal_directory:='+str(journal),
|
||||
'-p','execution_enabled:=true','-p','skill_timeout_ms:=5000',
|
||||
'-p','recovery_token:='+token,'-p','recovery_operator_id:='+operator]
|
||||
process = subprocess.Popen(args,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
backend = DropFinal()
|
||||
assert backend._planner.wait_for_server(timeout_sec=10), 'Planner Action not discovered'
|
||||
assert backend._execution.wait_for_server(timeout_sec=10), 'native ExecuteTask Action not discovered'
|
||||
time.sleep(.6)
|
||||
assert process.poll() is None, 'native executor failed at startup'
|
||||
coordinator = Coordinator(str(database),backend,['robot_01'],site=site)
|
||||
|
||||
def submit(request_id):
|
||||
return coordinator.submit({'client_request_id':request_id,'robot_id':'robot_01',
|
||||
'instruction':instruction,'known_info':copy.deepcopy(slots)})
|
||||
|
||||
def pump_until(condition, seconds=45):
|
||||
def check():
|
||||
assert process.poll() is None, 'native executor exited'
|
||||
coordinator.tick()
|
||||
return condition()
|
||||
eventually(check,seconds)
|
||||
|
||||
first = submit('normal-result')
|
||||
pump_until(lambda:coordinator.get(first['task_id'])['status'] in ('SUCCEEDED','FAILED','INTERVENTION_REQUIRED','NEEDS_CLARIFICATION'))
|
||||
first = coordinator.get(first['task_id'])
|
||||
assert first['status'] == 'SUCCEEDED', first
|
||||
assert first['completed_quantity'] == 1 and first['stop_confirmed'] is True
|
||||
assert counts()['execute_manipulation'] == 2
|
||||
events = coordinator.events(first['task_id'],limit=500)
|
||||
planning = [event for event in events if event['kind']=='planning_result']
|
||||
assert planning and planning[-1]['payload']['planning_record_ref'].startswith('sim://plan_task/')
|
||||
assert any(event['kind']=='delivery_committed' for event in events)
|
||||
report['normal'] = {'task_id':first['task_id'],'run_id':first['run_id'],
|
||||
'status':first['status'],'completed_quantity':first['completed_quantity'],
|
||||
'planning_record_ref':planning[-1]['payload']['planning_record_ref'],
|
||||
'counts':counts()}
|
||||
|
||||
second = submit('lost-final-result')
|
||||
backend.drop_task_id = second['task_id']
|
||||
pump_until(lambda:bool(backend.dropped))
|
||||
assert len(backend.dropped) == 1
|
||||
dropped = backend.dropped[0]
|
||||
assert dropped['status']=='SUCCEEDED' and dropped['completed_quantity']==1, dropped
|
||||
assert dropped['stop_confirmed'] is True and dropped['evidence']['safe_to_release'] is True
|
||||
before_restart = coordinator.get(second['task_id'])
|
||||
assert before_restart['status']=='EXECUTING' and before_restart['completed_quantity']==0
|
||||
assert coordinator.store.db.execute('SELECT COUNT(*) FROM deliveries WHERE task_id=?',(second['task_id'],)).fetchone()[0]==0
|
||||
native_receipts = [json.loads(line) for line in (journal/'deliveries.jsonl').read_text().splitlines()]
|
||||
assert len([row for row in native_receipts if row['task_id']==second['task_id']])==1
|
||||
before_recovery = counts()
|
||||
assert before_recovery['execute_manipulation']==4
|
||||
coordinator.close();coordinator=None
|
||||
|
||||
backend = RosBackend(node=bridge,config=config)
|
||||
coordinator = Coordinator(str(database),backend,['robot_01'],site=site)
|
||||
restarted = coordinator.get(second['task_id'])
|
||||
assert restarted['status']=='INTERVENTION_REQUIRED'
|
||||
assert restarted['run_id']==before_restart['run_id']
|
||||
assert backend._recovery.wait_for_service(timeout_sec=10), 'native recovery service not discovered'
|
||||
recovered = coordinator.intervene(second['task_id'],{
|
||||
'run_id':restarted['run_id'], 'evidence_ref':'regression/operator-observed-stop', 'resolution':'resume_task'})
|
||||
assert recovered['status']=='SUCCEEDED' and recovered['completed_quantity']==1, recovered
|
||||
for _ in range(10):
|
||||
coordinator.tick();time.sleep(.03)
|
||||
assert counts()==before_recovery, 'receipt recovery redispatched planning or robot motion'
|
||||
deliveries = coordinator.store.db.execute('SELECT * FROM deliveries WHERE task_id=?',(second['task_id'],)).fetchall()
|
||||
assert len(deliveries)==1
|
||||
evidence = json.loads(deliveries[0]['evidence'])
|
||||
assert evidence['evidence_id']==dropped['evidence']['evidence_id']
|
||||
assert all(evidence[key] is True for key in ('passed','empty_hand','in_destination','valid'))
|
||||
rows = journal_records(journal/'goal_registry.log')
|
||||
second_motion = [row for row in rows if row['run_id']==restarted['run_id'] and 'ExecuteManipulation' in row['type']]
|
||||
assert len(second_motion)==2, 'duplicate pick/place goals in recovered run'
|
||||
report['lost_result_recovery'] = {'task_id':second['task_id'],'run_id':restarted['run_id'],
|
||||
'status_before_restart':before_restart['status'],'status_after_restart':restarted['status'],
|
||||
'status_after_recovery':recovered['status'],'completed_quantity':recovered['completed_quantity'],
|
||||
'ledger_rows':len(deliveries),'manipulation_goal_count':len(second_motion),
|
||||
'counts_before':before_recovery,'counts_after':counts(),'receipt_evidence':evidence}
|
||||
report['passed']=True
|
||||
except Exception as exc:
|
||||
report['passed']=False;report['failure']=repr(exc)
|
||||
finally:
|
||||
if coordinator is not None:
|
||||
coordinator.close()
|
||||
if process is not None and process.poll() is None:
|
||||
os.killpg(process.pid,signal.SIGINT)
|
||||
try:process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid,signal.SIGKILL);process.wait()
|
||||
for path in journal.glob('*'):
|
||||
if path.is_file():(output/('coordinator-'+path.name)).write_bytes(path.read_bytes())
|
||||
if database.is_file():(output/'native-coordinator.sqlite').write_bytes(database.read_bytes())
|
||||
executor.shutdown(timeout_sec=2);thread.join(timeout=2)
|
||||
server.destroy_node();bridge.destroy_node();rclpy.shutdown();log.close()
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--repo-root',type=Path,default=Path(__file__).resolve().parents[2])
|
||||
parser.add_argument('--executor',type=Path)
|
||||
parser.add_argument('--output-dir',type=Path,required=True)
|
||||
args = parser.parse_args()
|
||||
root = args.repo_root.resolve()
|
||||
sys.path[:0] = [str(root/'coordinator'),str(root/'ros2/bt_mock_servers')]
|
||||
binary = (args.executor or root/'install/bt_executor/lib/bt_executor/bt_executor_node').resolve()
|
||||
args.output_dir.mkdir(parents=True,exist_ok=True)
|
||||
report = run(root,binary,args.output_dir)
|
||||
(args.output_dir/'native-coordinator-regression.json').write_text(json.dumps(report,indent=2))
|
||||
print(json.dumps({key:report[key] for key in ('passed','failure') if key in report}),flush=True)
|
||||
if not report['passed']:raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,326 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Actual ROS2 + native executor regression; run inside a built Humble overlay.
|
||||
|
||||
Uses only /sim/robot_01 fixture servers, never robot endpoints. Includes durable
|
||||
CDR evidence checks, executor restart receipt recovery, and cancellation. This
|
||||
is transport/orchestration evidence, not physical NAV/VLA acceptance.
|
||||
"""
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
|
||||
def wait(future, seconds=45):
|
||||
end = time.monotonic() + seconds
|
||||
while not future.done() and time.monotonic() < end:
|
||||
time.sleep(0.01)
|
||||
if not future.done():
|
||||
raise TimeoutError('ROS future deadline exceeded')
|
||||
return future.result()
|
||||
|
||||
|
||||
def eventually(predicate, seconds=8):
|
||||
end = time.monotonic() + seconds
|
||||
while time.monotonic() < end:
|
||||
if predicate():
|
||||
return
|
||||
time.sleep(0.02)
|
||||
raise AssertionError('expected native state transition did not occur')
|
||||
|
||||
|
||||
def journal_records(path):
|
||||
"""Independent decoder for durable registry v4 (does not call production)."""
|
||||
latest = {}
|
||||
for line in path.read_text().splitlines():
|
||||
row = shlex.split(line)
|
||||
assert row[0] in ('3', '4') and len(row) == (25 if row[0] == '4' else 23), 'missing durable evidence fields'
|
||||
decode = lambda index: bytes.fromhex(row[index]).decode('utf-8')
|
||||
latest[row[1]] = dict(goal_id=row[1], task_id=row[3], run_id=row[5],
|
||||
skill=int(row[10]), state=int(row[11]), sequence=int(row[14]),
|
||||
type=decode(18), wire=decode(19), feedback=decode(20),
|
||||
error_code=decode(21), execution_record_ref=decode(22), result_type=decode(23) if row[0] == '4' else '',
|
||||
result_wire=decode(24) if row[0] == '4' else '')
|
||||
return list(latest.values())
|
||||
|
||||
|
||||
def assert_manipulation_evidence(records, run_id, action_type, deserialize):
|
||||
selected = [row for row in records if row['run_id'] == run_id and
|
||||
'ExecuteManipulation' in row['type']]
|
||||
assert len(selected) == 2, 'one pick and one place wire snapshot required'
|
||||
decoded = []
|
||||
for row in selected:
|
||||
goal = deserialize(bytes.fromhex(row['wire']), action_type.Goal)
|
||||
assert goal.skill in ('pick', 'place')
|
||||
assert goal.trace.run_id == run_id and goal.trace.attempt == 1
|
||||
assert goal.target.object_ref == 'water' and goal.target.description
|
||||
assert goal.instruction == goal.skill + ' registered target water'
|
||||
assert goal.timeout.sec == 5 and goal.timeout.nanosec == 0
|
||||
if goal.skill == 'pick':
|
||||
assert goal.destination.region_ref == goal.destination.description == ''
|
||||
else:
|
||||
assert goal.destination.region_ref == 'tote_A' and goal.destination.description
|
||||
payload = json.loads(row['feedback'])
|
||||
feedback = deserialize(bytes.fromhex(payload['cdr_hex']), action_type.Feedback)
|
||||
assert feedback.sequence == row['sequence'] == payload['sequence'] > 0
|
||||
assert feedback.phase == payload['phase'] and feedback.message == payload['message']
|
||||
assert feedback.progress_valid == payload['progress_valid']
|
||||
assert feedback.message and payload['elapsed_time_ns'] >= 0
|
||||
assert row['execution_record_ref'].startswith('sim://execute_manipulation/')
|
||||
assert row['state'] == 4
|
||||
assert 'ExecuteManipulation' in row['result_type'] and row['result_wire']
|
||||
result = deserialize(bytes.fromhex(row['result_wire']), action_type.Result)
|
||||
assert result.result.status == 0 and result.result.stop_state == 1
|
||||
assert result.result.stop_evidence_ref and result.result.stopped_at.sec > 0
|
||||
assert result.execution_record_ref == row['execution_record_ref']
|
||||
decoded.append(dict(skill=goal.skill, sequence=feedback.sequence,
|
||||
phase=feedback.phase, execution_record_ref=row['execution_record_ref']))
|
||||
assert {row['skill'] for row in decoded} == {'pick', 'place'}
|
||||
return decoded
|
||||
|
||||
|
||||
def run_route(root, binary, output, suffix, recovery_enabled):
|
||||
import rclpy
|
||||
from rclpy.action import ActionClient
|
||||
from rclpy.executors import MultiThreadedExecutor
|
||||
from rclpy.node import Node
|
||||
from rclpy.serialization import deserialize_message
|
||||
from bt_skill_interfaces.action import ExecuteTask, ExecuteManipulation
|
||||
from bt_skill_interfaces.msg import RobotState
|
||||
from bt_skill_interfaces.srv import ReconcileTask
|
||||
from robot_bt_coordinator.plan_v2 import item_plan
|
||||
from bt_mock_servers.mock_skills import MockSkills, ACTION_TYPES
|
||||
|
||||
site_path = root / 'config' / ('sim_site_' + suffix + '.json')
|
||||
site = json.loads(site_path.read_text())
|
||||
plan = item_plan(json.loads((root / 'config' / ('demo_plan_' + suffix + '.json')).read_text()), 0)
|
||||
stops = ['shelf_A_stop', 'tote_A_stop'] if suffix == 'object_table' else ['observe_A', 'shelf_A_stop', 'tote_A_stop']
|
||||
scenarios = {'verify_state': [{'kind': 'passed'}],
|
||||
'locate_shelf_column': [{'kind': 'normal', 'shelf_id': 'shelf_A', 'side_id': 'FRONT', 'column_id': '1', 'tier_id': '2'}],
|
||||
'navigate_semantic': [{'kind': 'normal', 'final_pose': site['locations'][name]} for name in stops]}
|
||||
actions = [name for name in ACTION_TYPES if name != 'execute_task']
|
||||
rclpy.init(args=['--ros-args', '-p', 'initial_holding_state:=EMPTY',
|
||||
'-p', 'scenarios_json:=' + json.dumps(json.dumps(scenarios)),
|
||||
'-p', 'enabled_actions:=' + json.dumps(actions), '-p',
|
||||
'enabled_topics:=' + json.dumps(['robot_state', 'safety_state', 'visual_observation', 'dense_progress'])])
|
||||
server = MockSkills()
|
||||
node = Node('native_workflow_regression', namespace='/sim/robot_01')
|
||||
executor = MultiThreadedExecutor(num_threads=8)
|
||||
executor.add_node(server)
|
||||
executor.add_node(node)
|
||||
thread = threading.Thread(target=executor.spin, daemon=True)
|
||||
thread.start()
|
||||
client = ActionClient(node, ExecuteTask, 'tasks/execute')
|
||||
service = node.create_client(ReconcileTask, 'tasks/reconcile')
|
||||
feedback = []
|
||||
row = dict(route=plan['route'], scope='native ROS2 executor with simulated physical skills')
|
||||
process = None
|
||||
log = (output / ('native-' + suffix + '.log')).open('w')
|
||||
token = 'native-regression-recovery-token'
|
||||
operator = 'native-regression-operator'
|
||||
with tempfile.TemporaryDirectory(prefix='native-bt-regression-') as journal:
|
||||
def stop():
|
||||
nonlocal process
|
||||
if process is not None and process.poll() is None:
|
||||
os.killpg(process.pid, signal.SIGINT)
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
process = None
|
||||
|
||||
def start(expect_failure=False):
|
||||
nonlocal process
|
||||
args = [str(binary), '--ros-args', '-r', '__ns:=/sim/robot_01',
|
||||
'-p', 'robot_id:=robot_01', '-p', 'allowed_robots:=[robot_01]',
|
||||
'-p', 'site_config_file:=' + str(site_path), '-p', 'journal_directory:=' + journal,
|
||||
'-p', 'execution_enabled:=true', '-p', 'skill_timeout_ms:=5000',
|
||||
'-p', 'recovery_token:=' + token, '-p', 'recovery_operator_id:=' + operator]
|
||||
process = subprocess.Popen(args, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
if expect_failure:
|
||||
code = process.wait(timeout=5)
|
||||
assert code != 0, 'corrupt task journal accepted at startup'
|
||||
return
|
||||
assert client.wait_for_server(timeout_sec=12), 'native executor not discovered'
|
||||
time.sleep(0.6)
|
||||
assert process.poll() is None, 'native executor exited during startup'
|
||||
|
||||
def task_goal(tag):
|
||||
goal = ExecuteTask.Goal()
|
||||
goal.trace.task_id = 'native-' + suffix + '-' + tag
|
||||
goal.trace.run_id = 'run-' + suffix + '-' + tag
|
||||
goal.trace.subtask_id = 'execute_task'
|
||||
goal.trace.attempt = goal.trace.task_revision = goal.trace.plan_version = goal.trace.execution_generation = 1
|
||||
goal.approved_plan_json = json.dumps(plan)
|
||||
goal.context_json = json.dumps(dict(schema_version=1, registry_version=1, robot_id='robot_01', target_id='water', source_shelf='shelf_A', destination_id='tote_A', observe_location='observe_A', destination_location='tote_A_stop', task_revision=1, route=plan['route'], item_index=0))
|
||||
goal.timeout.sec = 30
|
||||
return goal
|
||||
|
||||
def reconcile(goal, authorization=token, stale=False, resolution='resume_task'):
|
||||
assert service.wait_for_service(timeout_sec=10), 'recovery service not discovered'
|
||||
request = ReconcileTask.Request()
|
||||
request.trace = copy.deepcopy(goal.trace)
|
||||
if stale:
|
||||
request.trace.execution_generation += 1
|
||||
request.operator_id, request.authorization = operator, authorization
|
||||
request.evidence_ref = 'regression/operator-observed-stop'
|
||||
request.resolution = resolution
|
||||
return wait(service.call_async(request), 15)
|
||||
|
||||
def motion_counts():
|
||||
with server.lock:
|
||||
return {name: server.counts[name] for name in ('navigate_semantic', 'execute_manipulation', 'execute_posture')}
|
||||
|
||||
try:
|
||||
start()
|
||||
goal = task_goal('complete')
|
||||
handle = wait(client.send_goal_async(goal, feedback_callback=lambda msg: feedback.append(msg.feedback.stage)), 8)
|
||||
assert handle.accepted, 'valid stage-one route was rejected'
|
||||
wrapped = wait(handle.get_result_async())
|
||||
result = wrapped.result
|
||||
row.update(native_status=wrapped.status, status=result.result.status,
|
||||
completed_quantity=result.completed_quantity, evidence=json.loads(result.evidence_json))
|
||||
assert wrapped.status == 4 and result.result.status == 0, result.result.message
|
||||
assert result.completed_quantity == 1 and result.result.stop_state == 1
|
||||
assert row['evidence']['safe_to_release'] is True
|
||||
row['decoded_manipulation'] = assert_manipulation_evidence(
|
||||
journal_records(Path(journal) / 'goal_registry.log'), goal.trace.run_id,
|
||||
ExecuteManipulation, deserialize_message)
|
||||
assert motion_counts()['execute_manipulation'] == 2
|
||||
if recovery_enabled:
|
||||
stop()
|
||||
time.sleep(0.8)
|
||||
start()
|
||||
before = motion_counts()
|
||||
wrong = reconcile(goal, authorization='wrong-token')
|
||||
assert not wrong.accepted, 'wrong authorization accepted'
|
||||
stale = reconcile(goal, stale=True)
|
||||
assert not stale.accepted, 'stale execution generation accepted'
|
||||
with server.lock:
|
||||
server.scenarios['verify_state'] = [{'kind': 'unknown'}]
|
||||
server.holding_state = RobotState.HOLDING_UNKNOWN
|
||||
time.sleep(0.25)
|
||||
unknown = reconcile(goal)
|
||||
assert not unknown.accepted, 'unknown holding was treated as safe recovery'
|
||||
with server.lock:
|
||||
server.scenarios['verify_state'] = [{'kind': 'passed'}]
|
||||
server.holding_state = RobotState.EMPTY
|
||||
time.sleep(0.25)
|
||||
recovered = reconcile(goal)
|
||||
assert recovered.accepted, recovered.error_code + ': ' + recovered.message
|
||||
state = json.loads(recovered.state_json)
|
||||
assert state['verified'] is True and state['stop_confirmed'] is True
|
||||
assert state['holding_state'] == 'EMPTY'
|
||||
receipts = state['receipts']
|
||||
assert len(receipts) == 1 and receipts[0]['task_id'] == goal.trace.task_id
|
||||
assert receipts[0]['completed_quantity'] == 1
|
||||
assert motion_counts() == before, 'receipt recovery dispatched motion'
|
||||
row['recovery'] = dict(wrong_token=wrong.error_code, stale_trace=stale.error_code,
|
||||
unknown_holding=unknown.error_code, recovered=state)
|
||||
if suffix == 'object_table':
|
||||
with server.lock:
|
||||
server.scenarios['navigate_semantic'] = [{'kind': 'normal', 'duration_seconds': 3., 'final_pose': site['locations']['shelf_A_stop']}]
|
||||
canceled_goal = task_goal('cancel')
|
||||
canceled_handle = wait(client.send_goal_async(canceled_goal), 8)
|
||||
assert canceled_handle.accepted
|
||||
eventually(lambda: motion_counts()['navigate_semantic'] > before['navigate_semantic'])
|
||||
wait(canceled_handle.cancel_goal_async(), 5)
|
||||
canceled_result = wait(canceled_handle.get_result_async(), 15)
|
||||
assert canceled_result.status == 5 and canceled_result.result.result.stop_state == 1
|
||||
assert motion_counts()['execute_manipulation'] == before['execute_manipulation']
|
||||
stop()
|
||||
time.sleep(0.8)
|
||||
start()
|
||||
# Absence of physical-goal history cannot prove no dispatch.
|
||||
stop()
|
||||
goal_log = Path(journal) / 'goal_registry.log'
|
||||
intact_goals = goal_log.read_bytes()
|
||||
goal_log.write_bytes(b'')
|
||||
start()
|
||||
incomplete = reconcile(canceled_goal, resolution='resume_task')
|
||||
assert not incomplete.accepted, 'missing dispatch history authorized an item replay'
|
||||
stop()
|
||||
goal_log.write_bytes(intact_goals)
|
||||
start()
|
||||
canceled_recovery = reconcile(canceled_goal, resolution='cancel_task')
|
||||
assert canceled_recovery.accepted, canceled_recovery.error_code
|
||||
row['cancel_recovery'] = json.loads(canceled_recovery.state_json)
|
||||
row['missing_history_rejected'] = incomplete.error_code
|
||||
stop()
|
||||
task_log = Path(journal) / 'task_runs.jsonl'
|
||||
intact_tasks = task_log.read_bytes()
|
||||
assert intact_tasks.endswith(b'\n')
|
||||
task_log.write_bytes(intact_tasks[:-1])
|
||||
start(expect_failure=True)
|
||||
row['unterminated_task_journal_rejected'] = True
|
||||
stop()
|
||||
task_log.write_bytes(intact_tasks)
|
||||
if suffix == 'shelf_cell':
|
||||
with server.lock:
|
||||
server.scenarios['locate_shelf_column'] = [{'kind': 'ambiguous'}]
|
||||
server.scenarios['navigate_semantic'] = [{'kind': 'normal', 'final_pose': site['locations']['observe_A']}]
|
||||
before_ask = motion_counts()
|
||||
ask_handle = wait(client.send_goal_async(task_goal('ambiguous')), 8)
|
||||
assert ask_handle.accepted
|
||||
ask_wrapped = wait(ask_handle.get_result_async(), 20)
|
||||
ask = ask_wrapped.result
|
||||
ask_evidence = json.loads(ask.evidence_json)
|
||||
assert ask_wrapped.status == 6 and ask.completed_quantity == 0
|
||||
assert ask_evidence['safe_to_release'] is True
|
||||
assert ask_evidence['needs_clarification'] is True
|
||||
assert ask_evidence['safe_to_retry'] is True
|
||||
assert motion_counts()['execute_manipulation'] == before_ask['execute_manipulation']
|
||||
row['runtime_clarification'] = ask_evidence
|
||||
row['mock_calls'] = dict(server.counts)
|
||||
row['feedback_stages'] = feedback
|
||||
row['passed'] = True
|
||||
except Exception as exc:
|
||||
row['passed'] = False
|
||||
row['failure'] = repr(exc)
|
||||
finally:
|
||||
stop()
|
||||
for path in Path(journal).glob('*'):
|
||||
if path.is_file():
|
||||
(output / ('native-' + suffix + '-' + path.name)).write_bytes(path.read_bytes())
|
||||
client.destroy()
|
||||
executor.shutdown(timeout_sec=2)
|
||||
thread.join(timeout=2)
|
||||
server.destroy_node()
|
||||
node.destroy_node()
|
||||
rclpy.shutdown()
|
||||
log.close()
|
||||
return row
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--repo-root', type=Path, default=Path(__file__).resolve().parents[2])
|
||||
parser.add_argument('--executor', type=Path)
|
||||
parser.add_argument('--output-dir', type=Path, required=True)
|
||||
parser.add_argument('--skip-recovery', action='store_true', help='Run workflow/evidence subset before recovery service is built')
|
||||
args = parser.parse_args()
|
||||
root = args.repo_root.resolve()
|
||||
sys.path[:0] = [str(root / 'coordinator'), str(root / 'ros2/bt_mock_servers')]
|
||||
binary = args.executor or root / 'install/bt_executor/lib/bt_executor/bt_executor_node'
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
rows = []
|
||||
for suffix in ('object_table', 'shelf_cell'):
|
||||
row = run_route(root, binary, args.output_dir, suffix, not args.skip_recovery)
|
||||
rows.append(row)
|
||||
(args.output_dir / 'native-workflow-regression.json').write_text(json.dumps(rows, indent=2))
|
||||
print(json.dumps({k: row[k] for k in ('route', 'passed', 'failure') if k in row}), flush=True)
|
||||
if not all(row['passed'] for row in rows):
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,190 @@
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path[:0]=[str(ROOT/'coordinator'),str(ROOT/'robobrain')]
|
||||
from robot_bt_coordinator.service import Coordinator,demo_site
|
||||
from robot_bt_coordinator.backends import ManualBackend,demo_plan
|
||||
from robot_bt_coordinator.errors import ApiError
|
||||
from robot_bt_coordinator.replay import replay_events
|
||||
from robot_bt_coordinator.store import Store
|
||||
from robot_bt_coordinator.plan_v2 import make_plan
|
||||
|
||||
|
||||
class TrustedBackend(ManualBackend):
|
||||
def reconcile(self,task,request):
|
||||
return dict(verified=True,stop_confirmed=True,holding_state='EMPTY',run_id=task['run_id'],evidence_ref=request['evidence_ref'],receipts=copy.deepcopy(self.receipts),safe_to_retry=self.retry)
|
||||
|
||||
|
||||
class RecoveryTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory();self.backend=TrustedBackend();self.backend.receipts=[];self.backend.retry=False
|
||||
self.site=demo_site();self.site['object_aliases']={'water':['water'],'doll':['doll']}
|
||||
self.c=Coordinator(self.tmp.name+'/tasks.db',self.backend,['r'],self.site)
|
||||
def tearDown(self):self.c.close();self.tmp.cleanup()
|
||||
def start(self):
|
||||
task=self.c.submit(dict(client_request_id='q',robot_id='r',instruction='move item',known_info={'target_name':'water','source_location':'shelf_A','destination':'tote_A'}))
|
||||
self.c.tick();task=self.c.get(task['task_id'])
|
||||
self.backend.emit(dict(type='plan',task_id=task['task_id'],task_revision=task['task_revision'],planning_generation=task['planning_generation'],status='PLAN_READY',plan=demo_plan(task['request']['known_info'])))
|
||||
self.c.tick();return self.c.get(task['task_id'])
|
||||
def quarantine(self,task):
|
||||
self.backend.emit(dict(type='execution_result',task_id=task['task_id'],run_id=task['run_id'],status='INTERVENTION_REQUIRED',completed_quantity=0,stop_confirmed=False))
|
||||
self.c.tick()
|
||||
def request(self,task,resolution='resume_task'):
|
||||
return dict(run_id=task['run_id'],evidence_ref='trusted-proof',resolution=resolution)
|
||||
def receipt(self,task):
|
||||
return dict(task_id=task['task_id'],item_index=0,completed_quantity=1,evidence=dict(evidence_id='delivery-proof',target_ref='water',destination_ref='tote_A',passed=True,empty_hand=True,in_destination=True,valid=True))
|
||||
def test_unsafe_resume_cannot_replay_an_uncertain_manipulation(self):
|
||||
task=self.start();self.quarantine(task)
|
||||
with self.assertRaises(ApiError) as error:self.c.intervene(task['task_id'],self.request(task))
|
||||
self.assertEqual(error.exception.code,'RECONCILIATION_REJECTED')
|
||||
self.assertEqual(len(self.backend.executions),1)
|
||||
def test_trusted_receipt_recovers_lost_result_without_redispatch(self):
|
||||
task=self.start();self.quarantine(task);self.backend.receipts=[self.receipt(task)]
|
||||
out=self.c.intervene(task['task_id'],self.request(task))
|
||||
self.assertEqual(out['status'],'SUCCEEDED');self.assertEqual(out['completed_quantity'],1)
|
||||
self.assertEqual(len(self.backend.executions),1)
|
||||
replay=replay_events(self.c.events(task['task_id'],limit=500))
|
||||
self.assertEqual(replay['completed_quantity'],1)
|
||||
def test_empty_hand_alone_cannot_credit_or_resume(self):
|
||||
task=self.start();self.quarantine(task)
|
||||
with self.assertRaises(ApiError):self.c.intervene(task['task_id'],self.request(task))
|
||||
out=self.c.intervene(task['task_id'],self.request(task,'cancel_task'))
|
||||
self.assertEqual(out['status'],'CANCELED');self.assertEqual(out['completed_quantity'],0)
|
||||
def clarification_result(self,task,**changes):
|
||||
evidence=dict(needs_clarification=True,safe_to_retry=True,empty_hand=True,valid=True,safe_to_release=True,questions=['Which registered shelf should be used?'])
|
||||
evidence.update(changes)
|
||||
self.backend.emit(dict(type='execution_result',task_id=task['task_id'],run_id=task['run_id'],status='FAILED',completed_quantity=0,stop_confirmed=True,evidence=evidence))
|
||||
self.c.tick();return self.c.get(task['task_id'])
|
||||
def test_verified_runtime_question_returns_to_planning_after_answer(self):
|
||||
task=self.start();out=self.clarification_result(task)
|
||||
self.assertEqual(out['status'],'NEEDS_CLARIFICATION');self.assertFalse(out['motion_dispatched'])
|
||||
self.assertEqual(out['question']['source_run_id'],task['run_id'])
|
||||
out=self.c.clarify(task['task_id'],dict(question_id=out['question']['question_id'],task_revision=out['task_revision'],known_info={'source_location':'shelf_A'}))
|
||||
self.c.tick();self.assertEqual(self.c.get(task['task_id'])['status'],'PLANNING')
|
||||
self.assertTrue(self.backend.plans[-1]['clarification_confirmed'])
|
||||
self.assertEqual(len(self.backend.executions),1)
|
||||
def test_runtime_question_without_no_manipulation_proof_is_quarantined(self):
|
||||
task=self.start();out=self.clarification_result(task,safe_to_retry=False)
|
||||
self.assertEqual(out['status'],'INTERVENTION_REQUIRED')
|
||||
self.assertIsNone(out['question'])
|
||||
def test_ordinary_failure_does_not_fabricate_question(self):
|
||||
task=self.start();out=self.clarification_result(task,needs_clarification=False)
|
||||
self.assertEqual(out['status'],'FAILED');self.assertIsNone(out['question'])
|
||||
def test_operator_replan_requires_no_manipulation_and_no_delivery(self):
|
||||
task=self.start();self.quarantine(task)
|
||||
with self.assertRaises(ApiError):self.c.intervene(task['task_id'],self.request(task,'replan_task'))
|
||||
self.backend.retry=True
|
||||
out=self.c.intervene(task['task_id'],self.request(task,'replan_task'))
|
||||
self.assertEqual(out['status'],'NEEDS_CLARIFICATION');self.assertFalse(out['motion_dispatched'])
|
||||
self.assertEqual(len(self.backend.executions),1)
|
||||
def test_operator_replan_cannot_rewrite_already_delivered_scope(self):
|
||||
task=self.start();self.quarantine(task);self.backend.retry=True;self.backend.receipts=[self.receipt(task)]
|
||||
with self.assertRaises(ApiError):self.c.intervene(task['task_id'],self.request(task,'replan_task'))
|
||||
self.assertEqual(self.c.get(task['task_id'])['status'],'INTERVENTION_REQUIRED')
|
||||
def test_question_cannot_hide_a_verified_physical_delivery(self):
|
||||
task=self.start();proof=self.receipt(task)['evidence']
|
||||
proof.update(needs_clarification=True,safe_to_retry=True,safe_to_release=True,questions=['Which shelf?'])
|
||||
self.backend.emit(dict(type='execution_result',task_id=task['task_id'],run_id=task['run_id'],status='FAILED',completed_quantity=1,stop_confirmed=True,evidence=proof));self.c.tick()
|
||||
out=self.c.get(task['task_id'])
|
||||
self.assertEqual(out['completed_quantity'],1)
|
||||
self.assertEqual(out['status'],'INTERVENTION_REQUIRED')
|
||||
def test_proven_pre_manipulation_resume_uses_new_run(self):
|
||||
task=self.start();self.quarantine(task);self.backend.retry=True
|
||||
out=self.c.intervene(task['task_id'],self.request(task))
|
||||
self.assertEqual(out['status'],'EXECUTING');self.assertNotEqual(out['run_id'],task['run_id'])
|
||||
self.assertEqual(len(self.backend.executions),2)
|
||||
def test_paused_execution_accepts_authenticated_reconciliation(self):
|
||||
task=self.start();self.c.control(task['task_id'],'pause')
|
||||
self.backend.emit(dict(type='execution_result',task_id=task['task_id'],run_id=task['run_id'],status='CANCELED',completed_quantity=0,stop_confirmed=True))
|
||||
self.c.tick();self.assertEqual(self.c.get(task['task_id'])['status'],'PAUSED')
|
||||
self.backend.retry=True
|
||||
self.assertEqual(self.c.intervene(task['task_id'],self.request(task))['status'],'EXECUTING')
|
||||
def test_unverified_resume_preserves_paused_state(self):
|
||||
task=self.start();self.c.control(task['task_id'],'pause')
|
||||
self.backend.emit(dict(type='execution_result',task_id=task['task_id'],run_id=task['run_id'],status='CANCELED',completed_quantity=0,stop_confirmed=True))
|
||||
self.c.tick()
|
||||
with self.assertRaises(ApiError) as error:self.c.control(task['task_id'],'resume')
|
||||
self.assertEqual(error.exception.code,'RECONCILIATION_REQUIRED')
|
||||
self.assertEqual(self.c.get(task['task_id'])['status'],'PAUSED')
|
||||
self.assertEqual(len(self.backend.executions),1)
|
||||
def test_completed_planning_releases_timer_state(self):
|
||||
self.start();self.assertEqual(self.c._planning_started,{})
|
||||
def test_duplicate_recovered_receipts_credit_once_then_start_next_item(self):
|
||||
self.site['execution_route']='OBJECT_TABLE'
|
||||
slots=dict(items=[dict(target_name='water',quantity=2,source_location='shelf_A')],destination='tote_A')
|
||||
task=self.c.submit(dict(client_request_id='multi',robot_id='r',instruction='move two',known_info=slots))
|
||||
self.c.tick();task=self.c.get(task['task_id'])
|
||||
self.backend.emit(dict(type='plan',task_id=task['task_id'],task_revision=1,planning_generation=task['planning_generation'],status='PLAN_READY',plan=make_plan('move two',slots,'OBJECT_TABLE')))
|
||||
self.c.tick();task=self.c.get(task['task_id']);self.quarantine(task)
|
||||
receipt=self.receipt(task);self.backend.receipts=[receipt,copy.deepcopy(receipt)]
|
||||
out=self.c.intervene(task['task_id'],self.request(task))
|
||||
self.assertEqual(out['status'],'EXECUTING');self.assertEqual(out['completed_quantity'],1)
|
||||
self.assertEqual(out['active_item_index'],1);self.assertNotEqual(out['run_id'],task['run_id'])
|
||||
self.assertEqual([e['item_index'] for e in self.c.events(task['task_id'],limit=500) if e['kind']=='delivery_committed'],[0])
|
||||
def test_empty_clarification_cannot_mark_intent_as_confirmed(self):
|
||||
task=self.c.submit(dict(client_request_id='intent',robot_id='r',instruction='1 water shelf_A tote_A',known_info=dict(target_name='doll')))
|
||||
with self.assertRaises(ApiError):
|
||||
self.c.clarify(task['task_id'],dict(question_id=task['question']['question_id'],task_revision=task['task_revision'],known_info={}))
|
||||
self.assertFalse(self.c.get(task['task_id'])['clarification_confirmed'])
|
||||
def test_invalid_receipts_are_atomic_and_do_not_release_quarantine(self):
|
||||
for mutate in [lambda r:r.update(task_id='other'),lambda r:r.update(item_index=True),lambda r:r.update(completed_quantity=2),lambda r:r['evidence'].update(destination_ref='wrong')]:
|
||||
task=self.start() if not hasattr(self,'current') else self.current
|
||||
self.current=task;self.quarantine(task)
|
||||
receipt=self.receipt(task);mutate(receipt);self.backend.receipts=[receipt]
|
||||
with self.assertRaises(ApiError):self.c.intervene(task['task_id'],self.request(task,'cancel_task'))
|
||||
self.assertEqual(self.c.get(task['task_id'])['completed_quantity'],0)
|
||||
self.assertEqual(self.c.get(task['task_id'])['status'],'INTERVENTION_REQUIRED')
|
||||
def test_tick_avoids_historical_task_scan(self):
|
||||
for n in range(30):
|
||||
t=self.c.submit(dict(client_request_id=str(n),robot_id='r',instruction='move'))
|
||||
self.c.control(t['task_id'],'cancel')
|
||||
with patch.object(self.c.store,'all',side_effect=AssertionError('historical scan')):
|
||||
self.c.tick()
|
||||
def test_progress_replay_preserves_stage_sequence_and_detail(self):
|
||||
task=self.start()
|
||||
self.backend.emit(dict(type='progress',task_id=task['task_id'],run_id=task['run_id'],sequence=1,stage='NavigateSource',detail='waiting obstacle'))
|
||||
self.c.tick()
|
||||
events=self.c.events(task['task_id'],limit=500)
|
||||
progress=next(e for e in events if e['kind']=='progress')
|
||||
self.assertEqual(progress['stage'],'NavigateSource');self.assertEqual(progress['sequence'],1)
|
||||
self.assertEqual(progress['detail'],'waiting obstacle')
|
||||
self.assertEqual(replay_events(events)['progress'][-1]['stage'],'NavigateSource')
|
||||
def test_supplied_slots_conflicting_with_instruction_require_clarification(self):
|
||||
task=self.c.submit(dict(client_request_id='intent',robot_id='r',instruction='1 water shelf_A tote_A',known_info=dict(target_name='doll',quantity=1,source_location='shelf_A',destination='tote_A')))
|
||||
self.assertEqual(task['status'],'NEEDS_CLARIFICATION');self.assertFalse(task['clarification_confirmed'])
|
||||
self.c.tick();self.assertEqual(self.backend.plans,[])
|
||||
task=self.c.clarify(task['task_id'],dict(question_id=task['question']['question_id'],task_revision=task['task_revision'],known_info=dict(target_name='doll')))
|
||||
self.assertTrue(task['clarification_confirmed'])
|
||||
self.c.close();self.c=Coordinator(self.tmp.name+'/tasks.db',self.backend,['r'],self.site)
|
||||
self.assertTrue(self.c.get(task['task_id'])['clarification_confirmed'])
|
||||
self.c.tick();self.assertTrue(self.backend.plans[-1]['clarification_confirmed'])
|
||||
def test_approved_plan_is_independently_checked_against_instruction(self):
|
||||
task=self.c.submit(dict(client_request_id='intent',robot_id='r',instruction='1 water shelf_A tote_A',known_info={}))
|
||||
self.c.tick();task=self.c.get(task['task_id'])
|
||||
plan=demo_plan(dict(target_name='doll',quantity=1,source_location='shelf_A',destination='tote_A'))
|
||||
self.backend.emit(dict(type='plan',task_id=task['task_id'],task_revision=1,planning_generation=task['planning_generation'],status='PLAN_READY',plan=plan));self.c.tick()
|
||||
self.assertEqual(self.c.get(task['task_id'])['status'],'NEEDS_CLARIFICATION');self.assertEqual(self.backend.executions,[])
|
||||
|
||||
|
||||
class MigrationTest(unittest.TestCase):
|
||||
def test_legacy_database_migrates_active_status_without_losing_history(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path=folder+'/old.db';db=sqlite3.connect(path)
|
||||
db.execute('CREATE TABLE tasks(seq INTEGER PRIMARY KEY AUTOINCREMENT,task_id TEXT UNIQUE NOT NULL,robot_id TEXT NOT NULL,request_id TEXT NOT NULL,request_hash TEXT NOT NULL,data TEXT NOT NULL,UNIQUE(robot_id,request_id))')
|
||||
for n,status in enumerate(('SUCCEEDED','QUEUED','PAUSED')):
|
||||
db.execute('INSERT INTO tasks(task_id,robot_id,request_id,request_hash,data) VALUES(?,?,?,?,?)',(str(n),'r',str(n),'hash',json.dumps(dict(task_id=str(n),status=status))))
|
||||
db.commit();db.close();store=Store(path)
|
||||
try:
|
||||
self.assertEqual([t['status'] for t in store.active()],['QUEUED','PAUSED'])
|
||||
self.assertEqual(len(store.all()),3)
|
||||
finally:store.close()
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -3,6 +3,9 @@ import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import socket
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from unittest.mock import patch
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'coordinator'))
|
||||
@@ -35,4 +38,54 @@ class HttpTest(unittest.TestCase):
|
||||
self.assertEqual(self.request('POST','/v1/tasks',payload)[0],400)
|
||||
self.assertEqual(self.request('GET','/not-found')[0],404)
|
||||
|
||||
def test_listener_can_queue_ten_fresh_connections_before_accept(self):
|
||||
# Model a burst while the accept loop is busy. This is a capacity check,
|
||||
# not a throughput or latency assertion on the host running the suite.
|
||||
listener=make_server(self.c,'127.0.0.1',0,'test-secret')
|
||||
clients=[]
|
||||
try:
|
||||
for _ in range(10):
|
||||
try:
|
||||
clients.append(socket.create_connection(('127.0.0.1',listener.server_port),timeout=2))
|
||||
except OSError as error:
|
||||
self.fail('listener could not queue ten clients: '+str(error))
|
||||
finally:
|
||||
for client in clients:client.close()
|
||||
listener.server_close()
|
||||
|
||||
def test_ten_concurrent_submissions_are_independently_accepted_and_deduplicated(self):
|
||||
gate=threading.Barrier(10)
|
||||
def submit(index):
|
||||
body=json.dumps(dict(client_request_id='burst-'+str(index),robot_id='robot_01',instruction='move water',known_info={}))
|
||||
gate.wait(timeout=10)
|
||||
return self.request('POST','/v1/tasks',body)
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
results=list(pool.map(submit,range(10)))
|
||||
self.assertEqual([code for code,_ in results],[202]*10)
|
||||
self.assertEqual(len({task['task_id'] for _,task in results}),10)
|
||||
self.assertTrue(all(task['status']=='QUEUED' for _,task in results))
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
repeats=list(pool.map(submit,range(10)))
|
||||
self.assertEqual([task['task_id'] for _,task in repeats],[task['task_id'] for _,task in results])
|
||||
self.assertTrue(all(code==202 and task['deduplicated'] for code,task in repeats))
|
||||
self.assertEqual(len(self.c.store.all()),10)
|
||||
|
||||
def test_recovery_backend_failure_is_sanitized_service_unavailable(self):
|
||||
with patch.object(self.c,'intervene',side_effect=RuntimeError('private-token-123 from ROS')):
|
||||
code,body=self.request('POST','/v1/tasks/task/interventions','{}',token='operator-secret')
|
||||
self.assertEqual(code,503);self.assertEqual(body['error_code'],'RECONCILIATION_UNAVAILABLE')
|
||||
self.assertNotIn('private-token',json.dumps(body))
|
||||
|
||||
def test_recovery_timeout_returns_json_instead_of_dropping_connection(self):
|
||||
with patch.object(self.c,'intervene',side_effect=TimeoutError('private-token-123 outcome unknown')):
|
||||
try:code,body=self.request('POST','/v1/tasks/task/interventions','{}',token='operator-secret')
|
||||
except http.client.RemoteDisconnected:self.fail('recovery timeout dropped the HTTP connection without a JSON result')
|
||||
self.assertEqual(code,504);self.assertEqual(body['error_code'],'RECONCILIATION_TIMEOUT')
|
||||
self.assertNotIn('private-token',json.dumps(body))
|
||||
|
||||
def test_recovery_backend_is_not_called_with_ordinary_api_token(self):
|
||||
with patch.object(self.c,'intervene') as reconcile:
|
||||
self.assertEqual(self.request('POST','/v1/tasks/task/interventions','{}')[0],401)
|
||||
reconcile.assert_not_called()
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path[:0] = [str(ROOT/'coordinator'), str(ROOT/'robobrain'), str(ROOT/'ros2/robobrain_services')]
|
||||
from robot_robobrain.backends import FixtureBackend
|
||||
from robot_robobrain.service import BrainService
|
||||
from robot_robobrain.observations import Observation
|
||||
from robot_bt_coordinator.plan_v2 import make_plan
|
||||
|
||||
class ConsistencyTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.site = json.loads((ROOT/'config/sim_site_object_table.json').read_text(encoding='utf-8'))
|
||||
self.slots = {'items': [{'target_name':'water', 'quantity':2, 'source_location':'shelf_A'}, {'target_name':'doll', 'quantity':1, 'source_location':'shelf_A'}], 'destination':'tote_A'}
|
||||
self.instruction = '把货架A的两瓶水和一个玩偶放到周转箱A'
|
||||
|
||||
def run_plan(self, slots, revision=1, confirmed=False, instruction=None):
|
||||
text = instruction or self.instruction
|
||||
service = BrainService(FixtureBackend(json.dumps(make_plan(text, slots, 'OBJECT_TABLE'))), self.tmp.name)
|
||||
return service.plan(dict(task_id='t', task_revision=revision, planning_generation=1,
|
||||
instruction=text, known_info=slots, context=self.site,
|
||||
constraints={'route':'OBJECT_TABLE', 'clarification_confirmed':confirmed}, timeout=1))
|
||||
|
||||
def test_initial_explicit_quantity_order_item_destination_conflicts_fail(self):
|
||||
variants=[]
|
||||
p=copy.deepcopy(self.slots);p['items'][0]['quantity']=1;variants.append(p)
|
||||
p=copy.deepcopy(self.slots);p['items'].reverse();variants.append(p)
|
||||
p=copy.deepcopy(self.slots);p['items'][0]['target_name']='doll';variants.append(p)
|
||||
p=copy.deepcopy(self.slots);p['destination']='tote_B';variants.append(p)
|
||||
for slots in variants:
|
||||
with self.subTest(slots=slots):
|
||||
result=self.run_plan(slots)
|
||||
self.assertEqual(result['status'], 'FAILED')
|
||||
self.assertEqual(result['error_code'], 'SEMANTIC_MISMATCH')
|
||||
self.assertTrue(Path(result['record_ref']).is_file())
|
||||
|
||||
def test_revision_alone_and_initial_confirmation_marker_do_not_override(self):
|
||||
slots=copy.deepcopy(self.slots);slots['items'][0]['quantity']=1
|
||||
for revision,confirmed in [(2,False),(1,True),(2,'true')]:
|
||||
with self.subTest(revision=revision,confirmed=confirmed):
|
||||
self.assertEqual(self.run_plan(slots,revision,confirmed)['status'],'FAILED')
|
||||
|
||||
def test_confirmed_clarification_can_override_original(self):
|
||||
slots=copy.deepcopy(self.slots);slots['items'][0]['quantity']=1
|
||||
self.assertEqual(self.run_plan(slots,2,True)['status'],'PLAN_READY')
|
||||
|
||||
def test_matching_and_unrecognized_language_are_not_false_proofs(self):
|
||||
self.assertEqual(self.run_plan(self.slots)['status'],'PLAN_READY')
|
||||
self.assertEqual(self.run_plan(self.slots,instruction='carry out the supplied item list')['status'],'PLAN_READY')
|
||||
|
||||
def test_shelf_non_success_retains_observation_identity(self):
|
||||
path=Path(self.tmp.name)/'image.png';path.write_bytes(b'fixture')
|
||||
obs=Observation('observation-7',100,'camera',str(path),'observe_A',1,'shelf_A')
|
||||
goal=dict(task_id='t',subtask_id='s',target_ref='water',source_region_ref='shelf_A',observation_station_id='observe_A',station_registry_version=1,capture_after=90,timeout=1)
|
||||
for status in ['NOT_FOUND','AMBIGUOUS']:
|
||||
result=BrainService(FixtureBackend(json.dumps({'status':status})),self.tmp.name).shelf(goal,obs,110)
|
||||
self.assertEqual(result.get('observation_id'),'observation-7')
|
||||
self.assertEqual(result.get('observed_at'),100)
|
||||
self.assertNotIn('column_id',result)
|
||||
|
||||
def test_unexpected_backend_failure_is_archived(self):
|
||||
def broken(_): raise RuntimeError('model driver crashed')
|
||||
service=BrainService(FixtureBackend(broken),self.tmp.name)
|
||||
goal=dict(task_id='t',task_revision=1,planning_generation=1,instruction=self.instruction,known_info=self.slots,context=self.site,constraints={'route':'OBJECT_TABLE'},timeout=1)
|
||||
try:
|
||||
result=service.plan(goal)
|
||||
except RuntimeError:
|
||||
self.fail('unexpected backend exception escaped without diagnostic result')
|
||||
self.assertEqual(result['status'],'FAILED')
|
||||
self.assertEqual(result['error_code'],'INFERENCE_FAILED')
|
||||
self.assertIn('model driver crashed',json.loads(Path(result['record_ref']).read_text())['result']['message'])
|
||||
|
||||
class AdapterConfigTests(unittest.TestCase):
|
||||
def test_missing_loader_configuration_has_actionable_error(self):
|
||||
from robot_robobrain.model_adapter import create
|
||||
for config in [{}, {'loader':'missing-separator','model':{},'task_mapping':{}}]:
|
||||
try:
|
||||
create(config)
|
||||
except Exception as ex:
|
||||
self.assertIsInstance(ex,ValueError)
|
||||
self.assertIn('loader',str(ex))
|
||||
else:self.fail('invalid loader configuration accepted')
|
||||
|
||||
def test_single_image_adapter_rejects_dense_frames_before_loader_import(self):
|
||||
from robot_robobrain.model_adapter import create
|
||||
config={'loader':'deployment_models:create_robobrain','model':{'checkpoint':'/not/present'},'task_mapping':{'dense_feedback':'general'}}
|
||||
try:create(config)
|
||||
except Exception as ex:
|
||||
self.assertIsInstance(ex,ValueError)
|
||||
self.assertIn('dense',str(ex))
|
||||
else:self.fail('single-image adapter accepted dense task mapping')
|
||||
|
||||
class ServiceBoundaryTests(unittest.TestCase):
|
||||
def test_malformed_ros_request_is_archived_without_parsing_it(self):
|
||||
from robobrain_services import nodes
|
||||
self.assertTrue(callable(getattr(nodes,'guarded_work',None)), 'ROS worker needs a failure-archiving boundary')
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
service=BrainService(FixtureBackend('{}'),tmp)
|
||||
request=types.SimpleNamespace(task_id='t',instruction='original text',known_info_json='{malformed',timeout=types.SimpleNamespace(sec=1,nanosec=0))
|
||||
def bad_request(): raise ValueError('invalid JSON input')
|
||||
result=nodes.guarded_work(service,'plan',request,bad_request)
|
||||
self.assertEqual(result['error_code'],'SERVICE_ERROR')
|
||||
record=json.loads(Path(result['record_ref']).read_text())
|
||||
self.assertEqual(record['input']['known_info_json'],'{malformed')
|
||||
self.assertEqual(record['input']['timeout']['sec'],1)
|
||||
self.assertIn('invalid JSON input',record['result']['message'])
|
||||
|
||||
def test_archive_failure_still_returns_terminal_failure(self):
|
||||
from robobrain_services import nodes
|
||||
self.assertTrue(callable(getattr(nodes,'guarded_work',None)), 'ROS worker needs a failure-archiving boundary')
|
||||
class UnavailableArchive:
|
||||
def _record(self,*args):raise OSError('disk full')
|
||||
def failure():raise RuntimeError('service crashed')
|
||||
result=nodes.guarded_work(UnavailableArchive(),'shelf',{'task_id':'t'},failure)
|
||||
self.assertEqual(result['status'],'FAILED')
|
||||
self.assertEqual(result.get('record_ref'),'')
|
||||
self.assertIn('archive',result['message'])
|
||||
self.assertIn('disk full',result['message'])
|
||||
|
||||
def test_success_result_is_not_rewritten(self):
|
||||
from robobrain_services import nodes
|
||||
self.assertTrue(callable(getattr(nodes,'guarded_work',None)), 'ROS worker needs a failure-archiving boundary')
|
||||
expected={'status':'PLAN_READY','record_ref':'existing'}
|
||||
self.assertIs(nodes.guarded_work(None,'plan',{},lambda:expected),expected)
|
||||
|
||||
class LocalizeDiagnosticTests(unittest.TestCase):
|
||||
def test_non_success_keeps_observation_without_inventing_point(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path=Path(tmp)/'image.png';path.write_bytes(b'fixture')
|
||||
obs=Observation('obs3d',100,'camera',str(path),'observe_A',1,'shelf_A','cal-1',7)
|
||||
goal=dict(task_id='t',subtask_id='s',target_ref='water',capture_after=90,timeout=1)
|
||||
for status in ('NOT_FOUND','AMBIGUOUS'):
|
||||
result=BrainService(FixtureBackend(json.dumps({'status':status})),tmp).localize(goal,obs,110)
|
||||
self.assertEqual(result.get('observation_id'),'obs3d')
|
||||
self.assertEqual(result.get('observed_at'),100)
|
||||
self.assertEqual(result.get('calibration_id'),'cal-1')
|
||||
self.assertEqual(result.get('geometry_epoch'),7)
|
||||
self.assertFalse(result['geometry_valid'])
|
||||
self.assertNotIn('target_point',result)
|
||||
|
||||
class DemoClarificationTests(unittest.TestCase):
|
||||
def test_demo_propagates_trusted_clarification_marker(self):
|
||||
from robot_robobrain.demo_backend import BrainDemoBackend
|
||||
backend=BrainDemoBackend.__new__(BrainDemoBackend)
|
||||
backend.plans=[];backend.site={'execution_route':'OBJECT_TABLE'}
|
||||
goals=[]
|
||||
backend.brain=types.SimpleNamespace(plan=lambda goal:(goals.append(goal) or {'status':'FAILED','record_ref':'r'}))
|
||||
backend.emit=lambda event:None
|
||||
task={'task_id':'t','task_revision':2,'planning_generation':3,'clarification_confirmed':True,'request':{'instruction':'original','known_info':{}}}
|
||||
backend.start_planning(task)
|
||||
self.assertIs(goals[0]['constraints'].get('clarification_confirmed'),True)
|
||||
@@ -0,0 +1,25 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
spec=importlib.util.spec_from_file_location('coverage_report',ROOT/'tools/coverage_report.py')
|
||||
coverage=importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(coverage)
|
||||
|
||||
|
||||
class CoverageCompatibilityTest(unittest.TestCase):
|
||||
def test_non_source_line_markers_do_not_enter_coverage_counts(self):
|
||||
# Python 3.14 trace may expose None for artificial bytecode positions.
|
||||
with patch.object(coverage.trace,'_find_executable_linenos',return_value={None:1,0:1,1:1,7:1}):
|
||||
actual=coverage.executable_lines('example.py')
|
||||
self.assertEqual(actual,{1,7})
|
||||
|
||||
def test_real_source_lines_remain_available(self):
|
||||
lines=coverage.executable_lines(__file__)
|
||||
self.assertTrue(lines)
|
||||
self.assertTrue(all(type(line) is int and line>0 for line in lines))
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in New Issue
Block a user