fix: harden task recovery and DR contract handling
This commit is contained in:
@@ -27,7 +27,10 @@ def main():
|
||||
backend=BrainDemoBackend(args.demo_executable,state/'simulator',site)
|
||||
else:
|
||||
from .ros_backend import RosBackend
|
||||
backend=RosBackend(namespace=args.ros_namespace,config={'planning_context':site,'dense_progress_enabled':args.dense_progress})
|
||||
backend=RosBackend(namespace=args.ros_namespace,config={'planning_context':site,
|
||||
'dense_progress_enabled':args.dense_progress,
|
||||
'recovery_token':os.environ.get('ROBOT_BT_RECOVERY_TOKEN',''),
|
||||
'recovery_operator_id':os.environ.get('ROBOT_BT_RECOVERY_OPERATOR','')})
|
||||
if args.backend=='mock':backend.route=site.get('execution_route','OBJECT_TABLE')
|
||||
coordinator=Coordinator(str(state/'tasks.sqlite3'),backend,{args.robot_id},site)
|
||||
server=make_server(coordinator,args.host,args.port,token,os.environ.get('ROBOT_BT_OPERATOR_TOKEN',''))
|
||||
|
||||
@@ -8,6 +8,12 @@ from .plan import strict_json
|
||||
|
||||
MAX_BODY=65536
|
||||
|
||||
|
||||
class TaskHTTPServer(ThreadingHTTPServer):
|
||||
# Accept a burst of task submissions without the default five-slot listen
|
||||
# queue forcing clients into a TCP retransmission before admission.
|
||||
request_queue_size=64
|
||||
|
||||
def make_server(coordinator,host,port,api_token,operator_token=''):
|
||||
if not api_token:raise ValueError('API token must be explicitly configured')
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
@@ -60,7 +66,12 @@ def make_server(coordinator,host,port,api_token,operator_token=''):
|
||||
if data:raise ApiError('INVALID_CONTROL','control body must be empty')
|
||||
return 202,coordinator.control(tid,action)
|
||||
if action=='clarifications':return 202,coordinator.clarify(tid,data)
|
||||
if action=='interventions':return 202,coordinator.intervene(tid,data)
|
||||
if action=='interventions':
|
||||
try:return 202,coordinator.intervene(tid,data)
|
||||
except TimeoutError:
|
||||
raise ApiError('RECONCILIATION_TIMEOUT','recovery outcome is unknown; retry with the same evidence reference',504) from None
|
||||
except RuntimeError:
|
||||
raise ApiError('RECONCILIATION_UNAVAILABLE','recovery service is unavailable or rejected the request; task state remains queryable',503) from None
|
||||
raise ApiError('NOT_FOUND','endpoint not found',404)
|
||||
def handle_request(self):
|
||||
try:status,payload=self.dispatch();self.reply(status,payload)
|
||||
@@ -70,5 +81,5 @@ def make_server(coordinator,host,port,api_token,operator_token=''):
|
||||
self.reply(500,{'error_code':'INTERNAL_ERROR','message':'request failed; task state remains queryable'})
|
||||
do_GET=handle_request
|
||||
do_POST=handle_request
|
||||
server=ThreadingHTTPServer((host,port),Handler);server.daemon_threads=True
|
||||
server=TaskHTTPServer((host,port),Handler);server.daemon_threads=True
|
||||
return server
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Conservative independent intent gate for instructions without confirmed slots.
|
||||
Only registered aliases and explicit quantities are accepted. Other language asks
|
||||
for structured clarification; this parser is not claimed to cover arbitrary NLP.
|
||||
"""
|
||||
import re
|
||||
COUNTS={'一':1,'二':2,'两':2,'三':3,'四':4,'五':5,'六':6,'七':7,'八':8,'九':9,'十':10,'one':1,'two':2,'three':3}
|
||||
def matches(instruction,registry,aliases):
|
||||
candidates=[]
|
||||
for key in registry:
|
||||
for term in [key]+list(aliases.get(key,[])):
|
||||
if not term:continue
|
||||
for m in re.finditer(re.escape(term),instruction):candidates.append((m.start(),m.end(),key))
|
||||
chosen=[]
|
||||
for item in sorted(candidates,key=lambda x:(-(x[1]-x[0]),x[0])):
|
||||
if not any(item[0]<q[1] and q[0]<item[1] for q in chosen):chosen.append(item)
|
||||
return sorted(chosen)
|
||||
def extract(instruction,site):
|
||||
names=site.get('object_locations',{}) or site.get('object_aliases',{})
|
||||
objects=matches(instruction,names,site.get('object_aliases',{}))
|
||||
sources=matches(instruction,site.get('sources',{}),site.get('source_aliases',{}))
|
||||
destinations=matches(instruction,site.get('destinations',{}),site.get('destination_aliases',{}))
|
||||
if not objects or len({x[2] for x in sources})!=1 or len({x[2] for x in destinations})!=1:return None
|
||||
items=[]
|
||||
for start,end,name in objects:
|
||||
m=re.search(r'(\d+|一|二|两|三|四|五|六|七|八|九|十|one|two|three)\s*(?:瓶|个|件|盒|袋)?\s*$',instruction[:start])
|
||||
if not m:return None
|
||||
count=COUNTS.get(m[1],int(m[1]) if m[1].isdigit() else 0)
|
||||
if not 1<=count<=20:return None
|
||||
items.append(dict(target_name=name,quantity=count,source_location=sources[0][2]))
|
||||
return dict(items=items,destination=destinations[0][2])
|
||||
|
||||
def conflicts_with_instruction(instruction, known, site, *, clarification_confirmed=False):
|
||||
"""Compare recognized explicit intent with validated structured slots.
|
||||
|
||||
False means no *detected* conflict, not proof of arbitrary-language fidelity.
|
||||
The caller must derive clarification_confirmed from trusted clarification
|
||||
history; neither a client-supplied flag nor a revision counter is authority.
|
||||
"""
|
||||
if clarification_confirmed is True:
|
||||
return False
|
||||
expected = extract(instruction, site)
|
||||
if expected is None:
|
||||
return False
|
||||
if 'items' not in known:
|
||||
if set(known) - {'destination'}:
|
||||
if len(expected['items']) != 1:
|
||||
return True
|
||||
expected = dict(expected['items'][0], destination=expected['destination'])
|
||||
return any(expected.get(key) != value for key, value in known.items())
|
||||
@@ -11,7 +11,7 @@ from .plan import validate_plan
|
||||
from .plan_v2 import instances
|
||||
|
||||
def replay_events(events):
|
||||
status='RECEIVED';version=0;cursor=0;delivered={};stages=[];plans=[];errors=[]
|
||||
status='RECEIVED';version=0;cursor=0;delivered={};stages=[];plans=[];errors=[];progress=[];reconciliations=[]
|
||||
for e in events:
|
||||
if type(e['event_id']) is not int or e['event_id']<=cursor:raise ValueError('non-monotonic event cursor')
|
||||
cursor=e['event_id']
|
||||
@@ -19,6 +19,7 @@ def replay_events(events):
|
||||
if e['status_version']!=version+1:raise ValueError('missing or non-monotonic status version')
|
||||
version=e['status_version'];status=e['status']
|
||||
if e.get('error_code'):errors.append(e['error_code'])
|
||||
if e['kind']=='progress':progress.append({k:e[k] for k in ('event_id','run_id','stage','sequence','detail') if k in e})
|
||||
elif e['kind']=='plan_approved':
|
||||
plans.append({'run_id':e['run_id'],'plan':validate_plan(e['plan'])})
|
||||
elif e['kind']=='delivery_committed':
|
||||
@@ -27,8 +28,10 @@ def replay_events(events):
|
||||
delivered[key]=e['evidence']
|
||||
elif e['kind']=='execution_result':
|
||||
payload=e['payload'];stages.append({'run_id':payload.get('run_id'),'status':payload.get('status'),'stop_confirmed':payload.get('stop_confirmed')})
|
||||
elif e['kind']=='reconciliation':
|
||||
reconciliations.append({k:e[k] for k in ('event_id','run_id','resolution','evidence_ref','credited_items') if k in e})
|
||||
if status=='SUCCEEDED' and (not plans or set(delivered)!=set(range(len(instances(plans[-1]['plan']))))):raise ValueError('success without committed delivery')
|
||||
return {'mode':'offline_audit_replay','status':status,'status_version':version,'last_event_id':cursor,'completed_quantity':len(delivered),'plans':plans,'results':stages,'errors':errors,'connects_to_robot':False}
|
||||
return {'mode':'offline_audit_replay','status':status,'status_version':version,'last_event_id':cursor,'completed_quantity':len(delivered),'plans':plans,'results':stages,'progress':progress,'reconciliations':reconciliations,'errors':errors,'connects_to_robot':False}
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--db',required=True);p.add_argument('--task-id',required=True);args=p.parse_args()
|
||||
|
||||
@@ -27,6 +27,7 @@ class RosBackend:
|
||||
from rclpy.context import Context
|
||||
from rclpy.executors import SingleThreadedExecutor
|
||||
from bt_skill_interfaces.action import PlanTask, ExecuteTask
|
||||
from bt_skill_interfaces.srv import ReconcileTask
|
||||
|
||||
self.config = dict(config or {})
|
||||
if namespace is not None:
|
||||
@@ -50,11 +51,16 @@ class RosBackend:
|
||||
self._PlanTask, self._ExecuteTask = PlanTask, ExecuteTask
|
||||
self._planner = ActionClient(node, PlanTask, self.config.get('plan_action', 'tasks/plan'))
|
||||
self._execution = ActionClient(node, ExecuteTask, self.config.get('execute_action', 'tasks/execute'))
|
||||
self._ReconcileTask = ReconcileTask
|
||||
self._recovery = node.create_client(ReconcileTask, self.config.get('reconcile_service', 'tasks/reconcile'))
|
||||
self._events = deque()
|
||||
self._planning = {}
|
||||
self._runs = {}
|
||||
self._lock = threading.RLock()
|
||||
self._closed = False
|
||||
self._terminal_retention = int(self.config.get('terminal_retention', 256))
|
||||
if self._terminal_retention < 1:
|
||||
raise ValueError('terminal_retention must be positive')
|
||||
self._acceptance_timeout = float(self.config.get('acceptance_timeout', 3.0))
|
||||
self._feedback_timeout = float(self.config.get('feedback_timeout', 5.0))
|
||||
self._stop_timeout = float(self.config.get('stop_timeout', 7.0))
|
||||
@@ -124,13 +130,25 @@ class RosBackend:
|
||||
route=self.config.get('planning_context',{}).get('execution_route')
|
||||
if route in ('OBJECT_TABLE','SHELF_CELL'):
|
||||
goal.constraints_json=canonical({'schema_version':2,'route':route,'max_items':20,'execution':'sequential','robot_id':task['robot_id']})
|
||||
if task.get('clarification_confirmed') is True and task['task_revision'] > 1:
|
||||
constraints = strict_json(goal.constraints_json)
|
||||
constraints['clarification_confirmed'] = True
|
||||
goal.constraints_json = canonical(constraints)
|
||||
self._duration(goal.timeout, self.config.get('planning_timeout', 25))
|
||||
future = self._planner.send_goal_async(goal)
|
||||
future.add_done_callback(lambda completed: self._plan_accepted(key, completed))
|
||||
|
||||
def _plan_accepted(self, key, future):
|
||||
with self._lock:
|
||||
rec = self._planning[key]
|
||||
rec = self._planning.get(key)
|
||||
if rec is None:
|
||||
try:
|
||||
handle = future.result()
|
||||
if handle and handle.accepted:
|
||||
handle.cancel_goal_async()
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
try:
|
||||
handle = future.result()
|
||||
if not handle or not handle.accepted:
|
||||
@@ -147,12 +165,14 @@ class RosBackend:
|
||||
|
||||
def _plan_result(self, key, future):
|
||||
with self._lock:
|
||||
rec = self._planning[key]
|
||||
if rec['done']:
|
||||
rec = self._planning.get(key)
|
||||
if rec is None or rec['done']:
|
||||
return
|
||||
rec['done'] = True
|
||||
record_ref = ''
|
||||
try:
|
||||
wrapped = future.result()
|
||||
record_ref = getattr(wrapped.result, 'planning_record_ref', '')
|
||||
if wrapped.status != 4:
|
||||
raise ValueError('planning native result was not SUCCEEDED')
|
||||
result = wrapped.result
|
||||
@@ -168,11 +188,12 @@ class RosBackend:
|
||||
raise ValueError('clarification result has no structured questions')
|
||||
self._emit(self._planning_event(rec, 'NEEDS_CLARIFICATION', questions=questions, planning_record_ref=getattr(result,'planning_record_ref','')))
|
||||
elif result.status == 2:
|
||||
self._emit(self._planning_event(rec, 'FAILED', error_code=result.error_code, error=result.message))
|
||||
self._emit(self._planning_event(rec, 'FAILED', error_code=result.error_code, error=result.message,
|
||||
planning_record_ref=getattr(result, 'planning_record_ref', '')))
|
||||
else:
|
||||
raise ValueError('unknown planning status')
|
||||
except Exception as exc:
|
||||
self._emit(self._planning_event(rec, 'FAILED', error_code='PLANNER_PROTOCOL_ERROR', error=str(exc)))
|
||||
self._emit(self._planning_event(rec, 'FAILED', error_code='PLANNER_PROTOCOL_ERROR', error=str(exc), planning_record_ref=record_ref))
|
||||
|
||||
def start_execution(self, task_dict):
|
||||
with self._lock:
|
||||
@@ -244,7 +265,15 @@ class RosBackend:
|
||||
|
||||
def _execution_accepted(self, key, future):
|
||||
with self._lock:
|
||||
rec = self._runs[key]
|
||||
rec = self._runs.get(key)
|
||||
if rec is None:
|
||||
try:
|
||||
handle = future.result()
|
||||
if handle and handle.accepted:
|
||||
handle.cancel_goal_async()
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
try:
|
||||
handle = future.result()
|
||||
if not handle or not handle.accepted:
|
||||
@@ -261,8 +290,8 @@ class RosBackend:
|
||||
|
||||
def _feedback(self, key, wrapped):
|
||||
with self._lock:
|
||||
rec = self._runs[key]
|
||||
if rec['done']:
|
||||
rec = self._runs.get(key)
|
||||
if rec is None or rec['done']:
|
||||
return
|
||||
msg = wrapped.feedback
|
||||
at = msg.stamp.sec * 1_000_000_000 + msg.stamp.nanosec
|
||||
@@ -278,12 +307,12 @@ class RosBackend:
|
||||
return
|
||||
rec['sequence'], rec['last_feedback'] = int(msg.sequence), time.monotonic()
|
||||
self._emit(dict(type='progress', task_id=key[0], run_id=key[1],
|
||||
sequence=int(msg.sequence), stage=msg.stage))
|
||||
sequence=int(msg.sequence), stage=msg.stage, detail=status))
|
||||
|
||||
def _execution_result(self, key, future):
|
||||
with self._lock:
|
||||
rec = self._runs[key]
|
||||
if rec['done']:
|
||||
rec = self._runs.get(key)
|
||||
if rec is None or rec['done']:
|
||||
return
|
||||
try:
|
||||
wrapped = future.result()
|
||||
@@ -349,6 +378,54 @@ class RosBackend:
|
||||
|
||||
pause = cancel
|
||||
|
||||
def _trim_terminal(self):
|
||||
limit = getattr(self, '_terminal_retention', 256)
|
||||
for records in (self._planning, self._runs):
|
||||
terminal = [key for key, rec in records.items() if rec['done']]
|
||||
for key in terminal[:-limit]:
|
||||
del records[key]
|
||||
|
||||
def reconcile(self, task, request):
|
||||
token = self.config.get('recovery_token', '')
|
||||
operator = self.config.get('recovery_operator_id', '')
|
||||
if not isinstance(token, str) or len(token) < 16 or not isinstance(operator, str) or not operator:
|
||||
raise ValueError('recovery credentials are not configured')
|
||||
timeout = float(self.config.get('recovery_timeout', 8))
|
||||
if not math.isfinite(timeout) or not 0 < timeout <= 30:
|
||||
raise ValueError('invalid recovery timeout')
|
||||
if not self._recovery.service_is_ready():
|
||||
raise RuntimeError('recovery service unavailable')
|
||||
message = self._ReconcileTask.Request()
|
||||
message.trace.task_id = task['task_id']
|
||||
message.trace.run_id = task['run_id']
|
||||
message.trace.subtask_id = 'execute_task'
|
||||
message.trace.attempt = 1
|
||||
message.trace.task_revision = task['task_revision']
|
||||
message.trace.plan_version = task['plan']['plan_version']
|
||||
message.trace.execution_generation = task.get('execution_generation', task['planning_generation'])
|
||||
message.operator_id, message.authorization = operator, token
|
||||
message.evidence_ref = request['evidence_ref']
|
||||
message.resolution = request['resolution']
|
||||
future = self._recovery.call_async(message)
|
||||
complete = threading.Event()
|
||||
future.add_done_callback(lambda _: complete.set())
|
||||
if not complete.wait(timeout):
|
||||
future.cancel()
|
||||
raise TimeoutError('recovery outcome unknown; retry with same evidence reference')
|
||||
response = future.result()
|
||||
if response is None or not response.accepted:
|
||||
raise RuntimeError('recovery rejected: ' + (response.error_code if response else 'NO_RESPONSE'))
|
||||
result = strict_json(response.state_json)
|
||||
if (not isinstance(result, dict) or result.get('verified') is not True or
|
||||
result.get('stop_confirmed') is not True or result.get('holding_state') != 'EMPTY' or
|
||||
result.get('run_id') != task['run_id'] or result.get('evidence_ref') != request['evidence_ref']):
|
||||
raise ValueError('recovery response identity or verification mismatch')
|
||||
with self._lock:
|
||||
rec = self._runs.get((task['task_id'], task['run_id']))
|
||||
if rec is not None:
|
||||
rec['done'] = True
|
||||
return result
|
||||
|
||||
def poll(self):
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
@@ -369,6 +446,7 @@ class RosBackend:
|
||||
self._unknown(rec, 'STOP_TIMEOUT')
|
||||
events = list(self._events)
|
||||
self._events.clear()
|
||||
self._trim_terminal()
|
||||
return events
|
||||
|
||||
def close(self):
|
||||
@@ -391,6 +469,7 @@ class RosBackend:
|
||||
self._executor.remove_node(self.node)
|
||||
self._planner.destroy()
|
||||
self._execution.destroy()
|
||||
self.node.destroy_client(self._recovery)
|
||||
if self._own_node:
|
||||
self.node.destroy_node()
|
||||
self._context.shutdown()
|
||||
|
||||
@@ -9,6 +9,7 @@ from .errors import ApiError
|
||||
from .plan import canonical, text, validate_known, validate_plan
|
||||
from .store import Store
|
||||
from .plan_v2 import instances, item_plan
|
||||
from .intent import conflicts_with_instruction
|
||||
|
||||
TERMINAL = {'SUCCEEDED','FAILED','CANCELED','EXPIRED'}
|
||||
EXECUTION = {'READY','EXECUTING','PAUSING','CANCELING','INTERVENTION_REQUIRED'}
|
||||
@@ -30,7 +31,7 @@ class Coordinator:
|
||||
self.clock=clock;self.queue_timeout=queue_timeout;self.closed=False
|
||||
self.steady=steady;self.planning_timeout=planning_timeout;self._planning_started={}
|
||||
with self.lock,self.store.db:
|
||||
for t in self.store.all():
|
||||
for t in self.store.active():
|
||||
if t['status'] in EXECUTION or (t['status']=='PAUSED' and t.get('motion_dispatched')):
|
||||
self._set(t,'INTERVENTION_REQUIRED','RESTART_RECONCILIATION_REQUIRED')
|
||||
elif t['status']=='PLANNING':
|
||||
@@ -45,9 +46,19 @@ class Coordinator:
|
||||
if t is None:raise ApiError('NOT_FOUND','task not found',404)
|
||||
return t
|
||||
def _set(self,t,status,error='',kind='state'):
|
||||
if status!='PLANNING':self._planning_started.pop(t['task_id'],None)
|
||||
t['status']=status;t['error_code']=error;t['updated_at']=self.clock();t['status_version']+=1
|
||||
self.store.put(t)
|
||||
self.store.event(t,kind,{'status':status,'status_version':t['status_version'],'error_code':error},self.clock())
|
||||
event={'status':status,'status_version':t['status_version'],'error_code':error}
|
||||
if kind=='progress':event.update(run_id=t['run_id'],stage=t['stage'],sequence=t['last_sequence'],detail=t.get('detail',''))
|
||||
self.store.event(t,kind,event,self.clock())
|
||||
def _intent_question(self,t):
|
||||
t['question']={'question_id':str(uuid.uuid4()),'task_revision':t['task_revision'],'questions':['The instruction and structured item, quantity, source or destination disagree. Confirm the intended values.']}
|
||||
self._set(t,'NEEDS_CLARIFICATION','INTENT_MISMATCH')
|
||||
def _runtime_question(self,t,questions):
|
||||
t['question']={'question_id':str(uuid.uuid4()),'task_revision':t['task_revision'],'source_run_id':t['run_id'],'questions':questions}
|
||||
t['motion_dispatched']=False;t['stop_confirmed']=True
|
||||
self._set(t,'NEEDS_CLARIFICATION','EXECUTION_CLARIFICATION_REQUIRED')
|
||||
def get(self,tid):
|
||||
with self.lock:return self._task(tid)
|
||||
def events(self,tid,after=0,limit=100):
|
||||
@@ -74,8 +85,11 @@ class Coordinator:
|
||||
return dict(json.loads(existing[1]),deduplicated=True)
|
||||
now=self.clock()
|
||||
t=dict(task_id=str(uuid.uuid4()),robot_id=body['robot_id'],request=body,task_revision=1,planning_generation=0,planning_attempts=0,plan_version=0,run_id='',status='RECEIVED',status_version=0,accepted_at=now,updated_at=now,stage='',error_code='',completed_quantity=0,plan=None,context=None,question=None,motion_dispatched=False,stop_confirmed=True,last_sequence=0)
|
||||
t['clarification_confirmed']=False
|
||||
self.store.db.execute('INSERT INTO tasks(task_id,robot_id,request_id,request_hash,data) VALUES(?,?,?,?,?)',(t['task_id'],t['robot_id'],body['client_request_id'],fingerprint,canonical(t)))
|
||||
self._set(t,'QUEUED');return dict(t,deduplicated=False)
|
||||
if conflicts_with_instruction(body['instruction'],info,self.site):self._intent_question(t)
|
||||
else:self._set(t,'QUEUED')
|
||||
return dict(t,deduplicated=False)
|
||||
def control(self,tid,action):
|
||||
if action not in {'cancel','pause','resume'}:raise ApiError('INVALID_CONTROL','unsupported control')
|
||||
with self.lock:
|
||||
@@ -83,10 +97,10 @@ class Coordinator:
|
||||
if t['status'] in TERMINAL:return t
|
||||
if action=='resume':
|
||||
if t['status']!='PAUSED':raise ApiError('INVALID_STATE','task is not paused',409)
|
||||
if t['motion_dispatched']:
|
||||
raise ApiError('RECONCILIATION_REQUIRED','operator reconciliation with fresh physical evidence is required',409)
|
||||
with self.store.db:
|
||||
if t['motion_dispatched']:
|
||||
self._set(t,'INTERVENTION_REQUIRED','RESUME_REQUIRES_PHYSICAL_RECHECK')
|
||||
else:self._set(t,'QUEUED')
|
||||
self._set(t,'QUEUED')
|
||||
return t
|
||||
if t['status']=='INTERVENTION_REQUIRED':
|
||||
# A second cancel is useful, but never removes the physical quarantine.
|
||||
@@ -116,26 +130,64 @@ class Coordinator:
|
||||
if t['status']!='NEEDS_CLARIFICATION' or not t['question'] or answer['question_id']!=t['question']['question_id'] or answer['task_revision']!=t['task_revision']:
|
||||
raise ApiError('STALE_ANSWER','answer does not match current question/revision',409)
|
||||
info=validate_known(answer['known_info'])
|
||||
if not info:raise ApiError('INVALID_ANSWER','explicit confirmed information is required')
|
||||
if 'items' in info:
|
||||
previous=t['request']['known_info'];t['request']['known_info']={k:v for k,v in previous.items() if k=='destination'}
|
||||
elif 'items' in t['request']['known_info'] and set(info)-{'destination'}:raise ApiError('INVALID_ANSWER','use items representation for multi-item clarification')
|
||||
t['request']['known_info'].update(info);t['task_revision']+=1;t['planning_generation']+=1;t['planning_attempts']=0;t['question']=None
|
||||
t['clarification_confirmed']=True
|
||||
self._set(t,'QUEUED');return t
|
||||
def intervene(self,tid,data):
|
||||
"""Reconciliation delegates evidence validation to a trusted physical backend."""
|
||||
if not isinstance(data,dict) or set(data)!={'run_id','evidence_ref','resolution'} or data['resolution']!='cancel_task':
|
||||
raise ApiError('INVALID_RECONCILIATION','run_id, evidence_ref, resolution=cancel_task required')
|
||||
if not isinstance(data,dict) or set(data)!={'run_id','evidence_ref','resolution'} or data['resolution'] not in {'cancel_task','resume_task','replan_task'}:
|
||||
raise ApiError('INVALID_RECONCILIATION','run_id, evidence_ref, resolution=cancel_task/resume_task/replan_task required')
|
||||
text(data['evidence_ref'],'evidence_ref',512)
|
||||
with self.lock:
|
||||
t=self._task(tid)
|
||||
if t['status']!='INTERVENTION_REQUIRED' or data['run_id']!=t['run_id']:
|
||||
if t['status'] not in {'INTERVENTION_REQUIRED','PAUSED'} or not t['motion_dispatched'] or data['run_id']!=t['run_id']:
|
||||
raise ApiError('STALE_RECONCILIATION','task/run does not require this reconciliation',409)
|
||||
if not hasattr(self.backend,'reconcile'):raise ApiError('RECONCILIATION_UNAVAILABLE','trusted physical verification backend is required',503)
|
||||
verified=self.backend.reconcile(t,data)
|
||||
if not isinstance(verified,dict) or verified.get('stop_confirmed') is not True or verified.get('holding')!='EMPTY' or verified.get('run_id')!=t['run_id'] or verified.get('evidence_ref')!=data['evidence_ref']:
|
||||
if not isinstance(verified,dict) or verified.get('verified') is not True or verified.get('stop_confirmed') is not True or verified.get('holding_state')!='EMPTY' or verified.get('run_id')!=t['run_id'] or verified.get('evidence_ref')!=data['evidence_ref']:
|
||||
raise ApiError('RECONCILIATION_REJECTED','matching stop and empty-hand evidence required',409)
|
||||
receipts=verified.get('receipts')
|
||||
if not isinstance(receipts,list) or len(receipts)>20:
|
||||
raise ApiError('RECONCILIATION_REJECTED','bounded receipt list required',409)
|
||||
approved=instances(t['plan']);pending={}
|
||||
existing={row['item_index']:json.loads(row['evidence']) for row in self.store.db.execute('SELECT item_index,evidence FROM deliveries WHERE task_id=?',(tid,))}
|
||||
for receipt in receipts:
|
||||
if not isinstance(receipt,dict):raise ApiError('RECONCILIATION_REJECTED','invalid receipt',409)
|
||||
index=receipt.get('item_index');quantity=receipt.get('completed_quantity',1);evidence=receipt.get('evidence')
|
||||
if receipt.get('task_id')!=tid or type(index) is not int or not 0<=index<len(approved) or index>t.get('active_item_index',0) or type(quantity) is not int or quantity!=1:
|
||||
raise ApiError('RECONCILIATION_REJECTED','receipt task/item/quantity mismatch',409)
|
||||
expected=approved[index];destination=self.site['destinations'][expected['destination']]['region_ref']
|
||||
if not isinstance(evidence,dict) or any(evidence.get(k) is not True for k in ('passed','empty_hand','in_destination','valid')) or not isinstance(evidence.get('evidence_id'),str) or not evidence['evidence_id'] or evidence.get('target_ref')!=expected['target_name'] or evidence.get('destination_ref')!=destination:
|
||||
raise ApiError('RECONCILIATION_REJECTED','receipt delivery proof invalid',409)
|
||||
previous=pending.get(index,existing.get(index))
|
||||
if previous is not None and any(previous.get(k)!=evidence.get(k) for k in ('evidence_id','target_ref','destination_ref','passed','empty_hand','in_destination','valid')):
|
||||
raise ApiError('RECONCILIATION_REJECTED','receipt conflicts with committed delivery',409)
|
||||
pending[index]=evidence
|
||||
credited=set(existing)|set(pending)
|
||||
if credited!=set(range(len(credited))):raise ApiError('RECONCILIATION_REJECTED','delivery ledger has an item gap',409)
|
||||
remaining=next((i for i in range(len(approved)) if i not in credited),None)
|
||||
if data['resolution']=='resume_task' and remaining is not None and t.get('active_item_index',0) not in credited and verified.get('safe_to_retry') is not True:
|
||||
raise ApiError('RECONCILIATION_REJECTED','cannot replay item without proof no manipulation was dispatched',409)
|
||||
if data['resolution']=='replan_task' and (credited or verified.get('safe_to_retry') is not True):
|
||||
raise ApiError('RECONCILIATION_REJECTED','replanning requires zero deliveries and proof no manipulation was dispatched',409)
|
||||
with self.store.db:
|
||||
t['stop_confirmed']=True;self._set(t,'CANCELED','RECONCILED_BY_OPERATOR')
|
||||
for index,evidence in pending.items():
|
||||
if index not in existing:
|
||||
self.store.db.execute('INSERT INTO deliveries(task_id,item_index,evidence,created_at) VALUES(?,?,?,?)',(tid,index,canonical(evidence),self.clock()))
|
||||
self.store.event(t,'delivery_committed',{'item_index':index,'evidence':evidence},self.clock())
|
||||
t['completed_quantity']=len(credited);t['stop_confirmed']=True
|
||||
if pending:t['delivery_evidence']=pending[max(pending)]
|
||||
self.store.event(t,'reconciliation',{'run_id':t['run_id'],'resolution':data['resolution'],'evidence_ref':data['evidence_ref'],'credited_items':sorted(credited)},self.clock())
|
||||
if data['resolution']=='cancel_task':self._set(t,'CANCELED','RECONCILED_BY_OPERATOR')
|
||||
elif data['resolution']=='replan_task':self._runtime_question(t,['Confirm the item, quantity, source and destination before a new plan is requested.'])
|
||||
elif remaining is None:self._set(t,'SUCCEEDED')
|
||||
else:
|
||||
t['active_item_index']=remaining;t['execution_generation']=t.get('execution_generation',t['planning_generation'])+1
|
||||
self._dispatch_item(t)
|
||||
return t
|
||||
def tick(self):
|
||||
with self.lock:
|
||||
@@ -144,10 +196,10 @@ class Coordinator:
|
||||
t=self.store.get(e['task_id'])
|
||||
if not t:continue
|
||||
with self.store.db:self._event(t,e)
|
||||
for waiting in self.store.all():
|
||||
for waiting in self.store.active():
|
||||
if waiting['status']=='PLANNING' and self.steady()-self._planning_started.get(waiting['task_id'],self.steady())>=self.planning_timeout:
|
||||
with self.store.db:self._planning_failure(waiting,'PLANNING_TIMEOUT')
|
||||
all_tasks=self.store.all()
|
||||
all_tasks=self.store.active()
|
||||
for robot in sorted(self.robots):
|
||||
tasks=[t for t in all_tasks if t['robot_id']==robot]
|
||||
if any(t['status'] not in TERMINAL|{'QUEUED'} for t in tasks):continue
|
||||
@@ -186,6 +238,8 @@ class Coordinator:
|
||||
if p['subtasks'][0]['skill']=='ASK_USER':
|
||||
t['question']={'question_id':str(uuid.uuid4()),'task_revision':t['task_revision'],'questions':[p['subtasks'][0]['arguments']['question']]}
|
||||
self._set(t,'NEEDS_CLARIFICATION');return
|
||||
if conflicts_with_instruction(t['request']['instruction'],p['slots'],self.site,clarification_confirmed=t.get('clarification_confirmed') is True):
|
||||
self._intent_question(t);return
|
||||
if self.site.get('execution_route') and p['schema_version']!=2:raise ApiError('SCHEMA_MISMATCH','v2 deployment requires v2 planner')
|
||||
slots=p['slots'];known=t['request']['known_info'];compare=slots
|
||||
if p['schema_version']==2 and 'items' not in known and known:
|
||||
@@ -210,6 +264,7 @@ class Coordinator:
|
||||
seq=e.get('sequence')
|
||||
if type(seq) is not int or seq<=t['last_sequence'] or not isinstance(e.get('stage'),str):return
|
||||
t['last_sequence']=seq;t['stage']=e['stage'][:100]
|
||||
t['detail']=e.get('detail','')[:2000] if isinstance(e.get('detail',''),str) else ''
|
||||
self._set(t,t['status'],t['error_code'],'progress');return
|
||||
self.store.event(t,'execution_result',{'payload':e},self.clock())
|
||||
ev=e.get('evidence',{});quantity=e.get('completed_quantity',0)
|
||||
@@ -228,6 +283,13 @@ class Coordinator:
|
||||
if e.get('stop_confirmed') is not True:
|
||||
self._set(t,'INTERVENTION_REQUIRED','STOP_UNKNOWN');return
|
||||
t['stop_confirmed']=True
|
||||
if isinstance(ev,dict) and ev.get('needs_clarification') is True and t['status']=='EXECUTING':
|
||||
questions=ev.get('questions')
|
||||
valid_questions=isinstance(questions,list) and 1<=len(questions)<=8 and all(isinstance(q,str) and q.strip() and len(q)<=500 for q in questions)
|
||||
safe=all(ev.get(k) is True for k in ('safe_to_retry','empty_hand','valid','safe_to_release'))
|
||||
if quantity!=0 or t['completed_quantity']!=0 or not safe or not valid_questions:
|
||||
self._set(t,'INTERVENTION_REQUIRED','CLARIFICATION_REQUIRES_PHYSICAL_RECHECK');return
|
||||
self._runtime_question(t,questions);return
|
||||
if e.get('status')=='SUCCEEDED':
|
||||
if quantity!=1:
|
||||
self._set(t,'INTERVENTION_REQUIRED','DELIVERY_EVIDENCE_INVALID');return
|
||||
|
||||
@@ -24,13 +24,24 @@ class Store:
|
||||
CREATE INDEX IF NOT EXISTS events_task_cursor ON events(task_id,event_id);
|
||||
CREATE TABLE IF NOT EXISTS deliveries(task_id TEXT NOT NULL REFERENCES tasks(task_id),item_index INTEGER NOT NULL,evidence TEXT NOT NULL,created_at REAL NOT NULL,PRIMARY KEY(task_id,item_index));
|
||||
''')
|
||||
# Migrate old databases once; terminal history remains queryable without
|
||||
# being decoded on every scheduler tick.
|
||||
columns={row[1] for row in self.db.execute('PRAGMA table_info(tasks)')}
|
||||
with self.db:
|
||||
if 'status' not in columns:
|
||||
self.db.execute("ALTER TABLE tasks ADD COLUMN status TEXT NOT NULL DEFAULT 'RECEIVED'")
|
||||
for row in self.db.execute('SELECT task_id,data FROM tasks').fetchall():
|
||||
self.db.execute('UPDATE tasks SET status=? WHERE task_id=?',(json.loads(row['data'])['status'],row['task_id']))
|
||||
self.db.execute("CREATE INDEX IF NOT EXISTS tasks_active_seq ON tasks(seq) WHERE status NOT IN ('SUCCEEDED','FAILED','CANCELED','EXPIRED')")
|
||||
def get(self,tid):
|
||||
r=self.db.execute('SELECT data FROM tasks WHERE task_id=?',(tid,)).fetchone()
|
||||
return json.loads(r[0]) if r else None
|
||||
def all(self):
|
||||
return [json.loads(r[0]) for r in self.db.execute('SELECT data FROM tasks ORDER BY seq')]
|
||||
def active(self):
|
||||
return [json.loads(r[0]) for r in self.db.execute("SELECT data FROM tasks WHERE status NOT IN ('SUCCEEDED','FAILED','CANCELED','EXPIRED') ORDER BY seq")]
|
||||
def put(self,t):
|
||||
self.db.execute('UPDATE tasks SET data=? WHERE task_id=?',(canonical(t),t['task_id']))
|
||||
self.db.execute('UPDATE tasks SET data=?,status=? WHERE task_id=?',(canonical(t),t['status'],t['task_id']))
|
||||
def event(self,t,kind,data,now):
|
||||
self.db.execute('INSERT INTO events(task_id,kind,created_at,data) VALUES(?,?,?,?)',(t['task_id'],kind,now,canonical(data)))
|
||||
def events(self,tid,after,limit):
|
||||
|
||||
Reference in New Issue
Block a user