fix: harden task recovery and DR contract handling

This commit is contained in:
2026-09-20 13:36:48 +08:00
parent 492676344a
commit f9d8feb6f0
49 changed files with 2083 additions and 165 deletions
+4 -1
View File
@@ -27,7 +27,10 @@ def main():
backend=BrainDemoBackend(args.demo_executable,state/'simulator',site)
else:
from .ros_backend import RosBackend
backend=RosBackend(namespace=args.ros_namespace,config={'planning_context':site,'dense_progress_enabled':args.dense_progress})
backend=RosBackend(namespace=args.ros_namespace,config={'planning_context':site,
'dense_progress_enabled':args.dense_progress,
'recovery_token':os.environ.get('ROBOT_BT_RECOVERY_TOKEN',''),
'recovery_operator_id':os.environ.get('ROBOT_BT_RECOVERY_OPERATOR','')})
if args.backend=='mock':backend.route=site.get('execution_route','OBJECT_TABLE')
coordinator=Coordinator(str(state/'tasks.sqlite3'),backend,{args.robot_id},site)
server=make_server(coordinator,args.host,args.port,token,os.environ.get('ROBOT_BT_OPERATOR_TOKEN',''))
+13 -2
View File
@@ -8,6 +8,12 @@ from .plan import strict_json
MAX_BODY=65536
class TaskHTTPServer(ThreadingHTTPServer):
# Accept a burst of task submissions without the default five-slot listen
# queue forcing clients into a TCP retransmission before admission.
request_queue_size=64
def make_server(coordinator,host,port,api_token,operator_token=''):
if not api_token:raise ValueError('API token must be explicitly configured')
class Handler(BaseHTTPRequestHandler):
@@ -60,7 +66,12 @@ def make_server(coordinator,host,port,api_token,operator_token=''):
if data:raise ApiError('INVALID_CONTROL','control body must be empty')
return 202,coordinator.control(tid,action)
if action=='clarifications':return 202,coordinator.clarify(tid,data)
if action=='interventions':return 202,coordinator.intervene(tid,data)
if action=='interventions':
try:return 202,coordinator.intervene(tid,data)
except TimeoutError:
raise ApiError('RECONCILIATION_TIMEOUT','recovery outcome is unknown; retry with the same evidence reference',504) from None
except RuntimeError:
raise ApiError('RECONCILIATION_UNAVAILABLE','recovery service is unavailable or rejected the request; task state remains queryable',503) from None
raise ApiError('NOT_FOUND','endpoint not found',404)
def handle_request(self):
try:status,payload=self.dispatch();self.reply(status,payload)
@@ -70,5 +81,5 @@ def make_server(coordinator,host,port,api_token,operator_token=''):
self.reply(500,{'error_code':'INTERNAL_ERROR','message':'request failed; task state remains queryable'})
do_GET=handle_request
do_POST=handle_request
server=ThreadingHTTPServer((host,port),Handler);server.daemon_threads=True
server=TaskHTTPServer((host,port),Handler);server.daemon_threads=True
return server
@@ -0,0 +1,49 @@
"""Conservative independent intent gate for instructions without confirmed slots.
Only registered aliases and explicit quantities are accepted. Other language asks
for structured clarification; this parser is not claimed to cover arbitrary NLP.
"""
import re
COUNTS={'一':1,'二':2,'两':2,'三':3,'四':4,'五':5,'六':6,'七':7,'八':8,'九':9,'十':10,'one':1,'two':2,'three':3}
def matches(instruction,registry,aliases):
candidates=[]
for key in registry:
for term in [key]+list(aliases.get(key,[])):
if not term:continue
for m in re.finditer(re.escape(term),instruction):candidates.append((m.start(),m.end(),key))
chosen=[]
for item in sorted(candidates,key=lambda x:(-(x[1]-x[0]),x[0])):
if not any(item[0]<q[1] and q[0]<item[1] for q in chosen):chosen.append(item)
return sorted(chosen)
def extract(instruction,site):
names=site.get('object_locations',{}) or site.get('object_aliases',{})
objects=matches(instruction,names,site.get('object_aliases',{}))
sources=matches(instruction,site.get('sources',{}),site.get('source_aliases',{}))
destinations=matches(instruction,site.get('destinations',{}),site.get('destination_aliases',{}))
if not objects or len({x[2] for x in sources})!=1 or len({x[2] for x in destinations})!=1:return None
items=[]
for start,end,name in objects:
m=re.search(r'(\d+|一|二|两|三|四|五|六|七|八|九|十|one|two|three)\s*(?:瓶|个|件|盒|袋)?\s*$',instruction[:start])
if not m:return None
count=COUNTS.get(m[1],int(m[1]) if m[1].isdigit() else 0)
if not 1<=count<=20:return None
items.append(dict(target_name=name,quantity=count,source_location=sources[0][2]))
return dict(items=items,destination=destinations[0][2])
def conflicts_with_instruction(instruction, known, site, *, clarification_confirmed=False):
"""Compare recognized explicit intent with validated structured slots.
False means no *detected* conflict, not proof of arbitrary-language fidelity.
The caller must derive clarification_confirmed from trusted clarification
history; neither a client-supplied flag nor a revision counter is authority.
"""
if clarification_confirmed is True:
return False
expected = extract(instruction, site)
if expected is None:
return False
if 'items' not in known:
if set(known) - {'destination'}:
if len(expected['items']) != 1:
return True
expected = dict(expected['items'][0], destination=expected['destination'])
return any(expected.get(key) != value for key, value in known.items())
+5 -2
View File
@@ -11,7 +11,7 @@ from .plan import validate_plan
from .plan_v2 import instances
def replay_events(events):
status='RECEIVED';version=0;cursor=0;delivered={};stages=[];plans=[];errors=[]
status='RECEIVED';version=0;cursor=0;delivered={};stages=[];plans=[];errors=[];progress=[];reconciliations=[]
for e in events:
if type(e['event_id']) is not int or e['event_id']<=cursor:raise ValueError('non-monotonic event cursor')
cursor=e['event_id']
@@ -19,6 +19,7 @@ def replay_events(events):
if e['status_version']!=version+1:raise ValueError('missing or non-monotonic status version')
version=e['status_version'];status=e['status']
if e.get('error_code'):errors.append(e['error_code'])
if e['kind']=='progress':progress.append({k:e[k] for k in ('event_id','run_id','stage','sequence','detail') if k in e})
elif e['kind']=='plan_approved':
plans.append({'run_id':e['run_id'],'plan':validate_plan(e['plan'])})
elif e['kind']=='delivery_committed':
@@ -27,8 +28,10 @@ def replay_events(events):
delivered[key]=e['evidence']
elif e['kind']=='execution_result':
payload=e['payload'];stages.append({'run_id':payload.get('run_id'),'status':payload.get('status'),'stop_confirmed':payload.get('stop_confirmed')})
elif e['kind']=='reconciliation':
reconciliations.append({k:e[k] for k in ('event_id','run_id','resolution','evidence_ref','credited_items') if k in e})
if status=='SUCCEEDED' and (not plans or set(delivered)!=set(range(len(instances(plans[-1]['plan']))))):raise ValueError('success without committed delivery')
return {'mode':'offline_audit_replay','status':status,'status_version':version,'last_event_id':cursor,'completed_quantity':len(delivered),'plans':plans,'results':stages,'errors':errors,'connects_to_robot':False}
return {'mode':'offline_audit_replay','status':status,'status_version':version,'last_event_id':cursor,'completed_quantity':len(delivered),'plans':plans,'results':stages,'progress':progress,'reconciliations':reconciliations,'errors':errors,'connects_to_robot':False}
def main():
p=argparse.ArgumentParser();p.add_argument('--db',required=True);p.add_argument('--task-id',required=True);args=p.parse_args()
+90 -11
View File
@@ -27,6 +27,7 @@ class RosBackend:
from rclpy.context import Context
from rclpy.executors import SingleThreadedExecutor
from bt_skill_interfaces.action import PlanTask, ExecuteTask
from bt_skill_interfaces.srv import ReconcileTask
self.config = dict(config or {})
if namespace is not None:
@@ -50,11 +51,16 @@ class RosBackend:
self._PlanTask, self._ExecuteTask = PlanTask, ExecuteTask
self._planner = ActionClient(node, PlanTask, self.config.get('plan_action', 'tasks/plan'))
self._execution = ActionClient(node, ExecuteTask, self.config.get('execute_action', 'tasks/execute'))
self._ReconcileTask = ReconcileTask
self._recovery = node.create_client(ReconcileTask, self.config.get('reconcile_service', 'tasks/reconcile'))
self._events = deque()
self._planning = {}
self._runs = {}
self._lock = threading.RLock()
self._closed = False
self._terminal_retention = int(self.config.get('terminal_retention', 256))
if self._terminal_retention < 1:
raise ValueError('terminal_retention must be positive')
self._acceptance_timeout = float(self.config.get('acceptance_timeout', 3.0))
self._feedback_timeout = float(self.config.get('feedback_timeout', 5.0))
self._stop_timeout = float(self.config.get('stop_timeout', 7.0))
@@ -124,13 +130,25 @@ class RosBackend:
route=self.config.get('planning_context',{}).get('execution_route')
if route in ('OBJECT_TABLE','SHELF_CELL'):
goal.constraints_json=canonical({'schema_version':2,'route':route,'max_items':20,'execution':'sequential','robot_id':task['robot_id']})
if task.get('clarification_confirmed') is True and task['task_revision'] > 1:
constraints = strict_json(goal.constraints_json)
constraints['clarification_confirmed'] = True
goal.constraints_json = canonical(constraints)
self._duration(goal.timeout, self.config.get('planning_timeout', 25))
future = self._planner.send_goal_async(goal)
future.add_done_callback(lambda completed: self._plan_accepted(key, completed))
def _plan_accepted(self, key, future):
with self._lock:
rec = self._planning[key]
rec = self._planning.get(key)
if rec is None:
try:
handle = future.result()
if handle and handle.accepted:
handle.cancel_goal_async()
except Exception:
pass
return
try:
handle = future.result()
if not handle or not handle.accepted:
@@ -147,12 +165,14 @@ class RosBackend:
def _plan_result(self, key, future):
with self._lock:
rec = self._planning[key]
if rec['done']:
rec = self._planning.get(key)
if rec is None or rec['done']:
return
rec['done'] = True
record_ref = ''
try:
wrapped = future.result()
record_ref = getattr(wrapped.result, 'planning_record_ref', '')
if wrapped.status != 4:
raise ValueError('planning native result was not SUCCEEDED')
result = wrapped.result
@@ -168,11 +188,12 @@ class RosBackend:
raise ValueError('clarification result has no structured questions')
self._emit(self._planning_event(rec, 'NEEDS_CLARIFICATION', questions=questions, planning_record_ref=getattr(result,'planning_record_ref','')))
elif result.status == 2:
self._emit(self._planning_event(rec, 'FAILED', error_code=result.error_code, error=result.message))
self._emit(self._planning_event(rec, 'FAILED', error_code=result.error_code, error=result.message,
planning_record_ref=getattr(result, 'planning_record_ref', '')))
else:
raise ValueError('unknown planning status')
except Exception as exc:
self._emit(self._planning_event(rec, 'FAILED', error_code='PLANNER_PROTOCOL_ERROR', error=str(exc)))
self._emit(self._planning_event(rec, 'FAILED', error_code='PLANNER_PROTOCOL_ERROR', error=str(exc), planning_record_ref=record_ref))
def start_execution(self, task_dict):
with self._lock:
@@ -244,7 +265,15 @@ class RosBackend:
def _execution_accepted(self, key, future):
with self._lock:
rec = self._runs[key]
rec = self._runs.get(key)
if rec is None:
try:
handle = future.result()
if handle and handle.accepted:
handle.cancel_goal_async()
except Exception:
pass
return
try:
handle = future.result()
if not handle or not handle.accepted:
@@ -261,8 +290,8 @@ class RosBackend:
def _feedback(self, key, wrapped):
with self._lock:
rec = self._runs[key]
if rec['done']:
rec = self._runs.get(key)
if rec is None or rec['done']:
return
msg = wrapped.feedback
at = msg.stamp.sec * 1_000_000_000 + msg.stamp.nanosec
@@ -278,12 +307,12 @@ class RosBackend:
return
rec['sequence'], rec['last_feedback'] = int(msg.sequence), time.monotonic()
self._emit(dict(type='progress', task_id=key[0], run_id=key[1],
sequence=int(msg.sequence), stage=msg.stage))
sequence=int(msg.sequence), stage=msg.stage, detail=status))
def _execution_result(self, key, future):
with self._lock:
rec = self._runs[key]
if rec['done']:
rec = self._runs.get(key)
if rec is None or rec['done']:
return
try:
wrapped = future.result()
@@ -349,6 +378,54 @@ class RosBackend:
pause = cancel
def _trim_terminal(self):
limit = getattr(self, '_terminal_retention', 256)
for records in (self._planning, self._runs):
terminal = [key for key, rec in records.items() if rec['done']]
for key in terminal[:-limit]:
del records[key]
def reconcile(self, task, request):
token = self.config.get('recovery_token', '')
operator = self.config.get('recovery_operator_id', '')
if not isinstance(token, str) or len(token) < 16 or not isinstance(operator, str) or not operator:
raise ValueError('recovery credentials are not configured')
timeout = float(self.config.get('recovery_timeout', 8))
if not math.isfinite(timeout) or not 0 < timeout <= 30:
raise ValueError('invalid recovery timeout')
if not self._recovery.service_is_ready():
raise RuntimeError('recovery service unavailable')
message = self._ReconcileTask.Request()
message.trace.task_id = task['task_id']
message.trace.run_id = task['run_id']
message.trace.subtask_id = 'execute_task'
message.trace.attempt = 1
message.trace.task_revision = task['task_revision']
message.trace.plan_version = task['plan']['plan_version']
message.trace.execution_generation = task.get('execution_generation', task['planning_generation'])
message.operator_id, message.authorization = operator, token
message.evidence_ref = request['evidence_ref']
message.resolution = request['resolution']
future = self._recovery.call_async(message)
complete = threading.Event()
future.add_done_callback(lambda _: complete.set())
if not complete.wait(timeout):
future.cancel()
raise TimeoutError('recovery outcome unknown; retry with same evidence reference')
response = future.result()
if response is None or not response.accepted:
raise RuntimeError('recovery rejected: ' + (response.error_code if response else 'NO_RESPONSE'))
result = strict_json(response.state_json)
if (not isinstance(result, dict) or result.get('verified') is not True or
result.get('stop_confirmed') is not True or result.get('holding_state') != 'EMPTY' or
result.get('run_id') != task['run_id'] or result.get('evidence_ref') != request['evidence_ref']):
raise ValueError('recovery response identity or verification mismatch')
with self._lock:
rec = self._runs.get((task['task_id'], task['run_id']))
if rec is not None:
rec['done'] = True
return result
def poll(self):
with self._lock:
now = time.monotonic()
@@ -369,6 +446,7 @@ class RosBackend:
self._unknown(rec, 'STOP_TIMEOUT')
events = list(self._events)
self._events.clear()
self._trim_terminal()
return events
def close(self):
@@ -391,6 +469,7 @@ class RosBackend:
self._executor.remove_node(self.node)
self._planner.destroy()
self._execution.destroy()
self.node.destroy_client(self._recovery)
if self._own_node:
self.node.destroy_node()
self._context.shutdown()
+75 -13
View File
@@ -9,6 +9,7 @@ from .errors import ApiError
from .plan import canonical, text, validate_known, validate_plan
from .store import Store
from .plan_v2 import instances, item_plan
from .intent import conflicts_with_instruction
TERMINAL = {'SUCCEEDED','FAILED','CANCELED','EXPIRED'}
EXECUTION = {'READY','EXECUTING','PAUSING','CANCELING','INTERVENTION_REQUIRED'}
@@ -30,7 +31,7 @@ class Coordinator:
self.clock=clock;self.queue_timeout=queue_timeout;self.closed=False
self.steady=steady;self.planning_timeout=planning_timeout;self._planning_started={}
with self.lock,self.store.db:
for t in self.store.all():
for t in self.store.active():
if t['status'] in EXECUTION or (t['status']=='PAUSED' and t.get('motion_dispatched')):
self._set(t,'INTERVENTION_REQUIRED','RESTART_RECONCILIATION_REQUIRED')
elif t['status']=='PLANNING':
@@ -45,9 +46,19 @@ class Coordinator:
if t is None:raise ApiError('NOT_FOUND','task not found',404)
return t
def _set(self,t,status,error='',kind='state'):
if status!='PLANNING':self._planning_started.pop(t['task_id'],None)
t['status']=status;t['error_code']=error;t['updated_at']=self.clock();t['status_version']+=1
self.store.put(t)
self.store.event(t,kind,{'status':status,'status_version':t['status_version'],'error_code':error},self.clock())
event={'status':status,'status_version':t['status_version'],'error_code':error}
if kind=='progress':event.update(run_id=t['run_id'],stage=t['stage'],sequence=t['last_sequence'],detail=t.get('detail',''))
self.store.event(t,kind,event,self.clock())
def _intent_question(self,t):
t['question']={'question_id':str(uuid.uuid4()),'task_revision':t['task_revision'],'questions':['The instruction and structured item, quantity, source or destination disagree. Confirm the intended values.']}
self._set(t,'NEEDS_CLARIFICATION','INTENT_MISMATCH')
def _runtime_question(self,t,questions):
t['question']={'question_id':str(uuid.uuid4()),'task_revision':t['task_revision'],'source_run_id':t['run_id'],'questions':questions}
t['motion_dispatched']=False;t['stop_confirmed']=True
self._set(t,'NEEDS_CLARIFICATION','EXECUTION_CLARIFICATION_REQUIRED')
def get(self,tid):
with self.lock:return self._task(tid)
def events(self,tid,after=0,limit=100):
@@ -74,8 +85,11 @@ class Coordinator:
return dict(json.loads(existing[1]),deduplicated=True)
now=self.clock()
t=dict(task_id=str(uuid.uuid4()),robot_id=body['robot_id'],request=body,task_revision=1,planning_generation=0,planning_attempts=0,plan_version=0,run_id='',status='RECEIVED',status_version=0,accepted_at=now,updated_at=now,stage='',error_code='',completed_quantity=0,plan=None,context=None,question=None,motion_dispatched=False,stop_confirmed=True,last_sequence=0)
t['clarification_confirmed']=False
self.store.db.execute('INSERT INTO tasks(task_id,robot_id,request_id,request_hash,data) VALUES(?,?,?,?,?)',(t['task_id'],t['robot_id'],body['client_request_id'],fingerprint,canonical(t)))
self._set(t,'QUEUED');return dict(t,deduplicated=False)
if conflicts_with_instruction(body['instruction'],info,self.site):self._intent_question(t)
else:self._set(t,'QUEUED')
return dict(t,deduplicated=False)
def control(self,tid,action):
if action not in {'cancel','pause','resume'}:raise ApiError('INVALID_CONTROL','unsupported control')
with self.lock:
@@ -83,10 +97,10 @@ class Coordinator:
if t['status'] in TERMINAL:return t
if action=='resume':
if t['status']!='PAUSED':raise ApiError('INVALID_STATE','task is not paused',409)
if t['motion_dispatched']:
raise ApiError('RECONCILIATION_REQUIRED','operator reconciliation with fresh physical evidence is required',409)
with self.store.db:
if t['motion_dispatched']:
self._set(t,'INTERVENTION_REQUIRED','RESUME_REQUIRES_PHYSICAL_RECHECK')
else:self._set(t,'QUEUED')
self._set(t,'QUEUED')
return t
if t['status']=='INTERVENTION_REQUIRED':
# A second cancel is useful, but never removes the physical quarantine.
@@ -116,26 +130,64 @@ class Coordinator:
if t['status']!='NEEDS_CLARIFICATION' or not t['question'] or answer['question_id']!=t['question']['question_id'] or answer['task_revision']!=t['task_revision']:
raise ApiError('STALE_ANSWER','answer does not match current question/revision',409)
info=validate_known(answer['known_info'])
if not info:raise ApiError('INVALID_ANSWER','explicit confirmed information is required')
if 'items' in info:
previous=t['request']['known_info'];t['request']['known_info']={k:v for k,v in previous.items() if k=='destination'}
elif 'items' in t['request']['known_info'] and set(info)-{'destination'}:raise ApiError('INVALID_ANSWER','use items representation for multi-item clarification')
t['request']['known_info'].update(info);t['task_revision']+=1;t['planning_generation']+=1;t['planning_attempts']=0;t['question']=None
t['clarification_confirmed']=True
self._set(t,'QUEUED');return t
def intervene(self,tid,data):
"""Reconciliation delegates evidence validation to a trusted physical backend."""
if not isinstance(data,dict) or set(data)!={'run_id','evidence_ref','resolution'} or data['resolution']!='cancel_task':
raise ApiError('INVALID_RECONCILIATION','run_id, evidence_ref, resolution=cancel_task required')
if not isinstance(data,dict) or set(data)!={'run_id','evidence_ref','resolution'} or data['resolution'] not in {'cancel_task','resume_task','replan_task'}:
raise ApiError('INVALID_RECONCILIATION','run_id, evidence_ref, resolution=cancel_task/resume_task/replan_task required')
text(data['evidence_ref'],'evidence_ref',512)
with self.lock:
t=self._task(tid)
if t['status']!='INTERVENTION_REQUIRED' or data['run_id']!=t['run_id']:
if t['status'] not in {'INTERVENTION_REQUIRED','PAUSED'} or not t['motion_dispatched'] or data['run_id']!=t['run_id']:
raise ApiError('STALE_RECONCILIATION','task/run does not require this reconciliation',409)
if not hasattr(self.backend,'reconcile'):raise ApiError('RECONCILIATION_UNAVAILABLE','trusted physical verification backend is required',503)
verified=self.backend.reconcile(t,data)
if not isinstance(verified,dict) or verified.get('stop_confirmed') is not True or verified.get('holding')!='EMPTY' or verified.get('run_id')!=t['run_id'] or verified.get('evidence_ref')!=data['evidence_ref']:
if not isinstance(verified,dict) or verified.get('verified') is not True or verified.get('stop_confirmed') is not True or verified.get('holding_state')!='EMPTY' or verified.get('run_id')!=t['run_id'] or verified.get('evidence_ref')!=data['evidence_ref']:
raise ApiError('RECONCILIATION_REJECTED','matching stop and empty-hand evidence required',409)
receipts=verified.get('receipts')
if not isinstance(receipts,list) or len(receipts)>20:
raise ApiError('RECONCILIATION_REJECTED','bounded receipt list required',409)
approved=instances(t['plan']);pending={}
existing={row['item_index']:json.loads(row['evidence']) for row in self.store.db.execute('SELECT item_index,evidence FROM deliveries WHERE task_id=?',(tid,))}
for receipt in receipts:
if not isinstance(receipt,dict):raise ApiError('RECONCILIATION_REJECTED','invalid receipt',409)
index=receipt.get('item_index');quantity=receipt.get('completed_quantity',1);evidence=receipt.get('evidence')
if receipt.get('task_id')!=tid or type(index) is not int or not 0<=index<len(approved) or index>t.get('active_item_index',0) or type(quantity) is not int or quantity!=1:
raise ApiError('RECONCILIATION_REJECTED','receipt task/item/quantity mismatch',409)
expected=approved[index];destination=self.site['destinations'][expected['destination']]['region_ref']
if not isinstance(evidence,dict) or any(evidence.get(k) is not True for k in ('passed','empty_hand','in_destination','valid')) or not isinstance(evidence.get('evidence_id'),str) or not evidence['evidence_id'] or evidence.get('target_ref')!=expected['target_name'] or evidence.get('destination_ref')!=destination:
raise ApiError('RECONCILIATION_REJECTED','receipt delivery proof invalid',409)
previous=pending.get(index,existing.get(index))
if previous is not None and any(previous.get(k)!=evidence.get(k) for k in ('evidence_id','target_ref','destination_ref','passed','empty_hand','in_destination','valid')):
raise ApiError('RECONCILIATION_REJECTED','receipt conflicts with committed delivery',409)
pending[index]=evidence
credited=set(existing)|set(pending)
if credited!=set(range(len(credited))):raise ApiError('RECONCILIATION_REJECTED','delivery ledger has an item gap',409)
remaining=next((i for i in range(len(approved)) if i not in credited),None)
if data['resolution']=='resume_task' and remaining is not None and t.get('active_item_index',0) not in credited and verified.get('safe_to_retry') is not True:
raise ApiError('RECONCILIATION_REJECTED','cannot replay item without proof no manipulation was dispatched',409)
if data['resolution']=='replan_task' and (credited or verified.get('safe_to_retry') is not True):
raise ApiError('RECONCILIATION_REJECTED','replanning requires zero deliveries and proof no manipulation was dispatched',409)
with self.store.db:
t['stop_confirmed']=True;self._set(t,'CANCELED','RECONCILED_BY_OPERATOR')
for index,evidence in pending.items():
if index not in existing:
self.store.db.execute('INSERT INTO deliveries(task_id,item_index,evidence,created_at) VALUES(?,?,?,?)',(tid,index,canonical(evidence),self.clock()))
self.store.event(t,'delivery_committed',{'item_index':index,'evidence':evidence},self.clock())
t['completed_quantity']=len(credited);t['stop_confirmed']=True
if pending:t['delivery_evidence']=pending[max(pending)]
self.store.event(t,'reconciliation',{'run_id':t['run_id'],'resolution':data['resolution'],'evidence_ref':data['evidence_ref'],'credited_items':sorted(credited)},self.clock())
if data['resolution']=='cancel_task':self._set(t,'CANCELED','RECONCILED_BY_OPERATOR')
elif data['resolution']=='replan_task':self._runtime_question(t,['Confirm the item, quantity, source and destination before a new plan is requested.'])
elif remaining is None:self._set(t,'SUCCEEDED')
else:
t['active_item_index']=remaining;t['execution_generation']=t.get('execution_generation',t['planning_generation'])+1
self._dispatch_item(t)
return t
def tick(self):
with self.lock:
@@ -144,10 +196,10 @@ class Coordinator:
t=self.store.get(e['task_id'])
if not t:continue
with self.store.db:self._event(t,e)
for waiting in self.store.all():
for waiting in self.store.active():
if waiting['status']=='PLANNING' and self.steady()-self._planning_started.get(waiting['task_id'],self.steady())>=self.planning_timeout:
with self.store.db:self._planning_failure(waiting,'PLANNING_TIMEOUT')
all_tasks=self.store.all()
all_tasks=self.store.active()
for robot in sorted(self.robots):
tasks=[t for t in all_tasks if t['robot_id']==robot]
if any(t['status'] not in TERMINAL|{'QUEUED'} for t in tasks):continue
@@ -186,6 +238,8 @@ class Coordinator:
if p['subtasks'][0]['skill']=='ASK_USER':
t['question']={'question_id':str(uuid.uuid4()),'task_revision':t['task_revision'],'questions':[p['subtasks'][0]['arguments']['question']]}
self._set(t,'NEEDS_CLARIFICATION');return
if conflicts_with_instruction(t['request']['instruction'],p['slots'],self.site,clarification_confirmed=t.get('clarification_confirmed') is True):
self._intent_question(t);return
if self.site.get('execution_route') and p['schema_version']!=2:raise ApiError('SCHEMA_MISMATCH','v2 deployment requires v2 planner')
slots=p['slots'];known=t['request']['known_info'];compare=slots
if p['schema_version']==2 and 'items' not in known and known:
@@ -210,6 +264,7 @@ class Coordinator:
seq=e.get('sequence')
if type(seq) is not int or seq<=t['last_sequence'] or not isinstance(e.get('stage'),str):return
t['last_sequence']=seq;t['stage']=e['stage'][:100]
t['detail']=e.get('detail','')[:2000] if isinstance(e.get('detail',''),str) else ''
self._set(t,t['status'],t['error_code'],'progress');return
self.store.event(t,'execution_result',{'payload':e},self.clock())
ev=e.get('evidence',{});quantity=e.get('completed_quantity',0)
@@ -228,6 +283,13 @@ class Coordinator:
if e.get('stop_confirmed') is not True:
self._set(t,'INTERVENTION_REQUIRED','STOP_UNKNOWN');return
t['stop_confirmed']=True
if isinstance(ev,dict) and ev.get('needs_clarification') is True and t['status']=='EXECUTING':
questions=ev.get('questions')
valid_questions=isinstance(questions,list) and 1<=len(questions)<=8 and all(isinstance(q,str) and q.strip() and len(q)<=500 for q in questions)
safe=all(ev.get(k) is True for k in ('safe_to_retry','empty_hand','valid','safe_to_release'))
if quantity!=0 or t['completed_quantity']!=0 or not safe or not valid_questions:
self._set(t,'INTERVENTION_REQUIRED','CLARIFICATION_REQUIRES_PHYSICAL_RECHECK');return
self._runtime_question(t,questions);return
if e.get('status')=='SUCCEEDED':
if quantity!=1:
self._set(t,'INTERVENTION_REQUIRED','DELIVERY_EVIDENCE_INVALID');return
+12 -1
View File
@@ -24,13 +24,24 @@ class Store:
CREATE INDEX IF NOT EXISTS events_task_cursor ON events(task_id,event_id);
CREATE TABLE IF NOT EXISTS deliveries(task_id TEXT NOT NULL REFERENCES tasks(task_id),item_index INTEGER NOT NULL,evidence TEXT NOT NULL,created_at REAL NOT NULL,PRIMARY KEY(task_id,item_index));
''')
# Migrate old databases once; terminal history remains queryable without
# being decoded on every scheduler tick.
columns={row[1] for row in self.db.execute('PRAGMA table_info(tasks)')}
with self.db:
if 'status' not in columns:
self.db.execute("ALTER TABLE tasks ADD COLUMN status TEXT NOT NULL DEFAULT 'RECEIVED'")
for row in self.db.execute('SELECT task_id,data FROM tasks').fetchall():
self.db.execute('UPDATE tasks SET status=? WHERE task_id=?',(json.loads(row['data'])['status'],row['task_id']))
self.db.execute("CREATE INDEX IF NOT EXISTS tasks_active_seq ON tasks(seq) WHERE status NOT IN ('SUCCEEDED','FAILED','CANCELED','EXPIRED')")
def get(self,tid):
r=self.db.execute('SELECT data FROM tasks WHERE task_id=?',(tid,)).fetchone()
return json.loads(r[0]) if r else None
def all(self):
return [json.loads(r[0]) for r in self.db.execute('SELECT data FROM tasks ORDER BY seq')]
def active(self):
return [json.loads(r[0]) for r in self.db.execute("SELECT data FROM tasks WHERE status NOT IN ('SUCCEEDED','FAILED','CANCELED','EXPIRED') ORDER BY seq")]
def put(self,t):
self.db.execute('UPDATE tasks SET data=? WHERE task_id=?',(canonical(t),t['task_id']))
self.db.execute('UPDATE tasks SET data=?,status=? WHERE task_id=?',(canonical(t),t['status'],t['task_id']))
def event(self,t,kind,data,now):
self.db.execute('INSERT INTO events(task_id,kind,created_at,data) VALUES(?,?,?,?)',(t['task_id'],kind,now,canonical(data)))
def events(self,tid,after,limit):