fix: harden task recovery and DR contract handling

2026-09-20 13:36:48 +08:00
parent 492676344a
commit f9d8feb6f0
49 changed files with 2083 additions and 165 deletions
@@ -4,7 +4,10 @@ if [[ ! -f /opt/ros/humble/setup.bash ]]; then
  echo 'NOT RUN: ROS2 Humble is not installed; no ROS compilation claim.' >&2
  exit 2
 fi
+# Humble's generated environment hooks inspect optional unset variables.
+set +u
 source /opt/ros/humble/setup.bash
+set -u
 command -v colcon >/dev/null || { echo 'colcon is required' >&2; exit 2; }
 bt_repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
 cd "$bt_repo_root"
@@ -0,0 +1,16 @@
+#include <bt_executor/recovery_policy.hpp>
+#include <cassert>
+#include <string>
+int main() {
+  using namespace bt_executor;
+  assert(!recovery_authorized("", ""));
+  assert(!recovery_authorized("short", "short"));
+  assert(recovery_authorized("sixteen-byte-key!", "sixteen-byte-key!"));
+  assert(!recovery_authorized("sixteen-byte-key!", "sixteen-byte-key?"));
+  assert(!recovery_authorized("sixteen-byte-key!", "sixteen-byte-key!x"));
+  assert(!recovery_authorized(std::string(1025,'x'),std::string(1025,'x')));
+  assert(!recovery_resume_permitted(false,false,false));
+  assert(!recovery_resume_permitted(true,true,false));
+  assert(recovery_resume_permitted(true,false,false));
+  assert(recovery_resume_permitted(false,true,true));
+}
@@ -6,6 +6,7 @@ from collections import deque
 from types import SimpleNamespace as S
 import unittest
 import time
+import json

 sys.path.insert(0, str(Path(__file__).resolve().parents[3] / 'coordinator'))
 from robot_bt_coordinator.ros_backend import RosBackend
@@ -112,6 +113,105 @@ class BackendLifecycle(unittest.TestCase):
        self.backend._execution_result(self.key, self.result(evidence='{"stop_confirmed":false,"stop_confirmed":true}'))
        self.assertEqual(self.backend._events.pop()['status'], 'INTERVENTION_REQUIRED')

+    def test_failed_planning_preserves_archive_reference(self):
+        key = ('task', 1, 1)
+        self.backend._planning[key] = dict(task_id='task', task_revision=1,
+            planning_generation=1, done=False)
+        self.backend._plan_result(key, Future(S(status=4, result=S(status=2,
+            error_code='SEMANTIC_MISMATCH', message='conflict', planning_record_ref='archive/42'))))
+        self.assertEqual(self.backend._events.pop()['planning_record_ref'], 'archive/42')
+
+    def test_native_planner_protocol_failure_preserves_archive_reference(self):
+        for native in (5,6):
+            with self.subTest(native=native):
+                key=('task',1,native)
+                self.backend._planning[key]=dict(task_id='task',task_revision=1,planning_generation=native,done=False)
+                self.backend._plan_result(key,Future(S(status=native,result=S(planning_record_ref='archive/native-'+str(native)))))
+                event=self.backend._events.pop()
+                self.assertEqual(event['status'],'FAILED')
+                self.assertEqual(event['error_code'],'PLANNER_PROTOCOL_ERROR')
+                self.assertEqual(event['planning_record_ref'],'archive/native-'+str(native))
+
+    def test_invalid_planner_json_retains_archive_for_diagnosis(self):
+        key=('task',1,1)
+        self.backend._planning[key]=dict(task_id='task',task_revision=1,planning_generation=1,done=False)
+        self.backend._plan_result(key,Future(S(status=4,result=S(status=0,task_plan_json='{invalid',planning_record_ref='archive/raw'))))
+        event=self.backend._events.pop()
+        self.assertEqual(event['error_code'],'PLANNER_PROTOCOL_ERROR')
+        self.assertEqual(event['planning_record_ref'],'archive/raw')
+
+    def test_evicted_late_acceptance_is_canceled_without_recreating_tracking(self):
+        self.backend._terminal_retention=1
+        old_run=('old','run');old_plan=('old',1,1)
+        self.backend._runs[old_run]=dict(done=True)
+        self.backend._runs[('new','run')]=dict(done=True)
+        self.backend._planning[old_plan]=dict(done=True)
+        self.backend._planning[('new',1,1)]=dict(done=True)
+        self.backend._trim_terminal()
+        self.assertNotIn(old_run,self.backend._runs);self.assertNotIn(old_plan,self.backend._planning)
+        execution_handle=Handle();planning_handle=Handle()
+        self.backend._execution_accepted(old_run,Future(execution_handle))
+        self.backend._plan_accepted(old_plan,Future(planning_handle))
+        self.assertEqual(execution_handle.cancels,1);self.assertEqual(planning_handle.cancels,1)
+        self.assertNotIn(old_run,self.backend._runs);self.assertNotIn(old_plan,self.backend._planning)
+        self.assertFalse(self.backend._events)
+
+    def test_retention_never_evicts_unresolved_executions_or_plans(self):
+        self.backend._terminal_retention=2
+        pending_plan=('pending',1,1)
+        self.backend._planning[pending_plan]=dict(done=False)
+        self.backend._unknown(self.rec,'FEEDBACK_TIMEOUT')
+        for index in range(8):
+            self.backend._runs[('finished',str(index))]=dict(done=True)
+            self.backend._planning[('finished',1,index)]=dict(done=True)
+        self.backend._trim_terminal()
+        self.assertIs(self.backend._runs[self.key],self.rec)
+        self.assertFalse(self.rec['done']);self.assertIn(pending_plan,self.backend._planning)
+        self.assertEqual(len(self.backend._runs),3);self.assertEqual(len(self.backend._planning),3)
+
+    def test_feedback_retains_snapshot(self):
+        self.backend._feedback(self.key, S(feedback=S(stamp=S(sec=10, nanosec=0),
+            sequence=1, stage='PICK', status_json='{"holding_state":"UNKNOWN"}')))
+        self.assertEqual(self.backend._events.pop()['detail'], {'holding_state':'UNKNOWN'})
+
+    def test_retention_keeps_uncertain_run_and_ignores_late_evicted_callbacks(self):
+        self.backend._terminal_retention = 2
+        for i in range(6):
+            self.backend._runs[('old', str(i))] = dict(done=True)
+        self.backend._trim_terminal()
+        self.assertIn(self.key, self.backend._runs)
+        self.assertEqual(len(self.backend._runs), 3)
+        self.backend._feedback(('old', '0'), S())
+        self.backend._execution_result(('old', '0'), Future(None))
+
+    def test_recovery_missing_authorization_never_calls_service(self):
+        self.backend.config = {}
+        with self.assertRaisesRegex(ValueError, 'recovery'):
+            self.backend.reconcile({}, {})
+
+    def test_recovery_validates_physical_state_before_retiring_tracking(self):
+        task = dict(task_id='task',run_id='run',task_revision=2,planning_generation=3,
+                    execution_generation=4,plan={'plan_version':5})
+        request = dict(evidence_ref='operator-proof',resolution='resume_task')
+        self.backend.config = dict(recovery_token='x'*24,recovery_operator_id='operator')
+        self.backend._ReconcileTask = S(Request=lambda:S(trace=S()))
+        calls=[]
+        class ReadyFuture(Future):
+            def add_done_callback(self, cb): cb(self)
+        state = dict(verified=True,stop_confirmed=False,holding_state='EMPTY',
+                     run_id='run',evidence_ref='operator-proof',receipts=[])
+        def call(message):
+            calls.append(message)
+            return ReadyFuture(S(accepted=True,state_json=json.dumps(state)))
+        self.backend._recovery = S(service_is_ready=lambda:True,call_async=call)
+        with self.assertRaises(ValueError): self.backend.reconcile(task,request)
+        self.assertFalse(self.rec['done'])
+        state['stop_confirmed']=True
+        self.assertTrue(self.backend.reconcile(task,request)['verified'])
+        self.assertTrue(self.rec['done'])
+        self.assertEqual(calls[-1].trace.execution_generation,4)
+        self.assertEqual(calls[-1].trace.plan_version,5)
+

 if __name__ == '__main__':
    unittest.main()