fix: retain planning feedback and add acceptance probes
This commit is contained in:
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Monitor one Linux process for a real eight-hour duration/liveness qualification.
|
||||
|
||||
Examples (never selects robot endpoints):
|
||||
python tools/native_soak.py --pid 1234 --output /tmp/soak-unique \
|
||||
--progress-file /tmp/completed.json --journal /tmp/goals.log
|
||||
python tools/native_soak.py --output /tmp/qualification --duration-seconds 30 \
|
||||
--command python -u workload.py
|
||||
|
||||
The workload must atomically replace progress JSON {"completed": N} after actual
|
||||
business work completes. A timer/heartbeat is NOT a valid business progress metric.
|
||||
Counter reset, stale progress, process exit/replacement and sampling failure fail.
|
||||
An attached process is never signaled. A spawned command runs once (no shell or
|
||||
restart); after monitoring it receives SIGTERM, then SIGKILL after five seconds.
|
||||
Only that direct child is owned/monitored: use a foreground executor, not a launcher
|
||||
or shell with detached children. Logs/resources of descendants are not aggregated.
|
||||
|
||||
Exit 0: observed >=8h and progressing; NOT full DR acceptance. Resource growth and
|
||||
workload validity still need review. Exit 2: short/missing-metric/interrupted run.
|
||||
Exit 1: observed failure. Missing final summary means interrupted/incomplete, never
|
||||
pass. No simulated/ROS clock, injectable clock or shortened acceptance threshold.
|
||||
"""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
EIGHT_HOURS = 8 * 60 * 60
|
||||
|
||||
|
||||
def verdict(elapsed, progress_observed, failure=None, interrupted=False, sampling_gap=False):
|
||||
qualified = (elapsed >= EIGHT_HOURS and progress_observed and not failure
|
||||
and not interrupted and not sampling_gap)
|
||||
return dict(status='failed' if failure else 'duration_liveness_pass' if qualified else 'qualification_incomplete',
|
||||
eight_hour_pass=qualified, no_deadlock_claim=False,
|
||||
duration_liveness_pass=qualified, full_dr_acceptance=False,
|
||||
resource_growth_review_required=True,
|
||||
scope='Same process survival and declared business progress only; not proof of physical safety or absence of all deadlocks')
|
||||
|
||||
|
||||
def positive(text):
|
||||
value = float(text)
|
||||
if not math.isfinite(value) or value <= 0:
|
||||
raise argparse.ArgumentTypeError('must be finite and positive')
|
||||
return value
|
||||
|
||||
|
||||
def process_sample(pid):
|
||||
base = Path('/proc') / str(pid)
|
||||
raw = (base / 'stat').read_text()
|
||||
fields = raw[raw.rfind(')') + 2:].split()
|
||||
if fields[0] in ('Z', 'X', 'x'):
|
||||
raise RuntimeError('process exited (zombie/dead)')
|
||||
status = dict(line.split(':', 1) for line in (base / 'status').read_text().splitlines() if ':' in line)
|
||||
result = dict(pid=pid, start_ticks=int(fields[19]),
|
||||
rss_bytes=int(status['VmRSS'].split()[0]) * 1024,
|
||||
threads=int(status['Threads']), fd_count=len(list((base / 'fd').iterdir())),
|
||||
cpu_seconds=(int(fields[11]) + int(fields[12])) / os.sysconf('SC_CLK_TCK'))
|
||||
# Detect exit/replacement during multi-file sampling as well as between samples.
|
||||
again = (base / 'stat').read_text().rsplit(')', 1)[1].split()
|
||||
if int(again[19]) != result['start_ticks'] or again[0] in ('Z', 'X', 'x'):
|
||||
raise RuntimeError('process identity changed during sampling')
|
||||
return result
|
||||
|
||||
|
||||
def atomic_json(path, value):
|
||||
temporary = path.with_suffix('.tmp')
|
||||
with temporary.open('w') as stream:
|
||||
json.dump(value, stream, indent=2, allow_nan=False)
|
||||
stream.write('\n')
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
source = parser.add_mutually_exclusive_group(required=True)
|
||||
source.add_argument('--pid', type=int)
|
||||
source.add_argument('--command', nargs=argparse.REMAINDER, help='Foreground argv; must be last option; no shell')
|
||||
parser.add_argument('--output', type=Path, required=True, help='New evidence directory; existing paths refused')
|
||||
parser.add_argument('--duration-seconds', type=positive, default=EIGHT_HOURS)
|
||||
parser.add_argument('--sample-interval-seconds', type=positive, default=5.)
|
||||
parser.add_argument('--max-progress-gap-seconds', type=positive, default=300.)
|
||||
parser.add_argument('--progress-file', type=Path)
|
||||
parser.add_argument('--journal', type=Path, action='append', default=[])
|
||||
args = parser.parse_args(argv)
|
||||
if not sys.platform.startswith('linux'):
|
||||
parser.error('Linux /proc required')
|
||||
if args.pid is not None and args.pid <= 0:
|
||||
parser.error('--pid must be positive')
|
||||
if args.command == []:
|
||||
parser.error('--command requires argv')
|
||||
if args.progress_file and args.sample_interval_seconds >= args.max_progress_gap_seconds:
|
||||
parser.error('sample interval must be shorter than progress gap')
|
||||
args.output.mkdir(parents=True, exist_ok=False)
|
||||
summary_path = args.output / 'summary.json'
|
||||
report = dict(schema_version=1, started_utc=datetime.now(timezone.utc).isoformat(),
|
||||
duration_requested_seconds=args.duration_seconds, acceptance_duration_seconds=EIGHT_HOURS,
|
||||
clock='time.monotonic (host Linux; excludes suspend)',
|
||||
mode='attach' if args.pid else 'spawn', command=args.command,
|
||||
progress_file=str(args.progress_file) if args.progress_file else None,
|
||||
progress_contract='Atomic JSON completed counter incremented by actual workload completion',
|
||||
sample_interval_seconds=args.sample_interval_seconds,
|
||||
max_progress_gap_seconds=args.max_progress_gap_seconds,
|
||||
boot_id=Path('/proc/sys/kernel/random/boot_id').read_text().strip(),
|
||||
status='running', eight_hour_pass=False, full_dr_acceptance=False,
|
||||
resource_growth_review_required=True)
|
||||
atomic_json(summary_path, report)
|
||||
child = None
|
||||
interrupted = False
|
||||
failure = None
|
||||
gap = False
|
||||
start = time.monotonic()
|
||||
last_progress = start
|
||||
previous_counter = None
|
||||
progress_observed = False
|
||||
previous = None
|
||||
identity = None
|
||||
sample_count = 0
|
||||
peaks = dict(rss_bytes=0, threads=0, fd_count=0)
|
||||
old_handlers = {}
|
||||
|
||||
def request_stop(signum, frame):
|
||||
nonlocal interrupted
|
||||
interrupted = True
|
||||
|
||||
try:
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
old_handlers[sig] = signal.signal(sig, request_stop)
|
||||
with (args.output / 'process.log').open('w') as process_log, (args.output / 'samples.jsonl').open('w') as samples:
|
||||
if args.command:
|
||||
child = subprocess.Popen(args.command, stdin=subprocess.DEVNULL, stdout=process_log, stderr=subprocess.STDOUT)
|
||||
pid = child.pid if child else args.pid
|
||||
report['pid'] = pid
|
||||
start = time.monotonic()
|
||||
last_progress = start
|
||||
while not interrupted:
|
||||
if child and child.poll() is not None:
|
||||
raise RuntimeError('process exited before observation completed: ' + str(child.returncode))
|
||||
sample = process_sample(pid)
|
||||
now = time.monotonic()
|
||||
sample['elapsed_seconds'] = now - start
|
||||
if identity is None:
|
||||
identity = sample['start_ticks']
|
||||
report['start_ticks'] = identity
|
||||
if sample['start_ticks'] != identity:
|
||||
raise RuntimeError('process identity changed (PID reused)')
|
||||
sample['cpu_percent'] = None
|
||||
if previous:
|
||||
delta = sample['elapsed_seconds'] - previous['elapsed_seconds']
|
||||
sample['cpu_percent'] = 100 * (sample['cpu_seconds'] - previous['cpu_seconds']) / delta
|
||||
if delta > max(5., args.sample_interval_seconds * 3):
|
||||
gap = True
|
||||
sample['journal_bytes'] = {str(p.resolve()): p.stat().st_size for p in args.journal}
|
||||
sample['completed'] = None
|
||||
if args.progress_file:
|
||||
# Startup and subsequent progress share the same strict bound.
|
||||
# Late evidence cannot retroactively erase an observed gap.
|
||||
if now - last_progress > args.max_progress_gap_seconds:
|
||||
raise RuntimeError('progress deadline exceeded; process liveness is insufficient')
|
||||
try:
|
||||
value = json.loads(args.progress_file.read_text())['completed']
|
||||
except FileNotFoundError:
|
||||
if previous_counter is not None:
|
||||
raise RuntimeError('progress metric disappeared')
|
||||
value = None # Allow initial workload startup, bounded by the same deadline.
|
||||
if value is not None:
|
||||
if type(value) is not int or value < 0:
|
||||
raise RuntimeError('progress counter must be a nonnegative integer')
|
||||
if previous_counter is not None:
|
||||
if value < previous_counter:
|
||||
raise RuntimeError('progress counter regressed')
|
||||
if value > previous_counter:
|
||||
progress_observed = True
|
||||
last_progress = now
|
||||
previous_counter = value
|
||||
sample['completed'] = value
|
||||
samples.write(json.dumps(sample, allow_nan=False) + '\n')
|
||||
samples.flush()
|
||||
os.fsync(samples.fileno())
|
||||
sample_count += 1
|
||||
for name in peaks:
|
||||
peaks[name] = max(peaks[name], sample[name])
|
||||
previous = sample
|
||||
report.update(elapsed_seconds=now - start, samples=sample_count, resource_peaks=peaks,
|
||||
progress_observed=progress_observed, last_completed=previous_counter)
|
||||
atomic_json(summary_path, report)
|
||||
if now - start >= args.duration_seconds:
|
||||
break
|
||||
deadline = min(now + args.sample_interval_seconds, start + args.duration_seconds)
|
||||
while not interrupted and time.monotonic() < deadline:
|
||||
time.sleep(max(0., min(1., deadline - time.monotonic())))
|
||||
except (Exception, KeyboardInterrupt) as exc:
|
||||
failure = str(exc) or type(exc).__name__
|
||||
finally:
|
||||
elapsed = previous['elapsed_seconds'] if previous else 0.
|
||||
report.update(verdict(elapsed, progress_observed, failure, interrupted, gap))
|
||||
report.update(elapsed_seconds=elapsed, reason=failure or ('interrupted' if interrupted else 'monitoring completed'),
|
||||
sampling_gap=gap, samples=sample_count, progress_observed=progress_observed,
|
||||
resource_peaks=peaks, ended_utc=datetime.now(timezone.utc).isoformat(),
|
||||
child_cleanup='not applicable; attached process untouched')
|
||||
if child:
|
||||
if child.poll() is None:
|
||||
child.terminate()
|
||||
try:
|
||||
child.wait(timeout=5)
|
||||
report['child_cleanup'] = 'direct child terminated after monitoring'
|
||||
except subprocess.TimeoutExpired:
|
||||
child.kill()
|
||||
child.wait()
|
||||
report['child_cleanup'] = 'direct child killed after five-second termination timeout'
|
||||
else:
|
||||
report['child_cleanup'] = 'direct child had already exited'
|
||||
report['child_exit_code'] = child.returncode
|
||||
for sig, old in old_handlers.items():
|
||||
signal.signal(sig, old)
|
||||
atomic_json(summary_path, report)
|
||||
print(json.dumps(report))
|
||||
return 1 if failure else 0 if report['duration_liveness_pass'] else 2
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user