1940 lines
120 KiB
Python
1940 lines
120 KiB
Python
"""Execute only the task types currently confirmed by the analysis protocol."""
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
import hashlib
|
||
import marshal
|
||
import re
|
||
import sys
|
||
import time
|
||
import traceback
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from urllib.error import HTTPError, URLError
|
||
from urllib.parse import urlsplit
|
||
|
||
from nao_runtime_policy import FINALIZATION_RESERVE_SECONDS, remaining_seconds
|
||
|
||
|
||
_STAGE_LABELS = {'black': '黑片', 'bright': '亮片'}
|
||
_MEMBER_PAGE_SIZE = 200
|
||
_MEMBER_MAX_PAGES = 20
|
||
_MEMBER_SNAPSHOT_MAX_BYTES = 8 * 1024 * 1024
|
||
_QUERY_RETRY_DELAYS = (0.25, 0.75)
|
||
_TRANSIENT_QUERY_STATUSES = {429, 502, 503, 504}
|
||
|
||
|
||
def _composition_context(request):
|
||
return {'_autonomousAnswer': True} if request.get('_autonomousAnswer') is True else {}
|
||
|
||
|
||
def _platform_visual_context(result):
|
||
"""Expose generated chart contracts without duplicating numeric fact data."""
|
||
charts = []
|
||
for visual in result.get('visuals', []):
|
||
if not isinstance(visual, dict):
|
||
continue
|
||
chart = {key: copy.deepcopy(visual[key]) for key in (
|
||
'id','type','title','metric','dimension','stage','scope','period','comparisonPeriod','evidenceRefs') if key in visual}
|
||
chart['pointCount'] = len(visual.get('categories', []))
|
||
charts.append(chart)
|
||
return {'visualState': result.get('visualState', 'not_requested'), 'charts': charts,
|
||
'missingEvidence': copy.deepcopy(result.get('missingEvidence', []))}
|
||
|
||
|
||
def query_evidence_bindings(documents, proofs):
|
||
"""Bind public evidence references to this execution's signed API receipts.
|
||
|
||
Normal evidence references are random and must not be reverse-engineered
|
||
from query IDs. The backend independently checks these IDs and chart values.
|
||
"""
|
||
allowed = {proof.get('queryId') for proof in proofs if isinstance(proof, dict)}
|
||
bindings = {}
|
||
def query_ids(result):
|
||
if not isinstance(result, dict):
|
||
return set()
|
||
raw = result.get('protocolEvidence')
|
||
items = [raw] if isinstance(raw, dict) else raw if isinstance(raw, list) else []
|
||
ids = {item['queryId'] for item in items
|
||
if isinstance(item, dict) and isinstance(item.get('queryId'), str)}
|
||
for side in ('current', 'comparison'):
|
||
ids.update(query_ids(result.get(side)))
|
||
return ids
|
||
for document in documents:
|
||
ref = document.get('evidenceRef')
|
||
ids = query_ids(document.get('result'))
|
||
if not isinstance(ref, str) or not re.fullmatch(r'Q[A-F0-9]{8}', ref) or not ids:
|
||
continue
|
||
if not ids <= allowed:
|
||
raise ValueError('图表证据包含不属于本轮的查询证明')
|
||
if ref in bindings and bindings[ref] != ids:
|
||
raise ValueError('同一证据编号不能绑定不同正式查询')
|
||
bindings[ref] = ids
|
||
return [{'evidenceRef': ref, 'queryIds': sorted(ids)} for ref, ids in bindings.items()]
|
||
|
||
|
||
def _confirmed_scalar_selection(request, task):
|
||
"""Dispatch an already confirmed scalar choice without rewriting its task."""
|
||
resolution = task.get('productResolution') or {}
|
||
question = resolution.get('originalQuestion')
|
||
selected = resolution.get('selectedName')
|
||
period = task.get('period') or {}
|
||
if not (request.get('_autonomousAnswer') is True and task.get('kind') == 'analysis'
|
||
and isinstance(question, str) and question and task.get('rootGoal') == question
|
||
and isinstance(selected, str) and selected == (task.get('filters') or {}).get('product')
|
||
and selected in [row.get('name') for row in (resolution.get('productMatch') or {}).get('candidates', []) if isinstance(row, dict)]
|
||
and task.get('fields') == ['rate'] and task.get('dimensions') == []
|
||
and not task.get('stages') and not task.get('dimensionStages')
|
||
and task.get('executionMode') in (None, '') and not task.get('queryPlan')
|
||
and task.get('freshQuery') is True and not task.get('comparisonRequired')
|
||
and not task.get('comparisonPeriod') and not task.get('trendRequired')
|
||
and not task.get('fullDetails') and not task.get('verifySource')
|
||
and set(period) == {'year', 'month'}):
|
||
return False
|
||
# This is a narrow compatibility path for saved scalar selections, never a
|
||
# substitute for an investigation or a grouped/ranked business question.
|
||
if re.search(r'为什么|为何|原因|调查|深挖|分析|重点|关注|优先|最差|最低|最高|排名|趋势|比较|对比|下降|贡献', question):
|
||
return False
|
||
return bool(re.search(r'合格率.{0,6}(?:是多少|为多少|多少|数值)|(?:查|查看|查询).{0,20}合格率', question))
|
||
|
||
|
||
def _save_composition(final, actor_root, turn_id):
|
||
if 'rawModelResponses' in final:
|
||
from nao_quality_runtime import _atomic_json
|
||
_atomic_json(actor_root / 'private' / 'answer-composition.json', {
|
||
'turnId': turn_id, 'rawModelResponses': final['rawModelResponses'],
|
||
'compositionDiagnostics': final.get('compositionDiagnostics', {}),
|
||
'bindingInput': final.get('_privateBindingInput')})
|
||
|
||
|
||
def _retryable_query_error(error):
|
||
# The delegated transport owns classification. Never infer recoverability
|
||
# from arbitrary server detail text, which may describe a permission error.
|
||
classified = getattr(error, 'retryable', None)
|
||
if classified is not None:
|
||
status = getattr(error, 'http_status', None)
|
||
return classified is True and (status is None or status in _TRANSIENT_QUERY_STATUSES)
|
||
if isinstance(error, HTTPError):
|
||
return error.code in _TRANSIENT_QUERY_STATUSES
|
||
if isinstance(error, URLError):
|
||
return isinstance(error.reason, (TimeoutError, ConnectionError))
|
||
return isinstance(error, (TimeoutError, ConnectionError))
|
||
|
||
|
||
def _identity(authority):
|
||
identity = {key: authority[key] for key in ('attemptId', 'sessionId', 'turnId', 'revision')}
|
||
if 'executionId' in authority:
|
||
identity['executionId'] = authority['executionId']
|
||
return identity
|
||
|
||
|
||
def _member_proof(response, identity):
|
||
proof = response.get('protocolEvidence')
|
||
required = ('queryId', 'attemptId', 'sessionId', 'turnId', 'revision', 'executionId', 'responseHash')
|
||
if not isinstance(proof, dict) or any(key not in proof for key in required):
|
||
raise ValueError('成员清单页面缺少完整协议证明')
|
||
if ('executionId' not in identity or any(proof.get(key) != value for key, value in identity.items())
|
||
or not isinstance(proof.get('queryId'), str) or not proof['queryId']
|
||
or not isinstance(proof.get('responseHash'), str) or not proof['responseHash']):
|
||
raise ValueError('成员清单页面证明未绑定当前执行')
|
||
return copy.deepcopy(proof)
|
||
|
||
|
||
def _members_answer(task, total_count, rows, *, query_complete):
|
||
"""The server builds the full structured table from its recorded pages."""
|
||
import json
|
||
available = 0
|
||
byte_count = 0
|
||
for row in rows:
|
||
size = len(json.dumps(row, ensure_ascii=False, separators=(',', ':')).encode('utf-8')) + 1
|
||
if byte_count + size > _MEMBER_SNAPSHOT_MAX_BYTES - 16_384:
|
||
break
|
||
byte_count += size
|
||
available += 1
|
||
lines = [f"符合条件的成员清单:匹配 {total_count} 个去重项,本次已读取 {len(rows)} 项。"]
|
||
if any(value is None or value == '' for row in rows for value in row['values'].values()):
|
||
lines.append('部分去重项含空字段值,表中以空值标识。')
|
||
if not query_complete:
|
||
lines.append(f'清单超过4000项的读取上限,另有 {max(0, total_count - len(rows))} 项未读取;查询不完整。')
|
||
if available < len(rows):
|
||
lines.append(f'完整清单超出展示容量,另有 {len(rows)-available} 项已读取但未载入表格;结果为部分完成。')
|
||
elif query_complete:
|
||
lines[0] = f"符合条件的成员清单:共 {total_count} 个去重项,已查全,可在下方表格翻页查看全部。"
|
||
return '\n'.join(lines), available, available == len(rows)
|
||
|
||
|
||
def _execution_proofs(result, identity, *, expected_count):
|
||
proofs = result.get('protocolEvidence')
|
||
if isinstance(proofs, dict):proofs = [proofs]
|
||
required = ('queryId', 'attemptId', 'sessionId', 'turnId', 'revision', 'executionId', 'responseHash')
|
||
if not isinstance(proofs, list) or len(proofs) != expected_count:
|
||
raise ValueError('正式查询证据数量与执行计划不一致')
|
||
clean = []
|
||
for proof in proofs:
|
||
if (not isinstance(proof, dict) or any(key not in proof for key in required)
|
||
or any(proof.get(key) != value for key, value in identity.items())
|
||
or not isinstance(proof.get('queryId'), str) or not proof['queryId']
|
||
or not isinstance(proof.get('responseHash'), str) or not proof['responseHash']):
|
||
raise ValueError('正式查询证据未绑定当前执行')
|
||
clean.append(copy.deepcopy(proof))
|
||
return clean
|
||
|
||
|
||
class ScopedApi:
|
||
"""Forward all child API calls with the server-issued protocol scope envelope."""
|
||
def __init__(self, delegated, node_id, scope_id, *, cancel_event=None, emit=None,
|
||
recovery_log=None, read_limit=40):
|
||
self.delegated = delegated
|
||
self.node_id = node_id
|
||
self.scope_id = scope_id
|
||
self.metric_responses = []
|
||
self.investigation_responses = []
|
||
self.defect_responses = []
|
||
self.read_count = 0
|
||
self.read_limit = read_limit
|
||
self.cancel_event = cancel_event
|
||
self.emit = emit
|
||
self.recovery_log = recovery_log if recovery_log is not None else []
|
||
|
||
def _cancelled(self):
|
||
return self.cancel_event is not None and self.cancel_event.is_set()
|
||
|
||
def _read(self, path, payload, method, *, counted):
|
||
endpoint = urlsplit(path).path
|
||
# Each retry keeps the exact authorized node/scope and request. Only a
|
||
# successful response becomes a receipt; failed attempts consume budget.
|
||
frozen_payload = copy.deepcopy(payload)
|
||
for attempt in range(len(_QUERY_RETRY_DELAYS) + 1):
|
||
if self._cancelled():
|
||
raise RuntimeError('cancelled')
|
||
if counted and self.read_count >= self.read_limit:
|
||
raise ValueError('本轮已达到40次正式查询上限,未继续读取。')
|
||
if counted and remaining_seconds() <= FINALIZATION_RESERVE_SECONDS + 5:
|
||
raise ValueError('正式查询时间不足,已保留时间整理取得的证据。')
|
||
if counted:
|
||
self.read_count += 1
|
||
try:
|
||
# DelegatedApi clamps its socket timeout to this reserve. Keep
|
||
# that clamp local so later finalization calls retain their own
|
||
# allowance even when the query portion has expired.
|
||
clamp_timeout = counted and hasattr(self.delegated, 'deadline_reserve_seconds')
|
||
prior_reserve = getattr(self.delegated, 'deadline_reserve_seconds', None)
|
||
if clamp_timeout:
|
||
self.delegated.deadline_reserve_seconds = max(
|
||
prior_reserve or 0, FINALIZATION_RESERVE_SECONDS + 5)
|
||
try:
|
||
response = self.delegated.call(path, payload=copy.deepcopy(frozen_payload), method=method,
|
||
node_id=self.node_id, scope_id=self.scope_id)
|
||
finally:
|
||
if clamp_timeout:
|
||
self.delegated.deadline_reserve_seconds = prior_reserve
|
||
except Exception as error:
|
||
if not counted or not _retryable_query_error(error):
|
||
raise
|
||
retry = (attempt < len(_QUERY_RETRY_DELAYS) and not self._cancelled()
|
||
and self.read_count < self.read_limit
|
||
and remaining_seconds() > FINALIZATION_RESERVE_SECONDS + 5)
|
||
self.recovery_log.append({'nodeId': self.node_id, 'scopeId': self.scope_id,
|
||
'endpoint': endpoint, 'attempt': attempt + 1, 'errorType': type(error).__name__,
|
||
'httpStatus': getattr(error, 'http_status', getattr(error, 'code', None)),
|
||
'outcome': 'retry_scheduled' if retry else 'stopped'})
|
||
if not retry:
|
||
if self._cancelled():
|
||
raise RuntimeError('cancelled') from None
|
||
raise
|
||
if self.emit is not None:
|
||
self.emit({'type': 'progress', 'message':
|
||
f'当前查询暂未取得响应,正在重试({attempt+1}/{len(_QUERY_RETRY_DELAYS)});已完成范围和证据保留。'})
|
||
delay = _QUERY_RETRY_DELAYS[attempt]
|
||
if self.cancel_event is not None:
|
||
if self.cancel_event.wait(delay):
|
||
raise RuntimeError('cancelled') from None
|
||
elif delay:
|
||
time.sleep(delay)
|
||
else:
|
||
if attempt:
|
||
self.recovery_log.append({'nodeId': self.node_id, 'scopeId': self.scope_id,
|
||
'endpoint': endpoint, 'attempt': attempt + 1, 'outcome': 'recovered'})
|
||
return response
|
||
|
||
def login(self):
|
||
# The wrapped delegated client is already authenticated by the runtime;
|
||
# QualityTools calls login() as part of its Api-compatible factory shape.
|
||
return self
|
||
|
||
def call(self, path, payload=None, method=None):
|
||
is_metric_read = method == 'GET' and urlsplit(path).path == '/api/metrics'
|
||
is_investigation_read = method == 'POST' and urlsplit(path).path == '/api/analysis/investigate'
|
||
is_defect_read = method == 'POST' and urlsplit(path).path == '/api/analysis/defects'
|
||
is_changes_read = method == 'POST' and urlsplit(path).path == '/api/analysis/changes'
|
||
counted = (is_metric_read or is_investigation_read
|
||
or is_changes_read or is_defect_read and isinstance(payload, dict) and payload.get('mode') == 'top')
|
||
response = self._read(path, payload, method, counted=counted)
|
||
if is_metric_read and isinstance(response, dict):
|
||
# Keep only the actual server receipt needed to preserve a proven
|
||
# no-source comparison as PARTIAL. It is not KPI evidence and is
|
||
# never written as an evidence document.
|
||
response_context = response.get('context')
|
||
context_keys = ('year', 'month', 'quarter', 'monthStart', 'monthEnd', 'view',
|
||
'department', 'group', 'product', 'projectGroup', 'extraFilters',
|
||
'exclusions', 'includeUnassigned', 'filters', 'columnFilters')
|
||
response_coverage = response.get('coverage')
|
||
coverage_keys = ('rawRows', 'includedRows', 'sourceMonths', 'missingStages')
|
||
response_filters = response.get('filters')
|
||
self.metric_responses.append({
|
||
'release': {key: copy.deepcopy(response.get('release', {}).get(key))
|
||
for key in ('id', 'status')}
|
||
if isinstance(response.get('release'), dict) else None,
|
||
'context': {key: copy.deepcopy(response_context.get(key))
|
||
for key in context_keys if key in response_context}
|
||
if isinstance(response_context, dict) else None,
|
||
'filters': {key: copy.deepcopy(response_filters.get(key)) for key in ('columnFilters',)
|
||
if key in response_filters}
|
||
if isinstance(response_filters, dict) else None,
|
||
'coverage': {key: copy.deepcopy(response_coverage.get(key)) for key in coverage_keys
|
||
if key in response_coverage}
|
||
if isinstance(response_coverage, dict) else None,
|
||
'protocolEvidence': copy.deepcopy(response.get('protocolEvidence')),
|
||
'hasSummary': isinstance(response.get('summary'), dict),
|
||
})
|
||
if is_investigation_read and isinstance(response, dict):
|
||
response_context = response.get('context')
|
||
keys = ('releaseId','stage','category','sourceRevisionId','year','month','quarter',
|
||
'monthStart','monthEnd','department','group','product','projectGroup',
|
||
'extraFilters','columnFilters','exclusions','includeUnassigned')
|
||
self.investigation_responses.append({
|
||
'kind': response.get('kind'), 'action': response.get('action'),
|
||
'context': {key:copy.deepcopy(response_context.get(key)) for key in keys
|
||
if key in response_context} if isinstance(response_context,dict) else None,
|
||
'aggregateComplete': response.get('aggregateComplete'),
|
||
'totalRows': response.get('totalRows'),
|
||
'sourceVerification': copy.deepcopy(response.get('sourceVerification')),
|
||
'offset': response.get('offset'), 'returnedRows': response.get('returnedRows'),
|
||
'matchingRows': response.get('matchingRows'), 'hasMore': response.get('hasMore'),
|
||
'nextOffset': response.get('nextOffset'),
|
||
'protocolEvidence': copy.deepcopy(response.get('protocolEvidence')),
|
||
})
|
||
if is_defect_read and isinstance(response, dict):
|
||
self.defect_responses.append({
|
||
'kind': response.get('kind'), 'releaseId': response.get('releaseId'),
|
||
'context': copy.deepcopy(response.get('context')),
|
||
'coverage': copy.deepcopy(response.get('coverage')),
|
||
'totalGroups': response.get('totalGroups'),
|
||
'rowsTruncated': response.get('rowsTruncated'),
|
||
'protocolEvidence': copy.deepcopy(response.get('protocolEvidence')),
|
||
})
|
||
return response
|
||
|
||
|
||
def _verified_empty_comparison_receipts(scoped_api, *, scope, task, identity, release_id):
|
||
"""Validate actual metric response receipts for an empty comparison source.
|
||
|
||
QualityTools intentionally rejects empty metrics before writing an evidence
|
||
document. This narrow recovery preserves only the signed response proofs
|
||
and a coverage gap, never metrics or KPI facts.
|
||
"""
|
||
from nao_period import normalize_period
|
||
from nao_evidence_report import period as format_period
|
||
|
||
observations = scoped_api.metric_responses
|
||
if len(observations) != 2 or not task.get('comparisonRequired'):
|
||
return None
|
||
expected_periods = [normalize_period(task.get('period')),
|
||
normalize_period(task.get('comparisonPeriod'))]
|
||
if any(period is None for period in expected_periods) or expected_periods[0] == expected_periods[1]:
|
||
return None
|
||
expected_filters = task.get('filters') if isinstance(task.get('filters'), dict) else {}
|
||
expected_keys = ('year', 'month', 'quarter', 'monthStart', 'monthEnd')
|
||
observed_by_period = {}
|
||
proofs = []
|
||
gaps = []
|
||
diagnostics = []
|
||
zero_responses = 0
|
||
for observation in observations:
|
||
release = observation.get('release')
|
||
context = observation.get('context')
|
||
coverage = observation.get('coverage')
|
||
if (not isinstance(release, dict) or release.get('id') != release_id
|
||
or not isinstance(context, dict) or context.get('view') != 'overall'
|
||
or not observation.get('hasSummary') or not isinstance(coverage, dict)):
|
||
return None
|
||
actual_period = {key: context.get(key) for key in expected_keys
|
||
if key in context and context.get(key) is not None}
|
||
try:
|
||
actual_period = normalize_period(actual_period)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
if not isinstance(actual_period, dict):
|
||
return None
|
||
period_key = tuple((key, actual_period.get(key)) for key in expected_keys if key in actual_period)
|
||
if actual_period not in expected_periods or period_key in observed_by_period:
|
||
return None
|
||
for key in expected_keys:
|
||
if context.get(key) != actual_period.get(key):
|
||
return None
|
||
for key in ('department', 'group', 'product', 'projectGroup'):
|
||
if context.get(key) != expected_filters.get(key):
|
||
return None
|
||
if (context.get('extraFilters', {}) != expected_filters.get('extraFilters', {})
|
||
or context.get('exclusions', {}) != expected_filters.get('exclusions', {})
|
||
or context.get('includeUnassigned') != expected_filters.get('includeUnassigned', False)):
|
||
return None
|
||
column_filters = expected_filters.get('columnFilters')
|
||
if column_filters is not None:
|
||
receipts = [observation.get('filters'), context.get('filters')]
|
||
echoed = [item.get('columnFilters') for item in receipts
|
||
if isinstance(item, dict) and 'columnFilters' in item]
|
||
if 'columnFilters' in context:
|
||
echoed.append(context.get('columnFilters'))
|
||
if not echoed or any(item != column_filters for item in echoed):
|
||
return None
|
||
proof = observation.get('protocolEvidence')
|
||
try:
|
||
verified = _scoped_proofs({'protocolEvidence': proof}, identity, expected_count=1,
|
||
node_id=scope['nodeId'], scope_id=scope['scopeId'])
|
||
except (TypeError, ValueError):
|
||
return None
|
||
raw_rows, included_rows = coverage.get('rawRows'), coverage.get('includedRows')
|
||
source_months = coverage.get('sourceMonths')
|
||
if (type(raw_rows) is not int or raw_rows < 0
|
||
or type(included_rows) is not int or included_rows < 0
|
||
or included_rows > raw_rows or not isinstance(source_months, list)):
|
||
return None
|
||
is_empty = raw_rows == 0 or included_rows == 0
|
||
observed_by_period[period_key] = (actual_period, raw_rows, included_rows, is_empty)
|
||
proofs.extend(verified)
|
||
diagnostics.append({'nodeId': scope['nodeId'], 'scopeId': scope['scopeId'],
|
||
'label': str(scope.get('label') or scope['scopeId']),
|
||
'period': copy.deepcopy(actual_period), 'rawRows': raw_rows,
|
||
'includedRows': included_rows,
|
||
'sourceMonths': copy.deepcopy(source_months),
|
||
'queryId': verified[0]['queryId']})
|
||
if is_empty:
|
||
zero_responses += 1
|
||
try:
|
||
label = format_period(actual_period)
|
||
except (KeyError, TypeError, ValueError):
|
||
return None
|
||
scope_label = str(scope.get('label') or scope['scopeId'])
|
||
gaps.append(scope_label+':'+label+'没有可用来源记录,暂不能给出同比结论。')
|
||
if len(observed_by_period) != 2 or zero_responses == 0:
|
||
return None
|
||
return {'proofs': proofs, 'missingEvidence': list(dict.fromkeys(gaps)),
|
||
'diagnostics': diagnostics}
|
||
|
||
|
||
def _scoped_proofs(result, identity, *, expected_count, node_id, scope_id):
|
||
proofs = _execution_proofs(result, identity, expected_count=expected_count)
|
||
for proof in proofs:
|
||
if proof.get('nodeId') != node_id or proof.get('scopeId') != scope_id:
|
||
raise ValueError('正式查询证明未绑定当前queryPlan节点和服务器范围')
|
||
return proofs
|
||
|
||
|
||
def _verified_investigation_receipts(scoped_api, identity, node_id, scope_id, seen_ids):
|
||
"""Recover only actual protocol receipts captured from investigate HTTP responses."""
|
||
recovered = []
|
||
for receipt in scoped_api.investigation_responses:
|
||
proof = receipt.get('protocolEvidence') if isinstance(receipt,dict) else None
|
||
if not isinstance(proof,dict):
|
||
continue
|
||
verified = _scoped_proofs({'protocolEvidence':proof}, identity, expected_count=1,
|
||
node_id=node_id, scope_id=scope_id)
|
||
query_id = verified[0]['queryId']
|
||
if query_id in seen_ids:
|
||
continue
|
||
seen_ids.add(query_id)
|
||
recovered.append(verified[0])
|
||
return recovered
|
||
|
||
|
||
def _raw_side_summary(side, *, action):
|
||
"""Return bounded coverage metadata; raw row values never leave this function."""
|
||
if not isinstance(side,dict):
|
||
raise ValueError('调查响应缺少期间结果')
|
||
if action == 'source':
|
||
proof = side.get('sourceVerification')
|
||
total = side.get('totalRows')
|
||
complete = (side.get('aggregateComplete') is True and type(total) is int and total > 0
|
||
and isinstance(proof,dict) and proof.get('complete') is True
|
||
and proof.get('allSelectedValuesMatchStored') is True
|
||
and type(proof.get('expectedRows')) is int and proof.get('expectedRows') == total
|
||
and type(proof.get('verifiedRows')) is int and proof.get('verifiedRows') == total
|
||
and type(proof.get('missingRows')) is int and proof.get('missingRows') == 0
|
||
and type(proof.get('mismatchRows')) is int and proof.get('mismatchRows') == 0)
|
||
return {'totalRows':total,'verifiedRows':proof.get('verifiedRows') if isinstance(proof,dict) else None,
|
||
'missingRows':proof.get('missingRows') if isinstance(proof,dict) else None,
|
||
'mismatchRows':proof.get('mismatchRows') if isinstance(proof,dict) else None,
|
||
'complete':bool(complete)}
|
||
rows = side.get('rows')
|
||
offset, returned, matching = side.get('offset'), side.get('returnedRows'), side.get('matchingRows')
|
||
if (not isinstance(rows,list) or type(offset) is not int or type(returned) is not int
|
||
or type(matching) is not int or returned != len(rows) or returned < 0
|
||
or matching < 0 or offset < 0 or offset + returned > matching
|
||
or type(side.get('hasMore')) is not bool
|
||
or side.get('hasMore') != (offset + returned < matching)
|
||
or side.get('nextOffset') != (offset + returned if side.get('hasMore') else None)):
|
||
raise ValueError('调查分页回执与实际返回记录不一致')
|
||
return {'offset':offset,'returnedRows':returned,'matchingRows':matching,
|
||
'hasMore':side['hasMore'],'nextOffset':side.get('nextOffset'),
|
||
'complete':side['hasMore'] is False}
|
||
|
||
|
||
def _safe_private_error_summary(error, task):
|
||
"""Keep a short, private diagnostic without copying scoped values into it."""
|
||
if type(error).__name__ in {'QualityToolError', 'ValueError'}:
|
||
message = str(error).replace('\n', ' ').strip()
|
||
else:
|
||
message = '查询执行时发生内部错误。'
|
||
filters = task.get('filters') if isinstance(task, dict) else None
|
||
sensitive_values = []
|
||
def collect(value):
|
||
if isinstance(value, str):
|
||
if len(value) >= 3:sensitive_values.append(value)
|
||
elif isinstance(value, dict):
|
||
for child in value.values():collect(child)
|
||
elif isinstance(value, (list, tuple, set)):
|
||
for child in value:collect(child)
|
||
collect(filters)
|
||
for value in sorted(set(sensitive_values), key=len, reverse=True):
|
||
message = message.replace(value, '[筛选值已隐藏]')
|
||
message = re.sub(r'(?i)bearer\s+\S+', 'Bearer [已隐藏]', message)
|
||
message = re.sub(r'(?i)(?:token|password|secret|authorization)(\s*[=::]\s*)[^\s,,;;]+',
|
||
r'\1[已隐藏]', message)
|
||
message = re.sub(r'\b[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\b', '[标识已隐藏]',
|
||
message, flags=re.I)
|
||
message = re.sub(r'\s+', ' ', message).strip()
|
||
return message[:180] or '查询执行时发生内部错误。'
|
||
|
||
|
||
def _private_module_provenance():
|
||
"""Fingerprint the code objects actually loaded by this worker."""
|
||
modules = {
|
||
'nao_protocol_execution': ('_scoped_arguments', '_execute_query_plan'),
|
||
'nao_quality_tools': ('QualityTools.quality_data',),
|
||
'nao_weight_ranking': ('rank_response',),
|
||
'nao_combined_ranking': ('_stage_has_explicit_zero_source', 'combined_response'),
|
||
'nao_query_contract': ('bind_query',),
|
||
'nao_agent_task': ('dimension_stages',),
|
||
}
|
||
result = {'python': sys.version.split()[0], 'executable': sys.executable, 'modules': {}}
|
||
for name, symbols in modules.items():
|
||
module = sys.modules.get(name)
|
||
if module is None:
|
||
result['modules'][name] = {'loaded': False}
|
||
continue
|
||
entry = {'loaded': True, 'file': getattr(module, '__file__', None), 'functions': {}}
|
||
spec = getattr(module, '__spec__', None)
|
||
if spec is not None:
|
||
entry['origin'] = getattr(spec, 'origin', None)
|
||
source_path = getattr(module, '__file__', None)
|
||
if isinstance(source_path, str):
|
||
try:
|
||
source = Path(source_path)
|
||
if source.is_file():
|
||
entry['sourceSha256'] = hashlib.sha256(source.read_bytes()).hexdigest()
|
||
entry['sourceMtimeNs'] = source.stat().st_mtime_ns
|
||
except OSError:
|
||
pass
|
||
for dotted in symbols:
|
||
value = module
|
||
try:
|
||
for part in dotted.split('.'):
|
||
value = getattr(value, part)
|
||
code = getattr(value, '__code__', None)
|
||
if code is not None:
|
||
entry['functions'][dotted] = hashlib.sha256(marshal.dumps(code)).hexdigest()
|
||
except (AttributeError, TypeError, ValueError):
|
||
continue
|
||
result['modules'][name] = entry
|
||
return result
|
||
|
||
|
||
def _private_failure_diagnostic(error, diagnostic, task, arguments=None):
|
||
"""Retain sanitized first-party validation detail only in private diagnostics."""
|
||
from nao_quality_tools import QualityToolError
|
||
|
||
result = copy.deepcopy(diagnostic)
|
||
if isinstance(error, QualityToolError):
|
||
message = str(error)
|
||
# Never persist SQL, transport addresses, or credential-bearing text,
|
||
# even if a future caller wraps an upstream failure in QualityToolError.
|
||
unsafe = re.compile(
|
||
r'(?is)(?:\b(?:select|insert|update|delete|drop|alter|truncate|create)\b|\bsql\b|https?://|'
|
||
r'\b(?:authorization|cookie|set-cookie)\s*[:=]|\bbearer\s+\S+)')
|
||
result['errorSummary'] = (
|
||
'受控查询校验失败;详细内容已隐藏。'
|
||
if unsafe.search(message) else _safe_private_error_summary(error, task))
|
||
result['moduleProvenance'] = _private_module_provenance()
|
||
rank_trace = getattr(error, '_quality_private_rank_trace', None)
|
||
if isinstance(rank_trace, dict):
|
||
result['rankResponseTrace'] = copy.deepcopy(rank_trace)
|
||
frames = []
|
||
for frame in traceback.extract_tb(error.__traceback__)[-16:]:
|
||
filename = Path(frame.filename).name
|
||
function = frame.name
|
||
if (re.fullmatch(r'[A-Za-z0-9_.-]{1,100}', filename)
|
||
and re.fullmatch(r'[A-Za-z0-9_.<>-]{1,100}', function)
|
||
and type(frame.lineno) is int and frame.lineno > 0):
|
||
frames.append({'file': filename, 'function': function, 'line': frame.lineno})
|
||
if frames:
|
||
result['exceptionFrames'] = frames
|
||
if isinstance(arguments, dict):
|
||
allowed = {
|
||
'mode': {'query', 'compare', 'rank', 'changes', 'investigate', 'defects'},
|
||
'dimension': {'product', 'department', 'group', 'defect'},
|
||
'metric': {'rate', 'scrapRate', 'totalWeight', 'scrapWeight'},
|
||
'stage': {'black', 'bright'},
|
||
}
|
||
summary = {}
|
||
for key, choices in allowed.items():
|
||
value = arguments.get(key)
|
||
if value in choices or key == 'stage' and value is None:
|
||
summary[key] = value
|
||
limit = arguments.get('limit')
|
||
if type(limit) is int and 1 <= limit <= 100:
|
||
summary['limit'] = limit
|
||
if summary:
|
||
result['queryInput'] = summary
|
||
return result
|
||
|
||
|
||
def _controlled_failure_diagnostic(error, *, node_id=None, purpose=None):
|
||
"""Return bounded failure classification without copying upstream text or query values."""
|
||
from nao_quality_tools import QualityToolError, SourceCoverageUnavailable
|
||
from nao_quality_runtime import RuntimeErrorSafe
|
||
|
||
raw_status = getattr(error, 'http_status', getattr(error, 'code', None))
|
||
status = raw_status if type(raw_status) is int and 100 <= raw_status <= 599 else None
|
||
retryable = _retryable_query_error(error)
|
||
if isinstance(error, SourceCoverageUnavailable):
|
||
kind = 'source_coverage'
|
||
elif status is not None:
|
||
kind = 'transient_http' if retryable else 'http_error'
|
||
elif isinstance(error, QualityToolError):
|
||
kind = 'formal_query_rejected'
|
||
elif isinstance(error, RuntimeErrorSafe) and retryable or isinstance(error, (TimeoutError, ConnectionError, OSError)):
|
||
kind = 'transport_unavailable'
|
||
elif isinstance(error, ValueError):
|
||
kind = 'query_rejected'
|
||
else:
|
||
kind = 'internal_error'
|
||
error_type = type(error).__name__
|
||
if not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,63}', error_type):
|
||
error_type = 'Exception'
|
||
result = {'failureKind': kind, 'errorType': error_type, 'retryable': retryable}
|
||
if status is not None:
|
||
result['httpStatus'] = status
|
||
if isinstance(node_id, str) and re.fullmatch(r'[A-Za-z0-9_-]{1,64}', node_id):
|
||
result['nodeId'] = node_id
|
||
if purpose in {'metrics', 'ranking', 'trend', 'changes', 'defects', 'source', 'records'}:
|
||
result['purpose'] = purpose
|
||
return result
|
||
|
||
|
||
def _raw_period_label(value):
|
||
from nao_evidence_report import period
|
||
try:
|
||
return period(value)
|
||
except (KeyError,TypeError,ValueError):
|
||
return '所选期间'
|
||
|
||
|
||
def _raw_node_report(raw_diagnostics, scopes, initial_gaps):
|
||
"""Create source/records business facts from verified receipts and page coverage."""
|
||
report = {'sections':[],'facts':[],'missingEvidence':list(dict.fromkeys(initial_gaps)),
|
||
'coverageSufficient':False}
|
||
scope_by_pair = {_scope_pair(scope):scope for scope in scopes if isinstance(scope,dict)}
|
||
represented = set()
|
||
observed_stages = {}
|
||
for item in raw_diagnostics:
|
||
if not isinstance(item,dict):continue
|
||
pair=(item.get('nodeId'),item.get('scopeId'))
|
||
scope=scope_by_pair.get(pair)
|
||
if scope is None:continue
|
||
label=str(scope.get('label') or pair[0])
|
||
stage=item.get('stage')
|
||
observed_stages.setdefault(pair,set()).add(stage)
|
||
stage_label={'black':'黑片','bright':'亮片'}.get(stage,str(stage or '所选阶段'))
|
||
action=item.get('action')
|
||
selected=item.get('sides') if isinstance(item.get('sides'),dict) else {}
|
||
expected_periods=[scope['task'].get('period')]
|
||
if scope['task'].get('comparisonRequired'):
|
||
expected_periods.append(scope['task'].get('comparisonPeriod'))
|
||
section_rows=[]
|
||
for index,period_value in enumerate(expected_periods):
|
||
side_name='current' if index==0 else 'comparison'
|
||
side=selected.get(side_name)
|
||
period_label=_raw_period_label(period_value)
|
||
if action=='source':
|
||
complete=bool(isinstance(side,dict) and side.get('complete') is True)
|
||
if isinstance(side,dict):
|
||
detail=f"已核验{side.get('verifiedRows')}/{side.get('totalRows')}行,未核验{side.get('missingRows')}行,不一致{side.get('mismatchRows')}行"
|
||
reference=side.get('evidenceRef')
|
||
else:
|
||
detail='没有取得可验证的原表核验回执';reference=None
|
||
section_rows.append([period_label,stage_label,'原表核验',detail])
|
||
if complete and reference:
|
||
report['facts'].append({'id':'F'+str(len(report['facts'])+1),'kind':'source',
|
||
'label':label+' '+stage_label+'原表核验',
|
||
'text':f'{label};{period_label};{stage_label}原表核验完整,已核验{side.get("verifiedRows")}行,原表值与存储记录一致。',
|
||
'evidenceRef':reference,'nodeId':pair[0],'scopeId':pair[1]})
|
||
else:
|
||
report['missingEvidence'].append(f'{label}:{period_label}的{stage_label}原表核验未完整通过。')
|
||
else:
|
||
pages=side.get('pages',[]) if isinstance(side,dict) else []
|
||
total=side.get('matchingRows') if isinstance(side,dict) else None
|
||
end=0;complete=bool(pages)
|
||
ordered=sorted(pages,key=lambda page:page.get('offset',-1) if isinstance(page,dict) else -1)
|
||
if not ordered:complete=False
|
||
for page in ordered:
|
||
if (not isinstance(page,dict) or type(page.get('offset')) is not int
|
||
or page.get('offset')!=end or type(page.get('returnedRows')) is not int
|
||
or page.get('matchingRows')!=total or page.get('returnedRows')<0
|
||
or page.get('offset')+page.get('returnedRows')>total):
|
||
complete=False;break
|
||
if page.get('hasMore') is not (page.get('offset')+page.get('returnedRows')<total):
|
||
complete=False;break
|
||
end=page['offset']+page['returnedRows']
|
||
complete=bool(complete and end==total and ordered and ordered[-1].get('hasMore') is False)
|
||
reference=next((page.get('evidenceRef') for page in reversed(ordered)
|
||
if isinstance(page,dict) and isinstance(page.get('evidenceRef'),str)),None)
|
||
detail=(f'已读取{end}/{total}条原始记录' if type(total) is int else '没有取得有效分页总数')
|
||
section_rows.append([period_label,stage_label,'原始记录',detail])
|
||
if complete and reference:
|
||
report['facts'].append({'id':'F'+str(len(report['facts'])+1),'kind':'records',
|
||
'label':label+' '+stage_label+'原始记录',
|
||
'text':f'{label};{period_label};{stage_label}已按稳定分页读全{total}条符合条件的原始记录。',
|
||
'evidenceRef':reference,'nodeId':pair[0],'scopeId':pair[1]})
|
||
elif type(total) is int and total==0 and pages and pages[0].get('evidenceRef'):
|
||
# An empty selected range has no missing rows; keep the
|
||
# explicit zero result and its actual evidence reference.
|
||
report['facts'].append({'id':'F'+str(len(report['facts'])+1),'kind':'records',
|
||
'label':label+' '+stage_label+'原始记录',
|
||
'text':f'{label};{period_label};{stage_label}符合条件的原始记录为0条。',
|
||
'evidenceRef':pages[0]['evidenceRef'],'nodeId':pair[0],'scopeId':pair[1]})
|
||
complete=True
|
||
else:
|
||
report['missingEvidence'].append(f'{label}:{period_label}的{stage_label}原始记录未读全。')
|
||
if section_rows:
|
||
represented.add(pair)
|
||
header='### '+label+' '+stage_label
|
||
table='| 期间 | 阶段 | 读取 | 覆盖 |\n| --- | --- | --- | --- |\n'
|
||
table+='\n'.join('| '+' | '.join(str(cell) for cell in row)+' |' for row in section_rows)
|
||
report['sections'].append(header+'\n\n'+table)
|
||
for pair,scope in scope_by_pair.items():
|
||
if scope.get('purpose') in ('source','records') and pair not in represented:
|
||
label=str(scope.get('label') or pair[0])
|
||
report['missingEvidence'].append(f'节点{pair[0]}范围{label}没有取得原始读取回执。')
|
||
if scope.get('purpose') in ('source','records'):
|
||
expected_stages=set(scope.get('task',{}).get('stages',[]))
|
||
missing_stages=expected_stages-observed_stages.get(pair,set())
|
||
if missing_stages:
|
||
label=str(scope.get('label') or pair[0])
|
||
stage_names='、'.join({'black':'黑片','bright':'亮片'}.get(stage,stage) for stage in sorted(missing_stages))
|
||
report['missingEvidence'].append(f'节点{pair[0]}范围{label}缺少{stage_names}原始读取回执。')
|
||
report['missingEvidence']=list(dict.fromkeys(report['missingEvidence']))
|
||
report['coverageSufficient']=bool(any(scope.get('purpose') in ('source','records') for scope in scope_by_pair.values())
|
||
and not report['missingEvidence'])
|
||
return report
|
||
|
||
|
||
def _protocol_raw_coverage(task, report):
|
||
"""Expose only verified source/record counts to the KPI explanation model."""
|
||
required = {'source': 0, 'records': 0}
|
||
for node in task.get('queryPlan', []) if isinstance(task, dict) else []:
|
||
if not isinstance(node, dict) or node.get('purpose') not in required:
|
||
continue
|
||
selector = node.get('selector') if isinstance(node.get('selector'), dict) else {'kind':'base'}
|
||
selector_kind = selector.get('kind')
|
||
scope_count = {'catalog_members': len(selector.get('names', [])),
|
||
'ranking_top': selector.get('count'),
|
||
'previous_ranking': len(selector.get('ranks', []))}.get(selector_kind, 1)
|
||
if type(scope_count) is not int or scope_count < 1:
|
||
continue
|
||
stage_count = len(node.get('stages', [])) if isinstance(node.get('stages'), list) else 0
|
||
period_count = 2 if node.get('comparisonRequired') else 1
|
||
required[node['purpose']] += scope_count * stage_count * period_count
|
||
facts = report.get('facts', []) if isinstance(report, dict) else []
|
||
verified = {purpose: sum(1 for fact in facts
|
||
if isinstance(fact, dict) and fact.get('kind') == purpose)
|
||
for purpose in required}
|
||
if not any(required.values()):
|
||
return None
|
||
return {purpose: {'requiredChecks': required[purpose],
|
||
'verifiedChecks': verified[purpose],
|
||
'complete': bool(required[purpose] > 0
|
||
and verified[purpose] == required[purpose])}
|
||
for purpose in required if required[purpose] > 0}
|
||
|
||
|
||
def _protocol_query_snapshot(delegated, *, include_skipped=False):
|
||
result = delegated.call('/api/analysis/protocol/queries', payload={}, method='POST')
|
||
if not isinstance(result, dict) or result.get('version') != 1:
|
||
raise ValueError('queryPlan范围解析接口返回结构无效')
|
||
scopes, pending, gaps = result.get('scopes'), result.get('pending'), result.get('gaps')
|
||
if (not isinstance(scopes, list) or not isinstance(pending, list) or not isinstance(gaps, list)
|
||
or any(not isinstance(item, str) or not item.strip() for item in gaps)):
|
||
raise ValueError('queryPlan范围解析接口缺少scopes/pending/gaps')
|
||
skipped = result.get('skipped', [])
|
||
if not isinstance(skipped, list) or any(not isinstance(item, dict) for item in skipped):
|
||
raise ValueError('queryPlan条件停止回执无效')
|
||
values = (copy.deepcopy(scopes), copy.deepcopy(pending), list(gaps))
|
||
return (*values, copy.deepcopy(skipped)) if include_skipped else values
|
||
|
||
|
||
def _validate_protocol_scope(scope, node, root_task, release_id, *, catalog=None):
|
||
import re
|
||
from nao_period import normalize_period
|
||
from nao_evidence_report import _filters
|
||
if not isinstance(scope, dict):
|
||
raise ValueError('服务器queryPlan scope结构无效')
|
||
node_id, scope_id, purpose, child = (scope.get('nodeId'), scope.get('scopeId'),
|
||
scope.get('purpose'), scope.get('task'))
|
||
if (node_id != node.get('nodeId') or purpose != node.get('purpose')
|
||
or not isinstance(scope_id, str) or not scope_id or len(scope_id) > 180
|
||
or not isinstance(child, dict) or child.get('version') != 1 or 'queryPlan' in child):
|
||
raise ValueError('服务器scope与queryPlan节点不匹配')
|
||
try:
|
||
if normalize_period(child.get('period')) != normalize_period(root_task.get('period')):
|
||
raise ValueError('服务器scope不得改变root.period')
|
||
if node.get('comparisonRequired'):
|
||
if normalize_period(child.get('comparisonPeriod')) != normalize_period(root_task.get('comparisonPeriod')):
|
||
raise ValueError('服务器scope比较期间与root不匹配')
|
||
elif child.get('comparisonPeriod') is not None or child.get('comparisonRequired'):
|
||
raise ValueError('未请求比较的queryPlan节点不能新增比较期间')
|
||
except (TypeError, ValueError):
|
||
raise ValueError('服务器scope的period/comparisonPeriod无效或偏离root') from None
|
||
raw_purpose = node.get('purpose') if node.get('purpose') in ('source', 'records') else None
|
||
expected_kind = {'source':'source_verify','records':'details'}.get(raw_purpose)
|
||
if expected_kind is not None:
|
||
if child.get('kind') != expected_kind:
|
||
raise ValueError('服务器raw scope kind与节点purpose不一致')
|
||
if (child.get('verifySource') is not (raw_purpose == 'source')
|
||
or child.get('fullDetails') is not (raw_purpose == 'records')):
|
||
raise ValueError('服务器raw scope未保留root原始读取要求')
|
||
investigation_fields = child.get('investigationFields')
|
||
if node.get('sourceFields') and not isinstance(investigation_fields,dict):
|
||
raise ValueError('服务器scope未返回sourceFields对应的授权原始列映射')
|
||
if investigation_fields is not None:
|
||
if not isinstance(investigation_fields,dict) or set(investigation_fields) - set(node.get('stages',[])):
|
||
raise ValueError('服务器investigationFields阶段与原始节点不一致')
|
||
for stage, fields in investigation_fields.items():
|
||
if (not isinstance(fields,list) or not 1 <= len(fields) <= 32
|
||
or any(not isinstance(field,str) or not field.strip() or len(field)>200 for field in fields)
|
||
or len(set(fields)) != len(fields)):
|
||
raise ValueError('服务器investigationFields原始列映射结构无效')
|
||
selected = (node.get('sourceFields') or {}).get(stage)
|
||
if selected is not None and len(fields) != len(selected):
|
||
raise ValueError('服务器investigationFields未按选定stable keys完整映射原始列')
|
||
if node.get('sourceFields') and not set(node['sourceFields']) <= set(investigation_fields):
|
||
raise ValueError('服务器investigationFields未返回每个显式sourceFields阶段')
|
||
elif child.get('kind') not in ('analysis', 'metrics'):
|
||
raise ValueError('queryPlan临时scope task必须是analysis/metrics或对应raw kind')
|
||
if expected_kind is None and (child.get('verifySource') is True or child.get('fullDetails') is True):
|
||
raise ValueError('metrics scope不能继承原始读取标记')
|
||
conditional_stages = ((node.get('selector') or {}).get('kind') == 'changes_top'
|
||
and (node.get('selector') or {}).get('stage') == 'dominant')
|
||
stages_match = child.get('stages') == node.get('stages')
|
||
if conditional_stages and node.get('stages'):
|
||
stages_match = (isinstance(child.get('stages'), list) and bool(child['stages'])
|
||
and set(child['stages']) <= set(node['stages']))
|
||
if (child.get('fields') != node.get('fields') or not stages_match
|
||
or child.get('trendRequired') is not (node.get('purpose') == 'trend')
|
||
or child.get('comparisonRequired') is not node.get('comparisonRequired')):
|
||
raise ValueError('服务器scope task未保留节点指标、阶段或趋势/比较要求')
|
||
if child.get('periodCoverage') != root_task.get('periodCoverage'):
|
||
raise ValueError('服务器scope task未继承root periodCoverage')
|
||
root_filters, child_filters = _filters(root_task.get('filters')), _filters(child.get('filters'))
|
||
selector = node.get('selector') if isinstance(node.get('selector'), dict) else {}
|
||
conditional_dimensions = set()
|
||
if selector.get('kind') == 'changes_top':
|
||
by_id = {item.get('nodeId'):item for item in root_task.get('queryPlan', [])}
|
||
source_id, seen = selector.get('sourceNodeId'), set()
|
||
while source_id and source_id not in seen:
|
||
seen.add(source_id);source_node = by_id.get(source_id, {})
|
||
if source_node.get('purpose') != 'changes':
|
||
raise ValueError('结果下钻来源必须是已声明的changes节点')
|
||
conditional_dimensions.add(source_node.get('dimension'))
|
||
parent_selector=source_node.get('selector') or {}
|
||
source_id=parent_selector.get('sourceNodeId') if parent_selector.get('kind') == 'changes_top' else None
|
||
if 'group' in conditional_dimensions:conditional_dimensions.add('department')
|
||
from nao_changes import selection_contains
|
||
if any(child_filters.get(key) != value and not (key in conditional_dimensions
|
||
and selection_contains(value, child_filters.get(key))) for key, value in root_filters.items()):
|
||
raise ValueError('服务器scope task丢失或改写了root filters')
|
||
if 'columnFilters' in root_filters and child_filters.get('columnFilters') != root_filters['columnFilters']:
|
||
raise ValueError('服务器scope task丢失了root columnFilters')
|
||
selector_kind = selector.get('kind')
|
||
target_dimension = None
|
||
if selector_kind == 'catalog_members':
|
||
target_dimension = selector.get('dimension')
|
||
elif selector_kind in ('ranking_top', 'previous_ranking'):
|
||
target_dimension = 'product'
|
||
elif selector_kind == 'changes_top':
|
||
source_node = next((item for item in root_task.get('queryPlan', [])
|
||
if item.get('nodeId') == selector.get('sourceNodeId')), {})
|
||
target_dimension = source_node.get('dimension')
|
||
expected_scope_filters = copy.deepcopy(root_filters)
|
||
new_filter_keys = set(child_filters) - set(root_filters)
|
||
if selector_kind == 'base':
|
||
if child_filters != root_filters:
|
||
raise ValueError('base scope不能改变root filters')
|
||
elif target_dimension:
|
||
allowed_filter_keys = {target_dimension} | conditional_dimensions
|
||
if selector_kind == 'changes_top':
|
||
expected_scope_filters.update({key:child_filters[key] for key in conditional_dimensions
|
||
if key in child_filters})
|
||
# The server binds each group to its unique authorized parent. This
|
||
# narrows the selected group; it must match the same actor catalogue,
|
||
# rather than accepting an arbitrary extra department from a scope.
|
||
if target_dimension == 'group' and 'department' in new_filter_keys:
|
||
parents = catalog.get('groupDepartments') if isinstance(catalog, dict) else None
|
||
parent = parents.get(child_filters.get('group')) if isinstance(parents, dict) else None
|
||
if not isinstance(parent, str) or child_filters['department'] != parent:
|
||
raise ValueError('服务器scope业务分组的产业部归属与授权目录不一致')
|
||
allowed_filter_keys.add('department')
|
||
expected_scope_filters['department'] = parent
|
||
if new_filter_keys - allowed_filter_keys:
|
||
raise ValueError('服务器scope添加了选择器以外的筛选')
|
||
selected_value = child_filters.get(target_dimension)
|
||
if (target_dimension in root_filters and selected_value != root_filters[target_dimension]
|
||
and not (selector_kind == 'changes_top' and selection_contains(root_filters[target_dimension], selected_value))):
|
||
raise ValueError('服务器scope改写了root对象筛选')
|
||
if selector_kind == 'catalog_members':
|
||
if selected_value not in selector.get('names', []):
|
||
raise ValueError('服务器scope对象不在模型声明的目录成员选择内')
|
||
elif not isinstance(selected_value, str) or not selected_value.strip():
|
||
raise ValueError('服务器scope缺少由可信排名来源解析的产品规格')
|
||
expected_scope_filters[target_dimension] = selected_value
|
||
if child_filters != expected_scope_filters:
|
||
raise ValueError('服务器scope筛选与root范围及选择器不一致')
|
||
elif child_filters != root_filters:
|
||
raise ValueError('服务器scope selector未授权扩展root filters')
|
||
if child.get('visualPreference') != node.get('visualPreference') or child.get('visualType') != node.get('visualType'):
|
||
raise ValueError('服务器scope未保留节点图表偏好')
|
||
if node['purpose'] == 'changes':
|
||
query = {'dimension':node['dimension'], 'sort':node['sort'], 'limit':node['limit']}
|
||
if (child.get('changeQuery') != query or child.get('dimensions') != [node['dimension']]
|
||
or child.get('dimensionStages') != {node['dimension']:child.get('stages')}):
|
||
raise ValueError('服务器scope批量对比维度、排序或阶段与节点不一致')
|
||
elif node['purpose'] == 'ranking':
|
||
from nao_weight_ranking import ranking_task
|
||
if (child.get('dimensions') != [node['dimension']] or not ranking_task(child)
|
||
or node['dimension'] not in child.get('dimensionStages', {})
|
||
or child['dimensionStages'][node['dimension']] != node['stages']):
|
||
raise ValueError('服务器scope排名任务与节点维度/阶段不一致')
|
||
limit = scope.get('limit', node['limit'])
|
||
if type(limit) is not int or not 1 <= limit <= node['limit']:
|
||
raise ValueError('服务器scope排名展示数超过节点上限')
|
||
elif node['purpose'] == 'defects':
|
||
expected_query = {'dimension': node['dimension'], 'limit': node['limit']}
|
||
for key in ('defect', 'category'):
|
||
if key in node:
|
||
expected_query[key] = node[key]
|
||
if (child.get('kind') != 'analysis' or child.get('defectQuery') != expected_query
|
||
or child.get('fields') != ['scrapWeight']):
|
||
raise ValueError('服务器scope的defectQuery或正式analysis字段与节点不一致')
|
||
expected_dimensions = ['defect'] if node['dimension'] == 'defect' else ['product', 'defect']
|
||
expected_stages = {dimension: list(child['stages']) for dimension in expected_dimensions}
|
||
actual_stages = child.get('dimensionStages')
|
||
if (not isinstance(child.get('dimensions'), list)
|
||
or any(not isinstance(dimension, str) for dimension in child['dimensions'])
|
||
or len(child['dimensions']) != len(set(child['dimensions']))
|
||
or set(child['dimensions']) != set(expected_dimensions)
|
||
or not isinstance(actual_stages, dict)
|
||
or set(actual_stages) != set(expected_dimensions)
|
||
or {key: actual_stages.get(key) for key in expected_dimensions} != expected_stages):
|
||
raise ValueError('服务器scope缺陷输出维度或dimensionStages与节点不一致')
|
||
elif child.get('dimensions') not in ([], None):
|
||
raise ValueError('非ranking child task不得将排名维度带入根级过滤')
|
||
if child.get('period') is None or child.get('filters') is None:
|
||
raise ValueError('服务器scope缺少精确期间或筛选')
|
||
return copy.deepcopy(scope)
|
||
|
||
|
||
def _scoped_arguments(request, scope, node):
|
||
from nao_agent_task import dimension_stages
|
||
task = scope['task']
|
||
filters = copy.deepcopy(task.get('filters') or {})
|
||
column_filters = filters.pop('columnFilters', None)
|
||
arguments = []
|
||
if node['purpose'] == 'changes':
|
||
item = {'mode':'changes','releaseId':request['releaseId'],
|
||
'period':copy.deepcopy(task['period']), 'comparisonPeriod':copy.deepcopy(task['comparisonPeriod']),
|
||
'filters':copy.deepcopy(filters), 'dimension':node['dimension'],
|
||
'sort':node['sort'], 'limit':node['limit'],
|
||
'stage':(task.get('stages') or [None])[0]}
|
||
if column_filters is not None:item['columnFilters'] = copy.deepcopy(column_filters)
|
||
arguments.append(item)
|
||
elif node['purpose'] == 'ranking':
|
||
dimension = node['dimension']
|
||
for stage in dimension_stages(task, dimension, combined=False):
|
||
if stage not in node['stages'] and not (stage is None and not node['stages']):
|
||
raise ValueError('排名scope请求阶段超出queryPlan节点')
|
||
for field in node['fields']:
|
||
if stage is None and field == 'rate':continue
|
||
item = {'mode': 'rank', 'releaseId': request['releaseId'],
|
||
'period': copy.deepcopy(task['period']), 'filters': copy.deepcopy(filters),
|
||
'dimension': dimension, 'stage': stage, 'metric': field,
|
||
'limit': scope.get('limit', node['limit'])}
|
||
if column_filters is not None:item['columnFilters'] = copy.deepcopy(column_filters)
|
||
arguments.append(item)
|
||
elif node['purpose'] in ('source','records'):
|
||
action = 'source' if node['purpose'] == 'source' else 'rows'
|
||
investigation_fields = task.get('investigationFields')
|
||
for stage in task['stages']:
|
||
item = {'mode':'investigate','action':action,'releaseId':request['releaseId'],
|
||
'period':copy.deepcopy(task['period']),'filters':copy.deepcopy(filters),
|
||
'stage':stage,'limit':100}
|
||
if action == 'rows':
|
||
item.update(offset=0, recordFilter='all')
|
||
if node.get('comparisonRequired'):
|
||
item['comparisonPeriod'] = copy.deepcopy(task['comparisonPeriod'])
|
||
stage_fields = investigation_fields.get(stage) if isinstance(investigation_fields,dict) else None
|
||
if stage_fields is not None:
|
||
if not isinstance(stage_fields,list) or not stage_fields:
|
||
raise ValueError('服务器investigationFields缺少本阶段的原始列')
|
||
item['fields'] = copy.deepcopy(stage_fields)
|
||
if column_filters is not None:
|
||
item['columnFilters'] = copy.deepcopy(column_filters)
|
||
arguments.append(item)
|
||
elif node['purpose'] == 'defects':
|
||
query = task.get('defectQuery')
|
||
if not isinstance(query, dict):
|
||
raise ValueError('服务器scope缺少defectQuery')
|
||
limit = query.get('limit')
|
||
if type(limit) is not int or not 1 <= limit <= node['limit']:
|
||
raise ValueError('服务器scope缺陷Top展示数超过节点上限')
|
||
for stage in task['stages']:
|
||
item = {'mode':'defects','releaseId':request['releaseId'],
|
||
'period':copy.deepcopy(task['period']),'filters':copy.deepcopy(filters),
|
||
'stage':stage,'dimension':query['dimension'],'limit':limit}
|
||
if 'defect' in query:
|
||
item['defect'] = query['defect']
|
||
if 'category' in query:
|
||
item['category'] = query['category']
|
||
if node.get('comparisonRequired'):
|
||
item['comparisonPeriod'] = copy.deepcopy(task['comparisonPeriod'])
|
||
if column_filters is not None:
|
||
item['columnFilters'] = copy.deepcopy(column_filters)
|
||
arguments.append(item)
|
||
else:
|
||
arguments.append(_metric_arguments(request, task, column_filters=column_filters,
|
||
comparison=bool(node.get('comparisonRequired'))))
|
||
return arguments
|
||
|
||
|
||
def _scope_pair(scope):
|
||
return (scope.get('nodeId'), scope.get('scopeId'))
|
||
|
||
|
||
def _execute_query_plan(*, request, task, authority, actor_root, delegated, connection,
|
||
cancel_event, emit, timings):
|
||
"""Resolve, execute and finalize a bounded set of server-authorized child scopes."""
|
||
from nao_agent_task import bind_task
|
||
from nao_quality_runtime import public_tool_result
|
||
from nao_quality_tools import QualityToolError, QualityTools, SourceCoverageUnavailable
|
||
from nao_evidence_finalizer import (evidence_summaries, synthesize,
|
||
coverage_sufficient_scoped)
|
||
from analysis_visual_evidence import protocol_visual_result
|
||
from nao_agent_report import build_scoped_protocol_report
|
||
|
||
identity = _identity(authority)
|
||
plan_nodes = {node['nodeId']: node for node in task['queryPlan']}
|
||
all_scopes = {}
|
||
pending_latest = []
|
||
server_gaps = []
|
||
calls, proofs, accepted_refs = [], [], set()
|
||
query_diagnostics = []
|
||
failure_diagnostics = []
|
||
private_failure_diagnostics = []
|
||
raw_diagnostics = []
|
||
query_recovery = []
|
||
query_ids = set()
|
||
read_count = 0
|
||
raw_rows_read = 0
|
||
phase_started = time.monotonic()
|
||
emit({'type': 'progress', 'message': '任务已拆分,正在解析并执行有限范围查询节点…'})
|
||
|
||
scopes, pending, gaps, skipped = _protocol_query_snapshot(delegated, include_skipped=True)
|
||
skipped_nodes = {}
|
||
while True:
|
||
for receipt in skipped:
|
||
node = plan_nodes.get(receipt.get('nodeId'))
|
||
if not node or node.get('optional') is not True or not receipt.get('reason'):
|
||
raise ValueError('条件停止只能由服务器对已声明可选节点确认')
|
||
skipped_nodes[node['nodeId']] = receipt
|
||
for item in pending:
|
||
if isinstance(item, dict) and isinstance(item.get('nodeId'), str):
|
||
if item.get('nodeId') not in plan_nodes:
|
||
raise ValueError('queryPlan pending包含未声明nodeId')
|
||
elif not isinstance(item, dict):
|
||
raise ValueError('queryPlan pending结构无效')
|
||
pending_latest = pending
|
||
server_gaps.extend(gaps)
|
||
newly_ready = []
|
||
for raw_scope in scopes:
|
||
node_id = raw_scope.get('nodeId') if isinstance(raw_scope, dict) else None
|
||
node = plan_nodes.get(node_id)
|
||
if node is None:
|
||
raise ValueError('服务器返回queryPlan未声明nodeId')
|
||
scope = _validate_protocol_scope(raw_scope, node, task, request['releaseId'],
|
||
catalog=request.get('_scopeDirectory'))
|
||
key = _scope_pair(scope)
|
||
previous = all_scopes.get(key)
|
||
if previous is not None and previous != scope:
|
||
raise ValueError('同一nodeId/scopeId的服务器任务定义发生变化')
|
||
all_scopes[key] = scope
|
||
if previous is None:
|
||
newly_ready.append(scope)
|
||
if not newly_ready:
|
||
break
|
||
for scope in newly_ready:
|
||
if cancel_event.is_set():raise RuntimeError('cancelled')
|
||
node = plan_nodes[scope['nodeId']]
|
||
arguments_queue = []
|
||
active_arguments = None
|
||
scoped_api = None
|
||
active_raw_diagnostic = None
|
||
try:
|
||
arguments_queue = _scoped_arguments(request, scope, node)
|
||
scoped_reads = sum(2 if (item['mode'] == 'compare'
|
||
or item['mode'] in ('investigate', 'defects') and item.get('comparisonPeriod') is not None)
|
||
else 1 for item in arguments_queue)
|
||
if read_count + scoped_reads > 40:
|
||
server_gaps.append(f"节点{node['nodeId']}超出本轮40次正式查询上限,未执行其余范围。")
|
||
continue
|
||
context = bind_task({'releaseId': request['releaseId']}, scope['task'])
|
||
contract = {'task': scope['task'], 'releaseId': request['releaseId'],
|
||
'exactScope': True, 'nodeId': scope['nodeId'], 'scopeId': scope['scopeId']}
|
||
scoped_api = ScopedApi(delegated, scope['nodeId'], scope['scopeId'],
|
||
cancel_event=cancel_event, emit=emit, recovery_log=query_recovery,
|
||
read_limit=40-read_count)
|
||
tools = QualityTools(api_factory=lambda: scoped_api,
|
||
evidence_dir=actor_root / 'evidence', query_contract=contract)
|
||
if node['purpose'] in ('source','records'):
|
||
action = 'source' if node['purpose'] == 'source' else 'rows'
|
||
for base_arguments in arguments_queue:
|
||
if cancel_event.is_set():raise RuntimeError('cancelled')
|
||
started = datetime.now(timezone.utc).isoformat()
|
||
stage = base_arguments['stage']
|
||
diagnostic = {'nodeId':scope['nodeId'],'scopeId':scope['scopeId'],
|
||
'purpose':node['purpose'],'action':action,'stage':stage,
|
||
'sides':{}}
|
||
active_raw_diagnostic = diagnostic
|
||
if action == 'source':
|
||
expected_reads = 2 if node.get('comparisonRequired') else 1
|
||
if read_count + expected_reads > 40:
|
||
server_gaps.append(f"节点{node['nodeId']}范围{scope.get('label') or scope['scopeId']}达到40次正式查询上限,原表核验未执行完整。")
|
||
continue
|
||
reads_before = scoped_api.read_count
|
||
try:
|
||
result = tools.quality_data(base_arguments)
|
||
finally:
|
||
read_count += scoped_api.read_count - reads_before
|
||
expected_count = expected_reads
|
||
current_proofs = _scoped_proofs(result, identity, expected_count=expected_count,
|
||
node_id=scope['nodeId'], scope_id=scope['scopeId'])
|
||
if any(proof['queryId'] in query_ids for proof in current_proofs):
|
||
raise ValueError('原表核验证明queryId重复')
|
||
query_ids.update(proof['queryId'] for proof in current_proofs)
|
||
proofs.extend(current_proofs)
|
||
evidence_ref = result.get('evidenceRef')
|
||
if isinstance(evidence_ref,str):accepted_refs.add(evidence_ref)
|
||
for side_name in ('current','comparison'):
|
||
side = result.get(side_name)
|
||
if not isinstance(side,dict):continue
|
||
summary = _raw_side_summary(side,action='source')
|
||
summary['evidenceRef'] = evidence_ref
|
||
diagnostic['sides'][side_name] = summary
|
||
raw_diagnostics.append(diagnostic)
|
||
label = str(scope.get('label') or scope['scopeId'])+' '+stage+'原表核验'
|
||
calls.append({'tool':'quality_data','toolCallId':'protocol-'+current_proofs[0]['queryId'],
|
||
'label':label,'input':copy.deepcopy(base_arguments),
|
||
'nodeId':scope['nodeId'],'scopeId':scope['scopeId'],
|
||
'status':'completed','startedAt':started,
|
||
'finishedAt':datetime.now(timezone.utc).isoformat(),
|
||
'outputSummary':'原表核验回执已绑定服务器签发的node/scope;依据 '+str(evidence_ref or '正式调查证据')})
|
||
emit({'type':'progress','message':f'原始调查 {read_count}/40 已完成。',
|
||
'toolCalls':copy.deepcopy(calls),'toolCount':len(calls)})
|
||
continue
|
||
|
||
# Read full-detail rows in stable offset order. A
|
||
# comparison response reads both periods together;
|
||
# pages remain aligned and all values stay in evidence.
|
||
periods = [('current',scope['task'].get('period'))]
|
||
if node.get('comparisonRequired'):
|
||
periods.append(('comparison',scope['task'].get('comparisonPeriod')))
|
||
states = {name:{'period':copy.deepcopy(period),'pages':[],
|
||
'matchingRows':None,'done':False}
|
||
for name,period in periods}
|
||
raw_diagnostics.append(diagnostic)
|
||
offset = 0
|
||
while not all(state['done'] for state in states.values()):
|
||
if cancel_event.is_set():raise RuntimeError('cancelled')
|
||
active_count = sum(not state['done'] for state in states.values())
|
||
row_budget = 4000 - raw_rows_read
|
||
if row_budget < active_count:
|
||
server_gaps.append(f"节点{node['nodeId']}范围{scope.get('label') or scope['scopeId']}达到4000条原始记录上限,未缩小查询范围。")
|
||
break
|
||
page_limit = min(100, row_budget // active_count)
|
||
expected_reads = 2 if node.get('comparisonRequired') else 1
|
||
if read_count + expected_reads > 40:
|
||
server_gaps.append(f"节点{node['nodeId']}范围{scope.get('label') or scope['scopeId']}达到40次正式查询上限,原始记录未读全。")
|
||
break
|
||
page_arguments = {**base_arguments,'offset':offset,'limit':page_limit}
|
||
page_started = datetime.now(timezone.utc).isoformat()
|
||
reads_before = scoped_api.read_count
|
||
try:
|
||
result = tools.quality_data(page_arguments)
|
||
finally:
|
||
read_count += scoped_api.read_count - reads_before
|
||
current_proofs = _scoped_proofs(result, identity,
|
||
expected_count=expected_reads,node_id=scope['nodeId'],scope_id=scope['scopeId'])
|
||
if any(proof['queryId'] in query_ids for proof in current_proofs):
|
||
raise ValueError('原始记录分页证明queryId重复')
|
||
query_ids.update(proof['queryId'] for proof in current_proofs)
|
||
proofs.extend(current_proofs)
|
||
evidence_ref = result.get('evidenceRef')
|
||
if isinstance(evidence_ref,str):accepted_refs.add(evidence_ref)
|
||
next_offsets = []
|
||
for side_name, state in states.items():
|
||
side = result.get(side_name)
|
||
if not isinstance(side,dict):
|
||
raise ValueError('原始记录分页缺少comparison/current结果')
|
||
summary = _raw_side_summary(side,action='rows')
|
||
if summary['offset'] != offset:
|
||
raise ValueError('原始记录分页offset发生跳变')
|
||
if state['done']:
|
||
continue
|
||
if state['matchingRows'] is None:
|
||
state['matchingRows'] = summary['matchingRows']
|
||
elif state['matchingRows'] != summary['matchingRows']:
|
||
raise ValueError('原始记录分页期间matchingRows发生变化')
|
||
summary['evidenceRef'] = evidence_ref
|
||
state['pages'].append(summary)
|
||
raw_rows_read += summary['returnedRows']
|
||
if summary['hasMore']:
|
||
if summary['returnedRows'] != page_limit:
|
||
raise ValueError('仍有后续记录时当前页未填满,不能安全推进offset')
|
||
next_offsets.append(summary['nextOffset'])
|
||
else:
|
||
state['done'] = True
|
||
diagnostic['sides'] = {
|
||
name:{'matchingRows':state['matchingRows'],'pages':copy.deepcopy(state['pages'])}
|
||
for name,state in states.items()}
|
||
label = str(scope.get('label') or scope['scopeId'])+' '+stage+'原始记录第'+str(offset//page_limit+1)+'页'
|
||
calls.append({'tool':'quality_data','toolCallId':'protocol-'+current_proofs[0]['queryId'],
|
||
'label':label,'input':copy.deepcopy(page_arguments),
|
||
'nodeId':scope['nodeId'],'scopeId':scope['scopeId'],
|
||
'status':'completed','startedAt':page_started,
|
||
'finishedAt':datetime.now(timezone.utc).isoformat(),
|
||
'outputSummary':f"本页读取{sum((result.get(name) or {}).get('returnedRows',0) for name in states)}条;记录内容留在受限证据中。"})
|
||
if next_offsets:
|
||
if len(set(next_offsets)) != 1:
|
||
raise ValueError('比较期间原始记录分页游标不一致')
|
||
offset = next_offsets[0]
|
||
emit({'type':'progress','message':f'原始调查 {read_count}/40、已读取{raw_rows_read}/4000条。',
|
||
'toolCalls':copy.deepcopy(calls),'toolCount':len(calls)})
|
||
continue
|
||
for index, arguments in enumerate(arguments_queue):
|
||
active_arguments = arguments
|
||
if cancel_event.is_set():raise RuntimeError('cancelled')
|
||
started = datetime.now(timezone.utc).isoformat()
|
||
reads_before = scoped_api.read_count
|
||
try:
|
||
if node['purpose'] == 'changes':
|
||
from nao_changes import read_changes
|
||
result = read_changes(scoped_api, arguments, scope['task'], tools)
|
||
else:
|
||
result = tools.quality_data(arguments)
|
||
finally:
|
||
read_count += scoped_api.read_count - reads_before
|
||
expected_count = 2 if (arguments['mode'] == 'compare'
|
||
or arguments['mode'] == 'defects'
|
||
and arguments.get('comparisonPeriod') is not None) else 1
|
||
current_proofs = _scoped_proofs(result, identity, expected_count=expected_count,
|
||
node_id=scope['nodeId'], scope_id=scope['scopeId'])
|
||
if any(proof['queryId'] in query_ids for proof in current_proofs):
|
||
raise ValueError('正式查询证明queryId重复,不能合并不同scope读取')
|
||
query_ids.update(proof['queryId'] for proof in current_proofs)
|
||
proofs.extend(current_proofs)
|
||
evidence_ref = result.get('evidenceRef')
|
||
if isinstance(evidence_ref, str):accepted_refs.add(evidence_ref)
|
||
if node['purpose'] == 'changes':
|
||
label = '批量对比'+{'product':'产品规格','department':'产业部','group':'业务分组'}[node['dimension']]+',仅返回相关Top'
|
||
elif node['purpose'] == 'ranking':
|
||
label = '读取'+{'product':'产品规格','department':'产业部','group':'业务分组'}[node['dimension']]
|
||
label += '级' + {'black':'黑片','bright':'亮片'}.get(arguments.get('stage'), '')
|
||
label += {'totalWeight':'投入重量','scrapWeight':'报废重量'}.get(arguments.get('metric'), '')+'全量排名'
|
||
elif node['purpose'] == 'trend':
|
||
label = '读取范围'+str(scope.get('label') or scope['scopeId'])+'正式月度序列'
|
||
elif node['purpose'] == 'defects':
|
||
label = '读取范围'+str(scope.get('label') or scope['scopeId'])+'来源具体缺陷重量'
|
||
else:
|
||
label = '读取范围'+str(scope.get('label') or scope['scopeId'])+'正式指标'
|
||
calls.append({'tool': 'quality_data', 'toolCallId': 'protocol-' + current_proofs[0]['queryId'],
|
||
'label': label, 'input': copy.deepcopy(arguments),
|
||
'nodeId': scope['nodeId'], 'scopeId': scope['scopeId'],
|
||
'status': 'completed', 'startedAt': started,
|
||
'finishedAt': datetime.now(timezone.utc).isoformat(),
|
||
'outputSummary': public_tool_result(result)})
|
||
timings.setdefault('firstUsefulMs', sum(timings.get(k,0) for k in ('planningAndReview','taskConfirmation'))
|
||
+ round((time.monotonic()-phase_started)*1000))
|
||
emit({'type': 'progress', 'message': f'正式查询 {read_count}/40 已完成。',
|
||
'toolCalls': copy.deepcopy(calls), 'toolCount': len(calls)})
|
||
except Exception as error:
|
||
if isinstance(error, RuntimeError) and str(error) == 'cancelled':
|
||
raise
|
||
diagnostic = _controlled_failure_diagnostic(
|
||
error, node_id=scope.get('nodeId'), purpose=node.get('purpose'))
|
||
failure_diagnostics.append(diagnostic)
|
||
private_failure_diagnostics.append(
|
||
_private_failure_diagnostic(error, diagnostic, scope.get('task'), active_arguments))
|
||
recovered = None
|
||
if (scoped_api is not None and node['purpose'] == 'metrics' and arguments_queue
|
||
and len(arguments_queue) == 1 and arguments_queue[0].get('mode') == 'compare'
|
||
and isinstance(error, QualityToolError)
|
||
and (isinstance(error,SourceCoverageUnavailable)
|
||
or str(error) == '所选期间和筛选在该正式版本中没有可用覆盖数据;请检查 period、scope 和 filters。')):
|
||
recovered = _verified_empty_comparison_receipts(
|
||
scoped_api, scope=scope, task=scope['task'], identity=identity,
|
||
release_id=request['releaseId'])
|
||
if recovered:
|
||
recovered_proofs = recovered['proofs']
|
||
recovered_ids = [proof['queryId'] for proof in recovered_proofs]
|
||
if (len(set(recovered_ids)) == len(recovered_ids)
|
||
and not any(query_id in query_ids for query_id in recovered_ids)):
|
||
query_ids.update(recovered_ids)
|
||
proofs.extend(recovered_proofs)
|
||
server_gaps.extend(recovered['missingEvidence'])
|
||
query_diagnostics.extend(recovered.get('diagnostics', []))
|
||
proof_id = recovered_ids[0]
|
||
calls.append({'tool': 'quality_data', 'toolCallId': 'protocol-' + proof_id,
|
||
'label': '读取范围'+str(scope.get('label') or scope['scopeId'])+'正式指标',
|
||
'input': copy.deepcopy(arguments_queue[0]),
|
||
'nodeId': scope['nodeId'], 'scopeId': scope['scopeId'],
|
||
'status': 'failed', 'startedAt': started,
|
||
'finishedAt': datetime.now(timezone.utc).isoformat(),
|
||
'outputSummary': '已验证正式metrics期间、范围与协议证明;来源覆盖为空,未生成指标事实。'})
|
||
emit({'type': 'progress', 'message': f'正式查询 {read_count}/40 已完成,所选范围暂无可用来源覆盖。',
|
||
'toolCalls': copy.deepcopy(calls), 'toolCount': len(calls)})
|
||
continue
|
||
if node['purpose'] in ('source','records') and scoped_api is not None:
|
||
try:
|
||
recovered_raw = _verified_investigation_receipts(
|
||
scoped_api, identity, scope['nodeId'], scope['scopeId'], query_ids)
|
||
except (TypeError,ValueError):
|
||
recovered_raw = []
|
||
error_type = type(error).__name__
|
||
error_summary = _safe_private_error_summary(error, scope.get('task'))
|
||
if recovered_raw:
|
||
proofs.extend(recovered_raw)
|
||
for proof in recovered_raw:
|
||
receipt = next((item for item in reversed(scoped_api.investigation_responses)
|
||
if isinstance(item,dict) and isinstance(item.get('protocolEvidence'),dict)
|
||
and item['protocolEvidence'].get('queryId') == proof['queryId']),{})
|
||
query_diagnostics.append({'nodeId':scope['nodeId'],'scopeId':scope['scopeId'],
|
||
'purpose':node['purpose'],'action':receipt.get('action'),
|
||
'context':copy.deepcopy(receipt.get('context')),
|
||
'totalRows':receipt.get('totalRows'),
|
||
'offset':receipt.get('offset'),'returnedRows':receipt.get('returnedRows'),
|
||
'matchingRows':receipt.get('matchingRows'),'hasMore':receipt.get('hasMore'),
|
||
'queryId':proof['queryId'],'evidenceSaved':False,
|
||
'errorType':error_type,'errorSummary':error_summary})
|
||
else:
|
||
query_diagnostics.append({'nodeId':scope['nodeId'],'scopeId':scope['scopeId'],
|
||
'purpose':node['purpose'],'action':('source' if node['purpose']=='source' else 'rows'),
|
||
'stage':(arguments_queue[0].get('stage') if arguments_queue else None),
|
||
'queryId':None,'evidenceSaved':False,
|
||
'errorType':error_type,'errorSummary':error_summary})
|
||
label=str(scope.get('label') or scope['scopeId'])
|
||
message=('原表核验未取得完整且可验证的回执。' if node['purpose']=='source'
|
||
else '原始记录读取未取得完整且连续的分页回执。')
|
||
server_gaps.append(label+':'+message)
|
||
failed_id=recovered_raw[0]['queryId'] if recovered_raw else scope['scopeId']
|
||
calls.append({'tool':'quality_data','toolCallId':'protocol-'+str(failed_id),
|
||
'label':label+(' 原表核验' if node['purpose']=='source' else ' 原始记录读取'),
|
||
'input':copy.deepcopy(arguments_queue[0]) if arguments_queue else {},
|
||
'nodeId':scope['nodeId'],'scopeId':scope['scopeId'],
|
||
'status':'failed','startedAt':datetime.now(timezone.utc).isoformat(),
|
||
'finishedAt':datetime.now(timezone.utc).isoformat(),
|
||
'outputSummary':message})
|
||
continue
|
||
# A child failure remains visible and can never be hidden by a
|
||
# successful sibling scope.
|
||
safe_failure = {
|
||
'source_coverage': '所选期间和筛选没有可用的正式来源覆盖',
|
||
'formal_query_rejected': '正式查询响应未通过范围或完整性核验',
|
||
'transient_http': '正式查询服务暂时不可用,可在稍后重试',
|
||
'http_error': '正式查询服务拒绝或无法处理本次请求',
|
||
'transport_unavailable': '正式查询连接暂时不可用,可在稍后重试',
|
||
'query_rejected': '正式查询未通过已确认的范围或参数校验',
|
||
'internal_error': '正式查询执行未完成',
|
||
}.get(diagnostic['failureKind'], '正式查询执行未完成')
|
||
label = {'metrics':'正式指标','ranking':'正式排名','trend':'正式趋势','changes':'正式变化',
|
||
'defects':'正式缺陷','source':'原表核验','records':'原始记录'}.get(node.get('purpose'),'正式查询')
|
||
server_gaps.append(f"节点{node['nodeId']}:{label}{safe_failure}。")
|
||
if cancel_event.is_set():raise RuntimeError('cancelled')
|
||
scopes, pending, gaps, skipped = _protocol_query_snapshot(delegated, include_skipped=True)
|
||
|
||
# A node that was neither resolved nor left explicitly pending represents
|
||
# an unfulfilled part of the user's original compound request.
|
||
pending_nodes = {item.get('nodeId') for item in pending_latest if isinstance(item, dict)}
|
||
resolved_nodes = {node_id for node_id, _ in all_scopes}
|
||
for node_id in plan_nodes:
|
||
if node_id not in resolved_nodes and node_id not in pending_nodes and node_id not in skipped_nodes:
|
||
server_gaps.append(f'节点{node_id}没有取得服务器签发的可执行范围。')
|
||
for node_id, node in plan_nodes.items():
|
||
if node_id in skipped_nodes:
|
||
continue
|
||
selector = node.get('selector') or {'kind': 'base'}
|
||
expected_scope_count = {'catalog_members': len(selector.get('names', [])),
|
||
'ranking_top': selector.get('count'),
|
||
'previous_ranking': len(selector.get('ranks', []))}.get(selector.get('kind'), 1)
|
||
actual_scope_count = sum(1 for scope_node, _ in all_scopes if scope_node == node_id)
|
||
if (type(expected_scope_count) is int and expected_scope_count > 0
|
||
and actual_scope_count < expected_scope_count and node_id not in pending_nodes):
|
||
server_gaps.append(f'节点{node_id}仅取得{actual_scope_count}/{expected_scope_count}个所选对象范围。')
|
||
for item in pending_latest:
|
||
reason = item.get('reason') if isinstance(item, dict) else None
|
||
node_id = item.get('nodeId') if isinstance(item, dict) else None
|
||
server_gaps.append(f'节点{node_id or "?"}仍待完成:{reason or "服务器尚未解析所需范围"}。')
|
||
|
||
timings['formalQuery'] = round((time.monotonic() - phase_started) * 1000)
|
||
evidence_all = evidence_summaries(actor_root / 'evidence', request['releaseId'], allow_empty=True)
|
||
evidence = [document for document in evidence_all if document.get('evidenceRef') in accepted_refs]
|
||
server_scope_docs = list(all_scopes.values())
|
||
metric_scopes = [scope for scope in server_scope_docs
|
||
if scope.get('purpose') in ('ranking','metrics','trend','defects','changes')]
|
||
raw_scopes = [scope for scope in server_scope_docs if scope.get('purpose') in ('source','records')]
|
||
metric_evidence = [document for document in evidence
|
||
if document.get('mode') != 'investigate']
|
||
context = bind_task({'releaseId': request['releaseId'], **_composition_context(request)}, task)
|
||
context['_protocolScopes'] = {'scopes': server_scope_docs, 'pending': pending_latest, 'skipped':list(skipped_nodes.values()),
|
||
'gaps': list(dict.fromkeys(server_gaps))}
|
||
request['context'] = context
|
||
|
||
base, key, model = connection
|
||
visuals = None
|
||
if request.get('_autonomousAnswer') is True:
|
||
visual_started = time.monotonic()
|
||
visuals = (protocol_visual_result(metric_evidence, metric_scopes, request['question'], request['turnId'],
|
||
release_id=request['releaseId'], base_context=context)
|
||
if metric_scopes else {'visualState':'not_requested','visuals':[],'missingEvidence':[]})
|
||
timings['visuals'] = round((time.monotonic() - visual_started) * 1000)
|
||
context['_platformVisuals'] = _platform_visual_context(visuals)
|
||
emit({'type': 'progress', 'message': '范围证据已返回,正在按各子范围核对覆盖并整理答复…',
|
||
'toolCalls': calls, 'toolCount': len(calls)})
|
||
phase_started = time.monotonic()
|
||
synthesis_context = copy.deepcopy(context)
|
||
synthesis_context['_protocolScopes'] = {'scopes':metric_scopes,'pending':pending_latest,'gaps':[]}
|
||
has_synthesis_nodes = any(node.get('purpose') in ('ranking','metrics','trend','defects','changes')
|
||
for node in task.get('queryPlan',[]))
|
||
raw_report = _raw_node_report(raw_diagnostics, raw_scopes, [])
|
||
protocol_raw_coverage = _protocol_raw_coverage(task, raw_report)
|
||
if protocol_raw_coverage is not None:
|
||
synthesis_context['_protocolRawCoverage'] = protocol_raw_coverage
|
||
if raw_scopes and not has_synthesis_nodes:
|
||
answer_lines = ['已按本轮服务器授权范围核对原始数据。']
|
||
answer_lines.extend(fact.get('text','')+' ['+fact.get('evidenceRef','')+']'
|
||
for fact in raw_report.get('facts',[]) if fact.get('text'))
|
||
raw_missing=list(dict.fromkeys([*raw_report.get('missingEvidence',[]),*server_gaps]))
|
||
if raw_missing:
|
||
answer_lines.append('未覆盖内容:'+';'.join(raw_missing))
|
||
final = {'answer':'\n\n'.join(line for line in answer_lines if line),
|
||
'reportTitle':'原始数据核验与记录读取',
|
||
'evidenceSufficient':bool(raw_report.get('coverageSufficient')),
|
||
'missingEvidence':list(raw_report.get('missingEvidence',[])),
|
||
'modelDeclaredEvidenceSufficient':None,'usage':{},
|
||
'finishReason':'raw_scope_evidence'}
|
||
else:
|
||
final = synthesize(base_url=base, api_key=key, model=model, question=request['question'],
|
||
evidence=metric_evidence, cancel_event=cancel_event, context=synthesis_context)
|
||
_save_composition(final, actor_root, request['turnId'])
|
||
empty_scope_gaps = {}
|
||
for diagnostic in query_diagnostics:
|
||
if (not isinstance(diagnostic,dict)
|
||
or type(diagnostic.get('rawRows')) is not int or diagnostic['rawRows'] < 0
|
||
or type(diagnostic.get('includedRows')) is not int or diagnostic['includedRows'] < 0
|
||
or diagnostic['includedRows'] > diagnostic['rawRows']
|
||
or not (diagnostic['rawRows'] == 0 or diagnostic['includedRows'] == 0)):
|
||
continue
|
||
pair=(diagnostic.get('nodeId'),diagnostic.get('scopeId'))
|
||
label=str(diagnostic.get('label') or diagnostic.get('scopeId') or pair[0])
|
||
try:
|
||
from nao_evidence_report import period as format_period
|
||
period_label=format_period(diagnostic.get('period') or {})
|
||
except (KeyError,TypeError,ValueError):
|
||
period_label='所选对照期间'
|
||
gap=f'{label}:{period_label}没有可用来源记录,暂不能给出同比结论。'
|
||
empty_scope_gaps.setdefault(pair,[])
|
||
if gap not in empty_scope_gaps[pair]:empty_scope_gaps[pair].append(gap)
|
||
metric_scope_pairs={_scope_pair(scope) for scope in metric_scopes}
|
||
if (metric_scopes and not metric_evidence and metric_scope_pairs
|
||
and metric_scope_pairs <= set(empty_scope_gaps)):
|
||
specific_gaps=list(dict.fromkeys(gap for pair in metric_scope_pairs
|
||
for gap in empty_scope_gaps[pair]))
|
||
if specific_gaps:
|
||
final['answer']='所选比较期间没有可用来源记录,暂不能给出同比结论。\n\n未覆盖内容:'+';'.join(specific_gaps)
|
||
final['missingEvidence']=specific_gaps
|
||
final['evidenceSufficient']=False
|
||
final['modelDeclaredEvidenceSufficient']=None
|
||
final['completionAdjusted']=True
|
||
final['finishReason']='validated_empty_comparison_source'
|
||
timings['synthesis'] = round((time.monotonic() - phase_started) * 1000)
|
||
if cancel_event.is_set():raise RuntimeError('cancelled')
|
||
phase_started = time.monotonic()
|
||
if metric_scopes:
|
||
scoped_report = build_scoped_protocol_report(metric_evidence, metric_scopes,
|
||
question=request['question'],
|
||
release_id=request['releaseId'])
|
||
if empty_scope_gaps:
|
||
filtered=[]
|
||
for gap in scoped_report.get('missingEvidence',[]):
|
||
matched_empty=False
|
||
for scope in metric_scopes:
|
||
pair=_scope_pair(scope)
|
||
label=str(scope.get('label') or pair[0])
|
||
if pair in empty_scope_gaps and str(gap).startswith(f'节点{pair[0]}范围{label}没有同期间、同筛选且绑定node/scope'):
|
||
matched_empty=True
|
||
break
|
||
if not matched_empty:filtered.append(gap)
|
||
filtered.extend(gap for gaps in empty_scope_gaps.values() for gap in gaps)
|
||
scoped_report['missingEvidence']=list(dict.fromkeys(filtered))
|
||
metric_covered = bool(coverage_sufficient_scoped(metric_evidence, metric_scopes,
|
||
request['question'], request['releaseId'])
|
||
and scoped_report.get('coverageSufficient'))
|
||
else:
|
||
scoped_report = {'sections':[],'facts':[],'missingEvidence':[],
|
||
'coverageSufficient':not has_synthesis_nodes}
|
||
metric_covered = not has_synthesis_nodes
|
||
scoped_report['sections'].extend(raw_report.get('sections',[]))
|
||
next_fact = len(scoped_report.get('facts',[]))
|
||
for fact in raw_report.get('facts',[]):
|
||
scoped_report['facts'].append({**fact,'id':'F'+str(next_fact+1)})
|
||
next_fact += 1
|
||
scoped_report['missingEvidence'] = list(dict.fromkeys([
|
||
*scoped_report.get('missingEvidence',[]),*raw_report.get('missingEvidence',[]),*server_gaps]))
|
||
raw_covered = bool(not raw_scopes or raw_report.get('coverageSufficient'))
|
||
scoped_report['coverageSufficient'] = bool(metric_covered and raw_covered and not server_gaps)
|
||
if visuals is None:
|
||
visuals = (protocol_visual_result(metric_evidence, metric_scopes, request['question'], request['turnId'],
|
||
release_id=request['releaseId'], base_context=context)
|
||
if metric_scopes else {'visualState':'not_requested','visuals':[],'missingEvidence':[]})
|
||
timings['visuals'] = timings.get('visuals', 0) + round((time.monotonic() - phase_started) * 1000)
|
||
missing_visuals = visuals.get('missingEvidence', []) if isinstance(visuals.get('missingEvidence'), list) else []
|
||
covered = bool(metric_covered and raw_covered and not server_gaps)
|
||
raw_answer_facts = [fact for fact in raw_report.get('facts',[])
|
||
if isinstance(fact,dict) and isinstance(fact.get('text'),str)]
|
||
if has_synthesis_nodes and raw_answer_facts:
|
||
final['answer'] = str(final.get('answer') or '')+'\n\n原始数据读取:\n'+'\n'.join(
|
||
'- '+fact['text']+' ['+str(fact.get('evidenceRef') or '')+']' for fact in raw_answer_facts)
|
||
if has_synthesis_nodes and raw_report.get('missingEvidence'):
|
||
final['answer'] = (str(final.get('answer') or '')+'\n\n原始读取未覆盖:'
|
||
+';'.join(raw_report['missingEvidence']))
|
||
if has_synthesis_nodes and server_gaps:
|
||
final['answer'] = (str(final.get('answer') or '')+'\n\n未完成范围:'
|
||
+';'.join(server_gaps))
|
||
complete = bool(covered and final.get('evidenceSufficient') and not server_gaps and not missing_visuals
|
||
and not raw_report.get('missingEvidence'))
|
||
facts_complete = bool(covered and final.get('evidenceSufficient') and not raw_report.get('missingEvidence'))
|
||
missing_evidence = list(dict.fromkeys([*final.get('missingEvidence', []),
|
||
*scoped_report.get('missingEvidence', []),
|
||
*server_gaps, *missing_visuals]))
|
||
if not covered and not missing_evidence:
|
||
missing_evidence.append('queryPlan各子范围的正式证据未完整覆盖。')
|
||
status = 'COMPLETED' if complete else 'PARTIAL'
|
||
# This public agentContext is the root frame and must remain byte-for-byte
|
||
# shape-compatible with the ordinary protocol result. Child scope details
|
||
# stay in request.context for report/finalizer work.
|
||
agent_context = {'releaseId': request['releaseId'], 'filters': copy.deepcopy(task['filters']),
|
||
'period': copy.deepcopy(task['period'])}
|
||
if task.get('comparisonPeriod') is not None:
|
||
agent_context['comparisonPeriod'] = copy.deepcopy(task['comparisonPeriod'])
|
||
if not final.get('readingSummary') and any(node.get('purpose')=='changes' for node in task.get('queryPlan',[])):
|
||
# The model sees a concise fact set; queried statistical tables must
|
||
# still survive as normal key tables, without exposing audit appendices
|
||
# or performing further business reads.
|
||
sections=[section for section in scoped_report.get('sections',[])
|
||
if isinstance(section,str) and section.strip() and section.strip() not in final['answer']]
|
||
if sections:
|
||
final['answer'] += '\n\n## 重点数据\n\n'+'\n\n'.join(sections)
|
||
result = {'type': 'result', 'engine': 'NAO', 'status': status, 'answer': final['answer'],
|
||
'readingSummary': final.get('readingSummary'),
|
||
'answerQuality': {'version': 1, 'dataComplete': bool(covered),
|
||
'narrativeComplete': bool(final.get('evidenceSufficient')),
|
||
'chartsComplete': not missing_visuals,
|
||
'missingEvidence': missing_evidence},
|
||
'stopReason': 'evidence_finalized' if complete else 'evidence_incomplete',
|
||
'finishReason': 'stop', 'completionMethod': 'bounded_scoped_query_plan',
|
||
'agentTask': task, 'agentContext': agent_context, 'protocol': identity,
|
||
'actualModel': model, 'phaseTimingsMs': timings,
|
||
'protocolEvidence': proofs, 'queryEvidenceRefs': [d['evidenceRef'] for d in evidence],
|
||
'queryEvidenceBindings': query_evidence_bindings(evidence, proofs),
|
||
'toolCalls': calls, 'toolCount': len(calls), 'reportTitle': final.get('reportTitle'),
|
||
'evidenceCompletion': {'serverCoverageSufficient': covered,
|
||
'modelDeclaredEvidenceSufficient': final.get('modelDeclaredEvidenceSufficient', final.get('evidenceSufficient')),
|
||
'missingEvidence': missing_evidence},
|
||
'factsComplete': facts_complete,
|
||
'visualSchemaVersion': 1, 'visualState': visuals.get('visualState', 'unavailable'),
|
||
'visuals': visuals.get('visuals', []),
|
||
'scopeDiagnostics': copy.deepcopy(raw_diagnostics),
|
||
'failureDiagnostics': copy.deepcopy(failure_diagnostics),
|
||
'queryRecovery': copy.deepcopy(query_recovery),
|
||
'scopedReport': {'facts': scoped_report.get('facts', []),
|
||
'sections': scoped_report.get('sections', []),
|
||
'coverageSufficient': scoped_report.get('coverageSufficient', False)}}
|
||
from nao_quality_runtime import _atomic_json
|
||
_atomic_json(actor_root / 'private' / 'protocol-completion.json', {
|
||
'protocol': identity, 'model': model, 'status': status, 'usage': final.get('usage'),
|
||
'phaseTimingsMs': timings, 'queryIds': [proof['queryId'] for proof in proofs],
|
||
'protocolEvidence': proofs,
|
||
'queryDiagnostics': query_diagnostics,
|
||
'failureDiagnostics': private_failure_diagnostics,
|
||
'queryRecovery': query_recovery,
|
||
'rawScopeDiagnostics': raw_diagnostics,
|
||
'completion': {'serverCoverageSufficient': covered,
|
||
'modelDeclaredEvidenceSufficient': final.get('modelDeclaredEvidenceSufficient', final.get('evidenceSufficient')),
|
||
'missingEvidence': missing_evidence}})
|
||
emit(result)
|
||
return result
|
||
|
||
|
||
def _metric_arguments(request, task, *, column_filters, comparison):
|
||
filters = copy.deepcopy(task['filters'])
|
||
filters.pop('columnFilters', None)
|
||
arguments = {'mode': 'compare' if comparison else 'query',
|
||
'releaseId': request['releaseId'], 'period': copy.deepcopy(task['period']),
|
||
'filters': filters}
|
||
if comparison:
|
||
arguments['comparisonPeriod'] = copy.deepcopy(task['comparisonPeriod'])
|
||
else:
|
||
arguments['view'] = 'overall'
|
||
if column_filters is not None:arguments['columnFilters'] = copy.deepcopy(column_filters)
|
||
return arguments
|
||
|
||
|
||
def _execute_members(*, request, task, authority, actor_root, delegated, connection, cancel_event, emit, timings):
|
||
identity = _identity(authority)
|
||
period = task.get('period')
|
||
if not isinstance(period, dict):
|
||
raise ValueError('成员清单缺少已确认的查询期间')
|
||
member_fields = task.get('memberFields')
|
||
stages = task.get('stages')
|
||
filters = task.get('filters')
|
||
if (not isinstance(member_fields, list) or not 1 <= len(member_fields) <= 5
|
||
or not isinstance(stages, list) or not stages
|
||
or not isinstance(filters, dict)):
|
||
raise ValueError('成员清单任务结构未通过后台确认')
|
||
|
||
all_rows = []
|
||
page_evidence = []
|
||
calls = []
|
||
stable_total = None
|
||
offset = 0
|
||
started_all = time.monotonic()
|
||
complete = False
|
||
truncated = False
|
||
for page_index in range(_MEMBER_MAX_PAGES):
|
||
if cancel_event.is_set():
|
||
raise RuntimeError('cancelled')
|
||
payload = {'period': copy.deepcopy(period), 'filters': copy.deepcopy(filters),
|
||
'memberFields': list(member_fields), 'stages': list(stages),
|
||
'releaseId': request['releaseId'], 'limit': _MEMBER_PAGE_SIZE,
|
||
'offset': offset}
|
||
started = datetime.now(timezone.utc).isoformat()
|
||
emit({'type': 'progress', 'message': f'正在读取成员清单第 {page_index + 1} 页…'})
|
||
response = delegated.call('/api/analysis/source/members', payload=payload, method='POST')
|
||
if cancel_event.is_set():
|
||
raise RuntimeError('cancelled')
|
||
if not isinstance(response, dict):
|
||
raise ValueError('成员清单接口返回结构无效')
|
||
proof = _member_proof(response, identity)
|
||
if response.get('releaseId') != request['releaseId']:
|
||
raise ValueError('成员清单版本回执与请求不一致')
|
||
if response.get('period') != period or response.get('filters') != filters:
|
||
raise ValueError('成员清单期间或筛选回执与请求不一致')
|
||
if response.get('memberFields') != member_fields or response.get('selectedStages') != stages:
|
||
raise ValueError('成员清单字段或阶段回执与请求不一致')
|
||
total = response.get('totalCount')
|
||
page_offset = response.get('offset')
|
||
limit = response.get('limit')
|
||
rows = response.get('rows')
|
||
has_more = response.get('hasMore')
|
||
complete_receipt = response.get('complete')
|
||
page_complete = response.get('pageComplete')
|
||
if type(total) is not int or total < 0:
|
||
raise ValueError('成员清单totalCount无效')
|
||
if stable_total is None:
|
||
stable_total = total
|
||
elif total != stable_total:
|
||
raise ValueError('成员清单分页期间totalCount发生变化')
|
||
if (type(page_offset) is not int or page_offset != offset or type(limit) is not int
|
||
or limit != _MEMBER_PAGE_SIZE or not isinstance(rows, list)):
|
||
raise ValueError('成员清单分页offset/limit/rows回执不一致')
|
||
if len(rows) != min(_MEMBER_PAGE_SIZE, max(0, total - offset)):
|
||
raise ValueError('成员清单分页行数与totalCount/offset不一致')
|
||
expected_more = offset + len(rows) < total
|
||
expected_complete_receipt = offset == 0 and not expected_more
|
||
if (type(has_more) is not bool or has_more != expected_more
|
||
or page_complete is not True or type(complete_receipt) is not bool
|
||
or complete_receipt != expected_complete_receipt):
|
||
raise ValueError('成员清单complete/hasMore回执不一致')
|
||
for row in rows:
|
||
if not isinstance(row, dict) or not isinstance(row.get('values'), dict):
|
||
raise ValueError('成员清单行结构无效')
|
||
values = row['values']
|
||
if any(key not in values or values[key] is not None and not isinstance(values[key], str)
|
||
for key in member_fields):
|
||
raise ValueError('成员清单行未返回所选字段的文本值')
|
||
row_stages = row.get('stages')
|
||
if (not isinstance(row_stages, list) or not row_stages
|
||
or any(not isinstance(stage, str) or stage not in stages for stage in row_stages)):
|
||
raise ValueError('成员清单行阶段与请求不一致')
|
||
all_rows.append({'values': {key: values[key] for key in member_fields},
|
||
'stages': list(row_stages)})
|
||
page_evidence.append({'response': copy.deepcopy(response), 'protocolEvidence': proof})
|
||
calls.append({'tool': 'source_members', 'toolCallId': 'protocol-' + proof['queryId'],
|
||
'label': f'读取授权字段清单(第 {page_index + 1} 页)',
|
||
'input': payload, 'status': 'completed', 'startedAt': started,
|
||
'finishedAt': datetime.now(timezone.utc).isoformat(),
|
||
'outputSummary': f"本页返回 {len(rows)} 项,匹配总数 {total} 项"})
|
||
offset += len(rows)
|
||
if not expected_more:
|
||
complete = offset == total
|
||
break
|
||
if page_index == _MEMBER_MAX_PAGES - 1:
|
||
truncated = True
|
||
timings['memberQuery'] = round((time.monotonic() - started_all) * 1000)
|
||
if stable_total is None:
|
||
raise ValueError('成员清单没有返回页面')
|
||
truncated = truncated or offset < stable_total
|
||
query_complete = bool(complete and not truncated and offset == stable_total)
|
||
answer, displayed_count, display_complete = _members_answer(
|
||
task, stable_total, all_rows, query_complete=query_complete)
|
||
complete = bool(query_complete and display_complete)
|
||
status = 'COMPLETED' if complete else 'PARTIAL'
|
||
proofs = [item['protocolEvidence'] for item in page_evidence]
|
||
from nao_quality_runtime import _atomic_json
|
||
private_dir = actor_root / 'private'
|
||
private_dir.mkdir(parents=True, exist_ok=True)
|
||
_atomic_json(private_dir / 'protocol-members-evidence.json', {
|
||
'protocol': identity, 'releaseId': request['releaseId'], 'period': period,
|
||
'filters': filters, 'memberFields': member_fields, 'selectedStages': stages,
|
||
'totalCount': stable_total, 'offsetCount': offset, 'complete': query_complete,
|
||
'displayedCount': displayed_count, 'displayComplete': display_complete,
|
||
'pages': page_evidence})
|
||
completion = {'serverCoverageSufficient': query_complete,
|
||
'modelDeclaredEvidenceSufficient': query_complete,
|
||
'missingEvidence': ([] if complete else
|
||
['成员清单分页未查全。'] if not query_complete else
|
||
['成员清单已查全,但表格载入容量达到上限。'])}
|
||
_atomic_json(private_dir / 'protocol-completion.json', {
|
||
'protocol': identity, 'model': None, 'status': status, 'usage': None,
|
||
'phaseTimingsMs': timings, 'queryIds': [proof['queryId'] for proof in proofs],
|
||
'protocolEvidence': proofs, 'completion': completion})
|
||
result = {'type': 'result', 'engine': 'NAO', 'status': status, 'answer': answer,
|
||
'stopReason': 'members_list_complete' if complete else 'members_list_truncated',
|
||
'finishReason': 'stop', 'completionMethod': 'bounded_authorized_member_list',
|
||
'agentTask': task,
|
||
'agentContext': {'releaseId': request['releaseId'], 'filters': filters, 'period': period},
|
||
'protocol': identity, 'actualModel': connection[2],
|
||
'phaseTimingsMs': timings, 'protocolEvidence': proofs, 'queryEvidenceRefs': [],
|
||
'toolCalls': calls, 'toolCount': len(calls),
|
||
'evidenceCompletion': completion, 'visualSchemaVersion': 1,
|
||
'visualState': 'not_requested', 'visuals': []}
|
||
emit(result)
|
||
return result
|
||
|
||
|
||
def execute(*, request, task, authority, actor_root, delegated, connection, cancel_event, emit, phase_timings=None):
|
||
from nao_agent_task import bind_task
|
||
from nao_quality_runtime import public_tool_result
|
||
from nao_quality_tools import QualityTools
|
||
from nao_evidence_finalizer import evidence_summaries, synthesize, coverage_sufficient
|
||
from analysis_visual_evidence import visual_result
|
||
|
||
context = bind_task({'releaseId': request['releaseId'], **_composition_context(request)}, task)
|
||
timings = dict(phase_timings or {})
|
||
request['context'] = context
|
||
identity = _identity(authority)
|
||
if task['needsClarification']:
|
||
result = {'type': 'result', 'engine': 'NAO', 'status': 'NEEDS_CLARIFICATION',
|
||
'answer': task['clarification'], 'stopReason': 'needs_clarification',
|
||
'agentTask': task, 'protocol': identity, 'toolCalls': [],
|
||
'actualModel': connection[2], 'phaseTimingsMs': timings,
|
||
'visualSchemaVersion': 1, 'visualState': 'not_requested', 'visuals': []}
|
||
emit(result)
|
||
return result
|
||
if cancel_event.is_set():
|
||
raise RuntimeError('cancelled')
|
||
if isinstance(task.get('directResponse'), dict):
|
||
from nao_quality_runtime import ConversationAccumulator, _atomic_json, _record_non_query_turn, public_progress_text
|
||
answer = task['directResponse']['answer']
|
||
conversation = ConversationAccumulator()
|
||
conversation.record_event('assistant', text=public_progress_text(answer))
|
||
_atomic_json(actor_root / 'private' / 'last-conversation-answer.json', {
|
||
'turnId': request['turnId'], 'goal': task['goal'],
|
||
'purpose': task['directResponse']['purpose'], 'answer': answer})
|
||
_record_non_query_turn(actor_root, request)
|
||
result = {'type': 'result', 'engine': 'NAO', 'status': 'COMPLETED', 'answer': answer,
|
||
'agentTask': task, 'protocol': identity, 'completionMethod': 'direct_conversation',
|
||
'agentContext': {key: value for key, value in context.items()
|
||
if key in ('releaseId', 'period', 'comparisonPeriod', 'filters')},
|
||
'toolCalls': [], 'timeline': conversation.timeline, 'protocolEvidence': [],
|
||
'queryEvidenceRefs': [], 'visualSchemaVersion': 1, 'visualState': 'not_requested',
|
||
'visuals': [], 'stopReason': 'direct_conversation', 'finishReason': 'stop',
|
||
'actualModel': connection[2], 'phaseTimingsMs': timings}
|
||
emit(result)
|
||
return result
|
||
if task.get('kind') == 'report':
|
||
from nao_history_report import execute_history_report
|
||
return execute_history_report(request=request, task=task, authority=authority,
|
||
actor_root=actor_root, delegated=delegated,
|
||
connection=connection, cancel_event=cancel_event,
|
||
emit=emit, timings=timings)
|
||
if task.get('kind') == 'members':
|
||
# No model synthesis is used to create a source-member listing.
|
||
return _execute_members(request=request, task=task, authority=authority,
|
||
actor_root=actor_root, delegated=delegated,
|
||
connection=connection,
|
||
cancel_event=cancel_event, emit=emit, timings=timings)
|
||
if task.get('queryPlan'):
|
||
return _execute_query_plan(request=request, task=task, authority=authority,
|
||
actor_root=actor_root, delegated=delegated,
|
||
connection=connection, cancel_event=cancel_event,
|
||
emit=emit, timings=timings)
|
||
from nao_weight_ranking import ranking_intent, ranking_task
|
||
rank_intent = task.get('kind') in ('analysis', 'metrics') and ranking_intent(task)
|
||
if rank_intent and (task.get('trendRequired') or task.get('comparisonRequired') or task.get('comparisonPeriod')):
|
||
raise ValueError('单期重量排名不能与趋势或期间比较合并;请保留两项真实要求并拆分任务后再查。')
|
||
is_ranking = task.get('kind') in ('analysis', 'metrics') and ranking_task(task)
|
||
scalar_selection = _confirmed_scalar_selection(request, task)
|
||
if scalar_selection:
|
||
context['_confirmedScalarSelection'] = True
|
||
if task.get('kind') != 'metrics' and not is_ranking and not scalar_selection:
|
||
raise ValueError('当前协议执行器仅支持正式metrics和单期重量排名;保留原任务,不能转成正式指标。')
|
||
evidence_dir = actor_root / 'evidence'
|
||
tools = QualityTools(api_factory=lambda: delegated, evidence_dir=evidence_dir,
|
||
query_contract={'task': task, 'releaseId': request['releaseId']})
|
||
metric_filters = copy.deepcopy(task['filters'])
|
||
column_filters = metric_filters.pop('columnFilters', None)
|
||
comparison = bool(task.get('comparisonRequired') or task.get('comparisonPeriod'))
|
||
arguments_queue = []
|
||
if is_ranking:
|
||
from nao_agent_task import dimension_stages
|
||
for dimension in task['dimensions']:
|
||
for stage in dimension_stages(task, dimension, combined=False):
|
||
for metric in task['fields']:
|
||
arguments = {'mode': 'rank', 'releaseId': request['releaseId'],
|
||
'period': copy.deepcopy(task['period']),
|
||
'filters': copy.deepcopy(metric_filters),
|
||
'dimension': dimension, 'stage': stage, 'metric': metric, 'limit': 10}
|
||
if column_filters is not None:arguments['columnFilters'] = copy.deepcopy(column_filters)
|
||
arguments_queue.append(arguments)
|
||
else:
|
||
arguments_queue.append(_metric_arguments(request, task, column_filters=column_filters,
|
||
comparison=comparison))
|
||
|
||
calls = []
|
||
proofs = []
|
||
query_ids = set()
|
||
phase_started = time.monotonic()
|
||
emit({'type': 'progress', 'message': ('任务已确认,正在按维度、阶段和指标读取完整分组重量…'
|
||
if is_ranking else '任务已确认,正在读取同范围正式月度指标…')})
|
||
for index, arguments in enumerate(arguments_queue):
|
||
if cancel_event.is_set():raise RuntimeError('cancelled')
|
||
started = datetime.now(timezone.utc).isoformat()
|
||
result = tools.quality_data(arguments)
|
||
expected_count = 2 if arguments['mode'] == 'compare' else 1
|
||
current_proofs = _execution_proofs(result, identity, expected_count=expected_count)
|
||
if any(proof['queryId'] in query_ids for proof in current_proofs):
|
||
raise ValueError('正式查询证据queryId重复,不能合并不同读取结果')
|
||
query_ids.update(proof['queryId'] for proof in current_proofs)
|
||
proofs.extend(current_proofs)
|
||
if cancel_event.is_set():raise RuntimeError('cancelled')
|
||
if is_ranking:
|
||
label = '读取'+{'product':'产品规格','department':'产业部','group':'业务分组'}[arguments['dimension']]
|
||
label += '级'+{'black':'黑片','bright':'亮片'}[arguments['stage']]
|
||
label += {'totalWeight':'投入重量','scrapWeight':'报废重量'}[arguments['metric']]+'全量排名'
|
||
else:
|
||
label = '读取正式指标和月度序列'
|
||
evidence_ref = result.get('evidenceRef', '正式查询证据')
|
||
calls.append({'tool': 'quality_data', 'toolCallId': 'protocol-' + current_proofs[0]['queryId'],
|
||
'label': label, 'input': copy.deepcopy(arguments),
|
||
'status': 'completed', 'startedAt': started,
|
||
'finishedAt': datetime.now(timezone.utc).isoformat(),
|
||
'outputSummary': public_tool_result(result)})
|
||
timings.setdefault('firstUsefulMs', sum(timings.get(k,0) for k in ('planningAndReview','taskConfirmation'))
|
||
+ round((time.monotonic()-phase_started)*1000))
|
||
emit({'type': 'progress', 'message': f'正式读取 {index + 1}/{len(arguments_queue)} 已完成。',
|
||
'toolCalls': copy.deepcopy(calls), 'toolCount': len(calls)})
|
||
timings['formalQuery'] = round((time.monotonic() - phase_started) * 1000)
|
||
if not proofs:raise ValueError('执行计划未产生正式查询证据')
|
||
emit({'type': 'progress', 'message': '正式证据已返回,正在核对覆盖并整理答复…',
|
||
'toolCalls': calls, 'toolCount': len(calls)})
|
||
evidence = evidence_summaries(evidence_dir, request['releaseId'])
|
||
base, key, model = connection
|
||
visuals = None
|
||
if request.get('_autonomousAnswer') is True:
|
||
visual_started = time.monotonic()
|
||
visuals = visual_result(evidence_dir, request['releaseId'], request['question'], request['turnId'],
|
||
cancel_event=cancel_event, context=context)
|
||
timings['visuals'] = round((time.monotonic() - visual_started) * 1000)
|
||
context['_platformVisuals'] = _platform_visual_context(visuals)
|
||
phase_started = time.monotonic()
|
||
final = synthesize(base_url=base, api_key=key, model=model, question=request['question'],
|
||
evidence=evidence, cancel_event=cancel_event, context=context)
|
||
_save_composition(final, actor_root, request['turnId'])
|
||
timings['synthesis'] = round((time.monotonic() - phase_started) * 1000)
|
||
if cancel_event.is_set():
|
||
raise RuntimeError('cancelled')
|
||
covered = coverage_sufficient(evidence, request['question'], context)
|
||
complete = bool(covered and final['evidenceSufficient'])
|
||
facts_complete = complete
|
||
status = 'COMPLETED' if complete else 'PARTIAL'
|
||
phase_started = time.monotonic()
|
||
if visuals is None:
|
||
visuals = visual_result(evidence_dir, request['releaseId'], request['question'], request['turnId'],
|
||
cancel_event=cancel_event, context=context)
|
||
timings['visuals'] = timings.get('visuals', 0) + round((time.monotonic() - phase_started) * 1000)
|
||
ranking_visual_gaps = []
|
||
if is_ranking:
|
||
from nao_weight_ranking import missing_ranking_visuals
|
||
ranking_visual_gaps = missing_ranking_visuals(task, visuals.get('visuals', []), evidence)
|
||
if ranking_visual_gaps:
|
||
complete = False
|
||
status = 'PARTIAL'
|
||
if (task.get('trendRequired') and task.get('visualPreference') != 'none' or task.get('visualPreference') == 'required') and visuals.get('visualState') != 'ready':
|
||
complete = False
|
||
status = 'PARTIAL'
|
||
labels = task.get('sourceFieldLabels') if isinstance(task.get('sourceFieldLabels'), dict) else {}
|
||
if isinstance(visuals.get('scope'), dict) and labels:
|
||
visuals['scope']['columnLabels'] = copy.deepcopy(labels)
|
||
agent_context = {'releaseId': request['releaseId'], 'filters': task['filters'], 'period': task['period']}
|
||
if task.get('comparisonPeriod') is not None:
|
||
agent_context['comparisonPeriod'] = copy.deepcopy(task['comparisonPeriod'])
|
||
missing_evidence = list(final.get('missingEvidence', []))
|
||
missing_evidence.extend('重量排名图表未覆盖:' + gap for gap in ranking_visual_gaps)
|
||
completion = {'serverCoverageSufficient': bool(covered),
|
||
'modelDeclaredEvidenceSufficient': final.get('modelDeclaredEvidenceSufficient', final['evidenceSufficient']),
|
||
'missingEvidence': missing_evidence}
|
||
result = {'type': 'result', 'engine': 'NAO', 'status': status, 'answer': final['answer'],
|
||
'readingSummary': final.get('readingSummary'),
|
||
'stopReason': 'evidence_finalized' if complete else 'evidence_incomplete',
|
||
'finishReason': 'stop', 'completionMethod': 'bounded_evidence_finalization',
|
||
'agentTask': task, 'agentContext': agent_context, 'protocol': identity,
|
||
'actualModel': model, 'phaseTimingsMs': timings,
|
||
'protocolEvidence': proofs, 'queryEvidenceRefs': [d['evidenceRef'] for d in evidence],
|
||
'queryEvidenceBindings': query_evidence_bindings(evidence, proofs),
|
||
'toolCalls': calls, 'toolCount': len(calls), 'reportTitle': final.get('reportTitle'),
|
||
'evidenceCompletion': completion, 'factsComplete': facts_complete, **visuals}
|
||
from nao_quality_runtime import _atomic_json
|
||
_atomic_json(actor_root / 'private' / 'protocol-completion.json', {
|
||
'protocol': identity, 'model': model, 'status': status, 'usage': final.get('usage'), 'phaseTimingsMs': timings,
|
||
'queryIds': [p['queryId'] for p in proofs], 'protocolEvidence': proofs, 'completion': completion})
|
||
emit(result)
|
||
return result
|