优化:未分类连续维护与柏拉图预汇总,同步已发布修复

This commit is contained in:
2026-11-01 12:40:50 +08:00
parent 572873c6b8
commit 38290f5fec
135 changed files with 17613 additions and 663 deletions
+224 -24
View File
@@ -121,7 +121,7 @@ QUALITY_DATA_SCHEMA: dict[str, Any] = {
}, "required": ["field", "stages", "operator"]},
"description": "仅query/compare/rank/investigate/defects使用。必须是当前catalog.sourceFields授权的完整替换列表;null或[]清除全部。规则AND、values OR,文本大小写、空格及前导零按原文匹配。is_empty/not_empty不得带values。"},
"view": {"type": "string", "enum": ["overall", "department", "group", "product", "projectGroup"], "description": "query 的正式分组视图;changes 仅支持非 overall 维度。"},
"stage": {"type": "string", "enum": ["black", "bright"], "description": "rank/top/details/defects/investigate必填;changes可选且支持按black黑片或bright亮片单独比较排名,未传为跨阶段描述。query/compare不靠stage裁剪正式综合指标。TOP按阶段分别查询,不存在跨阶段默认TOP。"},
"stage": {"type": ["string", "null"], "enum": ["black", "bright", None], "description": "top/details/defects/investigate必须明确black/bright。rank单阶段传black/bright;rank的metric=scrapWeight且stage省略或null时,合计黑片+亮片报废总重全量排序,并保留正式综合合格率。投入重量rank仍必须指定阶段。changes可选,未传为综合范围。"},
"dimension": {"type": "string", "enum": ["category", "defect", "product", "department", "group", "projectGroup"], "description": "top模式仅category/product/department/group/projectGroup,不能用defect;具体缺陷TOP必须使用mode=defects、dimension=defect。product是规格。changes 使用 product/department/group/projectGroup。defects 使用 defect 或 product;未指定具体 defect 字段时,省略 dimension 默认按 defect 查询阶段具体缺陷。"},
"defect": {"type": "string", "minLength": 1, "description": "catalog 返回的具体缺陷源字段 key(不是显示名称);defects.dimension=product 时必填,用于按该字段排名规格。dimension=defect 可省略以查看具体缺陷聚合,也可指定 key 筛选。"},
"category": {
@@ -142,7 +142,7 @@ QUALITY_DATA_SCHEMA: dict[str, Any] = {
"if": {"properties": {"mode": {"not": {"const": "catalog"}}}, "required": ["mode"]},
"then": {"required": ["releaseId"]},
}, {
"if": {"properties": {"mode": {"enum": ["rank", "top", "defects", "details", "investigate"]}}, "required": ["mode"]},
"if": {"properties": {"mode": {"enum": ["top", "defects", "details", "investigate"]}}, "required": ["mode"]},
"then": {"required": ["stage"]},
}, {
"if": {"properties": {"mode": {"const": "top"}}, "required": ["mode"]},
@@ -213,6 +213,173 @@ def _coverage_counts(snapshot: dict[str, Any]) -> tuple[int, int, int]:
return int(raw or 0), int(included or 0), min(black_rows, bright_rows)
def _private_rank_response_trace(response: Any) -> dict[str, Any]:
"""Summarize the exact ranking snapshot for private failure diagnosis only.
Do not persist dimension keys/labels, filters, query strings, or arbitrary
response fields. The returned shape is enough to distinguish absent stage
sources from missing/invalid formal totals and grouping values.
"""
if not isinstance(response, dict):
return {'responseType': type(response).__name__}
def count_value(value):
return value if type(value) is int and value >= 0 else None
def count_state(mapping, key):
if not isinstance(mapping, dict) or key not in mapping:
return 'missing'
return 'known' if count_value(mapping.get(key)) is not None else 'invalid'
def numeric_value(value):
number = _number(value)
return str(number) if number is not None else None
coverage = response.get('coverage')
coverage = coverage if isinstance(coverage, dict) else {}
summary = response.get('summary')
summary = summary if isinstance(summary, dict) else {}
summary_coverage = summary.get('coverage')
summary_coverage = summary_coverage if isinstance(summary_coverage, dict) else {}
context = response.get('context')
context = context if isinstance(context, dict) else {}
source_months = []
raw_months = coverage.get('sourceMonths')
if isinstance(raw_months, list):
for item in raw_months[:36]:
if not isinstance(item, dict):
continue
entry = {key: count_value(item.get(key)) for key in ('year', 'month', 'blackRows', 'brightRows')}
entry['countStates'] = {key: count_state(item, key) for key in ('blackRows', 'brightRows')}
source_months.append(entry)
missing_stages = []
raw_missing = coverage.get('missingStages')
if isinstance(raw_missing, list):
for item in raw_missing[:36]:
if not isinstance(item, dict):
continue
stage = item.get('stage')
entry = {key: count_value(item.get(key)) for key in ('year', 'month')}
entry['stage'] = stage if stage in ('black', 'bright') else None
entry['stageState'] = 'known' if stage in ('black', 'bright') else 'invalid-or-missing'
missing_stages.append(entry)
stage_blocks = {}
for stage in ('black', 'bright'):
value = summary.get(stage)
block = value if isinstance(value, dict) else None
stage_blocks[stage] = {
'present': block is not None,
'keys': sorted(key for key in block if key in {
'rate', 'scrapRate', 'qualifiedWeight', 'scrapWeight', 'totalWeight',
'headquartersScrapWeight', 'rateWithoutHeadquartersScrap', 'defects'
}) if block is not None else [],
'values': ({key: numeric_value(block.get(key)) for key in (
'rate', 'scrapRate', 'qualifiedWeight', 'scrapWeight', 'totalWeight',
'headquartersScrapWeight', 'rateWithoutHeadquartersScrap'
) if key in block} if block is not None else {}),
}
rows = response.get('rows')
rows = rows if isinstance(rows, list) else None
row_shape = {'count': len(rows) if rows is not None else None,
'contextRowCount': count_value(context.get('rowCount')),
'stageCoverage': {'blackRows': 0, 'brightRows': 0,
'missing': {'blackRows': 0, 'brightRows': 0},
'invalid': {'blackRows': 0, 'brightRows': 0}},
'stageBlocks': {stage: {'missing': 0, 'null': 0, 'object': 0, 'other': 0}
for stage in ('black', 'bright')},
'sample': []}
if rows is not None:
for row in rows:
if not isinstance(row, dict):
for stage in ('black', 'bright'):
row_shape['stageCoverage']['invalid'][stage + 'Rows'] += 1
for stage in ('black', 'bright'):
row_shape['stageBlocks'][stage]['other'] += 1
continue
row_coverage = row.get('coverage')
for stage in ('black', 'bright'):
field = stage + 'Rows'
state = count_state(row_coverage, field)
if state == 'known':
row_shape['stageCoverage'][field] += count_value(row_coverage[field])
else:
row_shape['stageCoverage'][state][field] += 1
if len(row_shape['sample']) < 12:
sample = {'compositeRate': numeric_value(row.get('compositeRate')),
'coverage': {stage + 'Rows': count_value((row.get('coverage') or {}).get(stage + 'Rows'))
for stage in ('black', 'bright')} if isinstance(row.get('coverage'), dict) else None,
'coverageStates': {stage + 'Rows': count_state(row.get('coverage'), stage + 'Rows')
for stage in ('black', 'bright')},
'stages': {}}
for stage in ('black', 'bright'):
block = row.get(stage, '__missing__')
if block == '__missing__':
row_shape['stageBlocks'][stage]['missing'] += 1
sample['stages'][stage] = {'state': 'missing'}
elif block is None:
row_shape['stageBlocks'][stage]['null'] += 1
sample['stages'][stage] = {'state': 'null'}
elif isinstance(block, dict):
row_shape['stageBlocks'][stage]['object'] += 1
sample['stages'][stage] = {
'state': 'object',
'values': {key: numeric_value(block.get(key)) for key in (
'rate', 'scrapRate', 'qualifiedWeight', 'scrapWeight', 'totalWeight',
'headquartersScrapWeight', 'rateWithoutHeadquartersScrap'
) if key in block},
}
else:
row_shape['stageBlocks'][stage]['other'] += 1
sample['stages'][stage] = {'state': 'other'}
row_shape['sample'].append(sample)
else:
for stage in ('black', 'bright'):
block = row.get(stage, '__missing__')
if block == '__missing__':
row_shape['stageBlocks'][stage]['missing'] += 1
elif block is None:
row_shape['stageBlocks'][stage]['null'] += 1
elif isinstance(block, dict):
row_shape['stageBlocks'][stage]['object'] += 1
else:
row_shape['stageBlocks'][stage]['other'] += 1
evidence = response.get('protocolEvidence')
evidence = evidence if isinstance(evidence, dict) else {}
query_id = evidence.get('queryId')
if not isinstance(query_id, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,128}', query_id):
query_id = None
response_hash = evidence.get('responseHash')
if not isinstance(response_hash, str) or not re.fullmatch(r'[a-fA-F0-9]{64}', response_hash):
response_hash = None
return {
'queryId': query_id,
'responseHash': response_hash,
'responseKeys': sorted(key for key in response if key in {
'kind', 'release', 'releaseId', 'context', 'summary', 'rows', 'coverage', 'protocolEvidence'
}),
'context': {key: context.get(key) for key in ('view', 'year', 'month', 'quarter', 'monthStart', 'monthEnd')
if type(context.get(key)) is int or context.get(key) in ('overall', 'product', 'department', 'group')},
'coverage': {
key: count_value(coverage.get(key)) for key in (
'rawRows', 'includedRows', 'excludedBlankRows', 'unmappedRows', 'projectGroupUnconfiguredRows'
) if key in coverage
},
'coverageSourceMonths': source_months,
'coverageMissingStages': missing_stages,
'summaryCoverage': {key: count_value(summary_coverage.get(key))
for key in ('blackRows', 'brightRows') if key in summary_coverage},
'summaryCompositeRate': numeric_value(summary.get('compositeRate')),
'summaryStages': stage_blocks,
'rows': row_shape,
}
def _valid_period_coverage(snapshot: dict[str, Any], period: dict[str, int]) -> tuple[bool, str | None]:
black_ok, black_reason = _valid_stage_rate_coverage(snapshot, period, "black")
bright_ok, bright_reason = _valid_stage_rate_coverage(snapshot, period, "bright")
@@ -271,7 +438,7 @@ def _valid_composite_rate_reference(snapshot: dict[str, Any], period: dict[str,
if not isinstance(summary, dict):
# Comparison snapshots carry the same formal fields at their top level.
summary = snapshot
composite = _number(summary.get("compositeRate"), minimum=Decimal(0), maximum=Decimal(1))
composite = _number(summary.get("compositeRate"), minimum=Decimal(0))
if composite is None:
return False, "服务端正式compositeRate缺失或不合法。"
@@ -288,6 +455,25 @@ def _valid_composite_rate_reference(snapshot: dict[str, Any], period: dict[str,
return True, None
def _row_composite_rate(row: dict[str, Any]) -> Decimal | None:
"""Reference an official row rate with either measured stage as its source.
Coverage must be known for both stages. The server owns stage substitution;
this check neither rebuilds that formula nor copies stage weights.
"""
coverage = row.get("coverage")
if not isinstance(coverage, dict):
return None
counts = {stage: coverage.get(stage + "Rows") for stage in ("black", "bright")}
if any(type(count) is not int or count < 0 for count in counts.values()):
return None
if not any(counts[stage] > 0 and isinstance(row.get(stage), dict) and
(_number((row.get(stage) or {}).get("totalWeight")) or Decimal(0)) > 0
for stage in counts):
return None
return _number(row.get("compositeRate"), minimum=Decimal(0))
def _valid_stage_rate_coverage(snapshot: dict[str, Any], period: dict[str, int], stage: str) -> tuple[bool, str | None]:
coverage_ok, reason = _valid_stage_period_coverage(snapshot, period, stage)
if not coverage_ok:
@@ -295,8 +481,8 @@ def _valid_stage_rate_coverage(snapshot: dict[str, Any], period: dict[str, int],
rate = _stage_metric(snapshot, stage, "rate")
total_weight = _stage_metric(snapshot, stage, "totalWeight")
qualified_weight = _stage_metric(snapshot, stage, "qualifiedWeight")
if rate is None or rate < 0 or rate > 1:
return False, f"{stage}阶段率缺失或不在0到1范围内,不能比较。"
if rate is None or rate < 0:
return False, f"{stage}阶段正式率缺失或为负,不能比较。"
if total_weight is None or total_weight <= 0:
return False, f"{stage}阶段总投入缺失或为0,不能比较率。"
if qualified_weight is None or qualified_weight < 0:
@@ -317,8 +503,8 @@ def _valid_stage_rate_reference(snapshot: dict[str, Any], period: dict[str, int]
return False, reason
rate = _stage_metric(snapshot, stage, "rate")
total_weight = _stage_metric(snapshot, stage, "totalWeight")
if rate is None or rate < 0 or rate > 1:
return False, f"{stage}阶段率缺失或不在0到1范围内,不能引用。"
if rate is None or rate < 0:
return False, f"{stage}阶段正式率缺失或为负,不能引用。"
if total_weight is None or total_weight <= 0:
return False, f"{stage}阶段总投入缺失或为0,不能引用率。"
return True, None
@@ -400,14 +586,14 @@ def symmetric_stage_decomposition(
) -> dict[str, Any]:
"""Return symmetric black/bright rate contributions in percentage points."""
values = [
_number(black_current, minimum=Decimal(0), maximum=Decimal(1)),
_number(bright_current, minimum=Decimal(0), maximum=Decimal(1)),
_number(black_comparison, minimum=Decimal(0), maximum=Decimal(1)),
_number(bright_comparison, minimum=Decimal(0), maximum=Decimal(1)),
_number(black_current, minimum=Decimal(0)),
_number(bright_current, minimum=Decimal(0)),
_number(black_comparison, minimum=Decimal(0)),
_number(bright_comparison, minimum=Decimal(0)),
_number(composite_delta),
]
if any(value is None for value in values):
return {"status": "unavailable", "reason": "阶段率缺失或超出 0 到 1 的合法范围,不能拆解。", "blackPP": None, "brightPP": None, "sumPP": None}
return {"status": "unavailable", "reason": "阶段正式率缺失、为负或不是有限数值,不能拆解。", "blackPP": None, "brightPP": None, "sumPP": None}
denominators = [
_number(black_current_total_weight, minimum=Decimal(0)),
_number(bright_current_total_weight, minimum=Decimal(0)),
@@ -623,7 +809,9 @@ class QualityTools:
if isinstance(item, dict) and 'columnFilters' in item]
if 'columnFilters' in context:
echoed.append(context.get('columnFilters'))
if not echoed or any(item != column_filters for item in echoed):
# An empty replacement clears all rules. The metrics API omits an
# empty list from its echo; absence is equivalent only in this case.
if not echoed and column_filters or any(item != column_filters for item in echoed):
raise QualityToolError('正式指标接口未原样确认 filters.columnFilters;结果已拒绝。')
if context.get("includeUnassigned") != filters.get("includeUnassigned", False):
raise QualityToolError("正式指标接口未确认 includeUnassigned;结果已拒绝。")
@@ -716,8 +904,8 @@ class QualityTools:
current_ok, current_reason = _valid_composite_rate_reference(current, current["_period"])
comparison_ok, comparison_reason = _valid_composite_rate_reference(comparison, comparison["_period"])
valid = current_ok and comparison_ok
now = _number((current.get("summary") or {}).get("compositeRate"), minimum=Decimal(0), maximum=Decimal(1))
old = _number((comparison.get("summary") or {}).get("compositeRate"), minimum=Decimal(0), maximum=Decimal(1))
now = _number((current.get("summary") or {}).get("compositeRate"), minimum=Decimal(0))
old = _number((comparison.get("summary") or {}).get("compositeRate"), minimum=Decimal(0))
if not valid or now is None or old is None:
return {"available": False, "rate": None, "percentagePoints": None,
"reason": current_reason if not current_ok else comparison_reason if not comparison_ok else "正式综合率缺失或不合法。"}
@@ -769,7 +957,7 @@ class QualityTools:
**{key: value for key, value in contribution.items() if key not in ("blackPP", "brightPP")},
"blackContributionToCompositePP": contribution.get("blackPP"),
"brightContributionToCompositePP": contribution.get("brightPP"),
"meaning": "对综合合格率变化的数学拆解,不是黑片/亮片本身的率变化,也不是生产根因。",
"meaning": "对正式综合合格率变化的阶段拆解,不是黑片/亮片本身的率变化,也不是生产根因。",
}
result["stageRateChangesPP"] = {
stage: ((_stage_metric(current_raw, stage, "rate") - _stage_metric(comparison_raw, stage, "rate")) * Decimal(100)
@@ -885,6 +1073,8 @@ class QualityTools:
def _row_rate(cls, row: dict[str, Any] | None, stage: str | None) -> Decimal | None:
if row is None:
return None
if stage is None:
return _row_composite_rate(row)
coverage = row.get("coverage")
if not isinstance(coverage, dict):
return None
@@ -901,13 +1091,13 @@ class QualityTools:
return None
if _number(block.get("qualifiedWeight"), minimum=Decimal(0)) is None:
return None
if _number(block.get("rate"), minimum=Decimal(0), maximum=Decimal(1)) is None:
if _number(block.get("rate"), minimum=Decimal(0)) is None:
return None
if stage:
value = (row.get(stage) or {}).get("rate") if isinstance(row.get(stage), dict) else None
else:
value = row.get("compositeRate")
return _number(value, minimum=Decimal(0), maximum=Decimal(1))
return _number(value, minimum=Decimal(0))
@classmethod
def _row_weight(cls, row: dict[str, Any] | None, field: str, stage: str | None) -> Decimal | None:
@@ -1001,8 +1191,8 @@ class QualityTools:
current_index = self._index_rows(current_raw, dimension, "current")
comparison_index = self._index_rows(comparison_raw, dimension, "comparison")
if stage is None:
current_coverage_ok, current_coverage_reason = _valid_period_coverage(current_raw, current_period)
comparison_coverage_ok, comparison_coverage_reason = _valid_period_coverage(comparison_raw, comparison_period)
current_coverage_ok, current_coverage_reason = _valid_composite_rate_reference(current_raw, current_period)
comparison_coverage_ok, comparison_coverage_reason = _valid_composite_rate_reference(comparison_raw, comparison_period)
else:
current_coverage_ok, current_coverage_reason = _valid_stage_rate_coverage(current_raw, current_period, stage)
comparison_coverage_ok, comparison_coverage_reason = _valid_stage_rate_coverage(comparison_raw, comparison_period, stage)
@@ -1103,7 +1293,7 @@ class QualityTools:
"weightBasis": (f"仅使用正式 {stage} 阶段重量字段。" if stage else
"scrapWeight 是 black + bright 两阶段报废重量和;投入分别见 blackInputWeight/brightInputWeight,不提供跨阶段投入总量。"),
"rateBasis": (f"使用正式 {stage} 阶段 rate;rateDrop 还要求该行阶段 totalWeight>0、qualifiedWeight 有效且 coverage 有效。" if stage else
"使用正式 compositeRate;rateDrop 还要求每行 black/bright 两阶段 totalWeight>0、qualifiedWeight 有效且 coverage 有效。"),
"使用正式 compositeRate,沿用双向阶段引用;rateDrop 要求两期来源覆盖已知且每行至少一个有来源阶段的正式总重为正。"),
"coverage": {"current": current_coverage, "comparison": comparison_coverage,
"rateComparable": rates_comparable, "rateUnavailableReason": reason},
"notice": "变化行是分组线索;投入/报废增量和分组率不能加总解释总体率。未指定 stage 时 black+bright 重量和仅作分组描述,不代表独立投入或分母;投入变化请分阶段分析。",
@@ -1596,7 +1786,7 @@ class QualityTools:
raise QualityToolError("catalog.stage 只能是 black 或 bright。")
elif mode == "rank":
if ({'view','comparisonPeriod','sort','category','defect'} & set(args)
or args.get('stage') not in CATEGORIES
or not (args.get('stage') in CATEGORIES or args.get('stage') is None and args.get('metric') == 'scrapWeight')
or args.get('dimension') not in {'product','department','group'}
or args.get('metric') not in {'totalWeight','scrapWeight'}):
raise QualityToolError('rank必须指定stage、dimension=product/department/group、metric=totalWeight/scrapWeight;不接受view、comparisonPeriod或报废类别。')
@@ -1793,8 +1983,18 @@ class QualityTools:
column_filters=column_filters)
ranking_filters = ({**filters, 'columnFilters': copy.deepcopy(column_filters)}
if column_filters is not None else filters)
result = rank_response(response, release, period, ranking_filters, arguments['dimension'],
arguments['stage'], arguments['metric'], arguments.get('limit', 10))
try:
result = rank_response(response, release, period, ranking_filters, arguments['dimension'],
arguments.get('stage'), arguments['metric'], arguments.get('limit', 10))
except Exception as error:
# The normal result and exception remain unchanged. Attach
# a bounded, identifier-free summary so the protocol layer
# can store the exact source shape in its private trace.
try:
error._quality_private_rank_trace = _private_rank_response_trace(response)
except Exception:
pass
raise
elif mode == "query":
period = normalize_period(arguments.get("period"))
view = arguments.get("view", "overall")