From edef747e3bf33797392da48bb0b77a466cbd4efb Mon Sep 17 00:00:00 2001 From: lyun Date: Sat, 24 Oct 2026 10:53:33 +0800 Subject: [PATCH] Pair black and bright defect evidence for priority products --- ai/nao-quality/agent/prompts/system.md | 1 + scripts/nao_evidence_finalizer.py | 34 +++++++-- scripts/nao_product_defect_pairing.py | 67 +++++++++++++++++ scripts/nao_quality_runtime.py | 43 +++++++++++ verification/test_nao_finalizer_fallback.py | 40 ++++++++++ .../test_nao_product_defect_pairing.py | 74 +++++++++++++++++++ 6 files changed, 252 insertions(+), 7 deletions(-) create mode 100644 scripts/nao_product_defect_pairing.py create mode 100644 verification/test_nao_finalizer_fallback.py create mode 100644 verification/test_nao_product_defect_pairing.py diff --git a/ai/nao-quality/agent/prompts/system.md b/ai/nao-quality/agent/prompts/system.md index bf8e861..c037067 100644 --- a/ai/nao-quality/agent/prompts/system.md +++ b/ai/nao-quality/agent/prompts/system.md @@ -15,6 +15,7 @@ releaseId、发布版本号、候选发布状态、Story ID/slug/version 仅作 - 整体变化调查须实际覆盖:①综合率、黑片和亮片率、各阶段投入与报废变化;②范围内各产业部;③范围内各业务分组;④黑片和亮片的正式报废大类及来源具体缺陷;⑤重点产品/规格。产业部与业务分组分别用 changes 的 department、group 维度查询完整可用集合(limit可取100,核对totalRows与truncated;仍截断时明确覆盖范围,不能声称查全)。不能只看最差组织或把产业部分析代替业务分组分析。既指出主要恶化对象,也保留改善、持平、新进/退出等抵消情况。 - 整体比较已返回两个阶段各报废大类的defects重量时,直接复用比较期与本期的数据分析大类增减,不重复为每个类别查询。具体缺陷用 defects 按黑片、亮片分别带 comparisonPeriod 查询;既关注报废增量,也结合当前重量识别主要存量问题,不把大类当小类。某阶段不适用或没有细分数据时如实说明,继续其余维度;原始细分为零不自动当作系统故障,也不因此中断分析。 - 产品从报废增量、率变化、当前报废规模等角度找重点,核对阶段投入以区分放量、缩量与质量恶化;新增/退出规格单独解释。组织、产品和缺陷之间应有至少1至2条实际交叉查询支持的主要线索(如重点业务分组内TOP规格、重点规格的具体缺陷、某缺陷涉及的TOP规格),具体路线由前面结果决定。不要只并列几张互不关联的排名表,不要求逐个组织、产品全部下钻。 +- 在综合分析中决定展示某重点规格的来源具体缺陷时,对该规格按相同发布版本、期间、对照期间和筛选条件分别查询黑片与亮片,再比较两阶段线索;不要只查其中一阶段后把该规格当成完整阶段对比。某阶段查询失败或细分不适用时明确标出,不能补零。用户明确只问单阶段时保持其指定范围。 - 用户要深入分析时,最终通常需要具体的TOP规格和TOP报废分类,并对至少一个主要线索进一步核对。写清楚排名按报废重量、增量还是率降幅;看投入规模和缺失覆盖,避免把小批量最低率排成主要问题。没有适用数据时说明具体缺口。 - 用户说TOP缺陷类型时,不要停留在责废/料废等大类:使用 quality_data 的 defects 模式查询主要阶段的实际具体缺陷TOP。每次 defects 调用都必须显式传 dimension;查某规格的具体缺陷时传 dimension=defect、stage=black 或 bright,必要时带 comparisonPeriod 看增量;按某个缺陷找产品时才传 dimension=product 和 defect 原字段名。缺陷源字段名、覆盖与对账结果均要尊重工具返回。 - 正式指标引用API;接口返回的阶段数学拆解可用于解释率变化。产品报废增量和报废份额用于确定核查优先级,不擅自变成总体百分点贡献。当前报废大类不是裂纹、崩边等具体现象;设备、班次、工艺等原因要留作待验证假设。 diff --git a/scripts/nao_evidence_finalizer.py b/scripts/nao_evidence_finalizer.py index 3d5b277..c874188 100644 --- a/scripts/nao_evidence_finalizer.py +++ b/scripts/nao_evidence_finalizer.py @@ -259,18 +259,38 @@ def synthesize(*, base_url: str, api_key: str, model: str, question: str, request = Request(base_url.rstrip('/') + '/chat/completions', data=json.dumps(body, ensure_ascii=False).encode(), headers={'Content-Type': 'application/json', 'Authorization': 'Bearer ' + api_key}, method='POST') + def verified_fallback(): + # Focus selection is presentation-only. If its model response is malformed, + # render the already verified facts with a deterministic default selection. + # Never promote missing server evidence to a complete answer. + selected = [candidates[0]['id'] for candidates in pools.values() if candidates] + missing = list(dict.fromkeys(report.get('missingEvidence', [])))[:10] + return {'answer':render_report(report,selected,missing), + 'evidenceSufficient':not missing,'missingEvidence':missing, + 'modelDeclaredEvidenceSufficient':None,'completionAdjusted':True, + 'model':model,'usage':{},'finishReason':'selection_fallback', + 'selectedFactIds':selected,'factCount':len(report['facts']), + 'selectionAdjusted':True,'selectionFallback':True} try: with (opener or _open_model)(request, timeout=FINAL_TIMEOUT_SECONDS) as response: raw = response.read(2 * 1024 * 1024 + 1) if cancel_event.is_set(): raise FinalizationUnavailable('已取消') if len(raw) > 2 * 1024 * 1024: raise ValueError('oversize') - document = json.loads(raw) + try: + document = json.loads(raw) + except (ValueError, TypeError): + return verified_fallback() + if not isinstance(document,dict):return verified_fallback() if document.get('model') not in (None, model): raise ValueError('model changed') - choice = document['choices'][0] - answer = choice['message']['content'] - if choice.get('finish_reason') != 'stop' or not isinstance(answer, str) or not answer.strip(): - raise ValueError('incomplete final answer') - parsed=json.loads(answer) + try: + choice = document['choices'][0] + answer = choice['message']['content'] + if choice.get('finish_reason') != 'stop' or not isinstance(answer, str) or not answer.strip(): + return verified_fallback() + parsed=json.loads(answer) + except (ValueError, TypeError, KeyError, IndexError, AttributeError): + return verified_fallback() + if not isinstance(parsed,dict):return verified_fallback() selected=[] selection_adjusted=False for field,candidates in pools.items(): @@ -283,7 +303,7 @@ def synthesize(*, base_url: str, api_key: str, model: str, question: str, selected.extend(valid) sufficient=parsed.get('evidenceSufficient');missing=parsed.get('missingEvidence') if (not isinstance(sufficient,bool) or not isinstance(missing,list) - or any(not isinstance(x,str) for x in missing)):raise ValueError('invalid evidence completion') + or any(not isinstance(x,str) for x in missing)):return verified_fallback() missing=list(dict.fromkeys(item.strip() for item in missing if item.strip()))[:10] # 模型的孤立 false 不能证明数据缺失;具体缺项仍阻止完成,调用方另行检查服务端覆盖。 completion_adjusted = not sufficient and not missing diff --git a/scripts/nao_product_defect_pairing.py b/scripts/nao_product_defect_pairing.py new file mode 100644 index 0000000..96e7667 --- /dev/null +++ b/scripts/nao_product_defect_pairing.py @@ -0,0 +1,67 @@ +"""Plan missing black/bright source-defect queries for the same product scope. + +Only current-turn, release-bound evidence is passed in by the runtime. This +module never estimates a missing stage or reuses another product's evidence. +""" +from __future__ import annotations + +import json +import re +from typing import Any + + +def _single_stage_requested(question: str, context: Any) -> bool: + if isinstance(context, dict) and context.get("stage") in ("black", "bright"): + return True + text = question if isinstance(question, str) else "" + if re.search(r"(?:只|仅)(?:看|查|分析|要|关注)?(?:黑片|亮片)", text): + return True + stages = [stage for label, stage in (("黑片", "black"), ("亮片", "bright")) if label in text] + return len(stages) == 1 and not re.search(r"整体|总体|综合|全部|两阶段|重点产品|重点规格", text) + + +def plan_missing_product_defect_stages( + documents: list[dict[str, Any]], question: str, context: Any = None, +) -> list[dict[str, Any]]: + """Return the missing counterpart queries; identical scope/period is required.""" + if _single_stage_requested(question, context): + return [] + groups: dict[str, dict[str, Any]] = {} + for document in documents: + result = document.get("result") if isinstance(document, dict) else None + if not isinstance(result, dict) or result.get("kind") != "quality_fine_defects": + continue + if result.get("dimension") != "defect" or result.get("stage") not in ("black", "bright"): + continue + filters = result.get("filters") + product = filters.get("product") if isinstance(filters, dict) else None + if not isinstance(product, str) or not product.strip(): + continue + if result.get("category") is not None or result.get("defect") is not None: + continue + release = result.get("releaseId") + period = result.get("period") + comparison = result.get("comparisonPeriod") + if (not isinstance(release, str) or not release + or not isinstance(period, dict) or (comparison is not None and not isinstance(comparison, dict))): + continue + sort = result.get("sort") + if sort not in ("scrap_delta", "current_scrap"): + continue + identity = json.dumps([release, period, comparison, filters, sort], ensure_ascii=False, sort_keys=True) + group = groups.setdefault(identity, {"stages": set(), "source": result}) + group["stages"].add(result["stage"]) + + queries = [] + for group in groups.values(): + if group["stages"] == {"black", "bright"}: + continue + source = group["source"] + other = "bright" if "black" in group["stages"] else "black" + args = {"mode": "defects", "releaseId": source["releaseId"], + "period": source["period"], "filters": source["filters"], + "dimension": "defect", "stage": other, "sort": source["sort"], "limit": 20} + if source.get("comparisonPeriod") is not None: + args["comparisonPeriod"] = source["comparisonPeriod"] + queries.append(args) + return queries diff --git a/scripts/nao_quality_runtime.py b/scripts/nao_quality_runtime.py index 182f586..1ddbbd8 100644 --- a/scripts/nao_quality_runtime.py +++ b/scripts/nao_quality_runtime.py @@ -1980,6 +1980,49 @@ def run_turn(payload: dict[str, Any], *, stdout: Any = sys.stdout, stdin: Any = actor_root=actor_root, cancel_event=cancel_event, timeout=480 if model_name == "mimo-v2.6-flash" else TURN_TIMEOUT_SECONDS, ) + # The model may investigate one stage of a priority product and forget + # the other. Complete the same release/period/filter pair before the + # evidence-based answer and charts are assembled. Never borrow a result + # from another turn or infer a missing stage from the first one. + pairable_stop = status in ("COMPLETED", "PARTIAL") or ( + status == "FAILED" and stop_reason in ("length", "timeout", "超时停止") + and stop_succeeded is True) + if pairable_stop and not cancel_event.is_set(): + try: + from analysis_visual_evidence import load_visual_evidence + from nao_product_defect_pairing import plan_missing_product_defect_stages + from nao_quality_tools import QualityTools + documents = load_visual_evidence(actor_root / "evidence", request["releaseId"], + exclude_files=prior_evidence_files, budget_seconds=2.0) + missing_stages = plan_missing_product_defect_stages( + documents, request["question"], request["context"]) + if missing_stages: + pairing_api = DelegatedApi(access_path, runtime_root=manager.users_root, + expected_actor_id=request["actorId"], + expected_session_id=request["sessionId"], timeout=5).login() + quality_tools = QualityTools(api_factory=lambda: pairing_api, + evidence_dir=actor_root / "evidence") + for arguments in missing_stages: + if cancel_event.is_set(): + break + stage_name = "黑片" if arguments["stage"] == "black" else "亮片" + emit({"type": "progress", "message": f"正在补查同规格的{stage_name}具体缺陷…"}) + try: + paired = quality_tools.quality_data(arguments) + except (ValueError, OSError, RuntimeError): + emit({"type": "progress", "message": f"{stage_name}具体缺陷补查未成功,结论将保留该阶段的数据限制。"}) + continue + call = {"tool": "quality_data", "toolCallId": "pair-" + paired["evidenceRef"], + "label": f"补查{stage_name}具体缺陷", "status": "completed", + "startedAt": datetime.now(timezone.utc).isoformat(), + "finishedAt": datetime.now(timezone.utc).isoformat(), + "outputSummary": f"同规格、同期间补查完成;依据 {paired['evidenceRef']}"} + accumulator.tool_calls.append(call) + accumulator.record_event("tool_result", tool=call["tool"], + toolCallId=call["toolCallId"], label=call["label"], + status="completed", text=call["outputSummary"]) + except (ValueError, OSError, RuntimeError): + emit({"type": "progress", "message": "阶段配对检查未完成,已有查询结果仍按原范围呈现。"}) from nao_evidence_finalizer import eligible, evidence_summaries, synthesize, coverage_sufficient, FinalizationUnavailable finalized = False evidence_completion = None diff --git a/verification/test_nao_finalizer_fallback.py b/verification/test_nao_finalizer_fallback.py new file mode 100644 index 0000000..e72df3e --- /dev/null +++ b/verification/test_nao_finalizer_fallback.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import io +import json +import sys +import threading +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import nao_evidence_finalizer as finalizer + + +class FinalizerFallbackTests(unittest.TestCase): + def test_malformed_optional_focus_uses_verified_report_without_failure(self): + evidence = [{"evidenceRef": "Q00000001", "mode": "compare", "result": { + "kind": "quality_comparison", + "current": {"period": {"year": 2026, "month": 9}, "compositeRate": 0.95}, + "comparison": {"period": {"year": 2026, "month": 8}, "compositeRate": 0.96}, + "delta": {"percentagePoints": -1}}}] + bad_responses = [b"not-json", + json.dumps({"choices": [{"finish_reason": "length", "message": { + "content": '{"evidenceSufficient":'}}]}).encode(), + json.dumps({"choices": [{"finish_reason": "stop", "message": { + "content": "{}"}}]}).encode()] + for payload in bad_responses: + with self.subTest(payload=payload[:20]): + result = finalizer.synthesize( + base_url="https://service.test/v1", api_key="not-real", + model="mimo-v2.6-flash", question="质量如何", evidence=evidence, + cancel_event=threading.Event(), + opener=lambda *_args, **_kwargs: io.BytesIO(payload)) + self.assertTrue(result["selectionFallback"]) + self.assertTrue(result["evidenceSufficient"]) + self.assertIn("下降1.00个百分点", result["answer"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/verification/test_nao_product_defect_pairing.py b/verification/test_nao_product_defect_pairing.py new file mode 100644 index 0000000..d2620d2 --- /dev/null +++ b/verification/test_nao_product_defect_pairing.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +sys.path.insert(0, str(ROOT / "verification")) + +from nao_product_defect_pairing import plan_missing_product_defect_stages +from nao_quality_tools import QualityTools +from analysis_visual_evidence import load_visual_evidence +from test_nao_quality_tools import FakeApi + + +def evidence(product: str, stage: str, *, month: int = 9): + return {"result": {"kind": "quality_fine_defects", "releaseId": "release-a", + "dimension": "defect", "stage": stage, "period": {"year": 2026, "month": month}, + "comparisonPeriod": {"year": 2026, "month": 8}, + "filters": {"product": product, "includeUnassigned": False}, + "category": None, "defect": None, "sort": "scrap_delta"}} + + +class ProductDefectPairingTests(unittest.TestCase): + def test_fills_only_missing_stage_for_same_product_and_period(self): + docs = [evidence("16.33", "black"), evidence("16.33", "bright"), + evidence("R127", "black"), evidence("R127", "black")] + planned = plan_missing_product_defect_stages(docs, "分析近期重点产品") + self.assertEqual(len(planned), 1) + self.assertEqual(planned[0]["stage"], "bright") + self.assertEqual(planned[0]["filters"]["product"], "R127") + self.assertEqual(planned[0]["comparisonPeriod"], {"year": 2026, "month": 8}) + + def test_does_not_join_different_period_or_release(self): + docs = [evidence("R127", "black"), evidence("R127", "bright", month=10)] + planned = plan_missing_product_defect_stages(docs, "重点规格") + self.assertEqual({p["stage"] for p in planned}, {"black", "bright"}) + + def test_explicit_single_stage_scope_stays_single(self): + docs = [evidence("R127", "black")] + self.assertEqual(plan_missing_product_defect_stages(docs, "只看黑片来源缺陷"), []) + self.assertEqual(plan_missing_product_defect_stages(docs, "重点规格", {"stage": "black"}), []) + + def test_unscoped_or_category_specific_defects_do_not_trigger_product_pair(self): + doc = evidence("R127", "black") + doc["result"]["filters"].pop("product") + self.assertEqual(plan_missing_product_defect_stages([doc], "重点规格"), []) + doc["result"]["filters"]["product"] = "R127" + doc["result"]["category"] = "责废" + self.assertEqual(plan_missing_product_defect_stages([doc], "重点规格"), []) + + def test_counterpart_query_creates_separate_traceable_evidence(self): + with tempfile.TemporaryDirectory() as folder: + api = FakeApi() + tools = QualityTools(api_factory=lambda: api, evidence_dir=folder) + tools.quality_data({"mode": "defects", "releaseId": "release-a", + "period": {"year": 2026, "month": 3}, + "comparisonPeriod": {"year": 2026, "month": 1}, + "filters": {"product": "规格 A"}, "stage": "black", + "dimension": "defect", "limit": 10}) + docs = load_visual_evidence(Path(folder), "release-a") + planned = plan_missing_product_defect_stages(docs, "分析重点规格的具体缺陷") + self.assertEqual(len(planned), 1) + tools.quality_data(planned[0]) + paired_docs = load_visual_evidence(Path(folder), "release-a") + self.assertEqual({d["result"]["stage"] for d in paired_docs}, {"black", "bright"}) + self.assertEqual(len({d["evidenceRef"] for d in paired_docs}), 2) + self.assertEqual(plan_missing_product_defect_stages(paired_docs, "分析重点规格的具体缺陷"), []) + + +if __name__ == "__main__": + unittest.main()