Files
quality_analyse/scripts/nao_data_presentation.py

91 lines
4.8 KiB
Python

"""Pure presentation of service-verified statistics, never model table values."""
from __future__ import annotations
import copy
import re
def _table_ranges(lines):
ranges=[];index=0
while index<len(lines):
if '|' not in lines[index]:index+=1;continue
end=index
while end<len(lines) and '|' in lines[end] and lines[end].strip():end+=1
if any(re.fullmatch(r'\s*\|?\s*:?-{3,}:?\s*(?:\|\s*:?-{3,}:?\s*)+\|?\s*',line) for line in lines[index:end]):
ranges.append((index,end))
index=end
return ranges
def strip_model_tables(document, known_ids, diagnostics):
"""Discard untrusted table layout without parsing/coercing its quantities."""
if not isinstance(document,dict) or not isinstance(document.get('blocks'),list):return document
result=copy.deepcopy(document);kept=[]
for index,block in enumerate(result['blocks']):
if not isinstance(block,dict) or not isinstance(block.get('text'),str):kept.append(block);continue
ids=block.get('factIds')
if isinstance(ids,list) and any(not isinstance(value,str) or value not in known_ids for value in ids):
raise ValueError('数据表选择引用了未签发事实')
text=block['text'];lines=text.splitlines();ranges=_table_ranges(lines)
removed={line for start,end in ranges for line in range(start,end)}
remaining='\n'.join(line for line_no,line in enumerate(lines) if line_no not in removed)
remaining,count=re.subn(r'<table\b[^>]*>.*?</table\s*>','',remaining,flags=re.I|re.S)
if ranges or count:
diagnostics.setdefault('modelTablesReplacedByVerifiedData',[]).append({'block':index,'tables':len(ranges)+count})
if not any(line.strip() and not re.match(r'^\s*#{1,6}\s',line) for line in remaining.splitlines()):continue
block['text']=remaining.strip()
kept.append(block)
result['blocks']=kept
return result
def _native_table_markdown(markdown, refs):
lines=markdown.splitlines();ranges=_table_ranges(lines)
if not ranges:return None
table_lines={i for start,end in ranges for i in range(start,end)}
# Existing service captions retain period, ordering and displayed TopN.
# Anchored prose is redundant with the model's independent explanation.
shown=[line for i,line in enumerate(lines) if i in table_lines or not any('['+ref+']' in line for ref in refs)]
return '\n'.join(shown).strip()
def native_sections(sources, facts):
result=[]
for source in sources:
refs=source['evidenceRefs'];text=_native_table_markdown(source['markdown'],refs)
if not text:continue
ids=[fact['id'] for fact in facts if fact.get('evidenceRef') in refs]
if ids:result.append({'markdown':text,'factIds':ids,'evidenceRefs':list(refs),'scope':copy.deepcopy(source.get('scope') or {})})
return result
def history_sections(snapshot, facts, renderer):
result=[];seen=set()
for index,section in enumerate(snapshot.get('sections') or []):
text=renderer(snapshot,section)
if not text or text in seen:continue
ids=[fact['id'] for fact in facts if (fact.get('scope') or {}).get('sectionIndex')==index]
if not ids:continue
seen.add(text);result.append({'markdown':text,'factIds':ids,
'evidenceRefs':list(dict.fromkeys(fact['evidenceRef'] for fact in facts if fact['id'] in ids)),
'scope':copy.deepcopy({k:section[k] for k in ('period','filters','stage','dimension') if k in section})})
return result
def attach_verified_sections(blocks, sections, facts, summary_selection, diagnostics):
"""Append exact data only to accepted explanations that reference its source."""
result=copy.deepcopy(blocks);known={fact['id']:fact for fact in facts};used=set()
selected=({summary_selection['leadIndex'],*summary_selection['evidenceIndexes']} if summary_selection else set())
for block in result:
if block.get('index') not in selected:continue
narrative_ids=set(block.get('factIds') or []);chosen=[]
for index,section in enumerate(sections):
if index in used or not narrative_ids.intersection(section['factIds']):continue
used.add(index);chosen.append(section)
if not chosen:continue
block['text']+='\n\n'+'\n\n'.join(section['markdown'] for section in chosen)
table_ids=list(dict.fromkeys(key for section in chosen for key in section['factIds'] if key in known))
block['factIds']=list(dict.fromkeys([*block['factIds'],*table_ids]))
block['evidenceRefs']=list(dict.fromkeys([*block['evidenceRefs'],*(known[key]['evidenceRef'] for key in table_ids)]))
diagnostics['verifiedDataPresentation']={'displayedSections':len(used),'availableSections':len(sections),'fullDataRetained':True}
return result