优化:未分类连续维护与柏拉图预汇总,同步已发布修复

This commit is contained in:
2026-11-01 12:40:50 +08:00
parent 572873c6b8
commit 38290f5fec
135 changed files with 17613 additions and 663 deletions
+522
View File
@@ -0,0 +1,522 @@
"""Pure display and immutable deletion primitives for Native drafts.
No query, authorization, metric calculation or model call lives here.
"""
from __future__ import annotations
import hashlib
import re
from functools import lru_cache
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
MASS_FIELDS = frozenset({'qualifiedWeight','totalWeight','scrapWeight','qualifiedWeightDelta','totalWeightDelta',
'scrapWeightDelta','scrapDelta','deltaWeight','categoryWeight','sourceWeight','weight','difference'})
NUMBER_PATTERN=r'[+−-]?[0-9]+(?:,[0-9]{3})*(?:\.[0-9]+)?(?:%|%)?'
_NUM=re.compile(NUMBER_PATTERN)
_MONTH=re.compile(r'(?:(?:[0-9]{4})年)?[0-9]{1,2}月')
_SUBJECT_REFERENCE=re.compile(r'(?P<operator>首|第|另|其余|剩余)(?P<number>[0-9]+|[一二两三四五六七八九十]+)?(?P<classifier>位|名|项|个)(?:规格|产品|对象|部门|分组)?')
_COMPARISON_OPERATORS={
'不低于':'>=','不少于':'>=','不小于':'>=','至少':'>=','>=':'>=','≥':'>=',
'不高于':'<=','不超过':'<=','不大于':'<=','不多于':'<=','至多':'<=','<=':'<=','≤':'<=',
'高于':'>','大于':'>','多于':'>','超过':'>','>':'>',
'低于':'<','小于':'<','少于':'<','不足':'<','不到':'<','未达':'<','<':'<',
}
_COMPARISON_SUFFIXES={'及以上':'>=','以上':'>=','及以下':'<=','以下':'<=','之上':'>','之下':'<'}
def comparison_relation(text,start,end):
"""Parse a boundary operand; formatting it as an observation is forbidden."""
before=re.sub(r'[*_`\s]','',text[max(0,start-60):start])
after=re.sub(r'^[*_`\s]+','',text[end:])
prefix=next((word for word in sorted(_COMPARISON_OPERATORS,key=len,reverse=True) if before.endswith(word)),None)
suffix=next((word for word in sorted(_COMPARISON_SUFFIXES,key=len,reverse=True) if after.startswith(word)),None)
operators={_COMPARISON_OPERATORS[prefix]} if prefix else set()
if suffix:operators.add(_COMPARISON_SUFFIXES[suffix])
if not operators:return None
return {'operator':next(iter(operators)) if len(operators)==1 else None,'ambiguous':len(operators)!=1}
def heading_subject(label):
"""A nominal heading's possessor, not an entity found inside a claim."""
tail=re.split(r'[::]',label,maxsplit=1)[-1].strip()
possessive=re.search(r'内部|的',tail)
return tail[:possessive.start()].strip() if possessive else tail
def _contribution_target(text):
match=re.search(r'对(?P<target>[^,。;;::()()]*?)(?:贡献|影响)',text)
return (text[:match.start()]+text[match.end():],match['target']) if match else (text,None)
def _first_measure(text,start,end):
placeholders=list(re.finditer(r'\{\{[A-Z][0-9]+\}\}',text[start:end]))
candidates=[start+m.start() for m in placeholders if m[0].startswith('{{N')]
for match in _NUM.finditer(text,start,end):
if any(start+m.start()<=match.start()<start+m.end() for m in placeholders):continue
if re.match(r'\s*(?:年|月|日|季度)',text[match.end():]):continue
if any(reference.start()<=match.start()<reference.end() for reference in _SUBJECT_REFERENCE.finditer(text,start,end)):continue
if match.start()>start and re.match(r'[A-Za-z0-9_.]',text[match.start()-1]):continue
if match.end()<end and re.match(r'[A-Za-z0-9_.]',text[match.end()]):continue
candidates.append(match.start())
return min(candidates) if candidates else None
_METRIC_LABELS=re.compile(r'报废率|综合合格率|综合率|合格率|合格重量|投入(?:重量|规模|量)?|报废(?:重量|总重)?|占比|份额|比重|比例|百分比|记录数|件数|数量|次数')
def _metric_head(context,explicit=False):
"""A visible field head, including non-mass heads which stop inheritance."""
matches=list(_METRIC_LABELS.finditer(context))
if not matches:return None,None
label=matches[-1][0]
if label in ('占比','份额','比重','比例','百分比'):return 'share',label
if label in ('记录数','件数','数量','次数'):return 'count',label
if label=='报废率':return 'scrap_rate',label
if label.endswith('率'):return 'rate',label
if explicit and label in ('投入','报废'):return None,label
return ('qualified' if label=='合格重量' else 'input' if label.startswith('投入') else 'scrap'),label
def _metric_family(contexts):
"""Classify only a visible predicate label; tokens grant no semantics."""
for index,context in enumerate(contexts):
family,label=_metric_head(context)
if label:return family,index
return None,None
def _governing_predicate(text,predicates,index,prefix):
"""Forward domination by an explicit field in the same container.
Only the field label crosses the edge. Subjects, stages and calendar
anchors remain local, including when a continuation changes period.
"""
topic=None;origin=None
for position,(start,end) in enumerate(predicates[:index+1]):
visible=prefix if position==index else text[start:end]
family,label=_metric_head(visible,explicit=True)
if label:topic=label if family is not None else None;origin=(start,end)
if position==index:
if not label and topic:return {'kind':'governing_predicate','label':topic,'start':origin[0],'end':origin[1]}
return None
if visible.rstrip().endswith(tuple('。!?!?')):topic=None;origin=None
return None
def _reference_integer(text):
if text.isdigit():return int(text)
digits={'一':1,'二':2,'两':2,'三':3,'四':4,'五':5,'六':6,'七':7,'八':8,'九':9}
if text in digits:return digits[text]
if text.count('十')==1:
left,right=text.split('十')
if (not left or left in digits) and (not right or right in digits):return digits.get(left,1)*10+digits.get(right,0)
return None
def _subject_reference(contexts):
"""Structural selectors, not a guessed subject or an authorization."""
for text,offset,source in contexts:
matches=list(_SUBJECT_REFERENCE.finditer(text))
if not matches:continue
if len(matches)!=1:return {'kind':'ambiguous','source':source,'context':text}
match=matches[0];operator=match['operator'];number=_reference_integer(match['number']) if match['number'] else None
if operator=='首':
if match['number']:return {'kind':'ambiguous','source':source,'context':text}
kind,number='rank',1
elif operator=='第':kind='rank'
else:kind='complement'
return {'kind':kind,'rank' if kind=='rank' else 'count':number,
'start':offset+match.start(),'end':offset+match.end(),'source':source,'context':text}
return None
def _ordered_subject_values(text,predicates,index,start,end):
"""An explicit respectively-list associates positions, never equal values."""
left,right=predicates[index];marker=re.search('分别',text[left:right])
if marker is None or start<left+marker.end():return None
first=index
while first>0 and text[predicates[first-1][1]-1]=='、':first-=1
if first==index:return None
subjects=[]
for position in range(first,index+1):
a,b=predicates[position];b=left+marker.start() if position==index else b-1
subjects.append({'text':text[a:b].strip(),'start':a,'end':b})
values=[]
for match in re.finditer(r'\{\{N[1-9][0-9]*\}\}|'+NUMBER_PATTERN,text[left+marker.end():right]):
a=left+marker.end()+match.start();b=left+marker.end()+match.end()
if re.match(r'\s*(?:年|月|日|季度)',text[b:]):continue
values.append((a,b))
ordinal=next((i for i,(a,b) in enumerate(values) if a<=start and end<=b),None)
return {'subjects':subjects,'index':ordinal,'ambiguous':len(subjects)!=len(values) or ordinal is None}
@lru_cache(maxsize=512)
def _ordered_pairs(text,start,end):
values=list(_NUM.finditer(text,start,end));pairs=[]
for first,second in zip(values,values[1:]):
between=re.sub(r'[*`_\s]','',text[first.end():second.start()])
before=re.sub(r'[*`_\s]','',text[start:first.start()])
arrow=re.fullmatch(r'(?:%|%|kg|公斤|千克|吨|pp|个百分点)?(?:→|->)',between,re.I)
ordered=re.search(r'(?:从|由)$',before) and re.fullmatch(r'(?:%|%|kg|公斤|千克|吨|pp|个百分点)?(?:增|升|降|减|扩大|扩|提高|降低)?(?:至|到)',between,re.I)
if arrow or ordered:pairs.append((first.start(),first.end(),second.start(),second.end()))
return tuple(pairs)
def _change_annotation(text,start,end):
"""A signed annotation attached to an ordered pair labels its change."""
_,children=_parts(text,start,end)
for first,_,_,last in reversed(_ordered_pairs(text,start,end)):
for left,right in children:
if left<last:continue
between=re.sub(r'[*`_\s]','',text[last:left])
if not re.fullmatch(r'(?:%|%|kg|公斤|千克|吨|pp|个百分点)?',between,re.I):continue
if re.search(r'[+−-][0-9]',text[left+1:right-1]):
return {'kind':'change_of_pair','label':text[start:first],'start':start,'end':end}
return None
def native_document_ancestors(blocks):
"""Structural ancestry only, never inferred scope or metric authority."""
headings=[];introduction=None;result={}
for block in blocks:
labels=[]
for line in block['text'].splitlines():
heading=re.match(r'^\s*(#{1,6})\s+(.+)',line)
if heading:
level=len(heading[1]);headings=[item for item in headings if item[0]<level]
headings.append((level,heading[2]));introduction=None
elif line.strip() and not line.lstrip().startswith('|'):labels.append(line.strip())
result[block['sourceId']]=tuple([text for _,text in headings]+([introduction] if introduction else []))
if labels and labels[-1].endswith((':',':')):introduction=labels[-1]
return result
@lru_cache(maxsize=512)
def _parts(text,start,end):
"""Sibling predicates and nested annotations, with immutable offsets."""
nodes=[];children=[];stack=[];left=start
for index in range(start,end):
char=text[index]
if char in '(([【':stack.append(index)
elif char in '))]】' and stack:
opening=stack.pop()
if not stack:children.append((opening,index+1))
elif not stack and char in '。!?!?;;,,::、':
if char==',' and index>start and index+1<end and text[index-1].isdigit() and text[index+1].isdigit():continue
nodes.append((left,index+1));left=index+1
if left<end:nodes.append((left,end))
return tuple(nodes),tuple(children)
class NativeSpanStructure:
"""An offset index of the supported Markdown structure, with no facts.
locate() gives a predicate/cell and its structural ancestors. Consumers
resolve those labels against server-authorized facts; this index grants
neither a scope alias nor a numerical authorization.
"""
def __init__(self,text,ancestors=()):
self.text=text;self.ancestors=ancestors;self.lines=[];self.ordinals=[]
offset=0;headers=None;table_line=0;headings=[]
for line in text.splitlines(keepends=True):
end=offset+len(line.rstrip('\r\n'));kind='prose'
heading=re.match(r'^\s*#{1,6}\s+(?:([0-9]+)[.)、]\s+)?',line)
listing=re.match(r'^\s*(?:[-*+]\s+|([0-9]+)[.)、]\s+)',line)
if heading:
kind='heading'
if heading[1]:self.ordinals.append((offset+heading.start(1),offset+heading.end(1)))
label=re.match(r'^\s*(#{1,6})\s+(.+)',line)
level=len(label[1]);headings=[item for item in headings if item[0]<level]
headings.append((level,label[2].strip()))
elif listing:
kind='list_item'
if listing[1]:self.ordinals.append((offset+listing.start(1),offset+listing.end(1)))
record={'start':offset,'end':end,'kind':kind,'ancestors':tuple(label for _,label in headings)}
if kind=='list_item':
label=re.match(r'[^::\n]{1,80}[::]',line[listing.end():])
if label:record['containerLabel']=label[0]
if line.lstrip().startswith('|'):
kind='table';record['kind']=kind
bars=[m.start() for m in re.finditer(r'\|',line)]
cells=[(offset+a+1,offset+b) for a,b in zip(bars,bars[1:])]
if headers is None:headers=[text[a:b].strip() for a,b in cells];table_line=0
table_line+=1
record.update(cells=cells,headers=headers,tableLine=table_line)
else:headers=None;table_line=0
self.lines.append(record);offset+=len(line)
def mask_ordinals(self):
chars=list(self.text)
for start,end in self.ordinals:chars[start:end]=' '*(end-start)
return ''.join(chars)
@lru_cache(maxsize=4096)
def locate(self,start,end):
text=self.text
line=next((item for item in self.lines if item['start']<=start and end<=item['end']),None)
if line is None:return None
result={'kind':line['kind'],'structuralOrdinal':any(a<=start and end<=b for a,b in self.ordinals),
'header':'','rowLabel':'','ancestors':tuple(dict.fromkeys((*self.ancestors,*line['ancestors']))),'periodRole':None,'periodText':None,
'pair':False,'start':start,'end':end,'precedingPredicates':[],'observationDateText':None,'dateAmbiguous':False,
'containerLabel':line.get('containerLabel',''),'fieldAssociation':None,'rowCells':[],'columnIndex':None,
'subjectAssociation':None,'referencePredecessors':[],'subjectReference':None,'orderedAssociation':None}
left,right=line['start'],line['end'];inherited=[result['containerLabel']] if result['containerLabel'] else []
if line['kind']=='table':
cell=next(((i,a,b) for i,(a,b) in enumerate(line['cells']) if a<=start and end<=b),None)
if cell is None:return None
column,left,right=cell;headers=line['headers']
result.update(header=headers[column] if column<len(headers) else '',rowLabel=text[slice(*line['cells'][0])].strip(),
columnIndex=column,rowCells=[{'text':text[a:b],'start':a,'end':b} for a,b in line['cells']])
values=list(_NUM.finditer(text,left,right));dates=list(_MONTH.finditer(result['header']))
ordinal=next((i for i,m in enumerate(values) if m.start()==start),None)
if len(dates)==2 and len(values)==2 and ordinal is not None:
separator=text[values[0].end():values[1].start()]
if re.fullmatch(r'[\s*`%%]*(?:/|/|→|->)[\s*`]*',separator):
result.update(periodText=dates[ordinal][0],pair=True)
elif len(dates)==1:result['periodText']=dates[0][0]
else:
# Descend only through the containing annotation; sibling clauses
# and later subject changes never enter the numeric predicate.
while True:
predicates,children=_parts(text,left,right)
segment=next(((a,b) for a,b in predicates if a<=start and end<=b),None)
if segment is None:break
seg_left,seg_right=segment
siblings=list(predicates);index=siblings.index(segment);marker=index
ordered=_ordered_subject_values(text,siblings,index,start,end)
if ordered is not None:result['orderedAssociation']=ordered
while marker>0 and text[siblings[marker-1][1]-1:siblings[marker-1][1]]=='、':marker-=1
if marker<index and result['fieldAssociation'] is None:
a,b=siblings[marker];first_measure=_first_measure(text,a,b)
if first_measure is not None:
result['fieldAssociation']={'kind':'enumeration_predicate','label':text[a:first_measure],'start':a,'end':b}
result['subjectAssociation']={'text':text[a:first_measure],'start':a,'end':first_measure}
first_part=re.sub(r'[*_`\s]','',text[siblings[marker][0]:siblings[marker][1]])
if first_part.startswith('其中') and marker>0:
a,b=siblings[marker-1];relation=_change_annotation(text,a,b)
if relation:result['fieldAssociation']={**relation,'kind':'part_of_change'}
governing=_governing_predicate(text,siblings,index,text[seg_left:start])
existing=result['fieldAssociation']
if governing and (existing is None or existing['kind']=='enumeration_predicate' and _metric_head(existing['label'])[0] is None):
result['fieldAssociation']=governing
previous=[(a,b) for a,b in predicates if b<=seg_left]
if not result['referencePredecessors']:
for a,b in reversed(previous):
if text[a:b].endswith(tuple('。!?!?')):break
result['referencePredecessors'].append({'text':text[a:b],'start':a,'end':b})
for a,b in reversed(previous):
value=text[a:b]
if value.endswith(tuple('。!?!?;;')):break
result['precedingPredicates'].append(value)
if value.endswith((':',':')):
inherited.append(value);break
child=next(((a,b) for a,b in children if a<start and end<b),None)
if child is None:left,right=segment;break
opening,closing=child
relation=_change_annotation(text,seg_left,seg_right)
if relation:result['fieldAssociation']=relation
inherited.append(text[seg_left:opening])
first_number=_NUM.search(text,opening+1,closing-1)
if first_number and first_number.start()<start:inherited.append(text[opening+1:first_number.start()])
left,right=opening+1,closing-1
predicate=text[left:right];prefix=text[left:start]
# A from/to or arrow pair gives ordered *levels*, not a delta field.
for first,_,second,last in _ordered_pairs(text,left,right):
if first<=start<=last:
if start in (first,second):result.update(periodRole='comparison' if start==first else 'current',pair=True)
dates=list(_MONTH.finditer(text,left,first))
if len(dates)==1:result['observationDateText']=dates[0][0]
elif len(dates)>1:result['dateAmbiguous']=True
break
owner_prefix,target=_contribution_target(' '.join([*inherited,prefix]))
predicate_context=' '.join([*inherited,*([result['fieldAssociation']['label']] if result['fieldAssociation'] else []),prefix])
family_contexts=[result['rowLabel']+' '+result['header']] if result['kind']=='table' else [prefix,
*([result['fieldAssociation']['label']] if result['fieldAssociation'] else []),*reversed(inherited)]
stage_words,_=_contribution_target((result['rowLabel']+' '+result['header']) if result['kind']=='table' else owner_prefix)
stages=re.findall(r'黑片|亮片|综合',stage_words)
family,family_index=_metric_family(family_contexts)
reference_contexts=[(prefix,left,'current')]
if result['subjectAssociation']:
association=result['subjectAssociation'];reference_contexts.append((association['text'],association['start'],'enumeration'))
reference_contexts.extend((label,0,'annotation') for label in reversed(inherited))
result['subjectReference']=_subject_reference(reference_contexts) if line['kind']!='table' else None
result.update(predicate=predicate,prefix=prefix,subjectText=' '.join([*inherited,prefix]),ownerPrefix=owner_prefix,metricTarget=target,
comparison=comparison_relation(text,start,end),
predicateContext=predicate_context,
metricFamily=family,metricFamilySource='current' if family_index==0 else 'shared' if family_index==1 and result['fieldAssociation'] and result['fieldAssociation']['kind']!='governing_predicate' else 'parent' if family_index is not None else None,
ownerStage={'黑片':'black','亮片':'bright','综合':'overall'}.get(stages[-1]) if stages else None,
inheritedText=' '.join(inherited),fieldText=(result['rowLabel']+' '+result['header']) if line['kind']=='table' else prefix,
predicateStart=left,predicateEnd=right)
# A quantity inside the target of a contribution is a reference to
# that target metric, not the contribution's own operand. Inspect the
# closed local relation; the closing predicate can follow the number.
relation=re.search(r'对(?P<target>[^,。;;::()()]*?)(?:贡献|影响)',predicate)
directions=({name for name,pattern in (('decrease',r'减少|降低|下降|回落|减小|下滑'),
('increase',r'上升|提高|增加|升高|增长')) if re.search(pattern,relation['target'])} if relation else set())
result['quantityReference']=({'kind':'contribution_target','text':relation['target'],
'direction':next(iter(directions)) if len(directions)==1 else 'ambiguous' if directions else None,
'start':left+relation.start('target'),'end':left+relation.end('target')}
if relation and left+relation.start('target')<=start and end<=left+relation.end('target') else None)
return result
@lru_cache(maxsize=96)
def native_span_structure(text,ancestors=()):
return NativeSpanStructure(text,ancestors)
def rebase_native_bindings(source, remaining, bindings, removed):
"""Remove only bindings wholly inside authorized deleted nodes."""
if not isinstance(bindings,list) or len(bindings)>160:return None
raw_remaining=source
for start,end in reversed(removed):raw_remaining=raw_remaining[:start]+raw_remaining[end:]
leading=len(raw_remaining)-len(raw_remaining.lstrip(' \t\r\n,,;;::')) if removed else 0
result=[]
for item in bindings:
if (not isinstance(item,dict) or set(item)-{'span','tokenId','occurrence'} or not isinstance(item.get('span'),str)
or not item['span'] or not isinstance(item.get('tokenId'),str)):return None
old=list(re.finditer(re.escape(item['span']),source));new=list(re.finditer(re.escape(item['span']),remaining))
occurrence=item.get('occurrence',1)
if (type(occurrence) is not int or not 1<=occurrence<=len(old)
or len(old)>1 and 'occurrence' not in item):return None
selected=old[occurrence-1]
if any(a<=selected.start() and selected.end()<=b for a,b in removed):continue
moved=move_span(selected.start(),selected.end(),removed)
if moved is None:return None
remapped=next((i+1 for i,match in enumerate(new) if match.span()==(moved[0]-leading,moved[1]-leading)),None)
if remapped is None:return None
result.append({**item,'occurrence':remapped})
return result
def metric_role(field, stage=None):
if field == 'stageContribution': return 'ContributionPp'
if field == 'rateChange': return 'CompositeRateChangePp' if stage == '综合' else 'OwnStageRateChangePp'
if field == 'rate': return 'CompositeRate' if stage == '综合' else 'OwnStageRate' if stage in ('黑片','亮片') else None
if field == 'scrapRate': return 'ScrapRate'
return field
def formal_display_metadata(raw, field, unit, stage=None, *, signed=False):
"""One formatter for tool presentation and the final typed catalog."""
try:
value=Decimal(str(raw))
if not value.is_finite():return None
shown=format(value.quantize(Decimal('.01'),rounding=ROUND_HALF_UP),',.2f')
if signed and value>0:shown='+'+shown
return {'rawValue':str(value),'displayValue':shown+('%' if unit=='%' else ''),
'unit':unit,'metricRole':metric_role(field,stage),
**({'rawUnit':'kg','measureKind':'mass'} if unit=='kg' and field in MASS_FIELDS else {})}
except (InvalidOperation,ValueError,TypeError):return None
def mass_display_match(token, number, given_unit, *, exact=False):
"""Validate a physical kg value in the draft's unit; never infer mass."""
if (token.get('rawUnit') != 'kg' or token.get('measureKind') != 'mass'
or token.get('metricField') not in MASS_FIELDS or given_unit not in ('kg','ton')):
return None
try:
raw=Decimal(str(token['rawValue']));asked=Decimal(number.replace(',','').replace('−','-'))
if not raw.is_finite() or not asked.is_finite(): return None
scale=Decimal(1000) if given_unit == 'ton' else Decimal(1)
displayed=raw/scale
if given_unit == 'ton' and raw != 0 and asked == 0: return None
places=len(number.split('.',1)[1]) if '.' in number else 0
valid=displayed == asked if exact else displayed.quantize(Decimal(1).scaleb(-places),rounding=ROUND_HALF_UP) == asked
if not valid:return None
if exact:
value=format(raw,'f')
if number.startswith('+') and raw >= 0:value='+'+value
else:value=token['value'].rstrip('%%')
return {'value':value,'unit':'kg','converted':given_unit == 'ton'}
except (InvalidOperation,KeyError,ValueError): return None
def clause_catalog(source_id, text):
"""Issue legal whole-node spans against this exact display text version."""
digest=hashlib.sha256(text.encode('utf-8')).hexdigest()[:8]
nodes=[];seen=set()
def add(start,end,kind):
while start < end and text[start] in ' \t\r\n':start+=1
while end > start and text[end-1] in ' \t\r\n':end-=1
if end-start < 4 or (start,end) in seen:return
seen.add((start,end));nodes.append({'start':start,'end':end,'kind':kind,'text':text[start:end]})
add(0,len(text),'block')
offset=0;table_line=0
for line in text.splitlines(keepends=True):
content=line.rstrip('\r\n');stripped=content.strip()
if stripped.startswith('|'):
table_line+=1
if table_line > 2 and not re.fullmatch(r'\|[\s|:\-]+\|',stripped):add(offset,offset+len(content),'table_row')
offset+=len(line);continue
table_line=0
if re.match(r'^\s*#{1,6}\s',content):offset+=len(line);continue
is_list=re.match(r'^\s*(?:[-*+]|[0-9]+[.)、])\s+',content)
if is_list:add(offset,offset+len(content),'list_item')
# A complete outer annotation is independently removable. Inner
# parenthesized substrings never become their own deletion nodes.
stack=[]
for index,char in enumerate(content):
if char in '((':stack.append(index)
elif char in '))' and stack:
left=stack.pop()
if not stack:add(offset+left,offset+index+1,'annotation')
start=offset;depth=0;sentence_starts=[offset]
for index,char in enumerate(content):
if char in '(([【':depth+=1
elif char in '))]】':depth=max(0,depth-1)
if depth == 0 and char in '。!?!?;;,,':
# Thousands separators are parts of numeric values.
if char == ',' and index and index+1 < len(content) and content[index-1].isdigit() and content[index+1].isdigit():continue
terminal=char in '。!?!?'
add(start,offset+index+1,'sentence' if terminal else 'clause')
if terminal:
for left in sentence_starts:add(left,offset+index+1,'sentence')
sentence_starts=[offset+index+1]
else:sentence_starts.append(offset+index+1)
start=offset+index+1
add(start,offset+len(content),'clause')
offset+=len(line)
nodes.sort(key=lambda node:(node['start'],node['end']))
for index,node in enumerate(nodes,1):node['clauseId']=f'{source_id}-{digest}-C{index}'
return nodes
def selected_clause_spans(text, row, catalog):
"""IDs or exact legal structural aliases only; no arbitrary substring edits."""
by_id={node['clauseId']:node for node in catalog};chosen=[]
ids=row.get('unsupportedClauseIds') or []
strings=row.get('unsupportedClauses') or []
if not isinstance(ids,list) or not isinstance(strings,list) or len(ids)+len(strings)>128:return None
for identifier in ids:
if not isinstance(identifier,str) or identifier not in by_id:return None
chosen.append(by_id[identifier])
for literal in strings:
if not isinstance(literal,str) or len(literal.strip())<4:return None
matches=[]
for node in catalog:
aliases={node['text']}
if node['kind']=='list_item':aliases.add(re.sub(r'^\s*(?:[-*+]|[0-9]+[.)、])\s+','',node['text']))
value=node['text']
if (value.startswith('(') and value.endswith(')') or value.startswith('(') and value.endswith(')')):
aliases.add(value[1:-1])
if literal in aliases:matches.append(node)
spans={(node['start'],node['end']) for node in matches}
if len(spans)!=1:return None
chosen.append(matches[0])
spans=sorted({(node['start'],node['end']) for node in chosen})
if not spans:return None
if any(left[1]>right[0] for left,right in zip(spans,spans[1:])):return None
return spans
def remove_spans(text, spans):
for start,end in reversed(spans):text=text[:start]+text[end:]
return text.strip(' \t\r\n,,;;::')
def move_span(start,end,removed):
if any(start < right and left < end for left,right in removed):return None
shift=sum(right-left for left,right in removed if right <= start)
return start-shift,end-shift