Files
quality_analyse/scripts/nao_native_contract.py

523 lines
29 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Pure display and immutable deletion primitives for Native drafts.
No query, authorization, metric calculation or model call lives here.
"""
from __future__ import annotations
import hashlib
import re
from functools import lru_cache
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
MASS_FIELDS = frozenset({'qualifiedWeight','totalWeight','scrapWeight','qualifiedWeightDelta','totalWeightDelta',
'scrapWeightDelta','scrapDelta','deltaWeight','categoryWeight','sourceWeight','weight','difference'})
NUMBER_PATTERN=r'[+−-]?[0-9]+(?:,[0-9]{3})*(?:\.[0-9]+)?(?:%|%)?'
_NUM=re.compile(NUMBER_PATTERN)
_MONTH=re.compile(r'(?:(?:[0-9]{4})年)?[0-9]{1,2}月')
_SUBJECT_REFERENCE=re.compile(r'(?P<operator>首|第|另|其余|剩余)(?P<number>[0-9]+|[一二两三四五六七八九十]+)?(?P<classifier>位|名|项|个)(?:规格|产品|对象|部门|分组)?')
_COMPARISON_OPERATORS={
'不低于':'>=','不少于':'>=','不小于':'>=','至少':'>=','>=':'>=','≥':'>=',
'不高于':'<=','不超过':'<=','不大于':'<=','不多于':'<=','至多':'<=','<=':'<=','≤':'<=',
'高于':'>','大于':'>','多于':'>','超过':'>','>':'>',
'低于':'<','小于':'<','少于':'<','不足':'<','不到':'<','未达':'<','<':'<',
}
_COMPARISON_SUFFIXES={'及以上':'>=','以上':'>=','及以下':'<=','以下':'<=','之上':'>','之下':'<'}
def comparison_relation(text,start,end):
"""Parse a boundary operand; formatting it as an observation is forbidden."""
before=re.sub(r'[*_`\s]','',text[max(0,start-60):start])
after=re.sub(r'^[*_`\s]+','',text[end:])
prefix=next((word for word in sorted(_COMPARISON_OPERATORS,key=len,reverse=True) if before.endswith(word)),None)
suffix=next((word for word in sorted(_COMPARISON_SUFFIXES,key=len,reverse=True) if after.startswith(word)),None)
operators={_COMPARISON_OPERATORS[prefix]} if prefix else set()
if suffix:operators.add(_COMPARISON_SUFFIXES[suffix])
if not operators:return None
return {'operator':next(iter(operators)) if len(operators)==1 else None,'ambiguous':len(operators)!=1}
def heading_subject(label):
"""A nominal heading's possessor, not an entity found inside a claim."""
tail=re.split(r'[::]',label,maxsplit=1)[-1].strip()
possessive=re.search(r'内部|的',tail)
return tail[:possessive.start()].strip() if possessive else tail
def _contribution_target(text):
match=re.search(r'对(?P<target>[^,。;;::()()]*?)(?:贡献|影响)',text)
return (text[:match.start()]+text[match.end():],match['target']) if match else (text,None)
def _first_measure(text,start,end):
placeholders=list(re.finditer(r'\{\{[A-Z][0-9]+\}\}',text[start:end]))
candidates=[start+m.start() for m in placeholders if m[0].startswith('{{N')]
for match in _NUM.finditer(text,start,end):
if any(start+m.start()<=match.start()<start+m.end() for m in placeholders):continue
if re.match(r'\s*(?:年|月|日|季度)',text[match.end():]):continue
if any(reference.start()<=match.start()<reference.end() for reference in _SUBJECT_REFERENCE.finditer(text,start,end)):continue
if match.start()>start and re.match(r'[A-Za-z0-9_.]',text[match.start()-1]):continue
if match.end()<end and re.match(r'[A-Za-z0-9_.]',text[match.end()]):continue
candidates.append(match.start())
return min(candidates) if candidates else None
_METRIC_LABELS=re.compile(r'报废率|综合合格率|综合率|合格率|合格重量|投入(?:重量|规模|量)?|报废(?:重量|总重)?|占比|份额|比重|比例|百分比|记录数|件数|数量|次数')
def _metric_head(context,explicit=False):
"""A visible field head, including non-mass heads which stop inheritance."""
matches=list(_METRIC_LABELS.finditer(context))
if not matches:return None,None
label=matches[-1][0]
if label in ('占比','份额','比重','比例','百分比'):return 'share',label
if label in ('记录数','件数','数量','次数'):return 'count',label
if label=='报废率':return 'scrap_rate',label
if label.endswith('率'):return 'rate',label
if explicit and label in ('投入','报废'):return None,label
return ('qualified' if label=='合格重量' else 'input' if label.startswith('投入') else 'scrap'),label
def _metric_family(contexts):
"""Classify only a visible predicate label; tokens grant no semantics."""
for index,context in enumerate(contexts):
family,label=_metric_head(context)
if label:return family,index
return None,None
def _governing_predicate(text,predicates,index,prefix):
"""Forward domination by an explicit field in the same container.
Only the field label crosses the edge. Subjects, stages and calendar
anchors remain local, including when a continuation changes period.
"""
topic=None;origin=None
for position,(start,end) in enumerate(predicates[:index+1]):
visible=prefix if position==index else text[start:end]
family,label=_metric_head(visible,explicit=True)
if label:topic=label if family is not None else None;origin=(start,end)
if position==index:
if not label and topic:return {'kind':'governing_predicate','label':topic,'start':origin[0],'end':origin[1]}
return None
if visible.rstrip().endswith(tuple('。!?!?')):topic=None;origin=None
return None
def _reference_integer(text):
if text.isdigit():return int(text)
digits={'一':1,'二':2,'两':2,'三':3,'四':4,'五':5,'六':6,'七':7,'八':8,'九':9}
if text in digits:return digits[text]
if text.count('十')==1:
left,right=text.split('十')
if (not left or left in digits) and (not right or right in digits):return digits.get(left,1)*10+digits.get(right,0)
return None
def _subject_reference(contexts):
"""Structural selectors, not a guessed subject or an authorization."""
for text,offset,source in contexts:
matches=list(_SUBJECT_REFERENCE.finditer(text))
if not matches:continue
if len(matches)!=1:return {'kind':'ambiguous','source':source,'context':text}
match=matches[0];operator=match['operator'];number=_reference_integer(match['number']) if match['number'] else None
if operator=='首':
if match['number']:return {'kind':'ambiguous','source':source,'context':text}
kind,number='rank',1
elif operator=='第':kind='rank'
else:kind='complement'
return {'kind':kind,'rank' if kind=='rank' else 'count':number,
'start':offset+match.start(),'end':offset+match.end(),'source':source,'context':text}
return None
def _ordered_subject_values(text,predicates,index,start,end):
"""An explicit respectively-list associates positions, never equal values."""
left,right=predicates[index];marker=re.search('分别',text[left:right])
if marker is None or start<left+marker.end():return None
first=index
while first>0 and text[predicates[first-1][1]-1]=='、':first-=1
if first==index:return None
subjects=[]
for position in range(first,index+1):
a,b=predicates[position];b=left+marker.start() if position==index else b-1
subjects.append({'text':text[a:b].strip(),'start':a,'end':b})
values=[]
for match in re.finditer(r'\{\{N[1-9][0-9]*\}\}|'+NUMBER_PATTERN,text[left+marker.end():right]):
a=left+marker.end()+match.start();b=left+marker.end()+match.end()
if re.match(r'\s*(?:年|月|日|季度)',text[b:]):continue
values.append((a,b))
ordinal=next((i for i,(a,b) in enumerate(values) if a<=start and end<=b),None)
return {'subjects':subjects,'index':ordinal,'ambiguous':len(subjects)!=len(values) or ordinal is None}
@lru_cache(maxsize=512)
def _ordered_pairs(text,start,end):
values=list(_NUM.finditer(text,start,end));pairs=[]
for first,second in zip(values,values[1:]):
between=re.sub(r'[*`_\s]','',text[first.end():second.start()])
before=re.sub(r'[*`_\s]','',text[start:first.start()])
arrow=re.fullmatch(r'(?:%|%|kg|公斤|千克|吨|pp|个百分点)?(?:→|->)',between,re.I)
ordered=re.search(r'(?:从|由)$',before) and re.fullmatch(r'(?:%|%|kg|公斤|千克|吨|pp|个百分点)?(?:增|升|降|减|扩大|扩|提高|降低)?(?:至|到)',between,re.I)
if arrow or ordered:pairs.append((first.start(),first.end(),second.start(),second.end()))
return tuple(pairs)
def _change_annotation(text,start,end):
"""A signed annotation attached to an ordered pair labels its change."""
_,children=_parts(text,start,end)
for first,_,_,last in reversed(_ordered_pairs(text,start,end)):
for left,right in children:
if left<last:continue
between=re.sub(r'[*`_\s]','',text[last:left])
if not re.fullmatch(r'(?:%|%|kg|公斤|千克|吨|pp|个百分点)?',between,re.I):continue
if re.search(r'[+−-][0-9]',text[left+1:right-1]):
return {'kind':'change_of_pair','label':text[start:first],'start':start,'end':end}
return None
def native_document_ancestors(blocks):
"""Structural ancestry only, never inferred scope or metric authority."""
headings=[];introduction=None;result={}
for block in blocks:
labels=[]
for line in block['text'].splitlines():
heading=re.match(r'^\s*(#{1,6})\s+(.+)',line)
if heading:
level=len(heading[1]);headings=[item for item in headings if item[0]<level]
headings.append((level,heading[2]));introduction=None
elif line.strip() and not line.lstrip().startswith('|'):labels.append(line.strip())
result[block['sourceId']]=tuple([text for _,text in headings]+([introduction] if introduction else []))
if labels and labels[-1].endswith((':',':')):introduction=labels[-1]
return result
@lru_cache(maxsize=512)
def _parts(text,start,end):
"""Sibling predicates and nested annotations, with immutable offsets."""
nodes=[];children=[];stack=[];left=start
for index in range(start,end):
char=text[index]
if char in '(([【':stack.append(index)
elif char in '))]】' and stack:
opening=stack.pop()
if not stack:children.append((opening,index+1))
elif not stack and char in '。!?!?;;,,::、':
if char==',' and index>start and index+1<end and text[index-1].isdigit() and text[index+1].isdigit():continue
nodes.append((left,index+1));left=index+1
if left<end:nodes.append((left,end))
return tuple(nodes),tuple(children)
class NativeSpanStructure:
"""An offset index of the supported Markdown structure, with no facts.
locate() gives a predicate/cell and its structural ancestors. Consumers
resolve those labels against server-authorized facts; this index grants
neither a scope alias nor a numerical authorization.
"""
def __init__(self,text,ancestors=()):
self.text=text;self.ancestors=ancestors;self.lines=[];self.ordinals=[]
offset=0;headers=None;table_line=0;headings=[]
for line in text.splitlines(keepends=True):
end=offset+len(line.rstrip('\r\n'));kind='prose'
heading=re.match(r'^\s*#{1,6}\s+(?:([0-9]+)[.)、]\s+)?',line)
listing=re.match(r'^\s*(?:[-*+]\s+|([0-9]+)[.)、]\s+)',line)
if heading:
kind='heading'
if heading[1]:self.ordinals.append((offset+heading.start(1),offset+heading.end(1)))
label=re.match(r'^\s*(#{1,6})\s+(.+)',line)
level=len(label[1]);headings=[item for item in headings if item[0]<level]
headings.append((level,label[2].strip()))
elif listing:
kind='list_item'
if listing[1]:self.ordinals.append((offset+listing.start(1),offset+listing.end(1)))
record={'start':offset,'end':end,'kind':kind,'ancestors':tuple(label for _,label in headings)}
if kind=='list_item':
label=re.match(r'[^::\n]{1,80}[::]',line[listing.end():])
if label:record['containerLabel']=label[0]
if line.lstrip().startswith('|'):
kind='table';record['kind']=kind
bars=[m.start() for m in re.finditer(r'\|',line)]
cells=[(offset+a+1,offset+b) for a,b in zip(bars,bars[1:])]
if headers is None:headers=[text[a:b].strip() for a,b in cells];table_line=0
table_line+=1
record.update(cells=cells,headers=headers,tableLine=table_line)
else:headers=None;table_line=0
self.lines.append(record);offset+=len(line)
def mask_ordinals(self):
chars=list(self.text)
for start,end in self.ordinals:chars[start:end]=' '*(end-start)
return ''.join(chars)
@lru_cache(maxsize=4096)
def locate(self,start,end):
text=self.text
line=next((item for item in self.lines if item['start']<=start and end<=item['end']),None)
if line is None:return None
result={'kind':line['kind'],'structuralOrdinal':any(a<=start and end<=b for a,b in self.ordinals),
'header':'','rowLabel':'','ancestors':tuple(dict.fromkeys((*self.ancestors,*line['ancestors']))),'periodRole':None,'periodText':None,
'pair':False,'start':start,'end':end,'precedingPredicates':[],'observationDateText':None,'dateAmbiguous':False,
'containerLabel':line.get('containerLabel',''),'fieldAssociation':None,'rowCells':[],'columnIndex':None,
'subjectAssociation':None,'referencePredecessors':[],'subjectReference':None,'orderedAssociation':None}
left,right=line['start'],line['end'];inherited=[result['containerLabel']] if result['containerLabel'] else []
if line['kind']=='table':
cell=next(((i,a,b) for i,(a,b) in enumerate(line['cells']) if a<=start and end<=b),None)
if cell is None:return None
column,left,right=cell;headers=line['headers']
result.update(header=headers[column] if column<len(headers) else '',rowLabel=text[slice(*line['cells'][0])].strip(),
columnIndex=column,rowCells=[{'text':text[a:b],'start':a,'end':b} for a,b in line['cells']])
values=list(_NUM.finditer(text,left,right));dates=list(_MONTH.finditer(result['header']))
ordinal=next((i for i,m in enumerate(values) if m.start()==start),None)
if len(dates)==2 and len(values)==2 and ordinal is not None:
separator=text[values[0].end():values[1].start()]
if re.fullmatch(r'[\s*`%%]*(?:/|/|→|->)[\s*`]*',separator):
result.update(periodText=dates[ordinal][0],pair=True)
elif len(dates)==1:result['periodText']=dates[0][0]
else:
# Descend only through the containing annotation; sibling clauses
# and later subject changes never enter the numeric predicate.
while True:
predicates,children=_parts(text,left,right)
segment=next(((a,b) for a,b in predicates if a<=start and end<=b),None)
if segment is None:break
seg_left,seg_right=segment
siblings=list(predicates);index=siblings.index(segment);marker=index
ordered=_ordered_subject_values(text,siblings,index,start,end)
if ordered is not None:result['orderedAssociation']=ordered
while marker>0 and text[siblings[marker-1][1]-1:siblings[marker-1][1]]=='、':marker-=1
if marker<index and result['fieldAssociation'] is None:
a,b=siblings[marker];first_measure=_first_measure(text,a,b)
if first_measure is not None:
result['fieldAssociation']={'kind':'enumeration_predicate','label':text[a:first_measure],'start':a,'end':b}
result['subjectAssociation']={'text':text[a:first_measure],'start':a,'end':first_measure}
first_part=re.sub(r'[*_`\s]','',text[siblings[marker][0]:siblings[marker][1]])
if first_part.startswith('其中') and marker>0:
a,b=siblings[marker-1];relation=_change_annotation(text,a,b)
if relation:result['fieldAssociation']={**relation,'kind':'part_of_change'}
governing=_governing_predicate(text,siblings,index,text[seg_left:start])
existing=result['fieldAssociation']
if governing and (existing is None or existing['kind']=='enumeration_predicate' and _metric_head(existing['label'])[0] is None):
result['fieldAssociation']=governing
previous=[(a,b) for a,b in predicates if b<=seg_left]
if not result['referencePredecessors']:
for a,b in reversed(previous):
if text[a:b].endswith(tuple('。!?!?')):break
result['referencePredecessors'].append({'text':text[a:b],'start':a,'end':b})
for a,b in reversed(previous):
value=text[a:b]
if value.endswith(tuple('。!?!?;;')):break
result['precedingPredicates'].append(value)
if value.endswith((':',':')):
inherited.append(value);break
child=next(((a,b) for a,b in children if a<start and end<b),None)
if child is None:left,right=segment;break
opening,closing=child
relation=_change_annotation(text,seg_left,seg_right)
if relation:result['fieldAssociation']=relation
inherited.append(text[seg_left:opening])
first_number=_NUM.search(text,opening+1,closing-1)
if first_number and first_number.start()<start:inherited.append(text[opening+1:first_number.start()])
left,right=opening+1,closing-1
predicate=text[left:right];prefix=text[left:start]
# A from/to or arrow pair gives ordered *levels*, not a delta field.
for first,_,second,last in _ordered_pairs(text,left,right):
if first<=start<=last:
if start in (first,second):result.update(periodRole='comparison' if start==first else 'current',pair=True)
dates=list(_MONTH.finditer(text,left,first))
if len(dates)==1:result['observationDateText']=dates[0][0]
elif len(dates)>1:result['dateAmbiguous']=True
break
owner_prefix,target=_contribution_target(' '.join([*inherited,prefix]))
predicate_context=' '.join([*inherited,*([result['fieldAssociation']['label']] if result['fieldAssociation'] else []),prefix])
family_contexts=[result['rowLabel']+' '+result['header']] if result['kind']=='table' else [prefix,
*([result['fieldAssociation']['label']] if result['fieldAssociation'] else []),*reversed(inherited)]
stage_words,_=_contribution_target((result['rowLabel']+' '+result['header']) if result['kind']=='table' else owner_prefix)
stages=re.findall(r'黑片|亮片|综合',stage_words)
family,family_index=_metric_family(family_contexts)
reference_contexts=[(prefix,left,'current')]
if result['subjectAssociation']:
association=result['subjectAssociation'];reference_contexts.append((association['text'],association['start'],'enumeration'))
reference_contexts.extend((label,0,'annotation') for label in reversed(inherited))
result['subjectReference']=_subject_reference(reference_contexts) if line['kind']!='table' else None
result.update(predicate=predicate,prefix=prefix,subjectText=' '.join([*inherited,prefix]),ownerPrefix=owner_prefix,metricTarget=target,
comparison=comparison_relation(text,start,end),
predicateContext=predicate_context,
metricFamily=family,metricFamilySource='current' if family_index==0 else 'shared' if family_index==1 and result['fieldAssociation'] and result['fieldAssociation']['kind']!='governing_predicate' else 'parent' if family_index is not None else None,
ownerStage={'黑片':'black','亮片':'bright','综合':'overall'}.get(stages[-1]) if stages else None,
inheritedText=' '.join(inherited),fieldText=(result['rowLabel']+' '+result['header']) if line['kind']=='table' else prefix,
predicateStart=left,predicateEnd=right)
# A quantity inside the target of a contribution is a reference to
# that target metric, not the contribution's own operand. Inspect the
# closed local relation; the closing predicate can follow the number.
relation=re.search(r'对(?P<target>[^,。;;::()()]*?)(?:贡献|影响)',predicate)
directions=({name for name,pattern in (('decrease',r'减少|降低|下降|回落|减小|下滑'),
('increase',r'上升|提高|增加|升高|增长')) if re.search(pattern,relation['target'])} if relation else set())
result['quantityReference']=({'kind':'contribution_target','text':relation['target'],
'direction':next(iter(directions)) if len(directions)==1 else 'ambiguous' if directions else None,
'start':left+relation.start('target'),'end':left+relation.end('target')}
if relation and left+relation.start('target')<=start and end<=left+relation.end('target') else None)
return result
@lru_cache(maxsize=96)
def native_span_structure(text,ancestors=()):
return NativeSpanStructure(text,ancestors)
def rebase_native_bindings(source, remaining, bindings, removed):
"""Remove only bindings wholly inside authorized deleted nodes."""
if not isinstance(bindings,list) or len(bindings)>160:return None
raw_remaining=source
for start,end in reversed(removed):raw_remaining=raw_remaining[:start]+raw_remaining[end:]
leading=len(raw_remaining)-len(raw_remaining.lstrip(' \t\r\n,,;;::')) if removed else 0
result=[]
for item in bindings:
if (not isinstance(item,dict) or set(item)-{'span','tokenId','occurrence'} or not isinstance(item.get('span'),str)
or not item['span'] or not isinstance(item.get('tokenId'),str)):return None
old=list(re.finditer(re.escape(item['span']),source));new=list(re.finditer(re.escape(item['span']),remaining))
occurrence=item.get('occurrence',1)
if (type(occurrence) is not int or not 1<=occurrence<=len(old)
or len(old)>1 and 'occurrence' not in item):return None
selected=old[occurrence-1]
if any(a<=selected.start() and selected.end()<=b for a,b in removed):continue
moved=move_span(selected.start(),selected.end(),removed)
if moved is None:return None
remapped=next((i+1 for i,match in enumerate(new) if match.span()==(moved[0]-leading,moved[1]-leading)),None)
if remapped is None:return None
result.append({**item,'occurrence':remapped})
return result
def metric_role(field, stage=None):
if field == 'stageContribution': return 'ContributionPp'
if field == 'rateChange': return 'CompositeRateChangePp' if stage == '综合' else 'OwnStageRateChangePp'
if field == 'rate': return 'CompositeRate' if stage == '综合' else 'OwnStageRate' if stage in ('黑片','亮片') else None
if field == 'scrapRate': return 'ScrapRate'
return field
def formal_display_metadata(raw, field, unit, stage=None, *, signed=False):
"""One formatter for tool presentation and the final typed catalog."""
try:
value=Decimal(str(raw))
if not value.is_finite():return None
shown=format(value.quantize(Decimal('.01'),rounding=ROUND_HALF_UP),',.2f')
if signed and value>0:shown='+'+shown
return {'rawValue':str(value),'displayValue':shown+('%' if unit=='%' else ''),
'unit':unit,'metricRole':metric_role(field,stage),
**({'rawUnit':'kg','measureKind':'mass'} if unit=='kg' and field in MASS_FIELDS else {})}
except (InvalidOperation,ValueError,TypeError):return None
def mass_display_match(token, number, given_unit, *, exact=False):
"""Validate a physical kg value in the draft's unit; never infer mass."""
if (token.get('rawUnit') != 'kg' or token.get('measureKind') != 'mass'
or token.get('metricField') not in MASS_FIELDS or given_unit not in ('kg','ton')):
return None
try:
raw=Decimal(str(token['rawValue']));asked=Decimal(number.replace(',','').replace('−','-'))
if not raw.is_finite() or not asked.is_finite(): return None
scale=Decimal(1000) if given_unit == 'ton' else Decimal(1)
displayed=raw/scale
if given_unit == 'ton' and raw != 0 and asked == 0: return None
places=len(number.split('.',1)[1]) if '.' in number else 0
valid=displayed == asked if exact else displayed.quantize(Decimal(1).scaleb(-places),rounding=ROUND_HALF_UP) == asked
if not valid:return None
if exact:
value=format(raw,'f')
if number.startswith('+') and raw >= 0:value='+'+value
else:value=token['value'].rstrip('%%')
return {'value':value,'unit':'kg','converted':given_unit == 'ton'}
except (InvalidOperation,KeyError,ValueError): return None
def clause_catalog(source_id, text):
"""Issue legal whole-node spans against this exact display text version."""
digest=hashlib.sha256(text.encode('utf-8')).hexdigest()[:8]
nodes=[];seen=set()
def add(start,end,kind):
while start < end and text[start] in ' \t\r\n':start+=1
while end > start and text[end-1] in ' \t\r\n':end-=1
if end-start < 4 or (start,end) in seen:return
seen.add((start,end));nodes.append({'start':start,'end':end,'kind':kind,'text':text[start:end]})
add(0,len(text),'block')
offset=0;table_line=0
for line in text.splitlines(keepends=True):
content=line.rstrip('\r\n');stripped=content.strip()
if stripped.startswith('|'):
table_line+=1
if table_line > 2 and not re.fullmatch(r'\|[\s|:\-]+\|',stripped):add(offset,offset+len(content),'table_row')
offset+=len(line);continue
table_line=0
if re.match(r'^\s*#{1,6}\s',content):offset+=len(line);continue
is_list=re.match(r'^\s*(?:[-*+]|[0-9]+[.)、])\s+',content)
if is_list:add(offset,offset+len(content),'list_item')
# A complete outer annotation is independently removable. Inner
# parenthesized substrings never become their own deletion nodes.
stack=[]
for index,char in enumerate(content):
if char in '((':stack.append(index)
elif char in '))' and stack:
left=stack.pop()
if not stack:add(offset+left,offset+index+1,'annotation')
start=offset;depth=0;sentence_starts=[offset]
for index,char in enumerate(content):
if char in '(([【':depth+=1
elif char in '))]】':depth=max(0,depth-1)
if depth == 0 and char in '。!?!?;;,,':
# Thousands separators are parts of numeric values.
if char == ',' and index and index+1 < len(content) and content[index-1].isdigit() and content[index+1].isdigit():continue
terminal=char in '。!?!?'
add(start,offset+index+1,'sentence' if terminal else 'clause')
if terminal:
for left in sentence_starts:add(left,offset+index+1,'sentence')
sentence_starts=[offset+index+1]
else:sentence_starts.append(offset+index+1)
start=offset+index+1
add(start,offset+len(content),'clause')
offset+=len(line)
nodes.sort(key=lambda node:(node['start'],node['end']))
for index,node in enumerate(nodes,1):node['clauseId']=f'{source_id}-{digest}-C{index}'
return nodes
def selected_clause_spans(text, row, catalog):
"""IDs or exact legal structural aliases only; no arbitrary substring edits."""
by_id={node['clauseId']:node for node in catalog};chosen=[]
ids=row.get('unsupportedClauseIds') or []
strings=row.get('unsupportedClauses') or []
if not isinstance(ids,list) or not isinstance(strings,list) or len(ids)+len(strings)>128:return None
for identifier in ids:
if not isinstance(identifier,str) or identifier not in by_id:return None
chosen.append(by_id[identifier])
for literal in strings:
if not isinstance(literal,str) or len(literal.strip())<4:return None
matches=[]
for node in catalog:
aliases={node['text']}
if node['kind']=='list_item':aliases.add(re.sub(r'^\s*(?:[-*+]|[0-9]+[.)、])\s+','',node['text']))
value=node['text']
if (value.startswith('(') and value.endswith(')') or value.startswith('(') and value.endswith(')')):
aliases.add(value[1:-1])
if literal in aliases:matches.append(node)
spans={(node['start'],node['end']) for node in matches}
if len(spans)!=1:return None
chosen.append(matches[0])
spans=sorted({(node['start'],node['end']) for node in chosen})
if not spans:return None
if any(left[1]>right[0] for left,right in zip(spans,spans[1:])):return None
return spans
def remove_spans(text, spans):
for start,end in reversed(spans):text=text[:start]+text[end:]
return text.strip(' \t\r\n,,;;::')
def move_span(start,end,removed):
if any(start < right and left < end for left,right in removed):return None
shift=sum(right-left for left,right in removed if right <= start)
return start-shift,end-shift