完善:同月多文件导入与当前用户离线迁移更新
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
"""Portable migration validation; no source-machine paths or credentials are logged."""
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath, PureWindowsPath
|
||||
import re
|
||||
import tarfile
|
||||
import uuid
|
||||
|
||||
|
||||
def sha256(path):
|
||||
with Path(path).open('rb') as stream:
|
||||
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def sql_text(value):
|
||||
return 'CONVERT(0x' + value.encode('utf-8').hex() + ' USING utf8mb4)'
|
||||
|
||||
|
||||
def relative_name(name):
|
||||
if not isinstance(name, str) or not name or '\\' in name or ':' in name:
|
||||
raise ValueError('迁移文件路径无效。')
|
||||
path = PurePosixPath(name)
|
||||
if not path.parts or path.is_absolute() or path.as_posix() != name or any(part in ('.', '..', '') for part in path.parts):
|
||||
raise ValueError('迁移文件路径无效。')
|
||||
for part in path.parts:
|
||||
if part.endswith((' ', '.')) or re.fullmatch(r'(?i)(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(?:\..*)?', part):
|
||||
raise ValueError('迁移文件名不兼容 Windows。')
|
||||
return path
|
||||
|
||||
|
||||
def path_under_source(stored, source_root):
|
||||
# A Windows restore must interpret the old Mac path as POSIX, not Path().
|
||||
path, root = PurePosixPath(stored), PurePosixPath(source_root)
|
||||
if not path.is_absolute() or '..' in path.parts or not path.is_relative_to(root):
|
||||
raise ValueError('来源文件引用不在已验证的归档目录内。')
|
||||
return relative_name(path.relative_to(root).as_posix()).as_posix()
|
||||
|
||||
|
||||
def extract_checked(archive, destination, expected):
|
||||
destination = Path(destination)
|
||||
destination.mkdir(parents=True, exist_ok=False, mode=0o700)
|
||||
observed = set()
|
||||
for name in expected:
|
||||
relative_name(name)
|
||||
folded = [name.casefold() for name in expected]
|
||||
if len(folded) != len(set(folded)):
|
||||
raise ValueError('迁移文件在 Windows 上存在重名。')
|
||||
with tarfile.open(archive, 'r:gz') as source:
|
||||
for member in source:
|
||||
rel = relative_name(member.name)
|
||||
if not member.isfile() or member.name in observed or member.name not in expected:
|
||||
raise ValueError('迁移归档包含未登记或无效的文件。')
|
||||
entry = expected[member.name]
|
||||
target = destination.joinpath(*rel.parts)
|
||||
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
if not target.resolve().is_relative_to(destination.resolve()):
|
||||
raise ValueError('迁移文件路径越界。')
|
||||
content = source.extractfile(member)
|
||||
if content is None:
|
||||
raise ValueError('迁移文件无法读取。')
|
||||
digest = hashlib.sha256()
|
||||
with content, target.open('xb') as output:
|
||||
while chunk := content.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
output.write(chunk)
|
||||
target.chmod(0o600)
|
||||
if target.stat().st_size != entry['bytes'] or digest.hexdigest() != entry['sha256']:
|
||||
raise ValueError('迁移文件完整性检查失败。')
|
||||
observed.add(member.name)
|
||||
if observed != set(expected):
|
||||
raise ValueError('迁移归档缺少文件。')
|
||||
|
||||
|
||||
def table_counts(query):
|
||||
result = {}
|
||||
for name in query('SHOW TABLES;').splitlines():
|
||||
if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', name):
|
||||
raise ValueError('数据库表名无效。')
|
||||
result[name] = int(query('SELECT COUNT(*) FROM `' + name + '`;').strip())
|
||||
return result
|
||||
|
||||
|
||||
def query_evidence(query):
|
||||
current = query("SELECT release_id FROM current_release WHERE dataset_id='quality';").strip()
|
||||
if not current:
|
||||
return {'releaseId': None, 'months': []}
|
||||
if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current):
|
||||
raise ValueError('正式数据版本无效。')
|
||||
result = {'releaseId': current, 'months': []}
|
||||
components = ['black_responsibility_scrap', 'black_material_scrap', 'black_external_scrap',
|
||||
'bright_headquarters_scrap', 'bright_plating_scrap', 'bright_material_scrap',
|
||||
'bright_processing_scrap', 'bright_diffusion_scrap']
|
||||
for stage in ('black', 'bright'):
|
||||
pairs = "'stage','" + stage + "','month',f.month_key,'rows',COUNT(*),'qualified',SUM(f.qualified_weight),'scrap',SUM(f.scrap_weight),'total',SUM(f.total_weight)"
|
||||
for key in components:
|
||||
pairs += ", '" + key + "',SUM(CAST(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(f.components,'$." + key + "')),'null') AS DECIMAL(28,10)))"
|
||||
result['months'].extend(json.loads(line) for line in query("SELECT JSON_OBJECT(" + pairs + ") FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm WHERE rm.release_id='" + current + "' AND rm.transform_build_id=f.build_id) GROUP BY f.month_key ORDER BY f.month_key;").splitlines())
|
||||
result['members'] = [json.loads(line) for line in query("SELECT JSON_OBJECT('sourceDocumentId',m.source_document_id,'sourceRevisionId',m.source_revision_id,'buildId',m.transform_build_id,'sourceSha256',r.sha256) FROM release_member m JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id='" + current + "' ORDER BY m.source_document_id;").splitlines()]
|
||||
return result
|
||||
|
||||
|
||||
_VALUES_TOKENS = re.compile(rb"'(?:[^'\\]+|\\.)*'|(\),\()")
|
||||
|
||||
|
||||
def inserted_rows(line):
|
||||
if not line.startswith(b'INSERT INTO `') or b' VALUES (' not in line:
|
||||
raise ValueError('不支持的数据库导出行。')
|
||||
values = line.split(b' VALUES ', 1)[1]
|
||||
if not values.rstrip().endswith(b';'):
|
||||
raise ValueError('数据库导出行不完整。')
|
||||
return 1 + sum(match[1] is not None for match in _VALUES_TOKENS.finditer(values))
|
||||
|
||||
|
||||
def dump_counts(path):
|
||||
counts, uncompressed = {}, 0
|
||||
with gzip.open(path, 'rb') as source:
|
||||
for line in source:
|
||||
uncompressed += len(line)
|
||||
create = re.match(rb'^CREATE TABLE `([A-Za-z0-9_]+)`', line)
|
||||
if create:
|
||||
counts[create[1].decode()] = 0
|
||||
insert = re.match(rb'^INSERT INTO `([A-Za-z0-9_]+)`', line)
|
||||
if insert:
|
||||
name = insert[1].decode()
|
||||
if name not in counts:
|
||||
raise ValueError('导出数据缺少表结构。')
|
||||
counts[name] += inserted_rows(line)
|
||||
return counts, uncompressed
|
||||
|
||||
|
||||
def filter_models(value):
|
||||
profiles = value.get('profiles')
|
||||
if not isinstance(profiles, list):
|
||||
profiles = [{**value, 'id': 'migrated-profile', 'name': '迁移模型'}]
|
||||
def excluded(profile):
|
||||
return any('mimo' in str(profile.get(key, '')).lower() for key in ('model', 'name', 'baseUrl'))
|
||||
kept = [profile.copy() for profile in profiles if not excluded(profile)]
|
||||
removed = [profile for profile in profiles if excluded(profile)]
|
||||
if not kept:
|
||||
return None, removed
|
||||
selected = next((profile for profile in kept if profile.get('id') == value.get('activeProfileId')), kept[0])
|
||||
required = ('baseUrl', 'apiKey', 'model')
|
||||
if any(not isinstance(selected.get(key), str) or not selected[key] for key in required):
|
||||
raise ValueError('需要迁移的模型配置不完整。')
|
||||
filtered = {key: value[key] for key in ('schemaVersion',) if key in value}
|
||||
filtered.update(selected)
|
||||
filtered.pop('id', None)
|
||||
filtered.pop('name', None)
|
||||
filtered.update({'schemaVersion': 3, 'profiles': kept, 'activeProfileId': selected['id']})
|
||||
filtered['revision'] = value.get('revision') or str(uuid.uuid4())
|
||||
return filtered, removed
|
||||
Reference in New Issue
Block a user