"""Portable migration validation; no source-machine paths or credentials are logged.""" from __future__ import annotations import gzip import hashlib import json from pathlib import Path, PurePosixPath, PureWindowsPath import re import tarfile import uuid def sha256(path): with Path(path).open('rb') as stream: return hashlib.file_digest(stream, 'sha256').hexdigest() def sql_text(value): return 'CONVERT(0x' + value.encode('utf-8').hex() + ' USING utf8mb4)' def relative_name(name): if not isinstance(name, str) or not name or '\\' in name or ':' in name: raise ValueError('迁移文件路径无效。') path = PurePosixPath(name) if not path.parts or path.is_absolute() or path.as_posix() != name or any(part in ('.', '..', '') for part in path.parts): raise ValueError('迁移文件路径无效。') for part in path.parts: if part.endswith((' ', '.')) or re.fullmatch(r'(?i)(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(?:\..*)?', part): raise ValueError('迁移文件名不兼容 Windows。') return path def path_under_source(stored, source_root): # A Windows restore must interpret the old Mac path as POSIX, not Path(). path, root = PurePosixPath(stored), PurePosixPath(source_root) if not path.is_absolute() or '..' in path.parts or not path.is_relative_to(root): raise ValueError('来源文件引用不在已验证的归档目录内。') return relative_name(path.relative_to(root).as_posix()).as_posix() def extract_checked(archive, destination, expected): destination = Path(destination) destination.mkdir(parents=True, exist_ok=False, mode=0o700) observed = set() for name in expected: relative_name(name) folded = [name.casefold() for name in expected] if len(folded) != len(set(folded)): raise ValueError('迁移文件在 Windows 上存在重名。') with tarfile.open(archive, 'r:gz') as source: for member in source: rel = relative_name(member.name) if not member.isfile() or member.name in observed or member.name not in expected: raise ValueError('迁移归档包含未登记或无效的文件。') entry = expected[member.name] target = destination.joinpath(*rel.parts) target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) if not target.resolve().is_relative_to(destination.resolve()): raise ValueError('迁移文件路径越界。') content = source.extractfile(member) if content is None: raise ValueError('迁移文件无法读取。') digest = hashlib.sha256() with content, target.open('xb') as output: while chunk := content.read(1024 * 1024): digest.update(chunk) output.write(chunk) target.chmod(0o600) if target.stat().st_size != entry['bytes'] or digest.hexdigest() != entry['sha256']: raise ValueError('迁移文件完整性检查失败。') observed.add(member.name) if observed != set(expected): raise ValueError('迁移归档缺少文件。') def table_counts(query): result = {} for name in query('SHOW TABLES;').splitlines(): if not re.fullmatch(r'[A-Za-z0-9_]{1,64}', name): raise ValueError('数据库表名无效。') result[name] = int(query('SELECT COUNT(*) FROM `' + name + '`;').strip()) return result def query_evidence(query): current = query("SELECT release_id FROM current_release WHERE dataset_id='quality';").strip() if not current: return {'releaseId': None, 'months': []} if not re.fullmatch(r'[A-Za-z0-9-]{1,80}', current): raise ValueError('正式数据版本无效。') result = {'releaseId': current, 'months': []} components = ['black_responsibility_scrap', 'black_material_scrap', 'black_external_scrap', 'bright_headquarters_scrap', 'bright_plating_scrap', 'bright_material_scrap', 'bright_processing_scrap', 'bright_diffusion_scrap'] for stage in ('black', 'bright'): pairs = "'stage','" + stage + "','month',f.month_key,'rows',COUNT(*),'qualified',SUM(f.qualified_weight),'scrap',SUM(f.scrap_weight),'total',SUM(f.total_weight)" for key in components: pairs += ", '" + key + "',SUM(CAST(NULLIF(JSON_UNQUOTE(JSON_EXTRACT(f.components,'$." + key + "')),'null') AS DECIMAL(28,10)))" result['months'].extend(json.loads(line) for line in query("SELECT JSON_OBJECT(" + pairs + ") FROM fact_" + stage + " f WHERE EXISTS(SELECT 1 FROM release_member rm WHERE rm.release_id='" + current + "' AND rm.transform_build_id=f.build_id) GROUP BY f.month_key ORDER BY f.month_key;").splitlines()) result['members'] = [json.loads(line) for line in query("SELECT JSON_OBJECT('sourceDocumentId',m.source_document_id,'sourceRevisionId',m.source_revision_id,'buildId',m.transform_build_id,'sourceSha256',r.sha256) FROM release_member m JOIN source_revision r ON r.id=m.source_revision_id WHERE m.release_id='" + current + "' ORDER BY m.source_document_id;").splitlines()] return result _VALUES_TOKENS = re.compile(rb"'(?:[^'\\]+|\\.)*'|(\),\()") def inserted_rows(line): if not line.startswith(b'INSERT INTO `') or b' VALUES (' not in line: raise ValueError('不支持的数据库导出行。') values = line.split(b' VALUES ', 1)[1] if not values.rstrip().endswith(b';'): raise ValueError('数据库导出行不完整。') return 1 + sum(match[1] is not None for match in _VALUES_TOKENS.finditer(values)) def dump_counts(path): counts, uncompressed = {}, 0 with gzip.open(path, 'rb') as source: for line in source: uncompressed += len(line) create = re.match(rb'^CREATE TABLE `([A-Za-z0-9_]+)`', line) if create: counts[create[1].decode()] = 0 insert = re.match(rb'^INSERT INTO `([A-Za-z0-9_]+)`', line) if insert: name = insert[1].decode() if name not in counts: raise ValueError('导出数据缺少表结构。') counts[name] += inserted_rows(line) return counts, uncompressed def filter_models(value): profiles = value.get('profiles') if not isinstance(profiles, list): profiles = [{**value, 'id': 'migrated-profile', 'name': '迁移模型'}] def excluded(profile): return any('mimo' in str(profile.get(key, '')).lower() for key in ('model', 'name', 'baseUrl')) kept = [profile.copy() for profile in profiles if not excluded(profile)] removed = [profile for profile in profiles if excluded(profile)] if not kept: return None, removed selected = next((profile for profile in kept if profile.get('id') == value.get('activeProfileId')), kept[0]) required = ('baseUrl', 'apiKey', 'model') if any(not isinstance(selected.get(key), str) or not selected[key] for key in required): raise ValueError('需要迁移的模型配置不完整。') filtered = {key: value[key] for key in ('schemaVersion',) if key in value} filtered.update(selected) filtered.pop('id', None) filtered.pop('name', None) filtered.update({'schemaVersion': 3, 'profiles': kept, 'activeProfileId': selected['id']}) filtered['revision'] = value.get('revision') or str(uuid.uuid4()) return filtered, removed