优化:统一来源筛选索引与后台构建
This commit is contained in:
@@ -13,6 +13,7 @@ from unittest.mock import patch
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
import backup_retention as br
|
||||
import managed_test_resources
|
||||
import revision_retention as revisions
|
||||
spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py'))
|
||||
ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret)
|
||||
@@ -26,12 +27,15 @@ class SourceRevisionTests(unittest.TestCase):
|
||||
cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root',
|
||||
password=cfg['rootPassword'], autocommit=True)
|
||||
cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
|
||||
managed_test_resources.create(ROOT, cls.name, cls.execute_admin)
|
||||
ddl = [
|
||||
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))',
|
||||
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))',
|
||||
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
|
||||
"CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)",
|
||||
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
|
||||
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext COLLATE utf8mb4_0900_as_cs NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
|
||||
'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||
'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
|
||||
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))',
|
||||
@@ -42,23 +46,34 @@ class SourceRevisionTests(unittest.TestCase):
|
||||
'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))']
|
||||
try:
|
||||
with cls.admin.cursor() as c:
|
||||
c.execute('CREATE DATABASE ' + ret.ident(cls.name)); c.execute('USE ' + ret.ident(cls.name))
|
||||
c.execute('USE ' + ret.ident(cls.name))
|
||||
for statement in ddl:
|
||||
c.execute(statement + ' ENGINE=InnoDB')
|
||||
except BaseException:
|
||||
with cls.admin.cursor() as c: c.execute('DROP DATABASE IF EXISTS ' + ret.ident(cls.name))
|
||||
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
|
||||
cls.admin.close(); raise
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
with cls.admin.cursor() as c: c.execute('DROP DATABASE ' + ret.ident(cls.name))
|
||||
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
|
||||
cls.admin.close()
|
||||
|
||||
@classmethod
|
||||
def execute_admin(cls, sql, args=None):
|
||||
result = ''
|
||||
statements = [part.strip() for part in sql.split(';') if part.strip()]
|
||||
with cls.admin.cursor() as cursor:
|
||||
for statement in statements:
|
||||
cursor.execute(statement, args if len(statements) == 1 else None)
|
||||
result = '\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
|
||||
return result
|
||||
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve()
|
||||
self.db = ret.Database(ROOT, self.name); self.db.root = self.root
|
||||
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
|
||||
for table in self.db.tables: self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
|
||||
for table in self.db.tables:
|
||||
if table != '__quality_test_owner': self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
|
||||
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
|
||||
self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')")
|
||||
self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')")
|
||||
@@ -121,6 +136,21 @@ class SourceRevisionTests(unittest.TestCase):
|
||||
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
|
||||
self.assertTrue(self.present()); self.assertTrue(path.exists())
|
||||
|
||||
def test_indexed_business_values_do_not_count_as_revision_metadata(self):
|
||||
path = self.add_revision('indexed-only', b'indexed-only workbook')
|
||||
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','客户',%s)", (b'1' * 32,))
|
||||
self.db.execute("INSERT INTO source_filter_value VALUES(1,1,%s,'indexed-only')", (b'1' * 32,))
|
||||
self.collect(100); result = self.collect(2000)
|
||||
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
||||
self.assertFalse(self.present('indexed-only')); self.assertFalse(path.exists())
|
||||
|
||||
def test_indexed_field_names_do_not_count_as_revision_metadata(self):
|
||||
path = self.add_revision('indexed-field-only', b'field-only workbook')
|
||||
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','indexed-field-only',%s)", (b'1' * 32,))
|
||||
self.collect(100); result = self.collect(2000)
|
||||
self.assertEqual(result['sourceRevisionsRemoved'], 1)
|
||||
self.assertFalse(self.present('indexed-field-only')); self.assertFalse(path.exists())
|
||||
|
||||
def test_new_reference_resets_continuous_grace(self):
|
||||
self.add_revision(); self.collect(100)
|
||||
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500)
|
||||
|
||||
Reference in New Issue
Block a user