优化:统一来源筛选索引与后台构建

This commit is contained in:
2026-11-13 17:56:18 +08:00
parent 5068b8fbc9
commit 7782e33bec
29 changed files with 997 additions and 59 deletions
+30 -4
View File
@@ -16,6 +16,7 @@ from unittest.mock import patch
ROOT=Path(__file__).resolve().parents[2]
sys.path.insert(0,str(ROOT/'scripts'))
import backup_retention as br
import managed_test_resources
spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py'))
ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret)
@@ -91,14 +92,18 @@ class DatabaseRetentionTests(unittest.TestCase):
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True)
cls.name='quality_retention_test_'+uuid.uuid4().hex[:12]
managed_test_resources.create(ROOT,cls.name,cls.execute_admin)
with cls.admin.cursor() as c:
c.execute('CREATE DATABASE '+ret.ident(cls.name))
c.execute('USE '+ret.ident(cls.name))
ddl=[
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)',
'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_build(build_id varchar(36) PRIMARY KEY,format_version int,FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
'CREATE TABLE source_filter_member(field_id bigint NOT NULL,fact_id bigint NOT NULL,value_id bigint NOT NULL,PRIMARY KEY(field_id,fact_id),KEY(value_id,fact_id),FOREIGN KEY(field_id) REFERENCES source_filter_field(id),FOREIGN KEY(value_id) REFERENCES source_filter_value(id))',
'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)',
@@ -115,14 +120,25 @@ class DatabaseRetentionTests(unittest.TestCase):
@classmethod
def tearDownClass(cls):
with cls.admin.cursor() as c:c.execute('DROP DATABASE '+ret.ident(cls.name))
managed_test_resources.drop(ROOT,cls.name,cls.execute_admin)
cls.admin.close()
@classmethod
def execute_admin(cls,sql,args=None):
result=''
statements=[part.strip() for part in sql.split(';') if part.strip()]
with cls.admin.cursor() as cursor:
for statement in statements:
cursor.execute(statement,args if len(statements)==1 else None)
result='\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
return result
def setUp(self):
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
self.db=ret.Database(ROOT,self.name);self.db.root=self.root
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
for table in self.db.tables:self.db.execute('TRUNCATE TABLE '+ret.ident(table))
for table in self.db.tables:
if table!='__quality_test_owner':self.db.execute('TRUNCATE TABLE '+ret.ident(table))
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')")
self.db.execute("INSERT INTO current_release VALUES('quality','current')")
@@ -187,14 +203,24 @@ class DatabaseRetentionTests(unittest.TestCase):
def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self):
self.retire_old();state={'builds':{'obsolete':1}}
self.db.execute("INSERT INTO source_filter_build VALUES('obsolete',1)")
self.db.execute("INSERT INTO source_filter_field VALUES(9,'obsolete','black','空字段',%s),(10,'obsolete','black','客户',%s),(11,'obsolete','black','工单',%s)", (b'0'*32,b'1'*32,b'2'*32))
self.db.execute("INSERT INTO source_filter_value VALUES(100,10,%s,'客户A'),(101,10,%s,'客户B'),(102,11,%s,'工单A')", (b'1'*32,b'2'*32,b'3'*32))
self.db.execute("INSERT INTO source_filter_member VALUES(10,1,100),(10,2,101),(11,3,102)")
with self.db.frozen('current'):
self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0))
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_member')[0]['n'],2)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_value')[0]['n'],3)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_field')[0]['n'],3)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_build')[0]['n'],1)
old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0]
self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64)
self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,))
for _ in range(4):
for _ in range(20):
with self.db.frozen('current'):self.db.build_step(state,1800,4000,1)
self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'"))
for table in ('source_filter_member','source_filter_value','source_filter_field','source_filter_build'):
self.assertFalse(self.db.rows('SELECT * FROM '+ret.ident(table)))
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'"))
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'"))