优化:统一来源筛选索引与后台构建

This commit is contained in:
2026-11-13 17:56:18 +08:00
parent 5068b8fbc9
commit 7782e33bec
29 changed files with 997 additions and 59 deletions
+19 -1
View File
@@ -26,6 +26,7 @@ SUPPORT = Path.home() / 'Library/Application Support/QualityCenterService'
TERMINAL = ('COMPLETED', 'FAILED', 'CANCELLED', 'SUCCEEDED', 'STOPPED', 'REJECTED', 'CLOSED', 'INTERRUPTED', 'INVALIDATED')
BUILD_CHILDREN = ('fine_defect_compact', 'fine_defect_compact_build', 'defect_component', 'fine_defect_rollup', 'metric_rollup', 'sales_order_metric_rollup',
'fine_defect_rollup_build', 'metric_rollup_build', 'sales_order_metric_rollup_build',
'source_filter_member', 'source_filter_value', 'source_filter_field', 'source_filter_build',
'fact_black', 'fact_bright')
OWNED_RELEASE = ('release_member', 'release_protocol_dependency', 'release_source_exclusion',
'release_action', 'candidate_reconciliation')
@@ -253,7 +254,24 @@ class Database:
self.execute("UPDATE transform_build SET status='RETIRED',build_key=%s WHERE id=%s", (tombstone,bid))
for t in BUILD_CHILDREN:
if t in self.columns:
count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,))
if t == 'source_filter_member':
# Members have no build_id; drain each owned field in
# order without aliases because the exclusion uses LOCK TABLES.
count = 0
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
count = self.execute('DELETE FROM source_filter_member WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
if count:
break
elif t == 'source_filter_value':
# Values are likewise owned through fields and are deleted
# only after all field memberships have been drained.
count = 0
for field in self.rows('SELECT id FROM source_filter_field WHERE build_id=%s ORDER BY id', (bid,)):
count = self.execute('DELETE FROM source_filter_value WHERE field_id=%s LIMIT ' + str(int(batch)), (field['id'],))
if count:
break
else:
count = self.execute('DELETE FROM ' + ident(t) + ' WHERE build_id=%s LIMIT ' + str(int(batch)), (bid,))
if count:
return count, 0
self.execute('DELETE FROM transform_build WHERE id=%s', (bid,))
+30 -4
View File
@@ -16,6 +16,7 @@ from unittest.mock import patch
ROOT=Path(__file__).resolve().parents[2]
sys.path.insert(0,str(ROOT/'scripts'))
import backup_retention as br
import managed_test_resources
spec=importlib.util.spec_from_file_location('quality_retention',Path(__file__).with_name('retention.py'))
ret=importlib.util.module_from_spec(spec);spec.loader.exec_module(ret)
@@ -91,14 +92,18 @@ class DatabaseRetentionTests(unittest.TestCase):
cfg=json.loads((ROOT/'.runtime/local-config.json').read_text())
cls.admin=ret.pymysql.connect(unix_socket=str(ROOT/'.runtime/mysql.sock'),user='root',password=cfg['rootPassword'],autocommit=True)
cls.name='quality_retention_test_'+uuid.uuid4().hex[:12]
managed_test_resources.create(ROOT,cls.name,cls.execute_admin)
with cls.admin.cursor() as c:
c.execute('CREATE DATABASE '+ret.ident(cls.name))
c.execute('USE '+ret.ident(cls.name))
ddl=[
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24),parent_release_id varchar(36),expected_current_release_id varchar(36),FOREIGN KEY(parent_release_id) REFERENCES data_release(id),FOREIGN KEY(expected_current_release_id) REFERENCES data_release(id))',
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id))',
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,status varchar(24),build_key char(64) UNIQUE)',
'CREATE TABLE release_member(release_id varchar(36),transform_build_id varchar(36),FOREIGN KEY(release_id) REFERENCES data_release(id),FOREIGN KEY(transform_build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_build(build_id varchar(36) PRIMARY KEY,format_version int,FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
'CREATE TABLE source_filter_member(field_id bigint NOT NULL,fact_id bigint NOT NULL,value_id bigint NOT NULL,PRIMARY KEY(field_id,fact_id),KEY(value_id,fact_id),FOREIGN KEY(field_id) REFERENCES source_filter_field(id),FOREIGN KEY(value_id) REFERENCES source_filter_value(id))',
'CREATE TABLE fact_black(id int PRIMARY KEY,build_id varchar(36),payload varchar(20),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE unknown_build_use(id int PRIMARY KEY,build_id varchar(36),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE task(id int PRIMARY KEY,status varchar(24),next_retry_at datetime,retry_count int,max_retries int)',
@@ -115,14 +120,25 @@ class DatabaseRetentionTests(unittest.TestCase):
@classmethod
def tearDownClass(cls):
with cls.admin.cursor() as c:c.execute('DROP DATABASE '+ret.ident(cls.name))
managed_test_resources.drop(ROOT,cls.name,cls.execute_admin)
cls.admin.close()
@classmethod
def execute_admin(cls,sql,args=None):
result=''
statements=[part.strip() for part in sql.split(';') if part.strip()]
with cls.admin.cursor() as cursor:
for statement in statements:
cursor.execute(statement,args if len(statements)==1 else None)
result='\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
return result
def setUp(self):
self.temp=tempfile.TemporaryDirectory();self.root=Path(self.temp.name)
self.db=ret.Database(ROOT,self.name);self.db.root=self.root
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
for table in self.db.tables:self.db.execute('TRUNCATE TABLE '+ret.ident(table))
for table in self.db.tables:
if table!='__quality_test_owner':self.db.execute('TRUNCATE TABLE '+ret.ident(table))
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
self.db.execute("INSERT INTO data_release VALUES('old','PUBLISHED',NULL,NULL),('current','PUBLISHED','old','old')")
self.db.execute("INSERT INTO current_release VALUES('quality','current')")
@@ -187,14 +203,24 @@ class DatabaseRetentionTests(unittest.TestCase):
def test_shared_build_is_preserved_and_partial_retirement_cannot_be_reused(self):
self.retire_old();state={'builds':{'obsolete':1}}
self.db.execute("INSERT INTO source_filter_build VALUES('obsolete',1)")
self.db.execute("INSERT INTO source_filter_field VALUES(9,'obsolete','black','空字段',%s),(10,'obsolete','black','客户',%s),(11,'obsolete','black','工单',%s)", (b'0'*32,b'1'*32,b'2'*32))
self.db.execute("INSERT INTO source_filter_value VALUES(100,10,%s,'客户A'),(101,10,%s,'客户B'),(102,11,%s,'工单A')", (b'1'*32,b'2'*32,b'3'*32))
self.db.execute("INSERT INTO source_filter_member VALUES(10,1,100),(10,2,101),(11,3,102)")
with self.db.frozen('current'):
self.assertEqual(self.db.build_step(state,1800,4000,1),(1,0))
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_member')[0]['n'],2)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_value')[0]['n'],3)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_field')[0]['n'],3)
self.assertEqual(self.db.rows('SELECT COUNT(*) AS n FROM source_filter_build')[0]['n'],1)
old=self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'")[0]
self.assertEqual(old['status'],'RETIRED');self.assertNotEqual(old['build_key'],'2'*64)
self.db.execute("INSERT INTO transform_build VALUES('fresh','COMPLETED',%s)",('2'*64,))
for _ in range(4):
for _ in range(20):
with self.db.frozen('current'):self.db.build_step(state,1800,4000,1)
self.assertFalse(self.db.rows("SELECT * FROM transform_build WHERE id='obsolete'"))
for table in ('source_filter_member','source_filter_value','source_filter_field','source_filter_build'):
self.assertFalse(self.db.rows('SELECT * FROM '+ret.ident(table)))
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='shared'"))
self.assertTrue(self.db.rows("SELECT * FROM transform_build WHERE id='fresh'"))
+34 -4
View File
@@ -13,6 +13,7 @@ from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
import backup_retention as br
import managed_test_resources
import revision_retention as revisions
spec = importlib.util.spec_from_file_location('source_retention_host', Path(__file__).with_name('retention.py'))
ret = importlib.util.module_from_spec(spec); spec.loader.exec_module(ret)
@@ -26,12 +27,15 @@ class SourceRevisionTests(unittest.TestCase):
cls.admin = ret.pymysql.connect(unix_socket=str(ROOT / '.runtime/mysql.sock'), user='root',
password=cfg['rootPassword'], autocommit=True)
cls.name = 'quality_retention_test_' + uuid.uuid4().hex[:12]
managed_test_resources.create(ROOT, cls.name, cls.execute_admin)
ddl = [
'CREATE TABLE data_release(id varchar(36) PRIMARY KEY,status varchar(24))',
'CREATE TABLE current_release(dataset_id varchar(20) PRIMARY KEY,release_id varchar(36))',
'CREATE TABLE source_document(id varchar(36) PRIMARY KEY,source_slot varchar(100))',
"CREATE TABLE source_revision(id varchar(36) PRIMARY KEY,source_document_id varchar(36),archive_path varchar(1000),sha256 char(64),file_size bigint,status varchar(24) DEFAULT 'TRANSFORMED',replaces_revision_id varchar(36),parser_config json)",
'CREATE TABLE transform_build(id varchar(36) PRIMARY KEY,source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE source_filter_field(id bigint AUTO_INCREMENT PRIMARY KEY,build_id varchar(36) NOT NULL,stage varchar(16) NOT NULL,column_name longtext COLLATE utf8mb4_0900_bin NOT NULL,name_hash binary(32) NOT NULL,UNIQUE KEY(build_id,stage,name_hash),FOREIGN KEY(build_id) REFERENCES transform_build(id))',
'CREATE TABLE source_filter_value(id bigint AUTO_INCREMENT PRIMARY KEY,field_id bigint NOT NULL,value_hash binary(32) NOT NULL,value longtext COLLATE utf8mb4_0900_as_cs NOT NULL,UNIQUE KEY(field_id,value_hash),FOREIGN KEY(field_id) REFERENCES source_filter_field(id))',
'CREATE TABLE release_member(release_id varchar(36),source_revision_id varchar(36),FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE fact_black(id int PRIMARY KEY,source_revision_id varchar(36),raw_values json,components json,FOREIGN KEY(source_revision_id) REFERENCES source_revision(id))',
'CREATE TABLE source_candidate(id varchar(36) PRIMARY KEY,status varchar(24),sha256 char(64),source_path varchar(1000),archive_path varchar(1000),replacement_revision_id varchar(36))',
@@ -42,23 +46,34 @@ class SourceRevisionTests(unittest.TestCase):
'CREATE TABLE unknown_reference(id int PRIMARY KEY,revision_ref varchar(36),FOREIGN KEY(revision_ref) REFERENCES source_revision(id))']
try:
with cls.admin.cursor() as c:
c.execute('CREATE DATABASE ' + ret.ident(cls.name)); c.execute('USE ' + ret.ident(cls.name))
c.execute('USE ' + ret.ident(cls.name))
for statement in ddl:
c.execute(statement + ' ENGINE=InnoDB')
except BaseException:
with cls.admin.cursor() as c: c.execute('DROP DATABASE IF EXISTS ' + ret.ident(cls.name))
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
cls.admin.close(); raise
@classmethod
def tearDownClass(cls):
with cls.admin.cursor() as c: c.execute('DROP DATABASE ' + ret.ident(cls.name))
managed_test_resources.drop(ROOT, cls.name, cls.execute_admin)
cls.admin.close()
@classmethod
def execute_admin(cls, sql, args=None):
result = ''
statements = [part.strip() for part in sql.split(';') if part.strip()]
with cls.admin.cursor() as cursor:
for statement in statements:
cursor.execute(statement, args if len(statements) == 1 else None)
result = '\n'.join(str(row[0]) for row in cursor.fetchall()) if cursor.description else str(cursor.rowcount)
return result
def setUp(self):
self.temp = tempfile.TemporaryDirectory(); self.root = Path(self.temp.name).resolve()
self.db = ret.Database(ROOT, self.name); self.db.root = self.root
self.db.execute('SET FOREIGN_KEY_CHECKS=0')
for table in self.db.tables: self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
for table in self.db.tables:
if table != '__quality_test_owner': self.db.execute('TRUNCATE TABLE ' + ret.ident(table))
self.db.execute('SET FOREIGN_KEY_CHECKS=1')
self.db.execute("INSERT INTO source_document VALUES('doc','black:2026:01')")
self.db.execute("INSERT INTO data_release VALUES('current','PUBLISHED')")
@@ -121,6 +136,21 @@ class SourceRevisionTests(unittest.TestCase):
self.collect(100); self.assertEqual(self.collect(2000)['sourceRevisionsRemoved'], 0)
self.assertTrue(self.present()); self.assertTrue(path.exists())
def test_indexed_business_values_do_not_count_as_revision_metadata(self):
path = self.add_revision('indexed-only', b'indexed-only workbook')
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','客户',%s)", (b'1' * 32,))
self.db.execute("INSERT INTO source_filter_value VALUES(1,1,%s,'indexed-only')", (b'1' * 32,))
self.collect(100); result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertFalse(self.present('indexed-only')); self.assertFalse(path.exists())
def test_indexed_field_names_do_not_count_as_revision_metadata(self):
path = self.add_revision('indexed-field-only', b'field-only workbook')
self.db.execute("INSERT INTO source_filter_field VALUES(1,'current-build','black','indexed-field-only',%s)", (b'1' * 32,))
self.collect(100); result = self.collect(2000)
self.assertEqual(result['sourceRevisionsRemoved'], 1)
self.assertFalse(self.present('indexed-field-only')); self.assertFalse(path.exists())
def test_new_reference_resets_continuous_grace(self):
self.add_revision(); self.collect(100)
self.db.execute("INSERT INTO unknown_reference VALUES(1,'old-rev-unique')"); self.collect(1500)