Skip to content

fix(semantic): rewrite dedup SQL with correlated subquery for DB compatibility #91

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lzq986 merged 1 commit into from
Aug 10, 2026
Merged

fix(semantic): rewrite dedup SQL with correlated subquery for DB compatibility #91

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,25 @@
</resultMap>

<sql id="DeduplicatedColumns">
SELECT *
FROM (
SELECT
column_info.*,
ROW_NUMBER() OVER (
PARTITION BY datasource_id, LOWER(table_name), LOWER(column_name)
ORDER BY
CASE
WHEN column_description IS NOT NULL AND column_description &lt;&gt; '' THEN 0
WHEN is_visible = 0 THEN 0
ELSE 1
END ASC,
update_time DESC,
create_time DESC,
id DESC
) AS row_num
FROM column_info
) deduplicated_columns
WHERE row_num = 1
SELECT t.*
FROM column_info t
WHERE t.id = (
SELECT t2.id
FROM column_info t2
WHERE t2.datasource_id = t.datasource_id
AND LOWER(t2.table_name) = LOWER(t.table_name)
AND LOWER(t2.column_name) = LOWER(t.column_name)
ORDER BY
CASE
WHEN t2.column_description IS NOT NULL AND t2.column_description &lt;&gt; '' THEN 0
WHEN t2.is_visible = 0 THEN 0
ELSE 1
END ASC,
t2.update_time DESC,
t2.create_time DESC,
t2.id DESC
LIMIT 1
)
</sql>

<select id="selectByDatasourceId" resultMap="BaseResultMap">
Expand Down
41 changes: 19 additions & 22 deletions data-agent-backend/src/main/resources/mapper/LogicalTableRelationMapper.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,28 +19,25 @@
</resultMap>

<sql id="DeduplicatedRelations">
SELECT *
FROM (
SELECT
logical_table_relation.*,
ROW_NUMBER() OVER (
PARTITION BY
datasource_id,
LOWER(source_table_name),
source_column_signature
ORDER BY
CASE
WHEN description IS NOT NULL AND description &lt;&gt; '' THEN 0
WHEN is_enabled = 0 THEN 0
ELSE 1
END ASC,
update_time DESC,
create_time DESC,
id DESC
) AS row_num
FROM logical_table_relation
) deduplicated_relations
WHERE row_num = 1
SELECT t.*
FROM logical_table_relation t
WHERE t.id = (
SELECT t2.id
FROM logical_table_relation t2
WHERE t2.datasource_id = t.datasource_id
AND LOWER(t2.source_table_name) = LOWER(t.source_table_name)
AND t2.source_column_signature = t.source_column_signature
ORDER BY
CASE
WHEN t2.description IS NOT NULL AND t2.description &lt;&gt; '' THEN 0
WHEN t2.is_enabled = 0 THEN 0
ELSE 1
END ASC,
t2.update_time DESC,
t2.create_time DESC,
t2.id DESC
LIMIT 1
)
</sql>

<select id="selectById" resultMap="BaseResultMap">
Expand Down
37 changes: 18 additions & 19 deletions data-agent-backend/src/main/resources/mapper/TableInfoMapper.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,24 @@
</resultMap>

<sql id="DeduplicatedTables">
SELECT *
FROM (
SELECT
table_info.*,
ROW_NUMBER() OVER (
PARTITION BY datasource_id, LOWER(table_name)
ORDER BY
CASE
WHEN table_description IS NOT NULL AND table_description &lt;&gt; '' THEN 0
WHEN is_visible = 0 THEN 0
ELSE 1
END ASC,
update_time DESC,
create_time DESC,
id DESC
) AS row_num
FROM table_info
) deduplicated_tables
WHERE row_num = 1
SELECT t.*
FROM table_info t
WHERE t.id = (
SELECT t2.id
FROM table_info t2
WHERE t2.datasource_id = t.datasource_id
AND LOWER(t2.table_name) = LOWER(t.table_name)
ORDER BY
CASE
WHEN t2.table_description IS NOT NULL AND t2.table_description &lt;&gt; '' THEN 0
WHEN t2.is_visible = 0 THEN 0
ELSE 1
END ASC,
t2.update_time DESC,
t2.create_time DESC,
t2.id DESC
LIMIT 1
)
</sql>

<select id="selectByDatasourceId" resultMap="BaseResultMap">
Expand Down
Loading