sql - Q : How to find same and similar string values in different tables in SQL

Question

我的数据库中有两个表，每个表都有带有名称的列。如何比较这些表列以及如何找到这些名称完全匹配的名称和这些名称，它们在 table1 和 table2 中相似？

例如:

表格1

column1
---------------------
Tom Hawky
Antony Hendric Formen
Cathy Cassy
Bill Gates
Mary Diore

表2

column1
----------------------
Christopher Fridricson
Ken Lovely
Tom Hawky
Anthony Foreman
Chati Cassei

结果应该是这样的:

table1 - table2
Tom Hawky - Tom Hawky
Antony Hendric Formen - Anthony Henrich Foreman
Cathy Cassy - Chati Cassei

Answer 1

一种解决方案是执行以下操作:

1) 将名称拆分到一些临时表中，以便更容易地进行比较

2) 构造一个查询，尝试根据某个距离查找匹配项

1) 分词

设置

-- drop table Table1
create table Table1 
(
    Id1 INT NOT NULL CONSTRAINT PK_Table1 PRIMARY KEY IDENTITY (1, 1),
    Column1 NVARCHAR(200) NOT NULL
)
GO

insert into Table1 (Column1)
VALUES ('Tom Hawky'), ('Antony Hendric Formen'), ('Cathy Cassy'), ('Bill Gates'), ('Mary Diore')
GO

-- drop table Table2
create table Table2
(
    Id2 INT NOT NULL CONSTRAINT PK_Table2 PRIMARY KEY IDENTITY (1, 1),
    Column2 NVARCHAR(200) NOT NULL
)
GO

insert into Table2 (Column2)
VALUES ('Christopher Fridricson'), ('Ken Lovely'), ('Tom Hawky'), ('Anthony Foreman'), ('Chati Cassei'), ('Tom X')
GO

select * from Table1
GO

select * from Table2
GO

拆分功能

有几个 split functions可以使用，我选择了 XML 方式:

CREATE FUNCTION dbo.SplitStrings_XML
(
   @List       NVARCHAR(MAX),
   @Delimiter  NVARCHAR(255)
)
RETURNS TABLE
WITH SCHEMABINDING
AS
   RETURN 
   (  
      SELECT Item = y.i.value('(./text())[1]', 'nvarchar(4000)')
      FROM 
      ( 
        SELECT x = CONVERT(XML, '<i>' 
          + REPLACE(@List, @Delimiter, '</i><i>') 
          + '</i>').query('.')
      ) AS a CROSS APPLY x.nodes('i') AS y(i)
   );
GO

分词

drop table #T1Words
Go

create table #T1Words (Id1 INT NOT NULL, Word NVARCHAR(100))
GO

insert into #T1Words 
select T1.Id1, X1.Item
from Table1 T1
    cross apply dbo.SplitStrings_XML (T1.Column1, N' ') X1
GO

drop table #T2Words
GO 

create table #T2Words (Id2 INT NOT NULL, Word NVARCHAR(200))
GO

insert into #T2Words 
select T2.Id2, X2.Item
from Table2 T2
    cross apply dbo.SplitStrings_XML (T2.Column2, N' ') X2
GO

选择使用一些距离

可以使用的一种距离是 Levenshtein .为了在SQL中使用，必须在CLR中实现，否则很慢。像这样的东西:

[Microsoft.SqlServer.Server.SqlFunction(IsDeterministic = true, IsPrecise = false)]
public static int Levenshtein(SqlString S1, SqlString S2)
{
    if (S1.IsNull)
        S1 = new SqlString("");

    if (S2.IsNull)
        S2 = new SqlString("");

    int maxLen = 4096;

    // keeping only the first part of the string (performance reasons)
    String SC1 = S1.Value.ToUpper();
    String SC2 = S2.Value.ToUpper();

    if (SC1.Length > maxLen)
        SC1 = SC1.Remove(maxLen);
    if (SC2.Length > maxLen)
        SC2 = SC2.Remove(maxLen);

    int n = SC1.Length;
    int m = SC2.Length;

    short[,] d = new short[n + 1, m + 1];
    int cost = 0;

    if (n + m == 0)
    {
        return 0;
    }
    else if (n == 0)
    {
        return 0;
    }
    else if (m == 0)
    {
        return 0;
    }

    for (short i = 0; i <= n; i++)
        d[i, 0] = i;

    for (short j = 0; j <= m; j++)
        d[0, j] = j;

    for (int i = 1; i <= n; i++)
    {
        for (int j = 1; j <= m; j++)
        {
            if (SC1[i - 1] == SC2[j - 1])
                cost = 0;
            else
                cost = 1;

            d[i, j] = (short) System.Math.Min(System.Math.Min(d[i - 1, j] + 1, d[i, j - 1] + 1), d[i - 1, j - 1] + cost);
        }
    }

    // double percentage = System.Math.Round((1.0 - ((double)d[n, m] / (double)System.Math.Max(n, m))) * 100.0, 2);
    // return percentage;
    return d[n, m];
}

-- SQL to actually create scalar function that calls CLR code
ALTER FUNCTION dbo.Levenshtein(@S1 nvarchar(max), @S2 nvarchar(max))
    RETURNS INT as EXTERNAL NAME ClrUtils.StoredFunctions.Levenshtein
GO

-- CLR must be enabled for database
sp_configure 'clr enabled', 1
GO
RECONFIGURE
GO

2)查询

select T1.Id1, T1.Column1, T2.Id2, T2.Column2
from Table1 T1
    cross join Table2 T2 
where EXISTS (
    SELECT 1
    FROM #T1Words T1W
        JOIN #T2Words T2W ON [dbo].[Levenshtein](T2W.Word, T1W.Word) < 3   -- T2W.Word = T1W.Word
    WHERE T1W.Id1 = T1.Id1
        AND T2W.Id2 = T2.Id2
) 

当然，您必须根据可接受的差异设置阈值。