除了计算距离之外,如何测量两个字符串的相似性

本文关键字:两个 字符串 相似性 测量 距离 何测量 计算 | 更新日期: 2023-09-27 18:23:52

我正在创建一个程序,该程序检查单词是否为简化单词(txt、msg等),如果经过简化,它会找到正确的拼写,如txt=text、msg=message。我正在使用c中的NHunsell建议方法,该方法建议了所有可能的结果。

问题是,如果我输入"txt",结果是text、tat、tot等。我不知道如何选择正确的单词。我使用了Levenstein距离(C#-比较字符串相似性),但结果仍然是1。

输入:txt结果:text=1,ext=1 tit=1

你能帮我如何理解简化单词的意思或正确拼写吗?示例:消息

除了计算距离之外,如何测量两个字符串的相似性

我用样本数据测试了您的输入,只有text的距离为25,而其他的距离为33。这是我的代码:

string input = "TXT";
string[] words = new[]{"text","tat","tot"};
var levenshtein = new Levenshtein();  
const int maxDistance = 30;
var distanceGroups = words
        .Select(w => new
        {
            Word = w,
            Distance = levenshtein.iLD(w.ToUpperInvariant(), input)
        })
        .Where(x => x.Distance <= maxDistance)
        .GroupBy(x => x.Distance)
        .OrderBy(g => g.Key)
        .ToList();
foreach (var topCandidate in distanceGroups.First())
    Console.WriteLine("Word:{0} Distance:{1}", topCandidate.Word, topCandidate.Distance);

这是Levenstein类:

public class Levenshtein
{
    ///*****************************
    /// Compute Levenshtein distance 
    /// Memory efficient version
    ///*****************************
    public int iLD(String sRow, String sCol)
    {
        int RowLen = sRow.Length;  // length of sRow
        int ColLen = sCol.Length;  // length of sCol
        int RowIdx;                // iterates through sRow
        int ColIdx;                // iterates through sCol
        char Row_i;                // ith character of sRow
        char Col_j;                // jth character of sCol
        int cost;                   // cost
        /// Test string length
        if (Math.Max(sRow.Length, sCol.Length) > Math.Pow(2, 31))
            throw (new Exception("'nMaximum string length in Levenshtein.iLD is " + Math.Pow(2, 31) + ".'nYours is " + Math.Max(sRow.Length, sCol.Length) + "."));
        // Step 1
        if (RowLen == 0)
        {
            return ColLen;
        }
        if (ColLen == 0)
        {
            return RowLen;
        }
        /// Create the two vectors
        int[] v0 = new int[RowLen + 1];
        int[] v1 = new int[RowLen + 1];
        int[] vTmp;

        /// Step 2
        /// Initialize the first vector
        for (RowIdx = 1; RowIdx <= RowLen; RowIdx++)
        {
            v0[RowIdx] = RowIdx;
        }
        // Step 3
        /// Fore each column
        for (ColIdx = 1; ColIdx <= ColLen; ColIdx++)
        {
            /// Set the 0'th element to the column number
            v1[0] = ColIdx;
            Col_j = sCol[ColIdx - 1];

            // Step 4
            /// Fore each row
            for (RowIdx = 1; RowIdx <= RowLen; RowIdx++)
            {
                Row_i = sRow[RowIdx - 1];

                // Step 5
                if (Row_i == Col_j)
                {
                    cost = 0;
                }
                else
                {
                    cost = 1;
                }
                // Step 6
                /// Find minimum
                int m_min = v0[RowIdx] + 1;
                int b = v1[RowIdx - 1] + 1;
                int c = v0[RowIdx - 1] + cost;
                if (b < m_min)
                {
                    m_min = b;
                }
                if (c < m_min)
                {
                    m_min = c;
                }
                v1[RowIdx] = m_min;
            }
            /// Swap the vectors
            vTmp = v0;
            v0 = v1;
            v1 = vTmp;
        }
        // Step 7
        /// Value between 0 - 100
        /// 0==perfect match 100==totaly different
        /// 
        /// The vectors where swaped one last time at the end of the last loop,
        /// that is why the result is now in v0 rather than in v1
        //System.Console.WriteLine("iDist=" + v0[RowLen]);
        int max = System.Math.Max(RowLen, ColLen);
        return ((100 * v0[RowLen]) / max);
    }

    ///*****************************
    /// Compute the min
    ///*****************************
    private int Minimum(int a, int b, int c)
    {
        int mi = a;
        if (b < mi)
        {
            mi = b;
        }
        if (c < mi)
        {
            mi = c;
        }
        return mi;
    }
    ///*****************************
    /// Compute Levenshtein distance         
    ///*****************************
    public int LD(String sNew, String sOld)
    {
        int[,] matrix;              // matrix
        int sNewLen = sNew.Length;  // length of sNew
        int sOldLen = sOld.Length;  // length of sOld
        int sNewIdx; // iterates through sNew
        int sOldIdx; // iterates through sOld
        char sNew_i; // ith character of sNew
        char sOld_j; // jth character of sOld
        int cost; // cost
        /// Test string length
        if (Math.Max(sNew.Length, sOld.Length) > Math.Pow(2, 31))
            throw (new Exception("'nMaximum string length in Levenshtein.LD is " + Math.Pow(2, 31) + ".'nYours is " + Math.Max(sNew.Length, sOld.Length) + "."));
        // Step 1
        if (sNewLen == 0)
        {
            return sOldLen;
        }
        if (sOldLen == 0)
        {
            return sNewLen;
        }
        matrix = new int[sNewLen + 1, sOldLen + 1];
        // Step 2
        for (sNewIdx = 0; sNewIdx <= sNewLen; sNewIdx++)
        {
            matrix[sNewIdx, 0] = sNewIdx;
        }
        for (sOldIdx = 0; sOldIdx <= sOldLen; sOldIdx++)
        {
            matrix[0, sOldIdx] = sOldIdx;
        }
        // Step 3
        for (sNewIdx = 1; sNewIdx <= sNewLen; sNewIdx++)
        {
            sNew_i = sNew[sNewIdx - 1];
            // Step 4
            for (sOldIdx = 1; sOldIdx <= sOldLen; sOldIdx++)
            {
                sOld_j = sOld[sOldIdx - 1];
                // Step 5
                if (sNew_i == sOld_j)
                {
                    cost = 0;
                }
                else
                {
                    cost = 1;
                }
                // Step 6
                matrix[sNewIdx, sOldIdx] = Minimum(matrix[sNewIdx - 1, sOldIdx] + 1, matrix[sNewIdx, sOldIdx - 1] + 1, matrix[sNewIdx - 1, sOldIdx - 1] + cost);
            }
        }
        // Step 7
        /// Value between 0 - 100
        /// 0==perfect match 100==totaly different
        //System.Console.WriteLine("Dist=" + matrix[sNewLen, sOldLen]);
        int max = System.Math.Max(sNewLen, sOldLen);
        return (100 * matrix[sNewLen, sOldLen]) / max;
    }
}

不是一个完整的解决方案,只是一个希望有用的建议。。。

在我看来,人们不太可能使用与正确单词一样长的简化,所以你至少可以过滤掉所有长度<=的结果输入的长度。

您确实需要实现SQL中存在的SOUNDEX例程。我在以下代码中做到了这一点:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Soundex
{
    class Program
    {
        static char[] ignoreChars = new char[] { 'a', 'e', 'h', 'i', 'o', 'u', 'w', 'y' };
        static Dictionary<char, int> charVals = new Dictionary<char, int>()
        {
            {'b',1},
            {'f',1},
            {'p',1},
            {'v',1},
            {'c',2},
            {'g',2},
            {'j',2},
            {'k',2},
            {'q',2},
            {'s',2},
            {'x',2},
            {'z',2},
            {'d',3},
            {'t',3},
            {'l',4},
            {'m',5},
            {'n',5},
            {'r',6}
        };
        static void Main(string[] args)
        {
            Console.WriteLine(Soundex("txt"));
            Console.WriteLine(Soundex("text"));
            Console.WriteLine(Soundex("ext"));
            Console.WriteLine(Soundex("tit"));
            Console.WriteLine(Soundex("Cammmppppbbbeeelll"));
        }
        static string Soundex(string s)
        {
            s = s.ToLower();
            StringBuilder sb = new StringBuilder();
            sb.Append(s.First());
            foreach (var c in s.Substring(1))
            {
                if (ignoreChars.Contains(c)) { continue; }
                // if the previous character yields the same integer then skip it
                if ((int)char.GetNumericValue(sb[sb.Length - 1]) == charVals[c]) { continue; }
                sb.Append(charVals[c]);
            }
            return string.Join("", sb.ToString().Take(4)).PadRight(4, '0');
        }
    }
}

请看,使用此代码,您给出的示例中唯一匹配的将是text。运行控制台应用程序,您将看到输出(即txt将与text匹配)。

我认为像word这样的程序用来纠正拼写的一种方法是使用NLP(自然语言处理)技术来获得拼写错误上下文中使用的名词/形容词的顺序。。然后将其与已知的句子结构进行比较,他们可以估计70%的拼写错误是名词,并使用这些信息来过滤更正的拼写。

SharpNLP看起来是一个很好的库,但我还没有机会摆弄它。为了建立一个已知句子结构BTW的库,在大学里,我们将我们的算法应用于公共领域的书籍。

查看我在SO上找到的sams-simMetrics库(此处下载,此处文档),除了Levenstein距离之外,还可以加载更多算法选项。

扩展我的注释,您可以使用regex搜索输入的"扩展"结果。类似这样的东西:

private int stringSimilarity(string input, string result)
{
    string regexPattern = ""
    foreach (char c in input)
        regexPattern += input + ".*"
    Match match = Regex.Match(result, regexPattern,
    RegexOptions.IgnoreCase);
    if (match.Success)
        return 1;
    else
        return 0;
}

忽略1和0-我不知道相似性评估是如何工作的。