Fix pinyin fuzzysearch (#131)

* fix typo

* make function obsolete 

it is not used in the code

* rewrite the function that converts chinese chars to pinyin

1. Only difference in this rewrite is instead of returning 2D array, return as a combined single string of all the possible pinyin combination. Since fuzzy search does character matching, this shouldn't be a problem.

2. Added a function that returns a custom language converter. In this case Pinyin converter. New converters can be added.

* Use new language converter param + strip out ScoreForPinyin method

* update

* Change parameter name

* fix failing tests

* WIP

* Remove todo

There should be some distinction between score after precision filter and actual raw score derived from FuzzySearch. Although so far RawScore is used in testing, but it seems to describe the structure. Originally it was to avoid assigning score directly as it would be hard to reason about that output of FuzzySearch score is.

* Add constructors, remove default to enforce required properties

* remove setting rawscore in SearchPrecision

* Change method name to reflect intention

* Change parameter name + update comment

* update

* Remove params comment

Co-authored-by: theClueless <14300910+theClueless@users.noreply.github.com>
This commit is contained in:
Jeremy Wu
2020-01-20 10:06:16 +11:00
committed by theClueless
parent ee93f7e018
commit 07310c7714
7 changed files with 133 additions and 122 deletions

View File

@@ -2,28 +2,35 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using hyjiacan.util.p4n;
using hyjiacan.util.p4n.format;
using JetBrains.Annotations;
using Wox.Infrastructure.Logger;
using Wox.Infrastructure.Storage;
using Wox.Infrastructure.UserSettings;
namespace Wox.Infrastructure
{
public static class Alphabet
public interface IAlphabet
{
private static readonly HanyuPinyinOutputFormat Format = new HanyuPinyinOutputFormat();
private static ConcurrentDictionary<string, string[][]> PinyinCache;
private static BinaryStorage<ConcurrentDictionary<string, string[][]>> _pinyinStorage;
private static Settings _settings;
string Translate(string stringToTranslate);
}
public class Alphabet : IAlphabet
{
private readonly HanyuPinyinOutputFormat Format = new HanyuPinyinOutputFormat();
private ConcurrentDictionary<string, string[][]> PinyinCache;
private BinaryStorage<ConcurrentDictionary<string, string[][]>> _pinyinStorage;
private Settings _settings;
public static void Initialize(Settings settings)
public void Initialize([NotNull] Settings settings)
{
_settings = settings;
_settings = settings ?? throw new ArgumentNullException(nameof(settings));
InitializePinyinHelpers();
}
private static void InitializePinyinHelpers()
private void InitializePinyinHelpers()
{
Format.setToneType(HanyuPinyinToneType.WITHOUT_TONE);
@@ -38,7 +45,35 @@ namespace Wox.Infrastructure
Log.Info($"|Wox.Infrastructure.Alphabet.Initialize|Number of preload pinyin combination<{PinyinCache.Count}>");
}
public static void Save()
public string Translate(string str)
{
return ConvertChineseCharactersToPinyin(str);
}
public string ConvertChineseCharactersToPinyin(string source)
{
if (!_settings.ShouldUsePinyin)
return source;
if (string.IsNullOrEmpty(source))
return source;
if (!ContainsChinese(source))
return source;
var combination = PinyinCombination(source);
var pinyinArray=combination.Select(x => string.Join("", x));
var acronymArray = combination.Select(Acronym).Distinct();
var joinedSingleStringCombination = new StringBuilder();
var all = acronymArray.Concat(pinyinArray);
all.ToList().ForEach(x => joinedSingleStringCombination.Append(x));
return joinedSingleStringCombination.ToString();
}
public void Save()
{
if (!_settings.ShouldUsePinyin)
{
@@ -50,11 +85,12 @@ namespace Wox.Infrastructure
private static string[] EmptyStringArray = new string[0];
private static string[][] Empty2DStringArray = new string[0][];
[Obsolete("Not accurate, eg 音乐 will not return yinyue but returns yinle ")]
/// <summary>
/// replace chinese character with pinyin, non chinese character won't be modified
/// <param name="word"> should be word or sentence, instead of single character. e.g. 微软 </param>
/// </summary>
public static string[] Pinyin(string word)
public string[] Pinyin(string word)
{
if (!_settings.ShouldUsePinyin)
{
@@ -76,7 +112,7 @@ namespace Wox.Infrastructure
/// e.g. 音乐 will return yinyue and yinle
/// <param name="characters"> should be word or sentence, instead of single character. e.g. 微软 </param>
/// </summmary>
public static string[][] PinyinComination(string characters)
public string[][] PinyinCombination(string characters)
{
if (!_settings.ShouldUsePinyin || string.IsNullOrEmpty(characters))
{
@@ -111,13 +147,13 @@ namespace Wox.Infrastructure
}
}
public static string Acronym(string[] pinyin)
public string Acronym(string[] pinyin)
{
var acronym = string.Join("", pinyin.Select(p => p[0]));
return acronym;
}
public static bool ContainsChinese(string word)
public bool ContainsChinese(string word)
{
if (!_settings.ShouldUsePinyin)
{
@@ -135,7 +171,7 @@ namespace Wox.Infrastructure
return chinese;
}
private static string[] Combination(string[] array1, string[] array2)
private string[] Combination(string[] array1, string[] array2)
{
if (!_settings.ShouldUsePinyin)
{