[PT Run] Split indexer plugin's queries into a fast and slow query (#5748)

* Added regex code

* Added regex based method to remove all LIKE queries

* Made regex readonly

* Added plugin interface and code to execute slower plugins after the fast plugins

* Added scoring for indexer and added statement to remove old indexer results

* Refactored from master and added thread sleep for debugging

* Removed lock from indexer plugin and added checks to avoid exceptions

* Remove debug statement

* Removed selected index update and fixed tests not building

* Added tests

* Removed scoring

* Removed lock

* Resolve merge conflicts

* Moved dispatcher code to function and add parallel foreach loop

* Removed DelayedExec metadata and modified QueryForPlugin to run only delayed exec plugins when bool param is true

* Removed metadata from plugin.json

src/modules/launcher/Plugins/Microsoft.Plugin.Indexer/Main.cs

@@ -20,7 +20,7 @@ using Wox.Plugin;
 
 namespace Microsoft.Plugin.Indexer
 {
-    internal class Main : ISettingProvider, IPlugin, ISavable, IPluginI18n, IContextMenu, IDisposable
+    internal class Main : ISettingProvider, IPlugin, ISavable, IPluginI18n, IContextMenu, IDisposable, IDelayedExecutionPlugin
     {
         // This variable contains metadata about the Plugin
         private PluginInitContext _context;
@@ -54,7 +54,7 @@ namespace Microsoft.Plugin.Indexer
 
         // This function uses the Windows indexer and returns the list of results obtained
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "We want to keep the process alive but will log the exception")]
-        public List<Result> Query(Query query)
+        public List<Result> Query(Query query, bool isFullQuery)
         {
             var results = new List<Result>();
 
@@ -95,7 +95,14 @@ namespace Microsoft.Plugin.Indexer
                             });
                         }
 
-                        var searchResultsList = _api.Search(searchQuery, maxCount: _settings.MaxSearchCount).ToList();
+                        var searchResultsList = _api.Search(searchQuery, isFullQuery, maxCount: _settings.MaxSearchCount).ToList();
+
+                        // If the delayed execution query is not required (since the SQL query is fast) return empty results
+                        if (searchResultsList.Count == 0 && isFullQuery)
+                        {
+                            return new List<Result>();
+                        }
+
                         foreach (var searchResult in searchResultsList)
                         {
                             var path = searchResult.Path;
@@ -161,6 +168,12 @@ namespace Microsoft.Plugin.Indexer
             return results;
         }
 
+        // This function uses the Windows indexer and returns the list of results obtained. This version is required to implement the interface
+        public List<Result> Query(Query query)
+        {
+            return Query(query, false);
+        }
+
         public void Init(PluginInitContext context)
         {
             // initialize the context of the plugin

src/modules/launcher/Plugins/Microsoft.Plugin.Indexer/SearchHelper/OleDBSearch.cs

@@ -33,9 +33,9 @@ namespace Microsoft.Plugin.Indexer.SearchHelper
                 {
                     using (wDSResults = command.ExecuteReader())
                     {
-                        if (wDSResults.HasRows)
+                        if (!wDSResults.IsClosed && wDSResults.HasRows)
                         {
-                            while (wDSResults.Read())
+                            while (!wDSResults.IsClosed && wDSResults.Read())
                             {
                                 List<object> fieldData = new List<object>();
                                 for (int i = 0; i < wDSResults.FieldCount; i++)

src/modules/launcher/Plugins/Microsoft.Plugin.Indexer/SearchHelper/WindowsSearchAPI.cs

@@ -4,6 +4,8 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
+using System.Text.RegularExpressions;
 using Microsoft.Search.Interop;
 
 namespace Microsoft.Plugin.Indexer.SearchHelper
@@ -13,8 +15,9 @@ namespace Microsoft.Plugin.Indexer.SearchHelper
         public bool DisplayHiddenFiles { get; set; }
 
         private readonly ISearch windowsIndexerSearch;
-        private readonly object _lock = new object();
+
         private const uint _fileAttributeHidden = 0x2;
+        private static readonly Regex _likeRegex = new Regex(@"[^\s(]+\s+LIKE\s+'([^']|'')*'\s+OR\s+", RegexOptions.Compiled);
 
         public WindowsSearchAPI(ISearch windowsIndexerSearch, bool displayHiddenFiles = false)
         {
@@ -22,7 +25,7 @@ namespace Microsoft.Plugin.Indexer.SearchHelper
             DisplayHiddenFiles = displayHiddenFiles;
         }
 
-        public List<SearchResult> ExecuteQuery(ISearchQueryHelper queryHelper, string keyword)
+        public List<SearchResult> ExecuteQuery(ISearchQueryHelper queryHelper, string keyword, bool isFullQuery = false)
         {
             if (queryHelper == null)
             {
@@ -33,6 +36,17 @@ namespace Microsoft.Plugin.Indexer.SearchHelper
 
             // Generate SQL from our parameters, converting the userQuery from AQS->WHERE clause
             string sqlQuery = queryHelper.GenerateSQLFromUserQuery(keyword);
+            var simplifiedQuery = SimplifyQuery(sqlQuery);
+
+            if (!isFullQuery)
+            {
+                sqlQuery = simplifiedQuery;
+            }
+            else if (simplifiedQuery.Equals(sqlQuery, StringComparison.CurrentCultureIgnoreCase))
+            {
+                // if a full query is requested but there is no difference between the queries, return empty results
+                return results;
+            }
 
             // execute the command, which returns the results as an OleDBResults.
             List<OleDBResult> oleDBResults = windowsIndexerSearch.Query(queryHelper.ConnectionString, sqlQuery);
@@ -121,15 +135,17 @@ namespace Microsoft.Plugin.Indexer.SearchHelper
             queryHelper.QuerySorting = "System.DateModified DESC";
         }
 
-        public IEnumerable<SearchResult> Search(string keyword, string pattern = "*", int maxCount = 30)
+        public IEnumerable<SearchResult> Search(string keyword, bool isFullQuery = false, string pattern = "*", int maxCount = 30)
         {
-            lock (_lock)
+            ISearchQueryHelper queryHelper;
-            {
+            InitQueryHelper(out queryHelper, maxCount);
-                ISearchQueryHelper queryHelper;
+            ModifyQueryHelper(ref queryHelper, pattern);
-                InitQueryHelper(out queryHelper, maxCount);
+            return ExecuteQuery(queryHelper, keyword, isFullQuery);
-                ModifyQueryHelper(ref queryHelper, pattern);
+        }
-                return ExecuteQuery(queryHelper, keyword);
+
-            }
+        public static string SimplifyQuery(string sqlQuery)
+        {
+            return _likeRegex.Replace(sqlQuery, string.Empty);
         }
     }
 }

src/modules/launcher/PowerLauncher/ViewModel/MainViewModel.cs

@@ -260,7 +260,6 @@ namespace PowerLauncher.ViewModel
                 if (!string.IsNullOrEmpty(QueryText))
                 {
                     ChangeQueryText(string.Empty, true);
-
                     // Push Event to UI SystemQuery has changed
                     OnPropertyChanged(nameof(SystemQueryText));
                 }
@@ -338,7 +337,6 @@ namespace PowerLauncher.ViewModel
                         QueryText = string.Empty;
                     }
                 }
-
                 _selectedResults.Visibility = Visibility.Visible;
             }
         }
@@ -362,6 +360,7 @@ namespace PowerLauncher.ViewModel
                 {
                     PowerToysTelemetry.Log.WriteEvent(new LauncherHideEvent());
                 }
+
             }
         }
 
@@ -507,23 +506,41 @@ namespace PowerLauncher.ViewModel
                             }
 
                             currentCancellationToken.ThrowIfCancellationRequested();
-                            Application.Current.Dispatcher.BeginInvoke(new Action(() =>
+                            UpdateResultsListViewAfterQuery(query);
-                            {
-                                if (query.RawQuery == _currentQuery.RawQuery)
-                                {
-                                    Results.Results.NotifyChanges();
-                                }
 
-                                if (Results.Results.Count > 0)
+                            // Run the slower query of the DelayedExecution plugins
+                            currentCancellationToken.ThrowIfCancellationRequested();
+                            Parallel.ForEach(plugins, (plugin) =>
                                 {
-                                    Results.Visibility = Visibility.Visible;
+                                    if (!plugin.Metadata.Disabled)
-                                    Results.SelectedIndex = 0;
+                                    {
-                                }
+                                        var results = PluginManager.QueryForPlugin(plugin, query, true);
-                                else
+                                        currentCancellationToken.ThrowIfCancellationRequested();
-                                {
+                                        if ((results?.Count ?? 0) != 0)
-                                    Results.Visibility = Visibility.Hidden;
+                                        {
-                                }
+                                            lock (_addResultsLock)
-                            }));
+                                            {
+                                                if (query.RawQuery == _currentQuery.RawQuery)
+                                                {
+                                                    currentCancellationToken.ThrowIfCancellationRequested();
+
+                                                    // Remove the original results from the plugin
+                                                    Results.Results.RemoveAll(r => r.Result.PluginID == plugin.Metadata.ID);
+                                                    currentCancellationToken.ThrowIfCancellationRequested();
+
+                                                    // Add the new results from the plugin
+                                                    UpdateResultView(results, query, currentCancellationToken);
+                                                    currentCancellationToken.ThrowIfCancellationRequested();
+                                                    Results.Sort();
+                                                }
+                                            }
+
+                                            currentCancellationToken.ThrowIfCancellationRequested();
+                                            UpdateResultsListViewAfterQuery(query, true);
+                                        }
+                                    }
+                                });
+
                         }
                         catch (OperationCanceledException)
                         {
@@ -538,6 +555,7 @@ namespace PowerLauncher.ViewModel
                             QueryLength = query.RawQuery.Length
                         };
                         PowerToysTelemetry.Log.WriteEvent(queryEvent);
+
                     }, currentCancellationToken);
                 }
             }
@@ -551,6 +569,30 @@ namespace PowerLauncher.ViewModel
             }
         }
 
+        private void UpdateResultsListViewAfterQuery(Query query, bool isDelayedInvoke = false)
+        {
+            Application.Current.Dispatcher.BeginInvoke(new Action(() =>
+            {
+                if (query.RawQuery == _currentQuery.RawQuery)
+                {
+                    Results.Results.NotifyChanges();
+                }
+
+                if (Results.Results.Count > 0)
+                {
+                    Results.Visibility = Visibility.Visible;
+                    if (!isDelayedInvoke)
+                    {
+                        Results.SelectedIndex = 0;
+                    }
+                }
+                else
+                {
+                    Results.Visibility = Visibility.Hidden;
+                }
+            }));
+        }
+
         private bool SelectedIsFromQueryResults()
         {
             var selected = SelectedResults == Results;
@@ -638,7 +680,6 @@ namespace PowerLauncher.ViewModel
                     {
                         StartHotkeyTimer();
                     }
-
                     if (_settings.LastQueryMode == LastQueryMode.Empty)
                     {
                         ChangeQueryText(string.Empty);
@@ -746,9 +787,7 @@ namespace PowerLauncher.ViewModel
                 {
                     var _ = PluginManager.QueryForPlugin(plugin, query);
                 }
-            }
+            };
-
-;
         }
 
         public void HandleContextMenu(Key AcceleratorKey, ModifierKeys AcceleratorModifiers)
@@ -795,7 +834,6 @@ namespace PowerLauncher.ViewModel
                         return query + input.Substring(query.Length);
                     }
                 }
-
                 return input;
             }
 
@@ -826,7 +864,6 @@ namespace PowerLauncher.ViewModel
                     {
                         _hotkeyManager?.UnregisterHotkey(_hotkeyHandle);
                     }
-
                     _hotkeyManager?.Dispose();
                     _updateSource?.Dispose();
                     _disposed = true;

src/modules/launcher/Wox.Core/Plugin/PluginManager.cs

@@ -152,7 +152,7 @@ namespace Wox.Core.Plugin
             }
         }
 
-        public static List<Result> QueryForPlugin(PluginPair pair, Query query)
+        public static List<Result> QueryForPlugin(PluginPair pair, Query query, bool delayedExecution = false)
         {
             try
             {
@@ -160,8 +160,19 @@ namespace Wox.Core.Plugin
                 var metadata = pair.Metadata;
                 var milliseconds = Stopwatch.Debug($"|PluginManager.QueryForPlugin|Cost for {metadata.Name}", () =>
                 {
-                    results = pair.Plugin.Query(query) ?? new List<Result>();
+                    if (delayedExecution && (pair.Plugin is IDelayedExecutionPlugin))
-                    UpdatePluginMetadata(results, metadata, query);
+                    {
+                        results = ((IDelayedExecutionPlugin)pair.Plugin).Query(query, delayedExecution) ?? new List<Result>();
+                    }
+                    else if (!delayedExecution)
+                    {
+                        results = pair.Plugin.Query(query) ?? new List<Result>();
+                    }
+
+                    if (results != null)
+                    {
+                        UpdatePluginMetadata(results, metadata, query);
+                    }
                 });
                 metadata.QueryCount += 1;
                 metadata.AvgQueryTime = metadata.QueryCount == 1 ? milliseconds : (metadata.AvgQueryTime + milliseconds) / 2;

src/modules/launcher/Wox.Plugin/IDelayedExecutionPlugin.cs

@@ -0,0 +1,13 @@
+// Copyright (c) Microsoft Corporation
+// The Microsoft Corporation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+
+namespace Wox.Plugin
+{
+    public interface IDelayedExecutionPlugin : IFeatures
+    {
+        List<Result> Query(Query query, bool delayedExecution);
+    }
+}

src/modules/launcher/Wox.Test/Plugins/WindowsIndexerTest.cs

@@ -316,5 +316,82 @@ namespace Wox.Test.Plugins
             // Act & Assert
             return driveDetection.DisplayWarning();
         }
+
+        [Test]
+        public void SimplifyQuery_ShouldRemoveLikeQuery_WhenSQLQueryUsesLIKESyntax()
+        {
+            // Arrange
+            string sqlQuery = "SELECT TOP 30 \"System.ItemUrl\", \"System.FileName\", \"System.FileAttributes\" FROM \"SystemIndex\" WHERE (System.FileName LIKE 'abcd.%' OR CONTAINS(System.FileName,'\"abcd.*\"',1033)) AND scope='file:' ORDER BY System.DateModified DESC";
+
+            // Act
+            var simplifiedSqlQuery = WindowsSearchAPI.SimplifyQuery(sqlQuery);
+
+            // Assert
+            string expectedSqlQuery = "SELECT TOP 30 \"System.ItemUrl\", \"System.FileName\", \"System.FileAttributes\" FROM \"SystemIndex\" WHERE (CONTAINS(System.FileName,'\"abcd.*\"',1033)) AND scope='file:' ORDER BY System.DateModified DESC";
+            Assert.IsFalse(simplifiedSqlQuery.Equals(sqlQuery, StringComparison.InvariantCultureIgnoreCase));
+            Assert.IsTrue(simplifiedSqlQuery.Equals(expectedSqlQuery, StringComparison.InvariantCultureIgnoreCase));
+        }
+
+        [Test]
+        public void SimplifyQuery_ShouldReturnArgument_WhenSQLQueryDoesNotUseLIKESyntax()
+        {
+            // Arrange
+            string sqlQuery = "SELECT TOP 30 \"System.ItemUrl\", \"System.FileName\", \"System.FileAttributes\" FROM \"SystemIndex\" WHERE CONTAINS(System.FileName,'\"abcd*\"',1033) AND scope='file:' ORDER BY System.DateModified DESC";
+
+            // Act
+            var simplifiedSqlQuery = WindowsSearchAPI.SimplifyQuery(sqlQuery);
+
+            // Assert
+            Assert.IsTrue(simplifiedSqlQuery.Equals(sqlQuery, StringComparison.InvariantCultureIgnoreCase));
+        }
+
+        [Test]
+        public void SimplifyQuery_ShouldRemoveAllOccurrencesOfLikeQuery_WhenSQLQueryUsesLIKESyntaxMultipleTimes()
+        {
+            // Arrange
+            string sqlQuery = "SELECT TOP 30 \"System.ItemUrl\", \"System.FileName\", \"System.FileAttributes\", \"System.FileExtension\" FROM \"SystemIndex\" WHERE (System.FileName LIKE 'ab.%' OR CONTAINS(System.FileName,'\"ab.*\"',1033)) AND (System.FileExtension LIKE '.cd%' OR CONTAINS(System.FileName,'\".cd*\"',1033)) AND scope='file:' ORDER BY System.DateModified DESC";
+
+            // Act
+            var simplifiedSqlQuery = WindowsSearchAPI.SimplifyQuery(sqlQuery);
+
+            // Assert
+            string expectedSqlQuery = "SELECT TOP 30 \"System.ItemUrl\", \"System.FileName\", \"System.FileAttributes\", \"System.FileExtension\" FROM \"SystemIndex\" WHERE (CONTAINS(System.FileName,'\"ab.*\"',1033)) AND (CONTAINS(System.FileName,'\".cd*\"',1033)) AND scope='file:' ORDER BY System.DateModified DESC";
+            Assert.IsFalse(simplifiedSqlQuery.Equals(sqlQuery, StringComparison.InvariantCultureIgnoreCase));
+            Assert.IsTrue(simplifiedSqlQuery.Equals(expectedSqlQuery, StringComparison.InvariantCultureIgnoreCase));
+        }
+
+        [Test]
+        public void SimplifyQuery_ShouldRemoveLikeQuery_WhenSQLQueryUsesLIKESyntaxAndContainsEscapedSingleQuotationMarks()
+        {
+            // Arrange
+            string sqlQuery = "SELECT TOP 30 \"System.ItemUrl\", \"System.FileName\", \"System.FileAttributes\" FROM \"SystemIndex\" WHERE (System.FileName LIKE '''ab.cd''%' OR CONTAINS(System.FileName,'\"'ab.cd'*\"',1033)) AND scope='file:' ORDER BY System.DateModified DESC";
+
+            // Act
+            var simplifiedSqlQuery = WindowsSearchAPI.SimplifyQuery(sqlQuery);
+
+            // Assert
+            string expectedSqlQuery = "SELECT TOP 30 \"System.ItemUrl\", \"System.FileName\", \"System.FileAttributes\" FROM \"SystemIndex\" WHERE (CONTAINS(System.FileName,'\"'ab.cd'*\"',1033)) AND scope='file:' ORDER BY System.DateModified DESC";
+            Assert.IsFalse(simplifiedSqlQuery.Equals(sqlQuery, StringComparison.InvariantCultureIgnoreCase));
+            Assert.IsTrue(simplifiedSqlQuery.Equals(expectedSqlQuery, StringComparison.InvariantCultureIgnoreCase));
+        }
+
+        [Test]
+        public void WindowsSearchAPI_ShouldReturnEmptyResults_WhenIsFullQueryIsTrueAndTheQueryDoesNotRequireLIKESyntax()
+        {
+            // Arrange
+            OleDBResult file1 = new OleDBResult(new List<object>() { "C:/test/path/file1.txt", DBNull.Value, (Int64)0x0 });
+            OleDBResult file2 = new OleDBResult(new List<object>() { "C:/test/path/file2.txt", "file2.txt", (Int64)0x0 });
+
+            List<OleDBResult> results = new List<OleDBResult>() { file1, file2 };
+            var mock = new Mock<ISearch>();
+            mock.Setup(x => x.Query(It.IsAny<string>(), It.IsAny<string>())).Returns(results);
+            WindowsSearchAPI _api = new WindowsSearchAPI(mock.Object, false);
+
+            // Act
+            var windowsSearchAPIResults = _api.Search("file", true);
+
+            // Assert
+            Assert.IsTrue(windowsSearchAPIResults.Count() == 0);
+        }
     }
 }