mirror of
https://github.com/n00mkrad/flowframes.git
synced 2025-12-16 16:37:48 +01:00
Smarter deduplication messages
This commit is contained in:
N00MKRAD
@@ -11,251 +11,253 @@ using Newtonsoft.Json;
|
|||||||
|
|
||||||
namespace Flowframes.Magick
|
namespace Flowframes.Magick
|
||||||
{
|
{
|
||||||
class Dedupe
|
class Dedupe
|
||||||
{
|
{
|
||||||
public enum Mode { None, Info, Enabled, Auto }
|
public enum Mode { None, Info, Enabled, Auto }
|
||||||
public static Mode currentMode;
|
public static Mode currentMode;
|
||||||
public static float currentThreshold;
|
public static float currentThreshold;
|
||||||
|
|
||||||
public static async Task Run(string path, bool testRun = false, bool setStatus = true)
|
public static async Task Run(string path, bool testRun = false, bool setStatus = true)
|
||||||
{
|
{
|
||||||
if (path == null || !Directory.Exists(path) || Interpolate.canceled)
|
if (path == null || !Directory.Exists(path) || Interpolate.canceled)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
currentMode = Mode.Auto;
|
currentMode = Mode.Auto;
|
||||||
|
|
||||||
if (setStatus)
|
if (setStatus)
|
||||||
Program.mainForm.SetStatus("Running frame de-duplication");
|
Program.mainForm.SetStatus("Running frame de-duplication");
|
||||||
|
|
||||||
currentThreshold = Config.GetFloat(Config.Key.dedupThresh);
|
currentThreshold = Config.GetFloat(Config.Key.dedupThresh);
|
||||||
Logger.Log("Running accurate frame de-duplication...");
|
Logger.Log("Running accurate frame de-duplication...");
|
||||||
|
|
||||||
if (currentMode == Mode.Enabled || currentMode == Mode.Auto)
|
if (currentMode == Mode.Enabled || currentMode == Mode.Auto)
|
||||||
await RemoveDupeFrames(path, currentThreshold, "*", testRun, false, (currentMode == Mode.Auto));
|
await RemoveDupeFrames(path, currentThreshold, "*", testRun, false, (currentMode == Mode.Auto));
|
||||||
}
|
}
|
||||||
|
|
||||||
static MagickImage GetImage(string path)
|
static MagickImage GetImage(string path)
|
||||||
{
|
{
|
||||||
return new MagickImage(path);
|
return new MagickImage(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async Task RemoveDupeFrames(string path, float threshold, string ext, bool testRun = false, bool debugLog = false, bool skipIfNoDupes = false)
|
public static async Task RemoveDupeFrames(string path, float threshold, string ext, bool testRun = false, bool debugLog = false, bool skipIfNoDupes = false)
|
||||||
{
|
{
|
||||||
Stopwatch sw = new Stopwatch();
|
Stopwatch sw = new Stopwatch();
|
||||||
sw.Restart();
|
sw.Restart();
|
||||||
Logger.Log("Removing duplicate frames - Threshold: " + threshold.ToString("0.00"));
|
Logger.Log("Removing duplicate frames - Threshold: " + threshold.ToString("0.00"));
|
||||||
|
|
||||||
FileInfo[] framePaths = IoUtils.GetFileInfosSorted(path, false, "*." + ext);
|
FileInfo[] framePaths = IoUtils.GetFileInfosSorted(path, false, "*." + ext);
|
||||||
List<string> framesToDelete = new List<string>();
|
List<string> framesToDelete = new List<string>();
|
||||||
|
|
||||||
int statsFramesKept = framePaths.Length > 0 ? 1 : 0; // always keep at least one frame
|
int statsFramesKept = framePaths.Length > 0 ? 1 : 0; // always keep at least one frame
|
||||||
int statsFramesDeleted = 0;
|
int statsFramesDeleted = 0;
|
||||||
|
|
||||||
Mutex mtx_framesToDelete = new Mutex();
|
Mutex mtx_framesToDelete = new Mutex();
|
||||||
Mutex mtx_debugLog = new Mutex();
|
Mutex mtx_debugLog = new Mutex();
|
||||||
Task[] workTasks = new Task[Environment.ProcessorCount];
|
Task[] workTasks = new Task[Environment.ProcessorCount];
|
||||||
|
|
||||||
bool threadAbort = false;
|
bool threadAbort = false;
|
||||||
|
|
||||||
Action<int, int> lamProcessFrames = (indStart, indEnd) =>
|
Action<int, int> lamProcessFrames = (indStart, indEnd) =>
|
||||||
{
|
{
|
||||||
MagickImage img1 = null;
|
MagickImage img1 = null;
|
||||||
MagickImage img2 = null;
|
MagickImage img2 = null;
|
||||||
|
|
||||||
for (int i = indStart; i < indEnd; i++) // Loop through frames
|
for (int i = indStart; i < indEnd; i++) // Loop through frames
|
||||||
{
|
{
|
||||||
string frame1_name = framePaths[i].FullName;
|
string frame1_name = framePaths[i].FullName;
|
||||||
|
|
||||||
// its likely we carried over an already loaded image from a previous iteration
|
// its likely we carried over an already loaded image from a previous iteration
|
||||||
if(!(img1 != null && img1.FileName == frame1_name))
|
if (!(img1 != null && img1.FileName == frame1_name))
|
||||||
img1 = GetImage(framePaths[i].FullName);
|
img1 = GetImage(framePaths[i].FullName);
|
||||||
|
|
||||||
if (img1 == null) continue;
|
if (img1 == null) continue;
|
||||||
|
|
||||||
for (int j = i + 1; j < framePaths.Length; j++)
|
for (int j = i + 1; j < framePaths.Length; j++)
|
||||||
{
|
{
|
||||||
if (threadAbort || Interpolate.canceled) return;
|
if (threadAbort || Interpolate.canceled) return;
|
||||||
|
|
||||||
//if (j % 3 == 0)
|
//if (j % 3 == 0)
|
||||||
//await Task.Delay(1);
|
//await Task.Delay(1);
|
||||||
|
|
||||||
string frame2_name = framePaths[j].FullName;
|
string frame2_name = framePaths[j].FullName;
|
||||||
|
|
||||||
if(j>=indEnd)
|
if (j >= indEnd)
|
||||||
{
|
{
|
||||||
// if we are already extending outside of this thread's range and j is already flagged, then we need to abort
|
// if we are already extending outside of this thread's range and j is already flagged, then we need to abort
|
||||||
bool isFlaggedForDeletion = false;
|
bool isFlaggedForDeletion = false;
|
||||||
mtx_framesToDelete.WaitOne();
|
mtx_framesToDelete.WaitOne();
|
||||||
isFlaggedForDeletion = framesToDelete.Contains(frame2_name);
|
isFlaggedForDeletion = framesToDelete.Contains(frame2_name);
|
||||||
mtx_framesToDelete.ReleaseMutex();
|
mtx_framesToDelete.ReleaseMutex();
|
||||||
if (isFlaggedForDeletion)
|
if (isFlaggedForDeletion)
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
img2 = GetImage(framePaths[j].FullName);
|
img2 = GetImage(framePaths[j].FullName);
|
||||||
if (img2 == null) continue;
|
if (img2 == null) continue;
|
||||||
|
|
||||||
|
|
||||||
float diff = GetDifference(img1, img2);
|
float diff = GetDifference(img1, img2);
|
||||||
|
|
||||||
if (diff < threshold) // Is a duped frame.
|
if (diff < threshold) // Is a duped frame.
|
||||||
{
|
{
|
||||||
if (!testRun)
|
if (!testRun)
|
||||||
{
|
{
|
||||||
mtx_framesToDelete.WaitOne();
|
mtx_framesToDelete.WaitOne();
|
||||||
framesToDelete.Add(frame2_name);
|
framesToDelete.Add(frame2_name);
|
||||||
mtx_framesToDelete.ReleaseMutex();
|
mtx_framesToDelete.ReleaseMutex();
|
||||||
if (debugLog)
|
if (debugLog)
|
||||||
{
|
{
|
||||||
mtx_debugLog.WaitOne();
|
mtx_debugLog.WaitOne();
|
||||||
Logger.Log("Deduplication: Deleted " + Path.GetFileName(frame2_name));
|
Logger.Log("Deduplication: Deleted " + Path.GetFileName(frame2_name));
|
||||||
mtx_debugLog.ReleaseMutex();
|
mtx_debugLog.ReleaseMutex();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Interlocked.Increment(ref statsFramesDeleted);
|
Interlocked.Increment(ref statsFramesDeleted);
|
||||||
|
|
||||||
if (j+1 == framePaths.Length)
|
if (j + 1 == framePaths.Length)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
continue; // test next frame
|
continue; // test next frame
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Interlocked.Increment(ref statsFramesKept);
|
Interlocked.Increment(ref statsFramesKept);
|
||||||
|
|
||||||
// this frame is different, stop testing agaisnt 'i'
|
// this frame is different, stop testing agaisnt 'i'
|
||||||
// all the frames between i and j are dupes, we can skip them
|
// all the frames between i and j are dupes, we can skip them
|
||||||
i = j - 1;
|
i = j - 1;
|
||||||
// keep the currently loaded in img for the next iteration
|
// keep the currently loaded in img for the next iteration
|
||||||
img1 = img2;
|
img1 = img2;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
Action lamUpdateInfoBox = () =>
|
Action lamUpdateInfoBox = () =>
|
||||||
{
|
{
|
||||||
int framesProcessed = statsFramesKept + statsFramesDeleted;
|
int framesProcessed = statsFramesKept + statsFramesDeleted;
|
||||||
Logger.Log($"Deduplication: Running de-duplication ({framesProcessed}/{framePaths.Length}), deleted {statsFramesDeleted} ({(((float)statsFramesDeleted / framePaths.Length) * 100f).ToString("0")}%) duplicate frames so far...", false, true);
|
Logger.Log($"Deduplication: Running de-duplication ({framesProcessed}/{framePaths.Length}), deleted {statsFramesDeleted} ({(((float)statsFramesDeleted / framePaths.Length) * 100f).ToString("0")}%) duplicate frames so far...", false, true);
|
||||||
Program.mainForm.SetProgress((int)Math.Round(((float)framesProcessed / framePaths.Length) * 100f));
|
Program.mainForm.SetProgress((int)Math.Round(((float)framesProcessed / framePaths.Length) * 100f));
|
||||||
};
|
};
|
||||||
|
|
||||||
// start the worker threads
|
// start the worker threads
|
||||||
for (int i = 0; i < workTasks.Length; i++)
|
for (int i = 0; i < workTasks.Length; i++)
|
||||||
{
|
{
|
||||||
int chunkSize = framePaths.Length / workTasks.Length;
|
int chunkSize = framePaths.Length / workTasks.Length;
|
||||||
int indStart = chunkSize * i;
|
int indStart = chunkSize * i;
|
||||||
int indEnd = indStart + chunkSize;
|
int indEnd = indStart + chunkSize;
|
||||||
if (i + 1 == workTasks.Length) indEnd = framePaths.Length;
|
if (i + 1 == workTasks.Length) indEnd = framePaths.Length;
|
||||||
|
|
||||||
workTasks[i] = Task.Run(() => lamProcessFrames(indStart, indEnd));
|
workTasks[i] = Task.Run(() => lamProcessFrames(indStart, indEnd));
|
||||||
}
|
|
||||||
|
|
||||||
// wait for all the worker threads to finish and update the info box
|
|
||||||
while (!Interpolate.canceled)
|
|
||||||
{
|
|
||||||
await Task.Delay(5);
|
|
||||||
|
|
||||||
|
|
||||||
bool anyThreadStillWorking = false;
|
|
||||||
// wait for the threads to finish
|
|
||||||
for (int i = 0; i < workTasks.Length; i++)
|
|
||||||
{
|
|
||||||
if (!workTasks[i].IsCompleted) anyThreadStillWorking = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (sw.ElapsedMilliseconds >= 250 || !anyThreadStillWorking) // Print every 0.25s (or when done)
|
|
||||||
{
|
|
||||||
sw.Restart();
|
|
||||||
lamUpdateInfoBox();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!anyThreadStillWorking) break;
|
|
||||||
}
|
|
||||||
|
|
||||||
threadAbort = true;
|
|
||||||
for (int i = 0; i < workTasks.Length; i++)
|
|
||||||
await workTasks[i];
|
|
||||||
|
|
||||||
lamUpdateInfoBox();
|
|
||||||
|
|
||||||
// int oldIndex = -1; // TODO: Compare with 1st to fix loops?
|
|
||||||
// if (i >= framePaths.Length) // If this is the last frame, compare with 1st to avoid OutOfRange error
|
|
||||||
// {
|
|
||||||
// oldIndex = i;
|
|
||||||
// i = 0;
|
|
||||||
// }
|
|
||||||
|
|
||||||
foreach (string frame in framesToDelete)
|
|
||||||
IoUtils.TryDeleteIfExists(frame);
|
|
||||||
|
|
||||||
string testStr = testRun ? " [TestRun]" : "";
|
|
||||||
|
|
||||||
if (Interpolate.canceled) return;
|
|
||||||
|
|
||||||
int framesLeft = IoUtils.GetAmountOfFiles(path, false, "*" + Interpolate.currentSettings.framesExt);
|
|
||||||
int framesDeleted = framePaths.Length - framesLeft;
|
|
||||||
float percentDeleted = ((float)framesDeleted / framePaths.Length) * 100f;
|
|
||||||
string keptPercent = $"{(100f - percentDeleted).ToString("0.0")}%";
|
|
||||||
|
|
||||||
Logger.Log($"[Deduplication]{testStr} Done. Kept {framesLeft} ({keptPercent}) frames, deleted {framesDeleted} frames.", false, true);
|
|
||||||
|
|
||||||
if (statsFramesKept <= 0)
|
|
||||||
{
|
|
||||||
Interpolate.Cancel("No frames were left after de-duplication!\n\nTry decreasing the de-duplication threshold.");
|
|
||||||
}
|
}
|
||||||
else
|
|
||||||
{
|
// wait for all the worker threads to finish and update the info box
|
||||||
// Interpolate.InterpProgressMultiplier = (framePaths.Length / (float)framesLeft);
|
while (!Interpolate.canceled)
|
||||||
|
{
|
||||||
|
await Task.Delay(5);
|
||||||
|
|
||||||
|
|
||||||
|
bool anyThreadStillWorking = false;
|
||||||
|
// wait for the threads to finish
|
||||||
|
for (int i = 0; i < workTasks.Length; i++)
|
||||||
|
{
|
||||||
|
if (!workTasks[i].IsCompleted) anyThreadStillWorking = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sw.ElapsedMilliseconds >= 250 || !anyThreadStillWorking) // Print every 0.25s (or when done)
|
||||||
|
{
|
||||||
|
sw.Restart();
|
||||||
|
lamUpdateInfoBox();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!anyThreadStillWorking) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
threadAbort = true;
|
||||||
|
for (int i = 0; i < workTasks.Length; i++)
|
||||||
|
await workTasks[i];
|
||||||
|
|
||||||
|
lamUpdateInfoBox();
|
||||||
|
|
||||||
|
// int oldIndex = -1; // TODO: Compare with 1st to fix loops?
|
||||||
|
// if (i >= framePaths.Length) // If this is the last frame, compare with 1st to avoid OutOfRange error
|
||||||
|
// {
|
||||||
|
// oldIndex = i;
|
||||||
|
// i = 0;
|
||||||
|
// }
|
||||||
|
|
||||||
|
foreach (string frame in framesToDelete)
|
||||||
|
IoUtils.TryDeleteIfExists(frame);
|
||||||
|
|
||||||
|
string testStr = testRun ? "[TESTRUN] " : "";
|
||||||
|
|
||||||
|
if (Interpolate.canceled) return;
|
||||||
|
|
||||||
|
int framesLeft = IoUtils.GetAmountOfFiles(path, false, "*" + Interpolate.currentSettings.framesExt);
|
||||||
|
int framesDeleted = framePaths.Length - framesLeft;
|
||||||
|
float percentDeleted = ((float)framesDeleted / framePaths.Length) * 100f;
|
||||||
|
string keptPercent = $"{(100f - percentDeleted).ToString("0.0")}%";
|
||||||
|
|
||||||
|
if (framesDeleted <= 0)
|
||||||
|
{
|
||||||
|
Logger.Log($"Deduplication: No duplicate frames detected on this video.", false, true);
|
||||||
|
}
|
||||||
|
else if (statsFramesKept <= 0)
|
||||||
|
{
|
||||||
|
Interpolate.Cancel("No frames were left after de-duplication!\n\nTry lowering the de-duplication threshold.");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Logger.Log($"{testStr}Deduplication: Kept {framesLeft} ({keptPercent}) frames, deleted {framesDeleted} frames.", false, true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
static float GetDifference(MagickImage img1, MagickImage img2)
|
static float GetDifference(MagickImage img1, MagickImage img2)
|
||||||
{
|
{
|
||||||
double err = img1.Compare(img2, ErrorMetric.Fuzz);
|
double err = img1.Compare(img2, ErrorMetric.Fuzz);
|
||||||
float errPercent = (float)err * 100f;
|
float errPercent = (float)err * 100f;
|
||||||
return errPercent;
|
return errPercent;
|
||||||
}
|
}
|
||||||
|
|
||||||
static float GetDifference(string img1Path, string img2Path)
|
static float GetDifference(string img1Path, string img2Path)
|
||||||
{
|
{
|
||||||
return GetDifference(GetImage(img1Path), GetImage(img2Path));
|
return GetDifference(GetImage(img1Path), GetImage(img2Path));
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async Task CreateDupesFile(string framesPath, int lastFrameNum, string ext)
|
public static async Task CreateDupesFile(string framesPath, int lastFrameNum, string ext)
|
||||||
{
|
{
|
||||||
bool debug = Config.GetBool("dupeScanDebug", false);
|
bool debug = Config.GetBool("dupeScanDebug", false);
|
||||||
|
|
||||||
FileInfo[] frameFiles = IoUtils.GetFileInfosSorted(framesPath, false, "*" + ext);
|
FileInfo[] frameFiles = IoUtils.GetFileInfosSorted(framesPath, false, "*" + ext);
|
||||||
|
|
||||||
if (debug)
|
if (debug)
|
||||||
Logger.Log($"Running CreateDupesFile for '{framesPath}' ({frameFiles.Length} files), lastFrameNum = {lastFrameNum}, ext = {ext}.", true, false, "dupes");
|
Logger.Log($"Running CreateDupesFile for '{framesPath}' ({frameFiles.Length} files), lastFrameNum = {lastFrameNum}, ext = {ext}.", true, false, "dupes");
|
||||||
|
|
||||||
Dictionary<string, List<string>> frames = new Dictionary<string, List<string>>();
|
Dictionary<string, List<string>> frames = new Dictionary<string, List<string>>();
|
||||||
|
|
||||||
|
|
||||||
for (int i = 0; i < frameFiles.Length; i++)
|
for (int i = 0; i < frameFiles.Length; i++)
|
||||||
{
|
{
|
||||||
bool isLastItem = (i + 1) == frameFiles.Length;
|
bool isLastItem = (i + 1) == frameFiles.Length;
|
||||||
|
|
||||||
String fnameCur = Path.GetFileNameWithoutExtension(frameFiles[i].Name);
|
String fnameCur = Path.GetFileNameWithoutExtension(frameFiles[i].Name);
|
||||||
int frameNumCur = fnameCur.GetInt();
|
int frameNumCur = fnameCur.GetInt();
|
||||||
|
|
||||||
frames[fnameCur] = new List<string>();
|
frames[fnameCur] = new List<string>();
|
||||||
|
|
||||||
if(!isLastItem)
|
if (!isLastItem)
|
||||||
{
|
{
|
||||||
String fnameNext = Path.GetFileNameWithoutExtension(frameFiles[i + 1].Name);
|
String fnameNext = Path.GetFileNameWithoutExtension(frameFiles[i + 1].Name);
|
||||||
int frameNumNext = fnameNext.GetInt();
|
int frameNumNext = fnameNext.GetInt();
|
||||||
|
|
||||||
for(int j = frameNumCur + 1; j < frameNumNext; j++)
|
for (int j = frameNumCur + 1; j < frameNumNext; j++)
|
||||||
{
|
{
|
||||||
frames[fnameCur].Add(j.ToString().PadLeft(9,'0'));
|
frames[fnameCur].Add(j.ToString().PadLeft(9, '0'));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
File.WriteAllText(Path.Combine(framesPath.GetParentDir(), "dupes.json"), JsonConvert.SerializeObject(frames, Formatting.Indented));
|
File.WriteAllText(Path.Combine(framesPath.GetParentDir(), "dupes.json"), JsonConvert.SerializeObject(frames, Formatting.Indented));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -146,12 +146,13 @@ namespace Flowframes
|
|||||||
float percentDeleted = ((float)framesDeleted / currentMediaFile.FrameCount) * 100f;
|
float percentDeleted = ((float)framesDeleted / currentMediaFile.FrameCount) * 100f;
|
||||||
string keptPercent = $"{(100f - percentDeleted).ToString("0.0")}%";
|
string keptPercent = $"{(100f - percentDeleted).ToString("0.0")}%";
|
||||||
|
|
||||||
if (QuickSettingsTab.trimEnabled)
|
if (framesDeleted > 0)
|
||||||
Logger.Log($"Deduplication: Kept {framesLeft} frames.");
|
{
|
||||||
else
|
if (QuickSettingsTab.trimEnabled)
|
||||||
Logger.Log($"Deduplication: Kept {framesLeft} ({keptPercent}) frames, deleted {framesDeleted} frames.");
|
Logger.Log($"Deduplication: Kept {framesLeft} frames.");
|
||||||
|
else
|
||||||
// InterpProgressMultiplier = (currentMediaFile.FrameCount / (float)framesLeft);
|
Logger.Log($"Deduplication: Kept {framesLeft} ({keptPercent}) frames, deleted {framesDeleted} frames.");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!Config.GetBool("allowConsecutiveSceneChanges", true))
|
if (!Config.GetBool("allowConsecutiveSceneChanges", true))
|
||||||
|
|||||||
Reference in New Issue
Block a user