Mirrors/flowframes
1
0
Fork 0
mirror of https://github.com/n00mkrad/flowframes.git synced 2026-09-01 19:51:28 +02:00
Code Issues Packages Projects Releases Wiki Activity

Finished improved magickdedupe code, added dynamic buffer size for dedupe

Browse Source
This commit is contained in:
N00MKRAD
2020-12-22 23:45:07 +01:00
parent 310f165653
commit a9592c02be
6 changed files with 80 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
Code/Data/InterpSettings.cs
View File

@@ -3,6 +3,7 @@ using Flowframes.IO;
using Flowframes.Main;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
namespace Flowframes
@@ -23,6 +24,8 @@ namespace Flowframes
public string interpFolder;
public bool inputIsFrames;
public string outFilename;
public Size inputResolution;
public Size scaledResolution;
public InterpSettings(string inPathArg, string outPathArg, AI aiArg, float inFpsArg, int interpFactorArg, Interpolate.OutMode outModeArg, int tilesizeArg = 512)
{
@@ -52,6 +55,9 @@ namespace Flowframes
inputIsFrames = false;
outFilename = "";
}
inputResolution = new Size(0, 0);
scaledResolution = new Size(0, 0);
}
public void UpdatePaths (string inPathArg, string outPathArg)
@@ -65,10 +71,25 @@ namespace Flowframes
outFilename = Path.Combine(outPath, Path.GetFileNameWithoutExtension(inPath) + IOUtils.GetAiSuffix(ai, interpFactor) + InterpolateUtils.GetExt(outMode));
}
public void SetFps(float inFpsArg)
public Size GetInputRes ()
{
inFps = inFpsArg;
outFps = inFps * interpFactor;
RefreshResolutions();
return inputResolution;
}
public Size GetScaledRes()
{
RefreshResolutions();
return scaledResolution;
}
void RefreshResolutions ()
{
if (inputResolution.IsEmpty || scaledResolution.IsEmpty)
{
inputResolution = IOUtils.GetVideoRes(inPath);
scaledResolution = InterpolateUtils.GetOutputResolution(inputResolution, false);
}
}
}
}

60
Code/Magick/MagickDedupe.cs
View File

@@ -12,6 +12,7 @@ using Microsoft.VisualBasic.Devices;
using ImageMagick;
using Flowframes.OS;
using Flowframes.Data;
using System.Drawing;
namespace Flowframes.Magick
{
@@ -35,7 +36,7 @@ namespace Flowframes.Magick
Logger.Log("Running accurate frame de-duplication...");
if (currentMode == Mode.Enabled || currentMode == Mode.Auto)
await RemoveDupeFrames(path, currentThreshold, "png", testRun, true, (currentMode == Mode.Auto));
await RemoveDupeFrames(path, currentThreshold, "png", testRun, false, (currentMode == Mode.Auto));
}
public static Dictionary<string, MagickImage> imageCache = new Dictionary<string, MagickImage>();
@@ -66,6 +67,8 @@ namespace Flowframes.Magick
FileInfo[] framePaths = IOUtils.GetFileInfosSorted(path, false, "*." + ext);
List<string> framesToDelete = new List<string>();
int bufferSize = GetBufferSize();
int currentOutFrame = 1;
int currentDupeCount = 0;
@@ -83,9 +86,6 @@ namespace Flowframes.Magick
if (hasReachedEnd)
break;
Logger.Log("Base Frame: #" + i);
//int thisFrameDupeCount = 0;
string frame1 = framePaths[i].FullName;
//if (!File.Exists(framePaths[i].FullName)) // Skip if file doesn't exist (already deleted / used to be a duped frame)
// continue;
@@ -103,7 +103,7 @@ namespace Flowframes.Magick
if (framesToDelete.Contains(framePaths[compareWithIndex].FullName) || !File.Exists(framePaths[compareWithIndex].FullName))
{
Logger.Log($"Frame {compareWithIndex} was already deleted - skipping");
//Logger.Log($"Frame {compareWithIndex} was already deleted - skipping");
compareWithIndex++;
}
else
@@ -111,14 +111,11 @@ namespace Flowframes.Magick
//if (compareWithIndex >= framePaths.Length)
// hasReachedEnd = true;
Logger.Log("Compare With: #" + compareWithIndex);
string frame2 = framePaths[compareWithIndex].FullName;
// if (oldIndex >= 0)
// i = oldIndex;
float diff = GetDifference(frame1, frame2);
Logger.Log("Diff: " + diff);
string delStr = "Keeping";
if (diff < threshold) // Is a duped frame.
@@ -133,7 +130,6 @@ namespace Flowframes.Magick
}
statsFramesDeleted++;
currentDupeCount++;
Logger.Log($"Frame {i} has {currentDupeCount} dupes");
}
else
{
@@ -143,40 +139,29 @@ namespace Flowframes.Magick
break;
}
if (i % 15 == 0 || true)
if (sw.ElapsedMilliseconds >= 1000 || (i+1) == framePaths.Length)
{
Logger.Log($"[FrameDedup] Difference from {Path.GetFileName(frame1)} to {Path.GetFileName(frame2)}: {diff.ToString("0.00")}% - {delStr}. Total: {statsFramesKept} kept / {statsFramesDeleted} deleted.", false, true);
sw.Restart();
Logger.Log($"[FrameDedup] Difference from {Path.GetFileName(frame1)} to {Path.GetFileName(frame2)}: {diff.ToString("0.00")}% - {delStr}. Total: {framePaths.Length - statsFramesDeleted} kept / {statsFramesDeleted} deleted.", false, true);
Program.mainForm.SetProgress((int)Math.Round(((float)i / framePaths.Length) * 100f));
if (imageCache.Count > 750 || (imageCache.Count > 50 && OSUtils.GetFreeRamMb() < 2500))
if (imageCache.Count > bufferSize || (imageCache.Count > 50 && OSUtils.GetFreeRamMb() < 2500))
ClearCache();
}
}
}
// int oldIndex = -1;
// int oldIndex = -1; // TODO: Compare with 1st to fix loops?
// if (i >= framePaths.Length) // If this is the last frame, compare with 1st to avoid OutOfRange error
// {
// oldIndex = i;
// i = 0;
// }
// while (!File.Exists(framePaths[i+1].FullName)) // If frame2 doesn't exist, keep stepping thru the array
// {
// if (i >= framePaths.Length)
// break;
// i++;
// }
if(i % 5 == 0)
await Task.Delay(1);
if (Interpolate.canceled) return;
foreach (string frame in framesToDelete)
IOUtils.TryDeleteIfExists(frame);
if (!testRun && skipIfNoDupes && !hasEncounteredAnyDupes && skipAfterNoDupesFrames > 0 && i >= skipAfterNoDupesFrames)
{
skipped = true;
@@ -184,20 +169,20 @@ namespace Flowframes.Magick
}
}
foreach (string frame in framesToDelete)
IOUtils.TryDeleteIfExists(frame);
string testStr = testRun ? " [TestRun]" : "";
if (Interpolate.canceled) return;
if (skipped)
{
Logger.Log($"[FrameDedup] First {skipAfterNoDupesFrames} frames did not have any duplicates - Skipping the rest!", false, true);
}
else
{
Logger.Log($"[FrameDedup]{testStr} Done. Kept {statsFramesKept} frames, deleted {statsFramesDeleted} frames.", false, true);
}
if (statsFramesKept <= 0)
Interpolate.Cancel("No frames were left after de-duplication.");
Interpolate.Cancel("No frames were left after de-duplication!\n\nTry decreasing the de-duplication threshold.");
}
static float GetDifference (string img1Path, string img2Path)
@@ -209,5 +194,20 @@ namespace Flowframes.Magick
float errPercent = (float)err * 100f;
return errPercent;
}
static int GetBufferSize ()
{
Size res = Interpolate.current.GetScaledRes();
long pixels = res.Width * res.Height; // 4K = 8294400, 1440p = 3686400, 1080p = 2073600, 720p = 921600, 540p = 518400, 360p = 230400
int bufferSize = 100;
if (pixels < 518400) bufferSize = 2800;
if (pixels >= 518400) return 2000;
if (pixels >= 921600) return 1200;
if (pixels >= 2073600) return 800;
if (pixels >= 3686400) return 400;
if (pixels >= 8294400) return 200;
Logger.Log($"Using magick dedupe buffer size {bufferSize} for frame resolution {res.Width}x{res.Height}", true);
return bufferSize;
}
}
}

2
Code/Main/Interpolate.cs
View File

@@ -82,7 +82,7 @@ namespace Flowframes
}
Program.mainForm.SetStatus("Extracting frames from video...");
await FFmpegCommands.VideoToFrames(inPath, outPath, Config.GetInt("dedupMode") == 2, false, Utils.GetOutputResolution(inPath));
await FFmpegCommands.VideoToFrames(inPath, outPath, Config.GetInt("dedupMode") == 2, false, Utils.GetOutputResolution(inPath, true));
if (extractAudio)
{

2
Code/Main/InterpolateSteps.cs
View File

@@ -83,7 +83,7 @@ namespace Flowframes.Main
AiProcess.filenameMap.Clear();
bool extractAudio = true;
Program.mainForm.SetStatus("Extracting frames from video...");
await FFmpegCommands.VideoToFrames(current.inPath, current.framesFolder, Config.GetInt("dedupMode") == 2, false, InterpolateUtils.GetOutputResolution(current.inPath));
await FFmpegCommands.VideoToFrames(current.inPath, current.framesFolder, Config.GetInt("dedupMode") == 2, false, InterpolateUtils.GetOutputResolution(current.inPath, true));
if (extractAudio)
{

19
Code/Main/InterpolateUtils.cs
View File

@@ -285,22 +285,27 @@ namespace Flowframes.Main
Logger.Log("Message: " + msg, true);
}
public static Size GetOutputResolution (string inputPath, bool print = true)
public static Size GetOutputResolution (string inputPath, bool print)
{
Size resolution = IOUtils.GetVideoRes(inputPath);
return GetOutputResolution(resolution, print);
}
public static Size GetOutputResolution(Size inputRes, bool print = false)
{
int maxHeight = RoundDiv2(Config.GetInt("maxVidHeight"));
if (resolution.Height > maxHeight)
if (inputRes.Height > maxHeight)
{
float factor = (float)maxHeight / resolution.Height;
Logger.Log($"Un-rounded downscaled size: {(resolution.Width * factor).ToString("0.00")}x{Config.GetInt("maxVidHeight")}", true);
int width = RoundDiv2((resolution.Width * factor).RoundToInt());
if(print)
float factor = (float)maxHeight / inputRes.Height;
Logger.Log($"Un-rounded downscaled size: {(inputRes.Width * factor).ToString("0.00")}x{Config.GetInt("maxVidHeight")}", true);
int width = RoundDiv2((inputRes.Width * factor).RoundToInt());
if (print)
Logger.Log($"Video is bigger than the maximum - Downscaling to {width}x{maxHeight}.");
return new Size(width, maxHeight);
}
else
{
return new Size(RoundDiv2(resolution.Width), RoundDiv2(resolution.Height));
return new Size(RoundDiv2(inputRes.Width), RoundDiv2(inputRes.Height));
}
}

17
Code/UI/MainUiFunctions.cs
View File

@@ -74,14 +74,21 @@ namespace Flowframes.UI
static async Task PrintResolution (string path)
{
Size res = new Size();
if (!IOUtils.IsPathDirectory(path)) // If path is video
if(path == Interpolate.current.inPath)
{
res = FFmpegCommands.GetSize(path);
res = Interpolate.current.GetInputRes();
}
else // Path is frame folder
else
{
Image thumb = await GetThumbnail(path);
res = new Size(thumb.Width, thumb.Height);
if (!IOUtils.IsPathDirectory(path)) // If path is video
{
res = FFmpegCommands.GetSize(path);
}
else // Path is frame folder
{
Image thumb = await GetThumbnail(path);
res = new Size(thumb.Width, thumb.Height);
}
}
if (res.Width > 1 && res.Height > 1)
Logger.Log($"Input Resolution: {res.Width}x{res.Height}");