Mirrors/flowframes
1
0
Fork 0
mirror of https://github.com/n00mkrad/flowframes.git synced 2026-09-01 19:51:28 +02:00
Code Issues Packages Projects Releases Wiki Activity

Always sort GetFiles for unsorted FS, WIP magickdedupe fix, fix scndetect with jpeg frames

Browse Source
This commit is contained in:
N00MKRAD
2020-12-22 19:52:37 +01:00
parent 3e190bb05c
commit 3af40825bd
9 changed files with 128 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
Code/AudioVideo/FFmpegCommands.cs
View File

@@ -4,6 +4,7 @@ using System;
using System.Drawing;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
@@ -23,7 +24,7 @@ namespace Flowframes
public static async Task ExtractSceneChanges(string inputFile, string frameFolderPath)
{
Logger.Log("Extracting scene changes...");
await VideoToFrames(inputFile, frameFolderPath, (Config.GetInt("dedupMode") == 2), false, new Size(320, 180), true, true);
await VideoToFrames(inputFile, frameFolderPath, false, false, new Size(320, 180), true, true);
bool hiddenLog = Interpolate.currentInputFrameCount <= 50;
Logger.Log($"Detected {IOUtils.GetAmountOfFiles(frameFolderPath, false)} scene changes.".Replace(" 0 ", " no "), false, !hiddenLog);
}
@@ -59,7 +60,7 @@ namespace Flowframes
IOUtils.CreateDir(outpath);
string concatFile = Path.Combine(Paths.GetDataPath(), "png-concat-temp.ini");
string concatFileContent = "";
foreach (string img in Directory.GetFiles(inpath))
foreach (string img in IOUtils.GetFilesSorted(inpath))
concatFileContent += $"file '{img.Replace(@"\", "/")}'\n";
File.WriteAllText(concatFile, concatFileContent);
@@ -85,19 +86,6 @@ namespace Flowframes
DeleteSource(inputFile);
}
public static async Task FramesToMp4(string inputDir, string outPath, bool useH265, int crf, float fps, string prefix, bool delSrc, int looptimes = -1, string imgFormat = "png")
{
Logger.Log($"Encoding MP4 video with CRF {crf}...");
int nums = IOUtils.GetFilenameCounterLength(Directory.GetFiles(inputDir, $"*.{imgFormat}")[0], prefix);
string enc = useH265 ? "libx265" : "libx264";
string loopStr = (looptimes > 0) ? $"-stream_loop {looptimes}" : "";
string presetStr = $"-preset {Config.Get("ffEncPreset")}";
string args = $" {loopStr} -framerate {fps.ToString().Replace(",", ".")} -i \"{inputDir}\\{prefix}%0{nums}d.{imgFormat}\" -c:v {enc} -crf {crf} {presetStr} {videoEncArgs} -threads {Config.GetInt("ffEncThreads")} -c:a copy {outPath.Wrap()}";
await AvProcess.RunFfmpeg(args, AvProcess.LogMode.OnlyLastLine);
if (delSrc)
DeleteSource(inputDir);
}
public static async Task FramesToMp4Vfr(string framesFile, string outPath, bool useH265, int crf, float fps, AvProcess.LogMode logMode = AvProcess.LogMode.OnlyLastLine)
{
if(logMode != AvProcess.LogMode.Hidden)
@@ -130,19 +118,9 @@ namespace Flowframes
DeleteSource(inputPath);
}
public static async void FramesToApng(string inputDir, bool opti, int fps, string prefix, bool delSrc = false)
{
int nums = IOUtils.GetFilenameCounterLength(Directory.GetFiles(inputDir, "*.png")[0], prefix);
string filter = opti ? "-vf \"split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse\"" : "";
string args = "-framerate " + fps + " -i \"" + inputDir + "\\" + prefix + "%0" + nums + "d.png\" -f apng -plays 0 " + filter + " \"" + inputDir + "-anim.png\"";
await AvProcess.RunFfmpeg(args, AvProcess.LogMode.OnlyLastLine);
if (delSrc)
DeleteSource(inputDir);
}
public static async void FramesToGif(string inputDir, bool palette, int fps, string prefix, bool delSrc = false)
{
int nums = IOUtils.GetFilenameCounterLength(Directory.GetFiles(inputDir, "*.png")[0], prefix);
int nums = IOUtils.GetFilenameCounterLength(IOUtils.GetFilesSorted(inputDir, false, "*.png")[0], prefix);
string filter = palette ? "-vf \"split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse\"" : "";
string args = "-framerate " + fps + " -i \"" + inputDir + "\\" + prefix + "%0" + nums + "d.png\" -f gif " + filter + " \"" + inputDir + ".gif\"";
await AvProcess.RunFfmpeg(args, AvProcess.LogMode.OnlyLastLine);

23
Code/IO/IOUtils.cs
View File

@@ -594,5 +594,28 @@ namespace Flowframes.IO
return false;
}
}
public static string[] GetFilesSorted (string path, bool recursive = false, string pattern = "*")
{
SearchOption opt = recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
return Directory.GetFiles(path, pattern, opt).OrderBy(x => Path.GetFileName(x)).ToArray();
}
public static string[] GetFilesSorted(string path, string pattern = "*")
{
return GetFilesSorted(path, false, pattern);
}
public static string[] GetFilesSorted(string path)
{
return GetFilesSorted(path, false, "*");
}
public static FileInfo[] GetFileInfosSorted(string path, bool recursive = false, string pattern = "*")
{
SearchOption opt = recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
DirectoryInfo dir = new DirectoryInfo(path);
return dir.GetFiles(pattern, opt).OrderBy(x => x.Name).ToArray();
}
}
}

5
Code/Magick/Converter.cs
View File

@@ -1,4 +1,5 @@
using Flowframes;
using Flowframes.IO;
using Flowframes.UI;
using ImageMagick;
using System;
@@ -16,7 +17,7 @@ namespace Flowframes.Magick
{
public static async Task Convert (string dir, MagickFormat format, int quality, string ext = "", bool print = true, bool setProgress = true)
{
var files = Directory.GetFiles(dir);
var files = IOUtils.GetFilesSorted(dir);
if(print) Logger.Log($"Converting {files.Length} files in {dir}");
int counter = 0;
foreach (string file in files)
@@ -37,7 +38,7 @@ namespace Flowframes.Magick
public static async Task Preprocess (string dir, bool setProgress = true)
{
var files = Directory.GetFiles(dir);
var files = IOUtils.GetFilesSorted(dir);
Logger.Log($"Preprocessing {files} files in {dir}");
int counter = 0;
foreach (string file in files)

285
Code/Magick/MagickDedupe.cs
View File

@@ -35,7 +35,7 @@ namespace Flowframes.Magick
Logger.Log("Running accurate frame de-duplication...");
if (currentMode == Mode.Enabled || currentMode == Mode.Auto)
await RemoveDupeFrames(path, currentThreshold, "png", testRun, false, (currentMode == Mode.Auto));
await RemoveDupeFrames(path, currentThreshold, "png", testRun, true, (currentMode == Mode.Auto));
}
public static Dictionary<string, MagickImage> imageCache = new Dictionary<string, MagickImage>();
@@ -62,114 +62,121 @@ namespace Flowframes.Magick
Stopwatch sw = new Stopwatch();
sw.Restart();
Logger.Log("Removing duplicate frames - Threshold: " + threshold.ToString("0.00"));
//Logger.Log("Analyzing frames...");
DirectoryInfo dirInfo = new DirectoryInfo(path);
FileInfo[] framePaths = dirInfo.GetFiles("*." + ext, SearchOption.TopDirectoryOnly);
Dictionary<int, int> framesDupesDict = new Dictionary<int, int>();
FileInfo[] framePaths = IOUtils.GetFileInfosSorted(path, false, "*." + ext);
List<string> framesToDelete = new List<string>();
int currentOutFrame = 1;
int currentDupeCount = 0;
string dupeInfoFile = Path.Combine(path, "..", "dupes.ini");
int lastFrameNum = 0;
int statsFramesKept = 0;
int statsFramesDeleted = 0;
IOUtils.TryDeleteIfExists(dupeInfoFile);
bool loopMode = Config.GetBool("enableLoop");
int skipAfterNoDupesFrames = Config.GetInt("autoDedupFrames");
bool hasEncounteredAnyDupes = false;
bool skipped = false;
int i = 0;
while (i < framePaths.Length)
bool hasReachedEnd = false;
for (int i = 0; i < framePaths.Length; i++) // Loop through frames
{
if (hasReachedEnd)
break;
Logger.Log("Base Frame: #" + i);
//int thisFrameDupeCount = 0;
string frame1 = framePaths[i].FullName;
if (!File.Exists(framePaths[i].FullName)) // Skip if file doesn't exist (used to be a duped frame)
{
i++;
continue;
}
//if (!File.Exists(framePaths[i].FullName)) // Skip if file doesn't exist (already deleted / used to be a duped frame)
// continue;
i++;
string frame2;
int compareWithIndex = i + 1;
int oldIndex = -1;
if (i >= framePaths.Length) // If this is the last frame, compare with 1st to avoid OutOfRange error
while (true) // Loop dupes
{
if (loopMode)
//compareWithIndex++;
if (compareWithIndex >= framePaths.Length)
{
framesDupesDict = UpdateDupeDict(framesDupesDict, currentOutFrame, 0);
hasReachedEnd = true;
break;
}
oldIndex = i;
i = 0;
}
while (!File.Exists(framePaths[i].FullName)) // If frame2 doesn't exist, keep stepping thru the array
{
if (i >= framePaths.Length)
break;
i++;
}
frame2 = framePaths[i].FullName;
if (oldIndex >= 0)
i = oldIndex;
//long msBeforeLoad = sw.ElapsedMilliseconds;
MagickImage img2 = GetImage(frame2);
MagickImage img1 = GetImage(frame1);
//MagickImage img1 = new MagickImage(frame1);
//MagickImage img2 = new MagickImage(frame2);
double err = img1.Compare(img2, ErrorMetric.Fuzz);
float errPercent = (float)err * 100f;
if (debugLog) Logger.Log("[dedup] current in frame: " + i);
if (debugLog) Logger.Log("[dedup] current out frame: " + currentOutFrame);
framesDupesDict = UpdateDupeDict(framesDupesDict, currentOutFrame, currentDupeCount);
lastFrameNum = currentOutFrame;
string delStr = "Keeping";
if (errPercent < threshold) // Is a duped frame.
{
if (!testRun)
if (framesToDelete.Contains(framePaths[compareWithIndex].FullName) || !File.Exists(framePaths[compareWithIndex].FullName))
{
delStr = "Deleting";
File.Delete(frame1);
if(debugLog) Logger.Log("[FrameDedup] Deleted " + Path.GetFileName(frame1));
hasEncounteredAnyDupes = true;
i--; // Turn the index back so we compare the same frame again, this time to the next one after the deleted frame
Logger.Log($"Frame {compareWithIndex} was already deleted - skipping");
compareWithIndex++;
}
else
{
//if (compareWithIndex >= framePaths.Length)
// hasReachedEnd = true;
Logger.Log("Compare With: #" + compareWithIndex);
string frame2 = framePaths[compareWithIndex].FullName;
// if (oldIndex >= 0)
// i = oldIndex;
float diff = GetDifference(frame1, frame2);
Logger.Log("Diff: " + diff);
string delStr = "Keeping";
if (diff < threshold) // Is a duped frame.
{
if (!testRun)
{
delStr = "Deleting";
//File.Delete(frame2);
framesToDelete.Add(frame2);
if (debugLog) Logger.Log("[FrameDedup] Deleted " + Path.GetFileName(frame2));
hasEncounteredAnyDupes = true;
}
statsFramesDeleted++;
currentDupeCount++;
Logger.Log($"Frame {i} has {currentDupeCount} dupes");
}
else
{
statsFramesKept++;
currentOutFrame++;
currentDupeCount = 0;
break;
}
if (i % 15 == 0 || true)
{
Logger.Log($"[FrameDedup] Difference from {Path.GetFileName(frame1)} to {Path.GetFileName(frame2)}: {diff.ToString("0.00")}% - {delStr}. Total: {statsFramesKept} kept / {statsFramesDeleted} deleted.", false, true);
Program.mainForm.SetProgress((int)Math.Round(((float)i / framePaths.Length) * 100f));
if (imageCache.Count > 750 || (imageCache.Count > 50 && OSUtils.GetFreeRamMb() < 2500))
ClearCache();
}
}
statsFramesDeleted++;
currentDupeCount++;
}
else
{
statsFramesKept++;
currentOutFrame++;
currentDupeCount = 0;
}
if(i % 15 == 0)
{
Logger.Log($"[FrameDedup] Difference from {Path.GetFileName(img1.FileName)} to {Path.GetFileName(img2.FileName)}: {errPercent.ToString("0.00")}% - {delStr}. Total: {statsFramesKept} kept / {statsFramesDeleted} deleted.", false, true);
Program.mainForm.SetProgress((int)Math.Round(((float)i / framePaths.Length) * 100f));
if (imageCache.Count > 750 || (imageCache.Count > 50 && OSUtils.GetFreeRamMb() < 2500))
ClearCache();
}
// int oldIndex = -1;
// if (i >= framePaths.Length) // If this is the last frame, compare with 1st to avoid OutOfRange error
// {
// oldIndex = i;
// i = 0;
// }
// while (!File.Exists(framePaths[i+1].FullName)) // If frame2 doesn't exist, keep stepping thru the array
// {
// if (i >= framePaths.Length)
// break;
// i++;
// }
if(i % 5 == 0)
await Task.Delay(1);
if (Interpolate.canceled) return;
foreach (string frame in framesToDelete)
IOUtils.TryDeleteIfExists(frame);
if (!testRun && skipIfNoDupes && !hasEncounteredAnyDupes && skipAfterNoDupesFrames > 0 && i >= skipAfterNoDupesFrames)
{
skipped = true;
@@ -186,131 +193,21 @@ namespace Flowframes.Magick
}
else
{
if(!testRun)
File.WriteAllLines(dupeInfoFile, framesDupesDict.Select(x => "frm" + x.Key + ":" + x.Value).ToArray());
Logger.Log($"[FrameDedup]{testStr} Done. Kept {statsFramesKept} frames, deleted {statsFramesDeleted} frames.", false, true);
}
if (statsFramesKept <= 0)
Interpolate.Cancel("No frames were left after de-duplication.");
//Logger.Log($"Finished in {FormatUtils.Time(sw.Elapsed)} - {framePaths.Length / (sw.ElapsedMilliseconds / 1000f)} Imgs/Sec");
//RenameCounterDir(path, "png");
//ZeroPadDir(path, ext, 8);
}
static Dictionary<int, int> UpdateDupeDict(Dictionary<int, int> dict, int frame, int amount)
static float GetDifference (string img1Path, string img2Path)
{
if (dict.ContainsKey(frame))
dict[frame] = amount;
else
dict.Add(frame, amount);
return dict;
}
MagickImage img2 = GetImage(img2Path);
MagickImage img1 = GetImage(img1Path);
public static async Task Reduplicate(string path, bool debugLog = false)
{
if (currentMode == Mode.None)
return;
string ext = InterpolateUtils.GetOutExt();
string dupeInfoFile = Path.Combine(Interpolate.current.tempFolder, "dupes.ini");
if (!File.Exists(dupeInfoFile)) return;
Logger.Log("Re-Duplicating frames to fix timing...");
RenameCounterDir(path, ext);
IOUtils.ZeroPadDir(path, ext, 8);
string[] dupeFrameLines = IOUtils.ReadLines(dupeInfoFile);
string tempSubFolder = Path.Combine(path, "temp");
Directory.CreateDirectory(tempSubFolder);
int interpFramesPerRealFrame = Interpolate.current.interpFactor - 1;
int sourceFrameNum = 0;
int outFrameNum = 1;
for (int i = 0; i < dupeFrameLines.Length; i++)
{
string line = dupeFrameLines[i];
sourceFrameNum++;
string[] kvp = line.Split(':');
int currentInFrame = kvp[0].GetInt();
int currentDupesAmount = kvp[1].GetInt();
// Copy Source Frame
string paddedFilename = sourceFrameNum.ToString().PadLeft(Padding.inputFrames, '0') + $".{ext}";
string sourceFramePath = Path.Combine(path, paddedFilename);
if(debugLog) Logger.Log("[Source] Moving " + Path.GetFileName(sourceFramePath) + " => " + outFrameNum + $".{ext}");
if (!IOUtils.TryCopy(sourceFramePath, Path.Combine(tempSubFolder, outFrameNum + $".{ext}")))
break;
outFrameNum++;
// Insert dupes for source frame
for (int copyTimes = 0; copyTimes < currentDupesAmount; copyTimes++)
{
paddedFilename = sourceFrameNum.ToString().PadLeft(Padding.inputFrames, '0') + $".{ext}";
sourceFramePath = Path.Combine(path, paddedFilename);
if (debugLog) Logger.Log("[Source Dupes] Moving " + Path.GetFileName(sourceFramePath) + " => " + outFrameNum + $".{ext}");
if (!IOUtils.TryCopy(sourceFramePath, Path.Combine(tempSubFolder, outFrameNum + $".{ext}")))
break;
outFrameNum++;
}
if (i == dupeFrameLines.Length - 1) // Break loop if this is the last input frame (as it has no interps)
break;
for(int interpFrames = 0; interpFrames < interpFramesPerRealFrame; interpFrames++)
{
sourceFrameNum++;
// Copy Interp Frame
paddedFilename = sourceFrameNum.ToString().PadLeft(Padding.inputFrames, '0') + $".{ext}";
sourceFramePath = Path.Combine(path, paddedFilename);
if (debugLog) Logger.Log("[Interp] Moving " + Path.GetFileName(sourceFramePath) + " => " + outFrameNum + $".{ext}");
if (!IOUtils.TryCopy(sourceFramePath, Path.Combine(tempSubFolder, outFrameNum + $".{ext}")))
break;
outFrameNum++;
// Insert dupes for interp frame
for (int copyTimes = 0; copyTimes < currentDupesAmount; copyTimes++)
{
paddedFilename = sourceFrameNum.ToString().PadLeft(Padding.inputFrames, '0') + $".{ext}";
sourceFramePath = Path.Combine(path, paddedFilename);
if (debugLog) if (debugLog) Logger.Log("[Interp Dupes] Moving " + Path.GetFileName(sourceFramePath) + " => " + outFrameNum + $".{ext}");
if (!IOUtils.TryCopy(sourceFramePath, Path.Combine(tempSubFolder, outFrameNum + $".{ext}")))
break;
outFrameNum++;
}
}
}
IOUtils.ZeroPadDir(tempSubFolder, ext, 8);
foreach (FileInfo file in new DirectoryInfo(path).GetFiles($"*.{ext}", SearchOption.TopDirectoryOnly))
file.Delete();
foreach (FileInfo file in new DirectoryInfo(tempSubFolder).GetFiles($"*.{ext}", SearchOption.TopDirectoryOnly))
file.MoveTo(Path.Combine(path, file.Name));
}
public static void RenameCounterDir(string path, string ext, int sortMode = 0)
{
int counter = 1;
FileInfo[] files = new DirectoryInfo(path).GetFiles($"*.{ext}", SearchOption.TopDirectoryOnly);
var filesSorted = files.OrderBy(n => n);
if (sortMode == 1)
filesSorted.Reverse();
foreach (FileInfo file in files)
{
string dir = new DirectoryInfo(file.FullName).Parent.FullName;
File.Move(file.FullName, Path.Combine(dir, counter.ToString()/*.PadLeft(filesDigits, '8')*/ + Path.GetExtension(file.FullName)));
counter++;
}
double err = img1.Compare(img2, ErrorMetric.Fuzz);
float errPercent = (float)err * 100f;
return errPercent;
}
}
}

4
Code/Main/AutoEncode.cs
View File

@@ -51,7 +51,7 @@ namespace Flowframes.Main
}
IOUtils.ZeroPadDir(Directory.GetFiles(interpFramesFolder, $"*.{InterpolateUtils.GetOutExt()}").ToList(), Padding.interpFrames, encodedFrames);
string[] interpFrames = Directory.GetFiles(interpFramesFolder, $"*.{InterpolateUtils.GetOutExt()}");
string[] interpFrames = IOUtils.GetFilesSorted(interpFramesFolder, $"*.{InterpolateUtils.GetOutExt()}");
unencodedFrames = interpFrames.ToList().Except(encodedFrames).ToList();
Directory.CreateDirectory(videoChunksFolder);
@@ -92,7 +92,7 @@ namespace Flowframes.Main
string concatFile = Path.Combine(interpFramesPath.GetParentDir(), "chunks-concat.ini");
string concatFileContent = "";
foreach (string vid in Directory.GetFiles(videoChunksFolder))
foreach (string vid in IOUtils.GetFilesSorted(videoChunksFolder))
concatFileContent += $"file '{Paths.chunksDir}/{Path.GetFileName(vid)}'\n";
File.WriteAllText(concatFile, concatFileContent);

4
Code/Main/FrameTiming.cs
View File

@@ -41,7 +41,7 @@ namespace Flowframes.Main
List<string> sceneFrames = new List<string>();
if (Directory.Exists(scnFramesPath))
sceneFrames = Directory.GetFiles(scnFramesPath).Select(file => Path.GetFileName(file)).ToList();
sceneFrames = Directory.GetFiles(scnFramesPath).Select(file => Path.GetFileNameWithoutExtension(file)).ToList();
int lastFrameDuration = 1;
@@ -65,7 +65,7 @@ namespace Flowframes.Main
int interpFramesAmount = interpFactor;
bool discardThisFrame = (sceneDetection && (i + 2) < frameFiles.Length && sceneFrames.Contains(frameFiles[i + 1].Name)); // i+2 is in scene detection folder, means i+1 is ugly interp frame
bool discardThisFrame = (sceneDetection && (i + 2) < frameFiles.Length && sceneFrames.Contains(Path.GetFileNameWithoutExtension(frameFiles[i + 1].Name))); // i+2 is in scene detection folder, means i+1 is ugly interp frame
// If loop is enabled, account for the extra frame added to the end for loop continuity
if (loopEnabled && i == (frameFiles.Length - 2))

2
Code/Main/InterpolateSteps.cs
View File

@@ -141,7 +141,7 @@ namespace Flowframes.Main
public static async Task CreateOutputVid()
{
string[] outFrames = Directory.GetFiles(current.interpFolder, $"*.{InterpolateUtils.GetOutExt()}");
string[] outFrames = IOUtils.GetFilesSorted(current.interpFolder, $"*.{InterpolateUtils.GetOutExt()}");
if (outFrames.Length > 0 && !IOUtils.CheckImageValid(outFrames[0]))
{
InterpolateUtils.ShowMessage("Invalid frame files detected!\n\nIf you used Auto-Encode, this is normal, and you don't need to run " +

2
Code/Main/InterpolateUtils.cs
View File

@@ -42,7 +42,7 @@ namespace Flowframes.Main
if (firstProgUpd && Program.mainForm.IsInFocus())
Program.mainForm.SetTab("preview");
firstProgUpd = false;
string[] frames = Directory.GetFiles(outdir, $"*.{GetOutExt()}");
string[] frames = IOUtils.GetFilesSorted(outdir, $"*.{GetOutExt()}");
if (frames.Length > 1)
UpdateInterpProgress(frames.Length, targetFrames, frames[frames.Length - 1]);
await Task.Delay(GetProgressWaitTime(frames.Length));

2
Code/UI/MainUiFunctions.cs
View File

@@ -99,7 +99,7 @@ namespace Flowframes.UI
}
else // Path is frame folder - Get first frame
{
return IOUtils.GetImage(Directory.GetFiles(path)[0]);
return IOUtils.GetImage(IOUtils.GetFilesSorted(path)[0]);
}
}
catch (Exception e)