Use path+filesize instead of xxHash for frame count caching

This commit is contained in:
N00MKRAD
2021-02-14 22:23:16 +01:00
parent ed2ef85bfd
commit bb143ddde1
4 changed files with 74 additions and 39 deletions

View File

@@ -0,0 +1,20 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Flowframes.Data
{
class PseudoUniqueFile
{
public string path;
public long filesize;
public PseudoUniqueFile (string pathArg, long filesizeArg)
{
path = pathArg;
filesize = filesizeArg;
}
}
}

View File

@@ -204,6 +204,7 @@
<Compile Include="Data\InterpSettings.cs" /> <Compile Include="Data\InterpSettings.cs" />
<Compile Include="Data\Networks.cs" /> <Compile Include="Data\Networks.cs" />
<Compile Include="Data\Padding.cs" /> <Compile Include="Data\Padding.cs" />
<Compile Include="Data\PseudoUniqueFile.cs" />
<Compile Include="Data\ResumeState.cs" /> <Compile Include="Data\ResumeState.cs" />
<Compile Include="Data\SubtitleTrack.cs" /> <Compile Include="Data\SubtitleTrack.cs" />
<Compile Include="Forms\BatchForm.cs"> <Compile Include="Forms\BatchForm.cs">

View File

@@ -184,43 +184,55 @@ namespace Flowframes.Main
bigPreviewForm.SetImage(img); bigPreviewForm.SetImage(img);
} }
public static Dictionary<string, int> frameCountCache = new Dictionary<string, int>(); public static Dictionary<PseudoUniqueFile, int> frameCountCache = new Dictionary<PseudoUniqueFile, int>();
public static async Task<int> GetInputFrameCountAsync(string path) public static async Task<int> GetInputFrameCountAsync(string path)
{ {
int maxMb = Config.GetInt("storeHashedFramecountMaxSizeMb", 256);
string hash = "";
if (IOUtils.GetFilesize(path) >= 0 && IOUtils.GetFilesize(path) < maxMb * 1024 * 1024) long filesize = IOUtils.GetFilesize(path);
hash = await IOUtils.GetHashAsync(path, IOUtils.Hash.xxHash); // Get checksum for caching
else
Logger.Log($"GetInputFrameCountAsync: File bigger than {maxMb}mb, won't hash.", true);
if (hash.Length > 1 && frameCountCache.ContainsKey(hash)) PseudoUniqueFile hash = new PseudoUniqueFile(path, filesize);
if (filesize > 0 && FrameCountCacheContains(hash))
{ {
Logger.Log($"FrameCountCache contains this hash ({hash}), using cached frame count.", true); Logger.Log($"FrameCountCache contains this hash, using cached frame count.", true);
return frameCountCache[hash]; return GetFrameCountFromCache(hash);
} }
else else
{ {
Logger.Log($"Hash ({hash}) not cached, reading frame count.", true); Logger.Log($"Hash not cached, reading frame count.", true);
} }
int frameCount = 0; int frameCount;
if (IOUtils.IsPathDirectory(path)) if (IOUtils.IsPathDirectory(path))
frameCount = IOUtils.GetAmountOfFiles(path, false); frameCount = IOUtils.GetAmountOfFiles(path, false);
else else
frameCount = await FfmpegCommands.GetFrameCountAsync(path); frameCount = await FfmpegCommands.GetFrameCountAsync(path);
if (hash.Length > 1 && frameCount > 5000) // Cache if >5k frames to avoid re-reading it every single time Logger.Log($"Adding hash with frame count {frameCount} to cache.", true);
{ frameCountCache.Add(hash, frameCount);
Logger.Log($"Adding hash ({hash}) with frame count {frameCount} to cache.", true);
frameCountCache[hash] = frameCount; // Use CRC32 instead of path to avoid using cached value if file was changed
}
return frameCount; return frameCount;
} }
private static bool FrameCountCacheContains (PseudoUniqueFile hash)
{
foreach(KeyValuePair<PseudoUniqueFile, int> entry in frameCountCache)
if (entry.Key.path == hash.path && entry.Key.filesize == hash.filesize)
return true;
return false;
}
private static int GetFrameCountFromCache(PseudoUniqueFile hash)
{
foreach (KeyValuePair<PseudoUniqueFile, int> entry in frameCountCache)
if (entry.Key.path == hash.path && entry.Key.filesize == hash.filesize)
return entry.Value;
return 0;
}
public static int GetProgressWaitTime(int numFrames) public static int GetProgressWaitTime(int numFrames)
{ {
float hddMultiplier = !Program.lastInputPathIsSsd ? 2f : 1f; float hddMultiplier = !Program.lastInputPathIsSsd ? 2f : 1f;

View File

@@ -71,6 +71,9 @@ namespace Flowframes
string trimmedLine = line.Remove("q=-0.0").Remove("size=N/A").Remove("bitrate=N/A").TrimWhitespaces(); string trimmedLine = line.Remove("q=-0.0").Remove("size=N/A").Remove("bitrate=N/A").TrimWhitespaces();
Logger.Log(trimmedLine, hidden, replaceLastLine, "ffmpeg"); Logger.Log(trimmedLine, hidden, replaceLastLine, "ffmpeg");
if (line.Contains(".srt: Invalid data found"))
Logger.Log($"Warning: Failed to encode subtitle track {line.Split(':')[1]}. This track will be missing in the output file.");
if (line.Contains("Could not open file")) if (line.Contains("Could not open file"))
Interpolate.Cancel($"FFmpeg Error: {line}"); Interpolate.Cancel($"FFmpeg Error: {line}");
@@ -98,7 +101,6 @@ namespace Flowframes
if (showProgressBar && line.Contains("time=")) if (showProgressBar && line.Contains("time="))
{ {
Logger.Log($"showProgressBar, contains: {line.Contains("time=")}", true, false, "ffmpeg");
Regex timeRegex = new Regex("(?<=time=).*(?= )"); Regex timeRegex = new Regex("(?<=time=).*(?= )");
UpdateFfmpegProgress(timeRegex.Match(line).Value); UpdateFfmpegProgress(timeRegex.Match(line).Value);
} }