Added model downloader including err handling & retrying

Code/Flowframes.csproj

@@ -239,6 +239,7 @@
     </Compile>
     <Compile Include="IO\ConfigParser.cs" />
     <Compile Include="Data\FlowPackage.cs" />
+    <Compile Include="IO\ModelDownloader.cs" />
     <Compile Include="IO\PkgInstaller.cs" />
     <Compile Include="Data\Packages.cs" />
     <Compile Include="IO\PkgUtils.cs" />

Code/IO/IOUtils.cs

@@ -537,7 +537,7 @@ namespace Flowframes.IO
 			string hashStr = "";
             if (IsPathDirectory(path))
             {
-				Logger.Log($"Path is directory! Returning empty hash for {Path.GetFileName(path)}", true);
+				Logger.Log($"Path '{path}' is directory! Returning empty hash.", true);
 				return hashStr;
             }
             try

Code/IO/ModelDownloader.cs

@@ -0,0 +1,169 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Flowframes.IO
+{
+    class ModelDownloader
+    {
+        static string baseUrl = "https://dl.nmkd.de/flowframes/mdl/";
+
+        public static async Task<Dictionary<string, string>> GetFilelist (string ai, string model)
+        {
+            var client = new WebClient();
+            string[] fileLines = client.DownloadString(GetMdlFileUrl(ai, model, "md5.txt")).SplitIntoLines();
+            Dictionary<string, string> filesDict = GetDict(fileLines);
+            return filesDict;
+        }
+
+        static string GetMdlUrl (string ai, string model)
+        {
+            return Path.Combine(baseUrl, ai.ToLower(), model);
+        }
+
+        static string GetMdlFileUrl(string ai, string model, string file)
+        {
+            return Path.Combine(GetMdlUrl(ai, model), file);
+        }
+
+        static string GetLocalPath(string ai, string model)
+        {
+            return Path.Combine(Paths.GetPkgPath(), ai, model);
+        }
+
+        static async Task DownloadTo (string url, string saveDir, int retries = 3)
+        {
+            string savePath = Path.Combine(saveDir, Path.GetFileName(url));
+            Logger.Log($"Downloading '{url}' to '{savePath}'", true);
+            Stopwatch sw = new Stopwatch();
+            sw.Restart();
+            bool completed = false;
+            int lastProgPercentage = -1;
+            var client = new WebClient();
+            client.DownloadProgressChanged += (sender, args) =>
+            {
+                if (sw.ElapsedMilliseconds > 200 && args.ProgressPercentage != lastProgPercentage)
+                {
+                    sw.Restart();
+                    lastProgPercentage = args.ProgressPercentage;
+                    Logger.Log($"Downloading model file '{Path.GetFileName(url)}'... {args.ProgressPercentage}%", false, true);
+                }
+            };
+            client.DownloadFileCompleted += (sender, args) =>
+            {
+                if (args.Error != null)
+                    Logger.Log("Download failed: " + args.Error.Message);
+                completed = true;
+            };
+            client.DownloadFileTaskAsync(url, savePath).ConfigureAwait(false);
+            while (!completed)
+            {
+                if (Interpolate.canceled)
+                {
+                    client.CancelAsync();
+                    return;
+                }
+                if (sw.ElapsedMilliseconds > 6000)
+                {
+                    client.CancelAsync();
+                    if(retries > 0)
+                    {
+                        await DownloadTo(url, saveDir, retries--);
+                    }
+                    else
+                    {
+                        Interpolate.Cancel("Model download failed.");
+                        return;
+                    }
+                }
+                await Task.Delay(500);
+            }
+            Logger.Log($"Downloaded '{Path.GetFileName(url)}' ({IOUtils.GetFilesize(savePath) / 1024} KB)", true);
+        }
+
+        public static async Task DownloadModelFiles (string ai, string model)
+        {
+            Logger.Log($"DownloadModelFiles(string ai = {ai}, string model = {model})", true);
+
+            try
+            {
+                string mdlDir = GetLocalPath(ai, model);
+
+                if (AreFilesValid(ai, model))
+                    return;
+
+                Logger.Log($"Downloading '{model}' model files...");
+                Directory.CreateDirectory(mdlDir);
+                await DownloadTo(GetMdlFileUrl(ai, model, "md5.txt"), mdlDir);
+
+                Dictionary<string, string> fileList = await GetFilelist(ai, model);
+
+                foreach (KeyValuePair<string, string> modelFile in fileList)
+                {
+                    //Logger.Log($"Downloading '{model}' model files: '{modelFile.Key}'...", false, true);
+                    await DownloadTo(GetMdlFileUrl(ai, model, modelFile.Key), mdlDir);
+                }
+
+                Logger.Log($"Downloaded \"{model}\" model files.", false, true);
+            }
+            catch (Exception e)
+            {
+                Logger.Log($"DownloadModelFiles Error: {e.Message}\nStack Trace:\n{e.StackTrace}");
+                Interpolate.Cancel($"Error downloading model files: {e.Message}");
+            }
+        }
+
+        public static bool AreFilesValid (string ai, string model)
+        {
+            string mdlDir = GetLocalPath(ai, model);
+
+            if (!Directory.Exists(mdlDir))
+            {
+                Logger.Log($"Files for model {model} not valid: {mdlDir} does not exist.", true);
+                return false;
+            }
+
+            string md5FilePath = Path.Combine(mdlDir, "md5.txt");
+
+            if (!File.Exists(md5FilePath))
+            {
+                Logger.Log($"Files for model {model} not valid: {mdlDir} does not exist.", true);
+                return false;
+            }
+
+            string[] md5Lines = IOUtils.ReadLines(md5FilePath);
+            Dictionary<string, string> filesDict = GetDict(md5Lines);
+
+            foreach(KeyValuePair<string, string> file in filesDict)
+            {
+                string md5 = IOUtils.GetHash(Path.Combine(mdlDir, file.Key), IOUtils.Hash.MD5);
+                if (md5.Trim() != file.Value.Trim())
+                {
+                    Logger.Log($"Files for model {model} not valid: Local MD5 ({md5.Trim()}) does not equal validation MD5 ({file.Value.Trim()}).", true);
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        static Dictionary<string, string> GetDict (string[] lines, char sep = ':')
+        {
+            Dictionary<string, string> dict = new Dictionary<string, string>();
+
+            foreach (string line in lines)
+            {
+                if (line.Length < 3) continue;
+                string[] keyValuePair = line.Split(':');
+                dict.Add(keyValuePair[0], keyValuePair[1]);
+            }
+
+            return dict;
+        }
+    }
+}

Code/Main/Interpolate.cs

@@ -55,7 +55,7 @@ namespace Flowframes
             await PostProcessFrames();
             if (canceled) return;
             Program.mainForm.SetStatus("Running AI...");
-            await RunAi(current.interpFolder, current.ai, current.model);
+            await RunAi(current.interpFolder, current.ai);
             if (canceled) return;
             Program.mainForm.SetProgress(100);
             if(!currentlyUsingAutoEnc)
@@ -140,8 +140,11 @@ namespace Flowframes
             AiProcess.filenameMap = IOUtils.RenameCounterDirReversible(current.framesFolder, "png", 1, 8);
         }
 
-        public static async Task RunAi(string outpath, AI ai, string model, bool stepByStep = false)
+        public static async Task RunAi(string outpath, AI ai, bool stepByStep = false)
         {
+            await ModelDownloader.DownloadModelFiles(Path.GetFileNameWithoutExtension(ai.pkg.fileName), current.model);
+            if (canceled) return;
+
             currentlyUsingAutoEnc = Utils.UseAutoEnc(stepByStep, current);
 
             IOUtils.CreateDir(outpath);

Code/Main/InterpolateSteps.cs

@@ -119,7 +119,7 @@ namespace Flowframes.Main
             int targetFrameCount = frames * current.interpFactor;
             if (canceled) return;
             Program.mainForm.SetStatus("Running AI...");
-            await RunAi(current.interpFolder, current.ai, current.model, true);
+            await RunAi(current.interpFolder, current.ai, true);
             Program.mainForm.SetProgress(0);
         }
 

Models/rife-cuda/RIFE15/md5.txt

@@ -1,3 +1,3 @@
 contextnet.pkl:44ec07a2199efaca3b632460a4a35ef6
 flownet.pkl:b17a3e7575ba46009147a9a6a329e58f
-unet.pkl839b8c23d02ff9f266f8642a0c48a1da
+unet.pkl:839b8c23d02ff9f266f8642a0c48a1da