From e3e9d1d6ae2b2507f494e60339484c6adfc606f1 Mon Sep 17 00:00:00 2001 From: McSchneidar <262792195+McSchneidar@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:11:53 +0100 Subject: [PATCH] Speed up program startup (#941) * Speed up program startup Several parts of the startup took much longer than necessary, especially with large song libraries (tested with ~2200 songs). - Cache compiled party mode assemblies in the PartyModeCache folder and reuse them as long as the sources and VocaluxeLib are unchanged. This saves 1.4-2.1 s per start. Cached assemblies are loaded directly from file so the virus scanner does not scan them in memory on every start. Loading and compiling of each party mode is logged with its duration. - Start listing and parsing song files right after the database is initialized, in parallel to the remaining initialization. Sorting, categories and cover loading still wait for the loading screen. - Read the Songs table of the highscore database into memory once while loading, insert new songs in a single transaction and add an index on Title and Artist. Previously the first start took more than 30 s. - Load covers with (cores - 1) threads at below normal priority instead of one task per song. Errors for a single cover are logged instead of stopping the cover loading. - Limit texture uploads to 8 ms per frame so many cached covers no longer stall single frames and delay the fade to the main menu by about 2 s. * Set DateTimeKind for song DateAdded values --- Vocaluxe/Base/CDataBase.cs | 16 +++ Vocaluxe/Base/CParty.cs | 101 +++++++++++++++++- Vocaluxe/Base/CSettings.cs | 1 + Vocaluxe/Base/CSongs.cs | 81 ++++++++++---- Vocaluxe/Lib/Database/CHighscoreDB.cs | 147 ++++++++++++++++++++++++++ Vocaluxe/Lib/Draw/CTextureProvider.cs | 5 +- Vocaluxe/Program.cs | 2 + Vocaluxe/Screens/CScreenLoad.cs | 6 +- 8 files changed, 330 insertions(+), 29 deletions(-) diff --git a/Vocaluxe/Base/CDataBase.cs b/Vocaluxe/Base/CDataBase.cs index 5e0bab2c..f0708c9c 100644 --- a/Vocaluxe/Base/CDataBase.cs +++ b/Vocaluxe/Base/CDataBase.cs @@ -65,6 +65,22 @@ namespace Vocaluxe.Base } } + public static void BeginSongInfoCache() + { + if (_HighscoreDB != null) + { + _HighscoreDB.BeginSongInfoCache(); + } + } + + public static void EndSongInfoCache() + { + if (_HighscoreDB != null) + { + _HighscoreDB.EndSongInfoCache(); + } + } + public static bool GetDataBaseSongInfos(string artist, string title, out int numPlayed, out DateTime dateAdded, out int highscoreId) { if (_HighscoreDB == null) diff --git a/Vocaluxe/Base/CParty.cs b/Vocaluxe/Base/CParty.cs index e5942a08..95c2989d 100644 --- a/Vocaluxe/Base/CParty.cs +++ b/Vocaluxe/Base/CParty.cs @@ -20,7 +20,10 @@ using System.CodeDom.Compiler; using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Reflection; +using System.Security.Cryptography; +using System.Text; using Vocaluxe.Base.ThemeSystem; using VocaluxeLib; using VocaluxeLib.Log; @@ -248,7 +251,7 @@ namespace Vocaluxe.Base var filesToCompile = new List(); filesToCompile.AddRange(CHelper.ListFiles(pathToCode, "*.cs", false, true)); - var output = _CompileFiles(filesToCompile.ToArray()); + var output = _LoadOrCompileFiles(filesToCompile.ToArray(), pm.Info.Folder); if (output == null) { return false; @@ -308,20 +311,97 @@ namespace Vocaluxe.Base return true; } - private static Assembly _CompileFiles(string[] files) + private static Assembly _LoadOrCompileFiles(string[] files, string partyModeName) { if (files == null || files.Length == 0) { return null; } + var cacheFolder = Path.Combine(CSettings.DataFolder, CSettings.FolderNamePartyModeCache); + var cacheFilePrefix = partyModeName + "_"; + var cachePath = Path.Combine(cacheFolder, cacheFilePrefix + _GetSourceHash(files) + ".dll"); + + if (File.Exists(cachePath)) + { + try + { + using (CBenchmark.Time("Load cached PartyMode " + partyModeName)) + { + return Assembly.LoadFrom(cachePath); + } + } + catch (Exception e) + { + CLog.Error("Error loading cached PartyMode assembly " + cachePath + ": " + e.Message); + } + } + + using (CBenchmark.Time("Compile PartyMode " + partyModeName)) + { + try + { + Directory.CreateDirectory(cacheFolder); + foreach (var oldFile in Directory.GetFiles(cacheFolder, cacheFilePrefix + "*")) + { + File.Delete(oldFile); + } + } + catch (Exception e) + { + CLog.Error("Error preparing PartyMode cache folder " + cacheFolder + ": " + e.Message); + return _CompileFiles(files, null); + } + + return _CompileFiles(files, cachePath); + } + } + + private static string _GetSourceHash(string[] files) + { + var sortedFiles = files.OrderBy(f => Path.GetFileName(f), StringComparer.OrdinalIgnoreCase).ToArray(); + + var key = _PartyModeSystemVersion + "|" + typeof(IPartyMode).Assembly.ManifestModule.ModuleVersionId; +#if DEBUG + key += "|Debug"; +#endif + + using (var sha = SHA256.Create()) + { + var header = Encoding.UTF8.GetBytes(key + "|"); + sha.TransformBlock(header, 0, header.Length, null, 0); + + foreach (var file in sortedFiles) + { + var name = Encoding.UTF8.GetBytes(Path.GetFileName(file) + "|"); + sha.TransformBlock(name, 0, name.Length, null, 0); + + var content = File.ReadAllBytes(file); + sha.TransformBlock(content, 0, content.Length, null, 0); + } + + sha.TransformFinalBlock(new byte[0], 0, 0); + return BitConverter.ToString(sha.Hash, 0, 16).Replace("-", ""); + } + } + + private static Assembly _CompileFiles(string[] files, string outputPath) + { var compilerParams = new CompilerParameters(); compilerParams.ReferencedAssemblies.Add("System.Windows.Forms.dll"); compilerParams.ReferencedAssemblies.Add("System.dll"); compilerParams.ReferencedAssemblies.Add("System.Core.dll"); compilerParams.ReferencedAssemblies.Add(Path.Combine("libs", "managed", "VocaluxeLib.dll")); - compilerParams.GenerateInMemory = true; + if (outputPath == null) + { + compilerParams.GenerateInMemory = true; + } + else + { + compilerParams.GenerateInMemory = false; + compilerParams.OutputAssembly = outputPath; + } #if DEBUG compilerParams.IncludeDebugInformation = true; #endif @@ -350,7 +430,20 @@ namespace Vocaluxe.Base return null; } - return compileResult.CompiledAssembly; + if (outputPath == null) + { + return compileResult.CompiledAssembly; + } + + try + { + return Assembly.LoadFrom(outputPath); + } + catch (Exception e) + { + CLog.Error("Error loading compiled PartyMode assembly " + outputPath + ": " + e.Message); + return null; + } } } diff --git a/Vocaluxe/Base/CSettings.cs b/Vocaluxe/Base/CSettings.cs index 5064c7d2..5d770c39 100644 --- a/Vocaluxe/Base/CSettings.cs +++ b/Vocaluxe/Base/CSettings.cs @@ -145,6 +145,7 @@ namespace Vocaluxe.Base public const string FolderNamePartyModeCode = "Code"; public const string FolderNamePartyModeLanguages = "Languages"; public const string FolderNamePartyModeFonts = "Fonts"; + public const string FolderNamePartyModeCache = "PartyModeCache"; public const string LinkAndroidApp = "https://build.phonegap.com/apps/639714/download/android/?qr_key=uY98ymvTr6K144RyTdhs"; public const string LinkSymbianApp = "https://build.phonegap.com/apps/639714/download/symbian/?qr_key=uY98ymvTr6K144RyTdhs"; diff --git a/Vocaluxe/Base/CSongs.cs b/Vocaluxe/Base/CSongs.cs index dbf91b2a..15b96f3d 100644 --- a/Vocaluxe/Base/CSongs.cs +++ b/Vocaluxe/Base/CSongs.cs @@ -43,6 +43,8 @@ namespace Vocaluxe.Base public static readonly CSongSorter Sorter = new CSongSorter(); public static readonly CSongCategorizer Categorizer = new CSongCategorizer(); + private static Thread _SongLoaderThread; + private static readonly ManualResetEvent _InitFinished = new ManualResetEvent(false); private static Thread _CoverLoaderThread; public static event CategoryChangedHandler OnCategoryChanged; @@ -390,6 +392,22 @@ namespace Vocaluxe.Base Categorizer.Tabs = tabs; } + public static void StartLoadSongs() + { + if (_SongLoaderThread != null) + { + return; + } + + _SongLoaderThread = new Thread(LoadSongs) { Name = "SongLoader", IsBackground = true }; + _SongLoaderThread.Start(); + } + + public static void SetInitFinished() + { + _InitFinished.Set(); + } + public static void LoadSongs() { using (CBenchmark.Time("Load Songs")) @@ -429,20 +447,28 @@ namespace Vocaluxe.Base MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount) }; - Parallel.ForEach(fileList, options, file => + CDataBase.BeginSongInfoCache(); + try { - var song = CSong.LoadSong(file); - if (song == null) + Parallel.ForEach(fileList, options, file => { - return; - } + var song = CSong.LoadSong(file); + if (song == null) + { + return; + } - if (song.LoadNotes()) - { - bag.Add(song); - Interlocked.Increment(ref _NumSongsLoaded); - } - }); + if (song.LoadNotes()) + { + bag.Add(song); + Interlocked.Increment(ref _NumSongsLoaded); + } + }); + } + finally + { + CDataBase.EndSongInfoCache(); + } var id = 0; foreach (var song in bag.OrderBy(s => s.Folder).ThenBy(s => s.FileName)) @@ -453,6 +479,8 @@ namespace Vocaluxe.Base } } + _InitFinished.WaitOne(); + using (CBenchmark.Time("Sorted Songs")) { Sorter.SongSorting = CConfig.Config.Game.SongSorting; @@ -496,23 +524,36 @@ namespace Vocaluxe.Base using (CBenchmark.Time("Loaded Covers")) { var songCount = Songs.Count; - var ev = new AutoResetEvent(songCount == 0); + var nextSong = -1; NumSongsWithCoverLoaded = 0; - foreach (var song in Songs) + var workers = new Thread[Math.Max(1, Math.Min(songCount, Environment.ProcessorCount - 1))]; + for (var i = 0; i < workers.Length; i++) { - var tmp = song; - Task.Factory.StartNew(() => + workers[i] = new Thread(() => { - tmp.LoadSmallCover(); - if (Interlocked.Increment(ref _NumSongsWithCoverLoaded) >= songCount) + int index; + while ((index = Interlocked.Increment(ref nextSong)) < songCount) { - ev.Set(); + try + { + Songs[index].LoadSmallCover(); + } + catch (Exception e) + { + CLog.Error(e, "Error loading cover for {SongFileName}: {ExceptionMessage}", CLog.Params(Songs[index].FileName, e.Message)); + } + + Interlocked.Increment(ref _NumSongsWithCoverLoaded); } - }); + }) { Name = "CoverLoader" + i, Priority = ThreadPriority.BelowNormal, IsBackground = true }; + workers[i].Start(); } - ev.WaitOne(); + foreach (var worker in workers) + { + worker.Join(); + } _CoverLoaded = true; CDataBase.CommitCovers(); } diff --git a/Vocaluxe/Lib/Database/CHighscoreDB.cs b/Vocaluxe/Lib/Database/CHighscoreDB.cs index d3dbbaf9..ede55158 100644 --- a/Vocaluxe/Lib/Database/CHighscoreDB.cs +++ b/Vocaluxe/Lib/Database/CHighscoreDB.cs @@ -57,6 +57,18 @@ namespace Vocaluxe.Lib.Database public string Str2; } + private struct SSongInfo + { + public int Id; + public int NumPlayed; + public DateTime DateAdded; + } + + private readonly object _SongInfoMutex = new object(); + private Dictionary<(string Title, string Artist), SSongInfo> _SongInfos; + private SqliteConnection _SongInfoConnection; + private SqliteTransaction _SongInfoTransaction; + public CHighscoreDB(string filePath) : base(filePath) { } public override bool Init() @@ -110,8 +122,143 @@ namespace Vocaluxe.Lib.Database //Do nothing } + public void BeginSongInfoCache() + { + lock (_SongInfoMutex) + { + if (_SongInfos != null) + { + return; + } + + var connection = new SqliteConnection("Data Source=" + _FilePath); + try + { + connection.Open(); + + using (var command = new SqliteCommand("CREATE INDEX IF NOT EXISTS IX_Songs_Title_Artist ON Songs (Title, Artist)", connection)) + { + command.ExecuteNonQuery(); + } + + var songInfos = new Dictionary<(string Title, string Artist), SSongInfo>(); + using (var command = new SqliteCommand("SELECT id, Title, Artist, NumPlayed, DateAdded FROM Songs ORDER BY id", connection)) + using (var reader = command.ExecuteReader()) + { + while (reader.Read()) + { + var key = (reader.GetString(1), reader.GetString(2)); + if (songInfos.ContainsKey(key)) + { + continue; + } + + songInfos.Add(key, new SSongInfo + { + Id = reader.GetInt32(0), + NumPlayed = reader.IsDBNull(3) ? 0 : reader.GetInt32(3), + DateAdded = reader.IsDBNull(4) ? DateTime.Today : new DateTime(reader.GetInt64(4), DateTimeKind.Local) + }); + } + } + + _SongInfos = songInfos; + _SongInfoConnection = connection; + } + catch (Exception e) + { + CLog.Error(e, "Error loading song infos from Highscore DB: {ExceptionMessage}", CLog.Params(e.Message)); + connection.Dispose(); + } + } + } + + public void EndSongInfoCache() + { + lock (_SongInfoMutex) + { + if (_SongInfoTransaction != null) + { + try + { + _SongInfoTransaction.Commit(); + } + catch (Exception e) + { + CLog.Error(e, "Error saving new songs to Highscore DB: {ExceptionMessage}", CLog.Params(e.Message)); + } + + _SongInfoTransaction.Dispose(); + _SongInfoTransaction = null; + } + + if (_SongInfoConnection != null) + { + _SongInfoConnection.Dispose(); + _SongInfoConnection = null; + } + + _SongInfos = null; + } + } + + private bool _GetCachedSongInfos(string artist, string title, out int numPlayed, out DateTime dateAdded, out int highscoreId) + { + var key = (title, artist); + if (!_SongInfos.TryGetValue(key, out var info)) + { + if (_SongInfoTransaction == null) + { + _SongInfoTransaction = _SongInfoConnection.BeginTransaction(); + } + + info = new SSongInfo { NumPlayed = 0, DateAdded = DateTime.Now }; + using (var command = new SqliteCommand()) + { + command.Connection = _SongInfoConnection; + command.Transaction = _SongInfoTransaction; + command.CommandText = "INSERT INTO Songs (Title, Artist, NumPlayed, DateAdded) VALUES (@title, @artist, @numplayed, @dateadded)"; + command.Parameters.AddWithValue("@title", title); + command.Parameters.AddWithValue("@artist", artist); + command.Parameters.AddWithValue("@numplayed", info.NumPlayed); + command.Parameters.AddWithValue("@dateadded", info.DateAdded.Ticks); + command.ExecuteNonQuery(); + + command.CommandText = "SELECT last_insert_rowid()"; + command.Parameters.Clear(); + info.Id = Convert.ToInt32(command.ExecuteScalar()); + } + + _SongInfos.Add(key, info); + } + + numPlayed = info.NumPlayed; + dateAdded = info.DateAdded; + highscoreId = info.Id; + return true; + } + public bool GetDataBaseSongInfos(string artist, string title, out int numPlayed, out DateTime dateAdded, out int highscoreId) { + lock (_SongInfoMutex) + { + if (_SongInfos != null) + { + try + { + return _GetCachedSongInfos(artist ?? string.Empty, title ?? string.Empty, out numPlayed, out dateAdded, out highscoreId); + } + catch (Exception e) + { + CLog.Error(e, "Error reading song infos from Highscore DB: {ExceptionMessage}", CLog.Params(e.Message)); + numPlayed = 0; + dateAdded = DateTime.Today; + highscoreId = -1; + return false; + } + } + } + string sArtist; string sTitle; int songId; diff --git a/Vocaluxe/Lib/Draw/CTextureProvider.cs b/Vocaluxe/Lib/Draw/CTextureProvider.cs index 8791ff5e..d8c46c02 100644 --- a/Vocaluxe/Lib/Draw/CTextureProvider.cs +++ b/Vocaluxe/Lib/Draw/CTextureProvider.cs @@ -106,6 +106,8 @@ namespace Vocaluxe.Lib.Draw // Maps texturePaths to textures. Texture may have already been disposed (RefCount<=0) private readonly Dictionary _TextureCache = new Dictionary(); private readonly Queue _TextureQueue = new Queue(); + private const int _MaxQueueTimePerFrameMs = 8; + private readonly Stopwatch _QueueTimer = new Stopwatch(); private readonly Dictionary> _BitmapsLoading = new Dictionary>(); private int _TextureCount; @@ -492,7 +494,8 @@ namespace Vocaluxe.Lib.Draw _EnsureMainThread(); lock (_TextureQueue) { - while (_TextureQueue.Count > 0) + _QueueTimer.Restart(); + while (_TextureQueue.Count > 0 && _QueueTimer.ElapsedMilliseconds < _MaxQueueTimePerFrameMs) { var q = _TextureQueue.Dequeue(); diff --git a/Vocaluxe/Program.cs b/Vocaluxe/Program.cs index cd5b8069..301f26a9 100644 --- a/Vocaluxe/Program.cs +++ b/Vocaluxe/Program.cs @@ -204,6 +204,8 @@ namespace Vocaluxe } } + CSongs.StartLoadSongs(); + Application.DoEvents(); //Init Webcam diff --git a/Vocaluxe/Screens/CScreenLoad.cs b/Vocaluxe/Screens/CScreenLoad.cs index 9380b197..562beefa 100644 --- a/Vocaluxe/Screens/CScreenLoad.cs +++ b/Vocaluxe/Screens/CScreenLoad.cs @@ -16,7 +16,6 @@ #endregion using System; -using System.Threading; using System.Windows.Forms; using Vocaluxe.Base; using VocaluxeLib; @@ -38,7 +37,6 @@ namespace Vocaluxe.Screens private readonly string[] _IntroVideo = new string[] { "IntroIn", "IntroMid", "IntroOut" }; - private Thread _SongLoaderThread; private bool _SkipIntro; private int _CurrentIntroVideo; private bool _IntroOutPlayed; @@ -139,8 +137,8 @@ namespace Vocaluxe.Screens _TimerLoadSongsFull = CBenchmark.Time("Loaded Songs Full"); - _SongLoaderThread = new Thread(CSongs.LoadSongs) { Name = "SongLoader", IsBackground = true }; - _SongLoaderThread.Start(); + CSongs.StartLoadSongs(); + CSongs.SetInitFinished(); CBackgroundMusic.OwnSongsAvailable = false; CBackgroundMusic.Play();