From 2a6be1b3fe7831f7eb6c7777c21f43a830d6dbbc Mon Sep 17 00:00:00 2001 From: McSchneidar <262792195+McSchneidar@users.noreply.github.com> Date: Fri, 16 Oct 2026 05:22:44 +0200 Subject: [PATCH] Speed up cover loading and keep the cover DB up to date (#905) * Recreate cover DB on version mismatch instead of throwing CCoverDB.Init threw NotImplementedException whenever the stored version was older than CSettings.DatabaseCoverVersion, so any future schema change would crash the game on start. The cover DB is only a cache. On any version mismatch, drop all tables and create an empty DB with the current layout instead. Covers are rebuilt on the next load. * Add indexes to cover DB and guard against duplicate entries Cover.Path and CoverData.CoverId had no index, so every cover lookup scanned the whole table. With thousands of songs this makes loading quadratic. Add unique indexes on both columns. Use INSERT OR IGNORE plus last_insert_rowid() when storing a cover. This prepares for loading covers outside the lock, where two threads may store the same cover concurrently. Bump DatabaseCoverVersion to 2 so existing cover DBs are recreated with the indexes. * Decode and resize covers outside of the cover DB lock The comment in GetCover claimed the lock was released before loading the bitmap, but the whole method ran inside lock(_Mutex). All cover tasks started by CSongs._LoadCovers therefore decoded and resized one after another, and the remaining thread pool threads just waited. Split GetCover into three steps: - _ReadCachedCover: DB lookup (locked) - decoding and resizing (not locked, runs in parallel) - _StoreCover: DB insert (locked) The duplicated resize code moves into _FitToMaxSize. If the DB connection has been closed in the meantime, GetCover now still returns true, because the texture has already been created. Before, it returned false, and the caller generated a second cover while the first texture was still queued. * Store resized covers and detect changed cover files The cover DB stored the original cover file. Every later load still had to decode the full-size image (often 1000-3000 px) and scale it down to CoverSize, so the cache only saved reading the file from disk. - Store the cover already scaled to CoverSize: JPEG (quality 90), or PNG if the original has transparency. Covers that are already small enough are stored unchanged to avoid recompressing them. - Entries were only keyed by path, so a replaced cover file with the same name was never picked up. Also store file size, last write time and CoverSize, and drop outdated entries on lookup. - _FitToMaxSize throws if Resize fails instead of passing null on. --- Vocaluxe/Base/CSettings.cs | 2 +- Vocaluxe/Lib/Database/CCoverDB.cs | 345 +++++++++++++++++------------- 2 files changed, 197 insertions(+), 150 deletions(-) diff --git a/Vocaluxe/Base/CSettings.cs b/Vocaluxe/Base/CSettings.cs index 8fae9294..5064c7d2 100644 --- a/Vocaluxe/Base/CSettings.cs +++ b/Vocaluxe/Base/CSettings.cs @@ -78,7 +78,7 @@ namespace Vocaluxe.Base } public const int DatabaseHighscoreVersion = 3; - public const int DatabaseCoverVersion = 1; + public const int DatabaseCoverVersion = 2; public const int DatabaseCreditsRessourcesVersion = 1; public const int MaxNumScreens = 6; diff --git a/Vocaluxe/Lib/Database/CCoverDB.cs b/Vocaluxe/Lib/Database/CCoverDB.cs index 2db128d1..74e6074d 100644 --- a/Vocaluxe/Lib/Database/CCoverDB.cs +++ b/Vocaluxe/Lib/Database/CCoverDB.cs @@ -18,6 +18,7 @@ using System; using System.Drawing.Imaging; using System.IO; +using System.Linq; using System.Runtime.InteropServices; using Microsoft.Data.Sqlite; using Vocaluxe.Base; @@ -29,6 +30,9 @@ namespace Vocaluxe.Lib.Database { public class CCoverDB : CDatabaseBase { + private const long _JpegQuality = 90; + private static readonly ImageCodecInfo _JpegCodec = ImageCodecInfo.GetImageEncoders().First(codec => codec.FormatID == ImageFormat.Jpeg.Guid); + private SqliteTransaction _TransactionCover; public CCoverDB(string filePath) : base(filePath) { } @@ -47,9 +51,11 @@ namespace Vocaluxe.Lib.Database return _CreateCoverDB(); } - if (_Version < CSettings.DatabaseCoverVersion) + if (_Version != CSettings.DatabaseCoverVersion) { - throw new NotImplementedException("Upgrading of cover DB not implemented"); + CLog.Information("Cover DB has version {OldVersion}, expected {NewVersion}. Recreating it.", + CLog.Params(_Version, CSettings.DatabaseCoverVersion)); + return _RecreateCoverDB(); } } @@ -78,11 +84,112 @@ namespace Vocaluxe.Lib.Database return false; } + var fileInfo = new FileInfo(coverPath); + var fileSize = fileInfo.Length; + var lastWrite = fileInfo.LastWriteTimeUtc.Ticks; + + var cachedData = _ReadCachedCover(coverPath, fileSize, lastWrite, maxSize); + if (cachedData != null) + { + try + { + using (var stream = new MemoryStream(cachedData)) + using (var source = new System.Drawing.Bitmap(stream)) + tex = CDraw.EnqueueTexture(_FitToMaxSize(source, maxSize)); + } + catch (Exception e) + { + CLog.Error("Can't decode cover from database: " + e); + return false; + } + + return true; + } + + // At this point we do not have a matching entry in the CoverDB + // (either no Data found and deleted or nothing at all) + byte[] data; + int width; + int height; + + try + { + using (var source = CHelper.LoadBitmap(coverPath)) + { + if (source == null) + return false; + + var bitmap = _FitToMaxSize(source, maxSize); + try + { + width = bitmap.Width; + height = bitmap.Height; + data = source.Width <= maxSize && source.Height <= maxSize + ? File.ReadAllBytes(coverPath) + : _EncodeCover(bitmap, source.PixelFormat); + } + catch + { + bitmap.Dispose(); + throw; + } + + tex = CDraw.EnqueueTexture(bitmap); + } + } + catch (Exception e) + { + CLog.Error("Error loading cover: " + coverPath + " - " + e); + return false; + } + + _StoreCover(coverPath, fileSize, lastWrite, maxSize, width, height, data); + return true; + } + + private static System.Drawing.Bitmap _FitToMaxSize(System.Drawing.Bitmap source, int maxSize) + { + var sourceSize = source.GetSize(); + if (sourceSize.Width <= maxSize && sourceSize.Height <= maxSize) + return new System.Drawing.Bitmap(source); + + var targetSize = CHelper.FitInBounds( + new SRectF(0, 0, maxSize, maxSize, 0), + (float)sourceSize.Width / sourceSize.Height, + EAspect.LetterBox + ).SizeI; + + return source.Resize(targetSize) ?? throw new InvalidOperationException("Resizing the cover failed"); + } + + private static byte[] _EncodeCover(System.Drawing.Bitmap bitmap, PixelFormat sourceFormat) + { + using (var stream = new MemoryStream()) + { + if (System.Drawing.Image.IsAlphaPixelFormat(sourceFormat)) + { + bitmap.Save(stream, ImageFormat.Png); + } + else + { + using (var parameters = new EncoderParameters(1)) + { + parameters.Param[0] = new EncoderParameter(System.Drawing.Imaging.Encoder.Quality, _JpegQuality); + bitmap.Save(stream, _JpegCodec, parameters); + } + } + + return stream.ToArray(); + } + } + + private byte[] _ReadCachedCover(string coverPath, long fileSize, long lastWrite, int maxSize) + { lock (_Mutex) { //Double check here because we may have just closed our connection if (_Connection == null) - return false; + return null; using (var command = new SqliteCommand()) { @@ -91,167 +198,78 @@ namespace Vocaluxe.Lib.Database if (_TransactionCover != null) command.Transaction = _TransactionCover; - command.CommandText = "SELECT id, width, height FROM Cover WHERE [Path] = @path"; + command.CommandText = "SELECT id, FileSize, LastWrite, MaxSize FROM Cover WHERE [Path] = @path"; command.Parameters.Clear(); command.Parameters.AddWithValue("@path", coverPath); - var reader = command.ExecuteReader(); - - if (reader != null && reader.HasRows) + int id; + bool isUpToDate; + using (var reader = command.ExecuteReader()) { - reader.Read(); - var id = reader.GetInt32(0); - var w = reader.GetInt32(1); - var h = reader.GetInt32(2); - reader.Close(); + if (!reader.Read()) + return null; + id = reader.GetInt32(0); + isUpToDate = reader.GetInt64(1) == fileSize && reader.GetInt64(2) == lastWrite && reader.GetInt32(3) == maxSize; + } + + command.Parameters.Clear(); + command.Parameters.AddWithValue("@id", id); + + if (isUpToDate) + { command.CommandText = "SELECT Data FROM CoverData WHERE CoverId = @id"; - command.Parameters.Clear(); - command.Parameters.AddWithValue("@id", id); - reader = command.ExecuteReader(); - - if (reader.HasRows) + using (var reader = command.ExecuteReader()) { - reader.Read(); - var data2 = _GetBytes(reader); - reader.Dispose(); - - try - { - using (var stream = new MemoryStream(data2)) - using (var source = new System.Drawing.Bitmap(stream)) - { - var sourceSize = source.GetSize(); - - System.Drawing.Bitmap bitmap; - - if (sourceSize.Width > maxSize || sourceSize.Height > maxSize) - { - var targetSize = CHelper.FitInBounds( - new SRectF(0, 0, maxSize, maxSize, 0), - (float)sourceSize.Width / sourceSize.Height, - EAspect.LetterBox - ).SizeI; - - bitmap = source.Resize(targetSize); - } - else - { - bitmap = new System.Drawing.Bitmap(source); - } - - tex = CDraw.EnqueueTexture(bitmap); - } - } - catch (Exception e) - { - CLog.Error("Can't decode cover from database: " + e); - return false; - } - - return true; + if (reader.Read()) + return _GetBytes(reader); } - - command.CommandText = "DELETE FROM Cover WHERE id = @id"; - command.Parameters.Clear(); - command.Parameters.AddWithValue("@id", id); - command.ExecuteNonQuery(); - - if (reader != null) - reader.Close(); } + + command.CommandText = "DELETE FROM CoverData WHERE CoverId = @id; DELETE FROM Cover WHERE id = @id;"; + command.ExecuteNonQuery(); + + return null; } + } + } - // At this point we do not have a matching entry in the CoverDB - // (either no Data found and deleted or nothing at all) - // We break out of the lock to do the bitmap loading here to allow multithreaded loading + private void _StoreCover(string coverPath, long fileSize, long lastWrite, int maxSize, int width, int height, byte[] data) + { + lock (_Mutex) + { + // Double check here because we may have just closed our connection + if (_Connection == null) + return; - byte[] data; - int width; - int height; + if (_TransactionCover == null) + _TransactionCover = _Connection.BeginTransaction(); - try + using (var command = new SqliteCommand()) { - data = File.ReadAllBytes(coverPath); + command.Connection = _Connection; + command.Transaction = _TransactionCover; + command.CommandText = "INSERT OR IGNORE INTO Cover (Path, width, height, FileSize, LastWrite, MaxSize) " + + "VALUES (@path, @w, @h, @size, @lastWrite, @maxSize)"; + command.Parameters.Clear(); + command.Parameters.AddWithValue("@w", width); + command.Parameters.AddWithValue("@h", height); + command.Parameters.AddWithValue("@path", coverPath); + command.Parameters.AddWithValue("@size", fileSize); + command.Parameters.AddWithValue("@lastWrite", lastWrite); + command.Parameters.AddWithValue("@maxSize", maxSize); + if (command.ExecuteNonQuery() == 0) + return; - using (var source = CHelper.LoadBitmap(coverPath)) - { - if (source == null) - return false; + command.CommandText = "SELECT last_insert_rowid()"; + command.Parameters.Clear(); + var id = Convert.ToInt32(command.ExecuteScalar()); - var sourceSize = source.GetSize(); - width = sourceSize.Width; - height = sourceSize.Height; - - System.Drawing.Bitmap bitmap; - - if (sourceSize.Width > maxSize || sourceSize.Height > maxSize) - { - var targetSize = CHelper.FitInBounds( - new SRectF(0, 0, maxSize, maxSize, 0), - (float)sourceSize.Width / sourceSize.Height, - EAspect.LetterBox - ).SizeI; - - bitmap = source.Resize(targetSize); - } - else - { - bitmap = new System.Drawing.Bitmap(source); - } - - tex = CDraw.EnqueueTexture(bitmap); - } - } - catch (Exception e) - { - CLog.Error("Error loading cover: " + coverPath + " - " + e); - return false; - } - - lock (_Mutex) - { - // Double check here because we may have just closed our connection - if (_Connection == null) - return false; - - if (_TransactionCover == null) - _TransactionCover = _Connection.BeginTransaction(); - - using (var command = new SqliteCommand()) - { - command.Connection = _Connection; - command.Transaction = _TransactionCover; - command.CommandText = "INSERT INTO Cover (Path, width, height) VALUES (@path, @w, @h)"; - command.Parameters.Clear(); - command.Parameters.AddWithValue("@w", width); - command.Parameters.AddWithValue("@h", height); - command.Parameters.AddWithValue("@path", coverPath); - command.ExecuteNonQuery(); - - command.CommandText = "SELECT id FROM Cover WHERE [Path] = @path"; - command.Parameters.Clear(); - command.Parameters.AddWithValue("@path", coverPath); - - var reader = command.ExecuteReader(); - - if (reader == null || !reader.Read()) - { - reader?.Dispose(); - return false; - } - - var id = reader.GetInt32(0); - reader.Dispose(); - - command.CommandText = "INSERT INTO CoverData (CoverId, Data) VALUES (@id, @data)"; - command.Parameters.Clear(); - command.Parameters.AddWithValue("@id", id); - command.Parameters.AddWithValue("@data", data); - command.ExecuteNonQuery(); - - return true; - } + command.CommandText = "INSERT INTO CoverData (CoverId, Data) VALUES (@id, @data)"; + command.Parameters.Clear(); + command.Parameters.AddWithValue("@id", id); + command.Parameters.AddWithValue("@data", data); + command.ExecuteNonQuery(); } } } @@ -295,12 +313,19 @@ namespace Vocaluxe.Lib.Database command.ExecuteNonQuery(); command.CommandText = "CREATE TABLE IF NOT EXISTS Cover ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, " + - "Path TEXT NOT NULL, width INTEGER NOT NULL, height INTEGER NOT NULL);"; + "Path TEXT NOT NULL, width INTEGER NOT NULL, height INTEGER NOT NULL, " + + "FileSize INTEGER NOT NULL, LastWrite INTEGER NOT NULL, MaxSize INTEGER NOT NULL);"; command.ExecuteNonQuery(); command.CommandText = "CREATE TABLE IF NOT EXISTS CoverData ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, " + "CoverId INTEGER NOT NULL, Data BLOB NOT NULL);"; command.ExecuteNonQuery(); + + command.CommandText = "CREATE UNIQUE INDEX IF NOT EXISTS IX_Cover_Path ON Cover (Path);"; + command.ExecuteNonQuery(); + + command.CommandText = "CREATE UNIQUE INDEX IF NOT EXISTS IX_CoverData_CoverId ON CoverData (CoverId);"; + command.ExecuteNonQuery(); } } catch (Exception e) @@ -311,5 +336,27 @@ namespace Vocaluxe.Lib.Database return true; } + + private bool _RecreateCoverDB() + { + try + { + using (var command = new SqliteCommand()) + { + command.Connection = _Connection; + command.CommandText = "DROP TABLE IF EXISTS CoverData; DROP TABLE IF EXISTS Cover; DROP TABLE IF EXISTS Version;"; + command.ExecuteNonQuery(); + command.CommandText = "VACUUM;"; + command.ExecuteNonQuery(); + } + } + catch (Exception e) + { + CLog.Error("Error dropping old Cover DB " + e); + return false; + } + + return _CreateCoverDB(); + } } }