Speed up cover loading and keep the cover DB up to date (#905)

* Recreate cover DB on version mismatch instead of throwing

CCoverDB.Init threw NotImplementedException whenever the stored
version was older than CSettings.DatabaseCoverVersion, so any future
schema change would crash the game on start.

The cover DB is only a cache. On any version mismatch, drop all
tables and create an empty DB with the current layout instead.
Covers are rebuilt on the next load.

* Add indexes to cover DB and guard against duplicate entries

Cover.Path and CoverData.CoverId had no index, so every cover lookup
scanned the whole table. With thousands of songs this makes loading
quadratic. Add unique indexes on both columns.

Use INSERT OR IGNORE plus last_insert_rowid() when storing a cover.
This prepares for loading covers outside the lock, where two threads
may store the same cover concurrently.

Bump DatabaseCoverVersion to 2 so existing cover DBs are recreated
with the indexes.

* Decode and resize covers outside of the cover DB lock

The comment in GetCover claimed the lock was released before loading
the bitmap, but the whole method ran inside lock(_Mutex). All cover
tasks started by CSongs._LoadCovers therefore decoded and resized
one after another, and the remaining thread pool threads just waited.

Split GetCover into three steps:
- _ReadCachedCover: DB lookup (locked)
- decoding and resizing (not locked, runs in parallel)
- _StoreCover: DB insert (locked)

The duplicated resize code moves into _FitToMaxSize.

If the DB connection has been closed in the meantime, GetCover now
still returns true, because the texture has already been created.
Before, it returned false, and the caller generated a second cover
while the first texture was still queued.

* Store resized covers and detect changed cover files

The cover DB stored the original cover file. Every later load still
had to decode the full-size image (often 1000-3000 px) and scale it
down to CoverSize, so the cache only saved reading the file from disk.

- Store the cover already scaled to CoverSize: JPEG (quality 90), or
  PNG if the original has transparency. Covers that are already small
  enough are stored unchanged to avoid recompressing them.
- Entries were only keyed by path, so a replaced cover file with the
  same name was never picked up. Also store file size, last write
  time and CoverSize, and drop outdated entries on lookup.
- _FitToMaxSize throws if Resize fails instead of passing null on.
This commit is contained in:
McSchneidar authored and GitHub committed 2026-10-16 05:22:44 +02:00
1 parent d48426b04d
commit 2a6be1b3fe
2 files changed
+160 -113

No files matched your search

+1 -1
View File
@@ -78,7 +78,7 @@ namespace Vocaluxe.Base
} }
public const int DatabaseHighscoreVersion = 3; public const int DatabaseHighscoreVersion = 3;
public const int DatabaseCoverVersion = 1; public const int DatabaseCoverVersion = 2;
public const int DatabaseCreditsRessourcesVersion = 1; public const int DatabaseCreditsRessourcesVersion = 1;
public const int MaxNumScreens = 6; public const int MaxNumScreens = 6;
+159 -112
View File
@@ -18,6 +18,7 @@
using System; using System;
using System.Drawing.Imaging; using System.Drawing.Imaging;
using System.IO; using System.IO;
using System.Linq;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using Microsoft.Data.Sqlite; using Microsoft.Data.Sqlite;
using Vocaluxe.Base; using Vocaluxe.Base;
@@ -29,6 +30,9 @@ namespace Vocaluxe.Lib.Database
{ {
public class CCoverDB : CDatabaseBase public class CCoverDB : CDatabaseBase
{ {
private const long _JpegQuality = 90;
private static readonly ImageCodecInfo _JpegCodec = ImageCodecInfo.GetImageEncoders().First(codec => codec.FormatID == ImageFormat.Jpeg.Guid);
private SqliteTransaction _TransactionCover; private SqliteTransaction _TransactionCover;
public CCoverDB(string filePath) : base(filePath) { } public CCoverDB(string filePath) : base(filePath) { }
@@ -47,9 +51,11 @@ namespace Vocaluxe.Lib.Database
return _CreateCoverDB(); return _CreateCoverDB();
} }
if (_Version < CSettings.DatabaseCoverVersion) if (_Version != CSettings.DatabaseCoverVersion)
{ {
throw new NotImplementedException("Upgrading of cover DB not implemented"); CLog.Information("Cover DB has version {OldVersion}, expected {NewVersion}. Recreating it.",
CLog.Params(_Version, CSettings.DatabaseCoverVersion));
return _RecreateCoverDB();
} }
} }
@@ -78,70 +84,18 @@ namespace Vocaluxe.Lib.Database
return false; return false;
} }
lock (_Mutex) var fileInfo = new FileInfo(coverPath);
var fileSize = fileInfo.Length;
var lastWrite = fileInfo.LastWriteTimeUtc.Ticks;
var cachedData = _ReadCachedCover(coverPath, fileSize, lastWrite, maxSize);
if (cachedData != null)
{ {
//Double check here because we may have just closed our connection
if (_Connection == null)
return false;
using (var command = new SqliteCommand())
{
command.Connection = _Connection;
// If we have an open transaction on this connection, all commands must use it.
if (_TransactionCover != null)
command.Transaction = _TransactionCover;
command.CommandText = "SELECT id, width, height FROM Cover WHERE [Path] = @path";
command.Parameters.Clear();
command.Parameters.AddWithValue("@path", coverPath);
var reader = command.ExecuteReader();
if (reader != null && reader.HasRows)
{
reader.Read();
var id = reader.GetInt32(0);
var w = reader.GetInt32(1);
var h = reader.GetInt32(2);
reader.Close();
command.CommandText = "SELECT Data FROM CoverData WHERE CoverId = @id";
command.Parameters.Clear();
command.Parameters.AddWithValue("@id", id);
reader = command.ExecuteReader();
if (reader.HasRows)
{
reader.Read();
var data2 = _GetBytes(reader);
reader.Dispose();
try try
{ {
using (var stream = new MemoryStream(data2)) using (var stream = new MemoryStream(cachedData))
using (var source = new System.Drawing.Bitmap(stream)) using (var source = new System.Drawing.Bitmap(stream))
{ tex = CDraw.EnqueueTexture(_FitToMaxSize(source, maxSize));
var sourceSize = source.GetSize();
System.Drawing.Bitmap bitmap;
if (sourceSize.Width > maxSize || sourceSize.Height > maxSize)
{
var targetSize = CHelper.FitInBounds(
new SRectF(0, 0, maxSize, maxSize, 0),
(float)sourceSize.Width / sourceSize.Height,
EAspect.LetterBox
).SizeI;
bitmap = source.Resize(targetSize);
}
else
{
bitmap = new System.Drawing.Bitmap(source);
}
tex = CDraw.EnqueueTexture(bitmap);
}
} }
catch (Exception e) catch (Exception e)
{ {
@@ -152,52 +106,32 @@ namespace Vocaluxe.Lib.Database
return true; return true;
} }
command.CommandText = "DELETE FROM Cover WHERE id = @id";
command.Parameters.Clear();
command.Parameters.AddWithValue("@id", id);
command.ExecuteNonQuery();
if (reader != null)
reader.Close();
}
}
// At this point we do not have a matching entry in the CoverDB // At this point we do not have a matching entry in the CoverDB
// (either no Data found and deleted or nothing at all) // (either no Data found and deleted or nothing at all)
// We break out of the lock to do the bitmap loading here to allow multithreaded loading
byte[] data; byte[] data;
int width; int width;
int height; int height;
try try
{ {
data = File.ReadAllBytes(coverPath);
using (var source = CHelper.LoadBitmap(coverPath)) using (var source = CHelper.LoadBitmap(coverPath))
{ {
if (source == null) if (source == null)
return false; return false;
var sourceSize = source.GetSize(); var bitmap = _FitToMaxSize(source, maxSize);
width = sourceSize.Width; try
height = sourceSize.Height;
System.Drawing.Bitmap bitmap;
if (sourceSize.Width > maxSize || sourceSize.Height > maxSize)
{ {
var targetSize = CHelper.FitInBounds( width = bitmap.Width;
new SRectF(0, 0, maxSize, maxSize, 0), height = bitmap.Height;
(float)sourceSize.Width / sourceSize.Height, data = source.Width <= maxSize && source.Height <= maxSize
EAspect.LetterBox ? File.ReadAllBytes(coverPath)
).SizeI; : _EncodeCover(bitmap, source.PixelFormat);
bitmap = source.Resize(targetSize);
} }
else catch
{ {
bitmap = new System.Drawing.Bitmap(source); bitmap.Dispose();
throw;
} }
tex = CDraw.EnqueueTexture(bitmap); tex = CDraw.EnqueueTexture(bitmap);
@@ -209,11 +143,104 @@ namespace Vocaluxe.Lib.Database
return false; return false;
} }
_StoreCover(coverPath, fileSize, lastWrite, maxSize, width, height, data);
return true;
}
private static System.Drawing.Bitmap _FitToMaxSize(System.Drawing.Bitmap source, int maxSize)
{
var sourceSize = source.GetSize();
if (sourceSize.Width <= maxSize && sourceSize.Height <= maxSize)
return new System.Drawing.Bitmap(source);
var targetSize = CHelper.FitInBounds(
new SRectF(0, 0, maxSize, maxSize, 0),
(float)sourceSize.Width / sourceSize.Height,
EAspect.LetterBox
).SizeI;
return source.Resize(targetSize) ?? throw new InvalidOperationException("Resizing the cover failed");
}
private static byte[] _EncodeCover(System.Drawing.Bitmap bitmap, PixelFormat sourceFormat)
{
using (var stream = new MemoryStream())
{
if (System.Drawing.Image.IsAlphaPixelFormat(sourceFormat))
{
bitmap.Save(stream, ImageFormat.Png);
}
else
{
using (var parameters = new EncoderParameters(1))
{
parameters.Param[0] = new EncoderParameter(System.Drawing.Imaging.Encoder.Quality, _JpegQuality);
bitmap.Save(stream, _JpegCodec, parameters);
}
}
return stream.ToArray();
}
}
private byte[] _ReadCachedCover(string coverPath, long fileSize, long lastWrite, int maxSize)
{
lock (_Mutex) lock (_Mutex)
{ {
//Double check here because we may have just closed our connection //Double check here because we may have just closed our connection
if (_Connection == null) if (_Connection == null)
return false; return null;
using (var command = new SqliteCommand())
{
command.Connection = _Connection;
// If we have an open transaction on this connection, all commands must use it.
if (_TransactionCover != null)
command.Transaction = _TransactionCover;
command.CommandText = "SELECT id, FileSize, LastWrite, MaxSize FROM Cover WHERE [Path] = @path";
command.Parameters.Clear();
command.Parameters.AddWithValue("@path", coverPath);
int id;
bool isUpToDate;
using (var reader = command.ExecuteReader())
{
if (!reader.Read())
return null;
id = reader.GetInt32(0);
isUpToDate = reader.GetInt64(1) == fileSize && reader.GetInt64(2) == lastWrite && reader.GetInt32(3) == maxSize;
}
command.Parameters.Clear();
command.Parameters.AddWithValue("@id", id);
if (isUpToDate)
{
command.CommandText = "SELECT Data FROM CoverData WHERE CoverId = @id";
using (var reader = command.ExecuteReader())
{
if (reader.Read())
return _GetBytes(reader);
}
}
command.CommandText = "DELETE FROM CoverData WHERE CoverId = @id; DELETE FROM Cover WHERE id = @id;";
command.ExecuteNonQuery();
return null;
}
}
}
private void _StoreCover(string coverPath, long fileSize, long lastWrite, int maxSize, int width, int height, byte[] data)
{
lock (_Mutex)
{
// Double check here because we may have just closed our connection
if (_Connection == null)
return;
if (_TransactionCover == null) if (_TransactionCover == null)
_TransactionCover = _Connection.BeginTransaction(); _TransactionCover = _Connection.BeginTransaction();
@@ -222,36 +249,27 @@ namespace Vocaluxe.Lib.Database
{ {
command.Connection = _Connection; command.Connection = _Connection;
command.Transaction = _TransactionCover; command.Transaction = _TransactionCover;
command.CommandText = "INSERT INTO Cover (Path, width, height) VALUES (@path, @w, @h)"; command.CommandText = "INSERT OR IGNORE INTO Cover (Path, width, height, FileSize, LastWrite, MaxSize) " +
"VALUES (@path, @w, @h, @size, @lastWrite, @maxSize)";
command.Parameters.Clear(); command.Parameters.Clear();
command.Parameters.AddWithValue("@w", width); command.Parameters.AddWithValue("@w", width);
command.Parameters.AddWithValue("@h", height); command.Parameters.AddWithValue("@h", height);
command.Parameters.AddWithValue("@path", coverPath); command.Parameters.AddWithValue("@path", coverPath);
command.ExecuteNonQuery(); command.Parameters.AddWithValue("@size", fileSize);
command.Parameters.AddWithValue("@lastWrite", lastWrite);
command.Parameters.AddWithValue("@maxSize", maxSize);
if (command.ExecuteNonQuery() == 0)
return;
command.CommandText = "SELECT id FROM Cover WHERE [Path] = @path"; command.CommandText = "SELECT last_insert_rowid()";
command.Parameters.Clear(); command.Parameters.Clear();
command.Parameters.AddWithValue("@path", coverPath); var id = Convert.ToInt32(command.ExecuteScalar());
var reader = command.ExecuteReader();
if (reader == null || !reader.Read())
{
reader?.Dispose();
return false;
}
var id = reader.GetInt32(0);
reader.Dispose();
command.CommandText = "INSERT INTO CoverData (CoverId, Data) VALUES (@id, @data)"; command.CommandText = "INSERT INTO CoverData (CoverId, Data) VALUES (@id, @data)";
command.Parameters.Clear(); command.Parameters.Clear();
command.Parameters.AddWithValue("@id", id); command.Parameters.AddWithValue("@id", id);
command.Parameters.AddWithValue("@data", data); command.Parameters.AddWithValue("@data", data);
command.ExecuteNonQuery(); command.ExecuteNonQuery();
return true;
}
} }
} }
} }
@@ -295,12 +313,19 @@ namespace Vocaluxe.Lib.Database
command.ExecuteNonQuery(); command.ExecuteNonQuery();
command.CommandText = "CREATE TABLE IF NOT EXISTS Cover ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, " + command.CommandText = "CREATE TABLE IF NOT EXISTS Cover ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, " +
"Path TEXT NOT NULL, width INTEGER NOT NULL, height INTEGER NOT NULL);"; "Path TEXT NOT NULL, width INTEGER NOT NULL, height INTEGER NOT NULL, " +
"FileSize INTEGER NOT NULL, LastWrite INTEGER NOT NULL, MaxSize INTEGER NOT NULL);";
command.ExecuteNonQuery(); command.ExecuteNonQuery();
command.CommandText = "CREATE TABLE IF NOT EXISTS CoverData ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, " + command.CommandText = "CREATE TABLE IF NOT EXISTS CoverData ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, " +
"CoverId INTEGER NOT NULL, Data BLOB NOT NULL);"; "CoverId INTEGER NOT NULL, Data BLOB NOT NULL);";
command.ExecuteNonQuery(); command.ExecuteNonQuery();
command.CommandText = "CREATE UNIQUE INDEX IF NOT EXISTS IX_Cover_Path ON Cover (Path);";
command.ExecuteNonQuery();
command.CommandText = "CREATE UNIQUE INDEX IF NOT EXISTS IX_CoverData_CoverId ON CoverData (CoverId);";
command.ExecuteNonQuery();
} }
} }
catch (Exception e) catch (Exception e)
@@ -311,5 +336,27 @@ namespace Vocaluxe.Lib.Database
return true; return true;
} }
private bool _RecreateCoverDB()
{
try
{
using (var command = new SqliteCommand())
{
command.Connection = _Connection;
command.CommandText = "DROP TABLE IF EXISTS CoverData; DROP TABLE IF EXISTS Cover; DROP TABLE IF EXISTS Version;";
command.ExecuteNonQuery();
command.CommandText = "VACUUM;";
command.ExecuteNonQuery();
}
}
catch (Exception e)
{
CLog.Error("Error dropping old Cover DB " + e);
return false;
}
return _CreateCoverDB();
}
} }
} }