Song server: search and download songs, mobile web UI

- Sources: "free" (Creative Commons songs with media from
  github.com/UltraStar-Deluxe/songs) and "usdb" (usdb.animux.de with an
  account; audio/video via yt-dlp from the song's #VIDEO hints or YouTube
  links in the comments, covers from fanart.tv/YouTube)
- Aggregated search (all sources in parallel, in-library flag), download
  queue into <library>/Downloads/<source>/<Artist - Title>, then rescan
- Web UI in wwwroot (plain JS, no build): search, downloads with progress,
  library; phone first, dark mode, installable
- Docker image with yt-dlp, ffmpeg and deno; library mounted writable
- Tests for matching, USDB parsing and song rewriting

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

VocaluxeSongServer/Sources/CGitHubSongSource.cs

@@ -0,0 +1,131 @@
+using System.Text.Json;
+using VocaluxeSongServer.Contracts;
+
+namespace VocaluxeSongServer.Sources
+{
+    /// <summary>
+    ///     Openly licensed songs with all media from a GitHub repository, by default UltraStar-Deluxe/songs
+    ///     (Creative Commons / public domain; one song per folder, license.txt in every folder).
+    ///     The repository tree is read with one API call and cached for a day (the API allows 60 calls/hour without token).
+    /// </summary>
+    public sealed class CGitHubSongSource : ISongSource
+    {
+        private sealed record CSong(string Folder, string Artist, string Title, string License, List<(string Path, long Size)> Files);
+
+        private readonly HttpClient _Http;
+        private readonly string _Repo;
+        private readonly string _Branch;
+        private readonly SemaphoreSlim _Lock = new(1, 1);
+        private List<CSong>? _Catalog;
+        private DateTime _CatalogTime;
+
+        public CGitHubSongSource(HttpClient http, string repo = "UltraStar-Deluxe/songs", string branch = "master")
+        {
+            _Http = http;
+            _Repo = repo;
+            _Branch = branch;
+        }
+
+        public string Id => "free";
+        public string Name => "Free songs";
+        public string Description => "Creative Commons and public domain songs with audio, from github.com/" + _Repo;
+        public bool Enabled => true;
+        public string Hint => "";
+
+        private async Task<List<CSong>> _GetCatalogAsync(CancellationToken ct)
+        {
+            await _Lock.WaitAsync(ct);
+            try
+            {
+                if (_Catalog != null && DateTime.UtcNow - _CatalogTime < TimeSpan.FromDays(1))
+                {
+                    return _Catalog;
+                }
+
+                using var request = new HttpRequestMessage(HttpMethod.Get, $"https://api.github.com/repos/{_Repo}/git/trees/{_Branch}?recursive=1");
+                request.Headers.UserAgent.ParseAdd("VocaluxeSongServer");
+                using var response = await _Http.SendAsync(request, ct);
+                response.EnsureSuccessStatusCode();
+                using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync(ct));
+
+                var files = doc.RootElement.GetProperty("tree").EnumerateArray()
+                               .Where(e => e.GetProperty("type").GetString() == "blob")
+                               .Select(e => (Path: e.GetProperty("path").GetString()!, Size: e.TryGetProperty("size", out var s) ? s.GetInt64() : 0))
+                               .ToList();
+
+                var songs = new List<CSong>();
+                foreach (var folder in files.GroupBy(f => f.Path.Contains('/') ? f.Path[..f.Path.LastIndexOf('/')] : ""))
+                {
+                    var hasSong = folder.Any(f => f.Path.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) &&
+                                                  !f.Path.EndsWith("/license.txt", StringComparison.OrdinalIgnoreCase));
+                    if (!hasSong || folder.Key == "")
+                    {
+                        continue;
+                    }
+
+                    // Folder names are "Artist - Title"; the category is the parent folder ("Creative Commons", ...).
+                    var name = folder.Key[(folder.Key.LastIndexOf('/') + 1)..];
+                    var category = folder.Key.Contains('/') ? folder.Key[..folder.Key.LastIndexOf('/')] : "";
+                    var dash = name.IndexOf(" - ", StringComparison.Ordinal);
+                    var artist = dash > 0 ? name[..dash] : "";
+                    var title = dash > 0 ? name[(dash + 3)..] : name;
+                    songs.Add(new CSong(folder.Key, artist, title, category, folder.ToList()));
+                }
+
+                _Catalog = songs;
+                _CatalogTime = DateTime.UtcNow;
+                return songs;
+            }
+            finally
+            {
+                _Lock.Release();
+            }
+        }
+
+        private string _RawUrl(string path)
+        {
+            return $"https://raw.githubusercontent.com/{_Repo}/{_Branch}/" + string.Join('/', path.Split('/').Select(Uri.EscapeDataString));
+        }
+
+        public async Task<List<SearchResultDto>> SearchAsync(string query, CancellationToken ct)
+        {
+            var normalized = CSourceHelper.Normalize(query);
+            var catalog = await _GetCatalogAsync(ct);
+            return catalog.Where(s => CSourceHelper.Matches(normalized, s.Artist + " " + s.Title))
+                          .Select(s =>
+                          {
+                              var cover = s.Files.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f.Path).Equals("cover", StringComparison.OrdinalIgnoreCase)).Path;
+                              return new SearchResultDto
+                              {
+                                  Source = Id,
+                                  SourceId = s.Folder,
+                                  Artist = s.Artist,
+                                  Title = s.Title,
+                                  Details = $"{s.License} · {s.Files.Sum(f => f.Size) / 1_000_000.0:0.#} MB",
+                                  CoverUrl = cover != null ? _RawUrl(cover) : ""
+                              };
+                          })
+                          .ToList();
+        }
+
+        public async Task DownloadAsync(string sourceId, string folder, IProgress<double> progress, CancellationToken ct)
+        {
+            var song = (await _GetCatalogAsync(ct)).FirstOrDefault(s => s.Folder == sourceId) ?? throw new InvalidOperationException("Song not found in " + _Repo);
+            var total = Math.Max(1, song.Files.Sum(f => f.Size));
+            long done = 0;
+            foreach (var file in song.Files)
+            {
+                var target = Path.Combine(folder, CSourceHelper.SafeName(Path.GetFileName(file.Path)));
+                using var response = await _Http.GetAsync(_RawUrl(file.Path), HttpCompletionOption.ResponseHeadersRead, ct);
+                response.EnsureSuccessStatusCode();
+                await using var source = await response.Content.ReadAsStreamAsync(ct);
+                await using var stream = File.Create(target);
+                await CSourceHelper.CopyAsync(source, stream, n =>
+                {
+                    done += n;
+                    progress.Report((double)done / total);
+                }, ct);
+            }
+        }
+    }
+}

VocaluxeSongServer/Sources/CSongSearch.cs

@@ -0,0 +1,223 @@
+using System.Collections.Concurrent;
+using System.Threading.Channels;
+using Microsoft.Extensions.Options;
+using VocaluxeSongServer.Contracts;
+using VocaluxeSongServer.Library;
+
+namespace VocaluxeSongServer.Sources
+{
+    /// <summary>Searches all enabled sources at once.</summary>
+    public sealed class CSongSearch
+    {
+        private readonly IReadOnlyList<ISongSource> _Sources;
+        private readonly CLibraryIndex _Index;
+        private readonly ILogger<CSongSearch> _Logger;
+
+        public CSongSearch(IEnumerable<ISongSource> sources, CLibraryIndex index, ILogger<CSongSearch> logger)
+        {
+            _Sources = sources.ToList();
+            _Index = index;
+            _Logger = logger;
+        }
+
+        public IReadOnlyList<ISongSource> Sources => _Sources;
+
+        public ISongSource? GetSource(string id)
+        {
+            return _Sources.FirstOrDefault(s => s.Id == id);
+        }
+
+        public static string SongKey(string artist, string title)
+        {
+            return CSourceHelper.Normalize(artist) + "|" + CSourceHelper.Normalize(title);
+        }
+
+        public async Task<SearchResponse> SearchAsync(string query, CancellationToken ct)
+        {
+            var response = new SearchResponse();
+            if (CSourceHelper.Normalize(query).Length < 2)
+            {
+                return response;
+            }
+
+            var inLibrary = new HashSet<string>((_Index.Current?.Dto.Songs ?? new List<SongDto>()).Select(s => SongKey(s.Artist, s.Title)));
+            var tasks = _Sources.Where(s => s.Enabled).Select(async source =>
+            {
+                using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct);
+                timeout.CancelAfter(TimeSpan.FromSeconds(20));
+                try
+                {
+                    return (source, results: await source.SearchAsync(query, timeout.Token), error: (string?)null);
+                }
+                catch (Exception e) when (!ct.IsCancellationRequested)
+                {
+                    _Logger.LogWarning("Search in {Source} failed: {Message}", source.Id, e.Message);
+                    return (source, results: new List<SearchResultDto>(), error: e is OperationCanceledException ? "timeout" : e.Message);
+                }
+            }).ToList();
+
+            foreach (var (source, results, error) in await Task.WhenAll(tasks))
+            {
+                if (error != null)
+                {
+                    response.Errors[source.Id] = error;
+                }
+
+                foreach (var r in results)
+                {
+                    r.InLibrary = inLibrary.Contains(SongKey(r.Artist, r.Title));
+                    response.Results.Add(r);
+                }
+            }
+
+            // Best matches first: artist/title starting with the query, then alphabetically.
+            var q = CSourceHelper.Normalize(query);
+            response.Results = response.Results
+                                       .OrderByDescending(r => CSourceHelper.Normalize(r.Artist + " " + r.Title).StartsWith(q) || CSourceHelper.Normalize(r.Title).StartsWith(q))
+                                       .ThenBy(r => r.Artist, StringComparer.OrdinalIgnoreCase)
+                                       .ThenBy(r => r.Title, StringComparer.OrdinalIgnoreCase)
+                                       .ToList();
+            return response;
+        }
+    }
+
+    /// <summary>
+    ///     Runs downloads one after another in the background. A song is downloaded into a temporary folder and moved to
+    ///     "&lt;library&gt;/&lt;DownloadFolder&gt;/&lt;source&gt;/&lt;Artist - Title&gt;" when complete; then the library is rescanned.
+    /// </summary>
+    public sealed class CDownloadQueue : BackgroundService
+    {
+        private sealed class CJob
+        {
+            public required DownloadJobDto Dto;
+            public required ISongSource Source;
+        }
+
+        private readonly Channel<CJob> _Queue = Channel.CreateUnbounded<CJob>();
+        private readonly ConcurrentDictionary<string, CJob> _Jobs = new();
+        private readonly CLibraryIndex _Index;
+        private readonly CSongSearch _Search;
+        private readonly CLibraryOptions _Options;
+        private readonly ILogger<CDownloadQueue> _Logger;
+
+        public CDownloadQueue(CLibraryIndex index, CSongSearch search, IOptions<CLibraryOptions> options, ILogger<CDownloadQueue> logger)
+        {
+            _Index = index;
+            _Search = search;
+            _Options = options.Value;
+            _Logger = logger;
+        }
+
+        public IEnumerable<DownloadJobDto> Jobs => _Jobs.Values.Select(j => j.Dto).OrderByDescending(j => j.Created);
+
+        /// <summary>Queues a download. Returns the existing job if the same song is already queued or running.</summary>
+        public DownloadJobDto Enqueue(DownloadRequest request)
+        {
+            var source = _Search.GetSource(request.Source);
+            if (source == null || !source.Enabled)
+            {
+                throw new ArgumentException("Unknown or disabled source: " + request.Source);
+            }
+
+            var existing = _Jobs.Values.FirstOrDefault(j => j.Dto.Source == request.Source && j.Dto.SourceId == request.SourceId && j.Dto.State is "queued" or "running");
+            if (existing != null)
+            {
+                return existing.Dto;
+            }
+
+            var job = new CJob
+            {
+                Source = source,
+                Dto = new DownloadJobDto
+                {
+                    Id = Guid.NewGuid().ToString("N")[..12],
+                    Source = request.Source,
+                    SourceId = request.SourceId,
+                    Artist = request.Artist,
+                    Title = request.Title,
+                    State = "queued",
+                    Created = DateTimeOffset.UtcNow.ToString("O")
+                }
+            };
+            _Jobs[job.Dto.Id] = job;
+            _Queue.Writer.TryWrite(job);
+
+            // Keep the list short: forget old finished jobs.
+            foreach (var old in _Jobs.Values.Where(j => j.Dto.State is "done" or "failed").OrderByDescending(j => j.Dto.Created).Skip(50).ToList())
+            {
+                _Jobs.TryRemove(old.Dto.Id, out _);
+            }
+
+            return job.Dto;
+        }
+
+        public string DownloadRoot => Path.Combine(_Index.Root, _Options.DownloadFolder);
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            await foreach (var job in _Queue.Reader.ReadAllAsync(stoppingToken))
+            {
+                await _RunAsync(job, stoppingToken);
+            }
+        }
+
+        private async Task _RunAsync(CJob job, CancellationToken ct)
+        {
+            var dto = job.Dto;
+            dto.State = "running";
+            var name = CSourceHelper.SafeName(dto.Artist.Length > 0 ? dto.Artist + " - " + dto.Title : dto.Title);
+            if (name.Length == 0)
+            {
+                name = dto.Source + "-" + CSourceHelper.SafeName(dto.SourceId);
+            }
+
+            var target = Path.Combine(DownloadRoot, dto.Source, name);
+            var temp = Path.Combine(DownloadRoot, ".incoming", dto.Id);
+            try
+            {
+                if (Directory.Exists(target))
+                {
+                    dto.State = "done";
+                    dto.Progress = 1;
+                    dto.Message = "Already downloaded";
+                    return;
+                }
+
+                Directory.CreateDirectory(temp);
+                await job.Source.DownloadAsync(dto.SourceId, temp, new Progress<double>(p => dto.Progress = Math.Clamp(p, 0, 1)), ct);
+                if (!Directory.EnumerateFiles(temp, "*.txt").Any(f => !Path.GetFileName(f).Equals("license.txt", StringComparison.OrdinalIgnoreCase)))
+                {
+                    throw new InvalidOperationException("The download contains no song file");
+                }
+
+                Directory.CreateDirectory(Path.GetDirectoryName(target)!);
+                Directory.Move(temp, target);
+                dto.State = "done";
+                dto.Progress = 1;
+                dto.Message = "";
+                _Logger.LogInformation("Downloaded {Artist} - {Title} from {Source}", dto.Artist, dto.Title, dto.Source);
+                _Index.RequestRescan();
+            }
+            catch (Exception e)
+            {
+                dto.State = "failed";
+                dto.Message = e is OperationCanceledException ? "cancelled" : e.Message;
+                _Logger.LogWarning("Download of {Artist} - {Title} from {Source} failed: {Message}", dto.Artist, dto.Title, dto.Source, e.Message);
+            }
+            finally
+            {
+                try
+                {
+                    if (Directory.Exists(temp))
+                    {
+                        Directory.Delete(temp, true);
+                    }
+                }
+                catch (IOException)
+                {
+                    // left for the next start
+                }
+            }
+        }
+    }
+}

VocaluxeSongServer/Sources/CUsdbSongSource.cs

@@ -0,0 +1,369 @@
+using System.Net;
+using System.Text;
+using System.Text.RegularExpressions;
+using VocaluxeSongServer.Contracts;
+
+namespace VocaluxeSongServer.Sources
+{
+    public sealed class CUsdbOptions
+    {
+        public string User { get; set; } = "";
+        public string Password { get; set; } = "";
+    }
+
+    /// <summary>
+    ///     USDB (usdb.animux.de), the largest UltraStar database. It has only the song text files and needs a (free)
+    ///     account. Audio, video and images come from the sources the song references: the #VIDEO tag
+    ///     ("v=&lt;YouTube&gt;,a=&lt;YouTube&gt;,co=&lt;fanart.tv&gt;,bg=...", the format of usdb_syncer) or YouTube links in the
+    ///     comments. Media are downloaded with yt-dlp.
+    /// </summary>
+    public sealed partial class CUsdbSongSource : ISongSource
+    {
+        public const string BaseUrl = "https://usdb.animux.de/";
+        private const string _NotLoggedIn = "You are not logged in";
+        private const string _LoginInvalid = "Login or Password invalid";
+
+        private readonly HttpClient _Http;
+        private readonly CUsdbOptions _Options;
+        private readonly CYtDlp _YtDlp;
+        private readonly ILogger<CUsdbSongSource> _Logger;
+        private readonly SemaphoreSlim _LoginLock = new(1, 1);
+        private bool _LoggedIn;
+
+        /// <param name="http">A client with its own cookie container (the USDB session is a cookie).</param>
+        public CUsdbSongSource(HttpClient http, CUsdbOptions options, CYtDlp ytDlp, ILogger<CUsdbSongSource> logger)
+        {
+            _Http = http;
+            _Options = options;
+            _YtDlp = ytDlp;
+            _Logger = logger;
+        }
+
+        public string Id => "usdb";
+        public string Name => "USDB";
+        public string Description => "usdb.animux.de, the largest UltraStar database (notes and lyrics); audio and video from the YouTube links of the song";
+
+        public bool Enabled => _Options.User.Length > 0 && _Options.Password.Length > 0;
+
+        public string Hint => Enabled
+            ? (_YtDlp.Available ? "" : "yt-dlp is not installed, songs can't be downloaded")
+            : "Needs a free account at usdb.animux.de (settings Usdb__User and Usdb__Password)";
+
+        #region HTTP
+        private async Task _LoginAsync(CancellationToken ct)
+        {
+            await _LoginLock.WaitAsync(ct);
+            try
+            {
+                if (_LoggedIn)
+                {
+                    return;
+                }
+
+                var form = new FormUrlEncodedContent(new Dictionary<string, string> { ["user"] = _Options.User, ["pass"] = _Options.Password, ["login"] = "Login" });
+                using var response = await _Http.PostAsync(BaseUrl, form, ct);
+                var html = await response.Content.ReadAsStringAsync(ct);
+                if (!response.IsSuccessStatusCode || html.Contains(_LoginInvalid))
+                {
+                    throw new InvalidOperationException("USDB login failed (check user and password)");
+                }
+
+                _LoggedIn = true;
+            }
+            finally
+            {
+                _LoginLock.Release();
+            }
+        }
+
+        /// <summary>Sends a request (logs in first and again when the session expired) and returns the page.</summary>
+        private async Task<string> _RequestAsync(Func<HttpRequestMessage> createRequest, CancellationToken ct)
+        {
+            for (var attempt = 0; attempt < 2; attempt++)
+            {
+                await _LoginAsync(ct);
+                using var request = createRequest();
+                using var response = await _Http.SendAsync(request, ct);
+                response.EnsureSuccessStatusCode();
+                var bytes = await response.Content.ReadAsByteArrayAsync(ct);
+                var html = Encoding.UTF8.GetString(bytes);
+                if (!html.Contains(_NotLoggedIn))
+                {
+                    return html;
+                }
+
+                _LoggedIn = false;
+            }
+
+            throw new InvalidOperationException("USDB: not logged in after login");
+        }
+        #endregion
+
+        #region Search
+        [GeneratedRegex(@"<tr class=""list_tr\d""[^>]*data-songid=""(\d+)""[^>]*>(.*?)</tr>", RegexOptions.Singleline)]
+        private static partial Regex _Row();
+
+        [GeneratedRegex(@"<td[^>]*>(.*?)</td>", RegexOptions.Singleline)]
+        private static partial Regex _Cell();
+
+        [GeneratedRegex(@"<[^>]+>")]
+        private static partial Regex _Tag();
+
+        private static string _Text(string html)
+        {
+            return WebUtility.HtmlDecode(_Tag().Replace(html, " ")).Trim();
+        }
+
+        /// <summary>Parses the result list (link=list with details=1): id, artist, title and the year if there is one.</summary>
+        public static List<SearchResultDto> ParseList(string html)
+        {
+            var results = new List<SearchResultDto>();
+            foreach (Match row in _Row().Matches(html))
+            {
+                // Cells: sample, cover (no text), artist, title, genre, year, edition, golden notes, language, creator, ...
+                var cells = _Cell().Matches(row.Groups[2].Value).Select(c => _Text(c.Groups[1].Value)).Where(t => t.Length > 0).ToList();
+                if (cells.Count < 2)
+                {
+                    continue;
+                }
+
+                var id = row.Groups[1].Value;
+                var year = cells.Skip(2).FirstOrDefault(c => Regex.IsMatch(c, @"^(19|20)\d\d$"));
+                results.Add(new SearchResultDto
+                {
+                    Source = "usdb",
+                    SourceId = id,
+                    Artist = cells[0],
+                    Title = cells[1],
+                    Details = "USDB #" + id + (year != null ? " · " + year : ""),
+                    CoverUrl = BaseUrl + "data/cover/" + id + ".jpg"
+                });
+            }
+
+            return results;
+        }
+
+        public async Task<List<SearchResultDto>> SearchAsync(string query, CancellationToken ct)
+        {
+            // USDB searches artist and title separately; a query matches either.
+            async Task<List<SearchResultDto>> Search(string field)
+            {
+                var html = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Post, BaseUrl + "index.php?link=list")
+                {
+                    Content = new FormUrlEncodedContent(new Dictionary<string, string>
+                    {
+                        [field] = query, ["order"] = "rating", ["ud"] = "desc", ["limit"] = "50", ["details"] = "1", ["start"] = "0"
+                    })
+                }, ct);
+                return ParseList(html);
+            }
+
+            var byArtist = await Search("interpret");
+            var byTitle = await Search("title");
+            return byArtist.Concat(byTitle).GroupBy(r => r.SourceId).Select(g => g.First()).ToList();
+        }
+        #endregion
+
+        #region Download
+        [GeneratedRegex(@"<textarea[^>]*>(.*?)</textarea>", RegexOptions.Singleline)]
+        private static partial Regex _TextArea();
+
+        /// <summary>Media hints of a song text file (#VIDEO tag in the usdb_syncer format).</summary>
+        public sealed record CMediaHints(string? Video, string? Audio, string? Cover, string? Background);
+
+        public static CMediaHints ParseVideoTag(string? value)
+        {
+            if (string.IsNullOrWhiteSpace(value) || !value.Contains('='))
+            {
+                return new CMediaHints(null, null, null, null);
+            }
+
+            var pairs = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+            foreach (var pair in value.Split(','))
+            {
+                var eq = pair.IndexOf('=');
+                if (eq > 0)
+                {
+                    pairs[pair[..eq].Trim()] = pair[(eq + 1)..].Trim().Replace("%2C", ",");
+                }
+            }
+
+            return new CMediaHints(pairs.GetValueOrDefault("v"), pairs.GetValueOrDefault("a"), pairs.GetValueOrDefault("co"), pairs.GetValueOrDefault("bg"));
+        }
+
+        /// <summary>URL of a cover/background hint: full URL, host/path, or a fanart.tv id.</summary>
+        public static string ImageUrl(string source)
+        {
+            if (source.Contains("://"))
+            {
+                return source.Replace("images.fanart.tv", "assets.fanart.tv");
+            }
+
+            return source.Contains('/') ? "https://" + source : "https://assets.fanart.tv/fanart/" + source;
+        }
+
+        /// <summary>Splits a song text file into header tags (in order) and the rest (notes).</summary>
+        public static (List<(string Tag, string Value)> Header, string Body) SplitSong(string text)
+        {
+            var header = new List<(string, string)>();
+            var lines = text.Replace("\r\n", "\n").Split('\n');
+            var i = 0;
+            for (; i < lines.Length; i++)
+            {
+                var line = lines[i].Trim();
+                if (line.Length == 0)
+                {
+                    continue;
+                }
+
+                if (line[0] != '#')
+                {
+                    break;
+                }
+
+                var colon = line.IndexOf(':');
+                header.Add(colon > 0 ? (line[1..colon].Trim().ToUpperInvariant(), line[(colon + 1)..].Trim()) : (line[1..].Trim().ToUpperInvariant(), ""));
+            }
+
+            return (header, string.Join("\n", lines.Skip(i)));
+        }
+
+        /// <summary>The song text with the media tags replaced by the downloaded files (null = tag removed).</summary>
+        public static string RewriteSong(string text, IReadOnlyDictionary<string, string?> media)
+        {
+            var (header, body) = SplitSong(text);
+            var sb = new StringBuilder();
+            foreach (var (tag, value) in header)
+            {
+                if (tag is "MP3" or "AUDIO" or "VIDEO" or "COVER" or "BACKGROUND" or "ENCODING")
+                {
+                    continue;
+                }
+
+                sb.Append('#').Append(tag).Append(':').Append(value).Append('\n');
+            }
+
+            foreach (var (tag, value) in media)
+            {
+                if (!string.IsNullOrEmpty(value))
+                {
+                    sb.Append('#').Append(tag).Append(':').Append(value).Append('\n');
+                }
+            }
+
+            return sb.Append(body.TrimEnd('\n')).Append('\n').ToString();
+        }
+
+        public async Task DownloadAsync(string sourceId, string folder, IProgress<double> progress, CancellationToken ct)
+        {
+            if (!int.TryParse(sourceId, out var id))
+            {
+                throw new ArgumentException("Invalid USDB id " + sourceId);
+            }
+
+            if (!_YtDlp.Available)
+            {
+                throw new InvalidOperationException("yt-dlp is not installed, USDB songs can't be downloaded");
+            }
+
+            var page = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Post, BaseUrl + "index.php?link=gettxt&id=" + id)
+            {
+                Content = new FormUrlEncodedContent(new Dictionary<string, string> { ["wd"] = "1" })
+            }, ct);
+            var area = _TextArea().Match(page);
+            if (!area.Success)
+            {
+                throw new InvalidOperationException("USDB returned no song text for #" + id);
+            }
+
+            var text = WebUtility.HtmlDecode(area.Groups[1].Value);
+            var (header, _) = SplitSong(text);
+            var hints = ParseVideoTag(header.FirstOrDefault(h => h.Tag == "VIDEO").Value);
+            var artist = header.FirstOrDefault(h => h.Tag == "ARTIST").Value ?? "";
+            var title = header.FirstOrDefault(h => h.Tag == "TITLE").Value ?? "";
+            progress.Report(0.02);
+
+            // Media: the #VIDEO hints, otherwise the YouTube links in the comments of the song page.
+            var videoId = hints.Video != null ? CYtDlp.YouTubeId(hints.Video) : null;
+            var audioId = hints.Audio != null ? CYtDlp.YouTubeId(hints.Audio) : null;
+            if (videoId == null && audioId == null)
+            {
+                var detail = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Get, BaseUrl + "index.php?link=detail&id=" + id), ct);
+                videoId = CYtDlp.FindYouTubeIds(detail).FirstOrDefault();
+            }
+
+            audioId ??= videoId;
+            if (audioId == null)
+            {
+                throw new InvalidOperationException("No audio source known for this song (no YouTube link in the song or its comments)");
+            }
+
+            var media = new Dictionary<string, string?>();
+            media["MP3"] = await _YtDlp.DownloadAsync("https://www.youtube.com/watch?v=" + audioId, folder, "audio",
+                "bestaudio[ext=m4a]/bestaudio", new CScaledProgress(progress, 0.05, 0.45), ct);
+            if (videoId != null)
+            {
+                try
+                {
+                    media["VIDEO"] = await _YtDlp.DownloadAsync("https://www.youtube.com/watch?v=" + videoId, folder, "video",
+                        "bestvideo[height<=720][ext=mp4]/bestvideo[height<=720]/best[height<=720]", new CScaledProgress(progress, 0.45, 0.9), ct);
+                }
+                catch (Exception e) when (e is not OperationCanceledException)
+                {
+                    _Logger.LogWarning("USDB #{Id}: video download failed, the song is saved without video: {Message}", id, e.Message);
+                }
+            }
+
+            media["COVER"] = await _DownloadImageAsync(hints.Cover != null ? ImageUrl(hints.Cover) : (videoId ?? audioId) is { } yt ? $"https://i.ytimg.com/vi/{yt}/hqdefault.jpg" : null,
+                folder, "cover", ct);
+            media["BACKGROUND"] = await _DownloadImageAsync(hints.Background != null ? ImageUrl(hints.Background) : null, folder, "background", ct);
+
+            var name = CSourceHelper.SafeName(artist + " - " + title);
+            await File.WriteAllTextAsync(Path.Combine(folder, name + ".txt"), RewriteSong(text, media), new UTF8Encoding(true), ct);
+            progress.Report(1);
+        }
+
+        private async Task<string?> _DownloadImageAsync(string? url, string folder, string baseName, CancellationToken ct)
+        {
+            if (url == null)
+            {
+                return null;
+            }
+
+            try
+            {
+                using var response = await _Http.GetAsync(url, ct);
+                response.EnsureSuccessStatusCode();
+                var ext = response.Content.Headers.ContentType?.MediaType == "image/png" ? ".png" : ".jpg";
+                await File.WriteAllBytesAsync(Path.Combine(folder, baseName + ext), await response.Content.ReadAsByteArrayAsync(ct), ct);
+                return baseName + ext;
+            }
+            catch (Exception e) when (e is not OperationCanceledException)
+            {
+                _Logger.LogInformation("Image {Url} not downloaded: {Message}", url, e.Message);
+                return null;
+            }
+        }
+        #endregion
+    }
+
+    /// <summary>Maps the progress of a part of a job (0..1) to a range of the whole job.</summary>
+    public sealed class CScaledProgress : IProgress<double>
+    {
+        private readonly IProgress<double> _Target;
+        private readonly double _From;
+        private readonly double _To;
+
+        public CScaledProgress(IProgress<double> target, double from, double to)
+        {
+            _Target = target;
+            _From = from;
+            _To = to;
+        }
+
+        public void Report(double value)
+        {
+            _Target.Report(_From + (_To - _From) * Math.Clamp(value, 0, 1));
+        }
+    }
+}

VocaluxeSongServer/Sources/CYtDlp.cs

@@ -0,0 +1,119 @@
+using System.Diagnostics;
+using System.Globalization;
+using System.Text.RegularExpressions;
+
+namespace VocaluxeSongServer.Sources
+{
+    /// <summary>Downloads audio/video with yt-dlp (installed in the Docker image; ffmpeg is needed for merging).</summary>
+    public sealed partial class CYtDlp
+    {
+        private readonly string _Executable;
+        private readonly ILogger<CYtDlp> _Logger;
+
+        public CYtDlp(IConfiguration config, ILogger<CYtDlp> logger)
+        {
+            _Executable = config["YtDlp:Path"] is { Length: > 0 } path ? path : "yt-dlp";
+            _Logger = logger;
+        }
+
+        private bool? _Available;
+
+        public bool Available
+        {
+            get
+            {
+                if (_Available == null)
+                {
+                    try
+                    {
+                        using var p = Process.Start(new ProcessStartInfo(_Executable, "--version") { RedirectStandardOutput = true, RedirectStandardError = true });
+                        p!.WaitForExit(15000);
+                        _Available = p.ExitCode == 0;
+                    }
+                    catch (Exception)
+                    {
+                        _Available = false;
+                    }
+                }
+
+                return _Available.Value;
+            }
+        }
+
+        [GeneratedRegex(@"(?:youtube\.com/(?:watch\?(?:.*&)?v=|embed/|shorts/|v/)|youtu\.be/)([A-Za-z0-9_-]{11})")]
+        private static partial Regex _YouTubeUrl();
+
+        [GeneratedRegex(@"^[A-Za-z0-9_-]{11}$")]
+        private static partial Regex _YouTubeIdOnly();
+
+        [GeneratedRegex(@"\[download\]\s+([\d.]+)%")]
+        private static partial Regex _ProgressLine();
+
+        /// <summary>A YouTube id from an id or URL, or null.</summary>
+        public static string? YouTubeId(string value)
+        {
+            value = value.Trim();
+            if (_YouTubeIdOnly().IsMatch(value))
+            {
+                return value;
+            }
+
+            var m = _YouTubeUrl().Match(value);
+            return m.Success ? m.Groups[1].Value : null;
+        }
+
+        /// <summary>All YouTube ids in a text (e.g. HTML of comments), in order, without duplicates.</summary>
+        public static List<string> FindYouTubeIds(string text)
+        {
+            return _YouTubeUrl().Matches(text).Select(m => m.Groups[1].Value).Distinct().ToList();
+        }
+
+        /// <summary>
+        ///     Downloads into <paramref name="folder" /> as "<paramref name="baseName" />.ext" and returns the file name.
+        ///     <paramref name="format" /> is a yt-dlp format selector.
+        /// </summary>
+        public async Task<string> DownloadAsync(string url, string folder, string baseName, string format, IProgress<double>? progress, CancellationToken ct)
+        {
+            var psi = new ProcessStartInfo(_Executable)
+            {
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                WorkingDirectory = folder
+            };
+            foreach (var arg in new[] { "--no-playlist", "--newline", "--no-mtime", "-f", format, "-o", baseName + ".%(ext)s", "--", url })
+            {
+                psi.ArgumentList.Add(arg);
+            }
+
+            using var process = Process.Start(psi) ?? throw new InvalidOperationException("yt-dlp could not be started");
+            var errors = new List<string>();
+            var stderr = Task.Run(async () =>
+            {
+                while (await process.StandardError.ReadLineAsync(ct) is { } line)
+                {
+                    errors.Add(line);
+                }
+            }, ct);
+            while (await process.StandardOutput.ReadLineAsync(ct) is { } line)
+            {
+                var m = _ProgressLine().Match(line);
+                if (m.Success && double.TryParse(m.Groups[1].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out var percent))
+                {
+                    progress?.Report(percent / 100);
+                }
+            }
+
+            await process.WaitForExitAsync(ct);
+            await stderr;
+            if (process.ExitCode != 0)
+            {
+                var error = errors.LastOrDefault(e => e.Contains("ERROR")) ?? errors.LastOrDefault() ?? "exit code " + process.ExitCode;
+                _Logger.LogWarning("yt-dlp failed for {Url}: {Error}", url, error);
+                throw new InvalidOperationException("yt-dlp: " + error);
+            }
+
+            var file = Directory.EnumerateFiles(folder, baseName + ".*").Select(Path.GetFileName).FirstOrDefault(f => !f!.EndsWith(".part"));
+            return file ?? throw new InvalidOperationException("yt-dlp finished but no file was written");
+        }
+    }
+}

VocaluxeSongServer/Sources/ISongSource.cs

@@ -0,0 +1,73 @@
+using System.Globalization;
+using System.Text;
+using VocaluxeSongServer.Contracts;
+
+namespace VocaluxeSongServer.Sources
+{
+    /// <summary>A place where new songs can be found and downloaded (song database, song repository, ...).</summary>
+    public interface ISongSource
+    {
+        string Id { get; }
+        string Name { get; }
+        string Description { get; }
+        bool Enabled { get; }
+        /// <summary>Why the source is disabled, or empty.</summary>
+        string Hint { get; }
+
+        Task<List<SearchResultDto>> SearchAsync(string query, CancellationToken ct);
+
+        /// <summary>
+        ///     Downloads the song into <paramref name="folder" /> (created by the caller, empty). Must leave a playable
+        ///     UltraStar song there (text file + audio) or throw.
+        /// </summary>
+        Task DownloadAsync(string sourceId, string folder, IProgress<double> progress, CancellationToken ct);
+    }
+
+    public static class CSourceHelper
+    {
+        /// <summary>Lower case, without diacritics and punctuation, single spaces: for matching artists and titles.</summary>
+        public static string Normalize(string s)
+        {
+            var sb = new StringBuilder();
+            foreach (var c in s.Normalize(NormalizationForm.FormD))
+            {
+                var cat = CharUnicodeInfo.GetUnicodeCategory(c);
+                if (cat == UnicodeCategory.NonSpacingMark)
+                {
+                    continue;
+                }
+
+                sb.Append(char.IsLetterOrDigit(c) ? char.ToLowerInvariant(c) : ' ');
+            }
+
+            return string.Join(' ', sb.ToString().Split(' ', StringSplitOptions.RemoveEmptyEntries));
+        }
+
+        /// <summary>True if every word of the query occurs in the text (both normalized).</summary>
+        public static bool Matches(string normalizedQuery, string text)
+        {
+            var normalized = " " + Normalize(text) + " ";
+            return normalizedQuery.Split(' ', StringSplitOptions.RemoveEmptyEntries).All(w => normalized.Contains(w));
+        }
+
+        /// <summary>A file or folder name without characters that are invalid on Windows or Linux.</summary>
+        public static string SafeName(string name)
+        {
+            var invalid = "<>:\"/\\|?*".ToCharArray();
+            var cleaned = new string(name.Select(c => invalid.Contains(c) || char.IsControl(c) ? '_' : c).ToArray()).Trim().TrimEnd('.');
+            return cleaned.Length > 120 ? cleaned[..120] : cleaned;
+        }
+
+        /// <summary>Copies a stream and reports the bytes copied.</summary>
+        public static async Task CopyAsync(Stream source, Stream target, Action<long> onBytes, CancellationToken ct)
+        {
+            var buffer = new byte[81920];
+            int read;
+            while ((read = await source.ReadAsync(buffer, ct)) > 0)
+            {
+                await target.WriteAsync(buffer.AsMemory(0, read), ct);
+                onBytes(read);
+            }
+        }
+    }
+}