using System.Net; using System.Text; using System.Text.RegularExpressions; using VocaluxeSongServer.Contracts; namespace VocaluxeSongServer.Sources { public sealed class CUsdbOptions { public string User { get; set; } = ""; public string Password { get; set; } = ""; } /// /// USDB (usdb.animux.de), the largest UltraStar database. It has only the song text files and needs a (free) /// account. Audio, video and images come from the sources the song references: the #VIDEO tag /// ("v=<YouTube>,a=<YouTube>,co=<fanart.tv>,bg=...", the format of usdb_syncer) or YouTube links in the /// comments. Media are downloaded with yt-dlp. /// public sealed partial class CUsdbSongSource : ISongSource { public const string BaseUrl = "https://usdb.animux.de/"; private const string _NotLoggedIn = "You are not logged in"; private const string _LoginInvalid = "Login or Password invalid"; private readonly HttpClient _Http; private readonly CUsdbOptions _Options; private readonly CYtDlp _YtDlp; private readonly ILogger _Logger; private readonly SemaphoreSlim _LoginLock = new(1, 1); private bool _LoggedIn; /// A client with its own cookie container (the USDB session is a cookie). public CUsdbSongSource(HttpClient http, CUsdbOptions options, CYtDlp ytDlp, ILogger logger) { _Http = http; _Options = options; _YtDlp = ytDlp; _Logger = logger; } public string Id => "usdb"; public string Name => "USDB"; public string Description => "usdb.animux.de, the largest UltraStar database (notes and lyrics); audio and video from the YouTube links of the song"; public bool Enabled => _Options.User.Length > 0 && _Options.Password.Length > 0; public string Hint => Enabled ? (_YtDlp.Available ? "" : "yt-dlp is not installed, songs can't be downloaded") : "Needs a free account at usdb.animux.de (settings Usdb__User and Usdb__Password)"; #region HTTP private async Task _LoginAsync(CancellationToken ct) { await _LoginLock.WaitAsync(ct); try { if (_LoggedIn) { return; } var form = new FormUrlEncodedContent(new Dictionary { ["user"] = _Options.User, ["pass"] = _Options.Password, ["login"] = "Login" }); using var response = await _Http.PostAsync(BaseUrl, form, ct); var html = await response.Content.ReadAsStringAsync(ct); if (!response.IsSuccessStatusCode || html.Contains(_LoginInvalid)) { throw new InvalidOperationException("USDB login failed (check user and password)"); } _LoggedIn = true; } finally { _LoginLock.Release(); } } /// Sends a request (logs in first and again when the session expired) and returns the page. private async Task _RequestAsync(Func createRequest, CancellationToken ct) { for (var attempt = 0; attempt < 2; attempt++) { await _LoginAsync(ct); using var request = createRequest(); using var response = await _Http.SendAsync(request, ct); response.EnsureSuccessStatusCode(); var bytes = await response.Content.ReadAsByteArrayAsync(ct); var html = Encoding.UTF8.GetString(bytes); if (!html.Contains(_NotLoggedIn)) { return html; } _LoggedIn = false; } throw new InvalidOperationException("USDB: not logged in after login"); } #endregion #region Search [GeneratedRegex(@"]*data-songid=""(\d+)""[^>]*>(.*?)", RegexOptions.Singleline)] private static partial Regex _Row(); [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline)] private static partial Regex _Cell(); [GeneratedRegex(@"<[^>]+>")] private static partial Regex _Tag(); private static string _Text(string html) { return WebUtility.HtmlDecode(_Tag().Replace(html, " ")).Trim(); } /// Parses the result list (link=list with details=1): id, artist, title and the year if there is one. public static List ParseList(string html) { var results = new List(); foreach (Match row in _Row().Matches(html)) { // Cells: sample, cover (no text), artist, title, genre, year, edition, golden notes, language, creator, ... var cells = _Cell().Matches(row.Groups[2].Value).Select(c => _Text(c.Groups[1].Value)).Where(t => t.Length > 0).ToList(); if (cells.Count < 2) { continue; } var id = row.Groups[1].Value; var year = cells.Skip(2).FirstOrDefault(c => Regex.IsMatch(c, @"^(19|20)\d\d$")); results.Add(new SearchResultDto { Source = "usdb", SourceId = id, Artist = cells[0], Title = cells[1], Details = "USDB #" + id + (year != null ? " ยท " + year : ""), CoverUrl = BaseUrl + "data/cover/" + id + ".jpg" }); } return results; } public async Task> SearchAsync(string query, CancellationToken ct) { // USDB searches artist and title separately; a query matches either. async Task> Search(string field) { var html = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Post, BaseUrl + "index.php?link=list") { Content = new FormUrlEncodedContent(new Dictionary { [field] = query, ["order"] = "rating", ["ud"] = "desc", ["limit"] = "50", ["details"] = "1", ["start"] = "0" }) }, ct); return ParseList(html); } var byArtist = await Search("interpret"); var byTitle = await Search("title"); return byArtist.Concat(byTitle).GroupBy(r => r.SourceId).Select(g => g.First()).ToList(); } #endregion #region Download [GeneratedRegex(@"]*>(.*?)", RegexOptions.Singleline)] private static partial Regex _TextArea(); /// Media hints of a song text file (#VIDEO tag in the usdb_syncer format). public sealed record CMediaHints(string? Video, string? Audio, string? Cover, string? Background); public static CMediaHints ParseVideoTag(string? value) { if (string.IsNullOrWhiteSpace(value) || !value.Contains('=')) { return new CMediaHints(null, null, null, null); } var pairs = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var pair in value.Split(',')) { var eq = pair.IndexOf('='); if (eq > 0) { pairs[pair[..eq].Trim()] = pair[(eq + 1)..].Trim().Replace("%2C", ","); } } return new CMediaHints(pairs.GetValueOrDefault("v"), pairs.GetValueOrDefault("a"), pairs.GetValueOrDefault("co"), pairs.GetValueOrDefault("bg")); } /// URL of a cover/background hint: full URL, host/path, or a fanart.tv id. public static string ImageUrl(string source) { if (source.Contains("://")) { return source.Replace("images.fanart.tv", "assets.fanart.tv"); } return source.Contains('/') ? "https://" + source : "https://assets.fanart.tv/fanart/" + source; } /// Splits a song text file into header tags (in order) and the rest (notes). public static (List<(string Tag, string Value)> Header, string Body) SplitSong(string text) { var header = new List<(string, string)>(); var lines = text.Replace("\r\n", "\n").Split('\n'); var i = 0; for (; i < lines.Length; i++) { var line = lines[i].Trim(); if (line.Length == 0) { continue; } if (line[0] != '#') { break; } var colon = line.IndexOf(':'); header.Add(colon > 0 ? (line[1..colon].Trim().ToUpperInvariant(), line[(colon + 1)..].Trim()) : (line[1..].Trim().ToUpperInvariant(), "")); } return (header, string.Join("\n", lines.Skip(i))); } /// The song text with the media tags replaced by the downloaded files (null = tag removed). public static string RewriteSong(string text, IReadOnlyDictionary media) { var (header, body) = SplitSong(text); var sb = new StringBuilder(); foreach (var (tag, value) in header) { if (tag is "MP3" or "AUDIO" or "VIDEO" or "COVER" or "BACKGROUND" or "ENCODING") { continue; } sb.Append('#').Append(tag).Append(':').Append(value).Append('\n'); } foreach (var (tag, value) in media) { if (!string.IsNullOrEmpty(value)) { sb.Append('#').Append(tag).Append(':').Append(value).Append('\n'); } } return sb.Append(body.TrimEnd('\n')).Append('\n').ToString(); } public async Task DownloadAsync(string sourceId, string folder, IProgress progress, CancellationToken ct) { if (!int.TryParse(sourceId, out var id)) { throw new ArgumentException("Invalid USDB id " + sourceId); } if (!_YtDlp.Available) { throw new InvalidOperationException("yt-dlp is not installed, USDB songs can't be downloaded"); } var page = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Post, BaseUrl + "index.php?link=gettxt&id=" + id) { Content = new FormUrlEncodedContent(new Dictionary { ["wd"] = "1" }) }, ct); var area = _TextArea().Match(page); if (!area.Success) { throw new InvalidOperationException("USDB returned no song text for #" + id); } var text = WebUtility.HtmlDecode(area.Groups[1].Value); var (header, _) = SplitSong(text); var hints = ParseVideoTag(header.FirstOrDefault(h => h.Tag == "VIDEO").Value); var artist = header.FirstOrDefault(h => h.Tag == "ARTIST").Value ?? ""; var title = header.FirstOrDefault(h => h.Tag == "TITLE").Value ?? ""; progress.Report(0.02); // Media: the #VIDEO hints, otherwise the YouTube links in the comments of the song page. var videoId = hints.Video != null ? CYtDlp.YouTubeId(hints.Video) : null; var audioId = hints.Audio != null ? CYtDlp.YouTubeId(hints.Audio) : null; if (videoId == null && audioId == null) { var detail = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Get, BaseUrl + "index.php?link=detail&id=" + id), ct); videoId = CYtDlp.FindYouTubeIds(detail).FirstOrDefault(); } audioId ??= videoId; if (audioId == null) { throw new InvalidOperationException("No audio source known for this song (no YouTube link in the song or its comments)"); } var media = new Dictionary(); media["MP3"] = await _YtDlp.DownloadAsync("https://www.youtube.com/watch?v=" + audioId, folder, "audio", "bestaudio[ext=m4a]/bestaudio", new CScaledProgress(progress, 0.05, 0.45), ct); if (videoId != null) { try { media["VIDEO"] = await _YtDlp.DownloadAsync("https://www.youtube.com/watch?v=" + videoId, folder, "video", "bestvideo[height<=720][ext=mp4]/bestvideo[height<=720]/best[height<=720]", new CScaledProgress(progress, 0.45, 0.9), ct); } catch (Exception e) when (e is not OperationCanceledException) { _Logger.LogWarning("USDB #{Id}: video download failed, the song is saved without video: {Message}", id, e.Message); } } media["COVER"] = await _DownloadImageAsync(hints.Cover != null ? ImageUrl(hints.Cover) : (videoId ?? audioId) is { } yt ? $"https://i.ytimg.com/vi/{yt}/hqdefault.jpg" : null, folder, "cover", ct); media["BACKGROUND"] = await _DownloadImageAsync(hints.Background != null ? ImageUrl(hints.Background) : null, folder, "background", ct); var name = CSourceHelper.SafeName(artist + " - " + title); await File.WriteAllTextAsync(Path.Combine(folder, name + ".txt"), RewriteSong(text, media), new UTF8Encoding(true), ct); progress.Report(1); } private async Task _DownloadImageAsync(string? url, string folder, string baseName, CancellationToken ct) { if (url == null) { return null; } try { using var response = await _Http.GetAsync(url, ct); response.EnsureSuccessStatusCode(); var ext = response.Content.Headers.ContentType?.MediaType == "image/png" ? ".png" : ".jpg"; await File.WriteAllBytesAsync(Path.Combine(folder, baseName + ext), await response.Content.ReadAsByteArrayAsync(ct), ct); return baseName + ext; } catch (Exception e) when (e is not OperationCanceledException) { _Logger.LogInformation("Image {Url} not downloaded: {Message}", url, e.Message); return null; } } #endregion } /// Maps the progress of a part of a job (0..1) to a range of the whole job. public sealed class CScaledProgress : IProgress { private readonly IProgress _Target; private readonly double _From; private readonly double _To; public CScaledProgress(IProgress target, double from, double to) { _Target = target; _From = from; _To = to; } public void Report(double value) { _Target.Report(_From + (_To - _From) * Math.Clamp(value, 0, 1)); } } }