Vocaluxe/VocaluxeSongServer/Sources/CUsdbSongSource.cs

using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using VocaluxeSongServer.Contracts;

namespace VocaluxeSongServer.Sources
{
    public sealed class CUsdbOptions
    {
        public string User { get; set; } = "";
        public string Password { get; set; } = "";
    }

    /// <summary>
    ///     USDB (usdb.animux.de), the largest UltraStar database. It has only the song text files and needs a (free)
    ///     account. Audio, video and images come from the sources the song references: the #VIDEO tag
    ///     ("v=&lt;YouTube&gt;,a=&lt;YouTube&gt;,co=&lt;fanart.tv&gt;,bg=...", the format of usdb_syncer) or YouTube links in the
    ///     comments. Media are downloaded with yt-dlp.
    /// </summary>
    public sealed partial class CUsdbSongSource : ISongSource
    {
        public const string BaseUrl = "https://usdb.animux.de/";
        private const string _NotLoggedIn = "You are not logged in";
        private const string _LoginInvalid = "Login or Password invalid";

        private readonly HttpClient _Http;
        private readonly CUsdbOptions _Options;
        private readonly CYtDlp _YtDlp;
        private readonly ILogger<CUsdbSongSource> _Logger;
        private readonly SemaphoreSlim _LoginLock = new(1, 1);
        private bool _LoggedIn;

        /// <param name="http">A client with its own cookie container (the USDB session is a cookie).</param>
        public CUsdbSongSource(HttpClient http, CUsdbOptions options, CYtDlp ytDlp, ILogger<CUsdbSongSource> logger)
        {
            _Http = http;
            _Options = options;
            _YtDlp = ytDlp;
            _Logger = logger;
        }

        public string Id => "usdb";
        public string Name => "USDB";
        public string Description => "usdb.animux.de, the largest UltraStar database (notes and lyrics); audio and video from the YouTube links of the song";

        public bool Enabled => _Options.User.Length > 0 && _Options.Password.Length > 0;

        public string Hint => Enabled
            ? (_YtDlp.Available ? "" : "yt-dlp is not installed, songs can't be downloaded")
            : "Needs a free account at usdb.animux.de (settings Usdb__User and Usdb__Password)";

        #region HTTP
        private async Task _LoginAsync(CancellationToken ct)
        {
            await _LoginLock.WaitAsync(ct);
            try
            {
                if (_LoggedIn)
                {
                    return;
                }

                var form = new FormUrlEncodedContent(new Dictionary<string, string> { ["user"] = _Options.User, ["pass"] = _Options.Password, ["login"] = "Login" });
                using var response = await _Http.PostAsync(BaseUrl, form, ct);
                var html = await response.Content.ReadAsStringAsync(ct);
                if (!response.IsSuccessStatusCode || html.Contains(_LoginInvalid))
                {
                    throw new InvalidOperationException("USDB login failed (check user and password)");
                }

                _LoggedIn = true;
            }
            finally
            {
                _LoginLock.Release();
            }
        }

        /// <summary>Sends a request (logs in first and again when the session expired) and returns the page.</summary>
        private async Task<string> _RequestAsync(Func<HttpRequestMessage> createRequest, CancellationToken ct)
        {
            for (var attempt = 0; attempt < 2; attempt++)
            {
                await _LoginAsync(ct);
                using var request = createRequest();
                using var response = await _Http.SendAsync(request, ct);
                response.EnsureSuccessStatusCode();
                var bytes = await response.Content.ReadAsByteArrayAsync(ct);
                var html = Encoding.UTF8.GetString(bytes);
                if (!html.Contains(_NotLoggedIn))
                {
                    return html;
                }

                _LoggedIn = false;
            }

            throw new InvalidOperationException("USDB: not logged in after login");
        }
        #endregion

        #region Search
        [GeneratedRegex(@"<tr class=""list_tr\d""[^>]*data-songid=""(\d+)""[^>]*>(.*?)</tr>", RegexOptions.Singleline)]
        private static partial Regex _Row();

        [GeneratedRegex(@"<td[^>]*>(.*?)</td>", RegexOptions.Singleline)]
        private static partial Regex _Cell();

        [GeneratedRegex(@"<[^>]+>")]
        private static partial Regex _Tag();

        private static string _Text(string html)
        {
            return WebUtility.HtmlDecode(_Tag().Replace(html, " ")).Trim();
        }

        /// <summary>Parses the result list (link=list with details=1): id, artist, title and the year if there is one.</summary>
        public static List<SearchResultDto> ParseList(string html)
        {
            var results = new List<SearchResultDto>();
            foreach (Match row in _Row().Matches(html))
            {
                // Cells: sample, cover (no text), artist, title, genre, year, edition, golden notes, language, creator, ...
                var cells = _Cell().Matches(row.Groups[2].Value).Select(c => _Text(c.Groups[1].Value)).Where(t => t.Length > 0).ToList();
                if (cells.Count < 2)
                {
                    continue;
                }

                var id = row.Groups[1].Value;
                var year = cells.Skip(2).FirstOrDefault(c => Regex.IsMatch(c, @"^(19|20)\d\d$"));
                results.Add(new SearchResultDto
                {
                    Source = "usdb",
                    SourceId = id,
                    Artist = cells[0],
                    Title = cells[1],
                    Details = "USDB #" + id + (year != null ? " · " + year : ""),
                    CoverUrl = BaseUrl + "data/cover/" + id + ".jpg"
                });
            }

            return results;
        }

        public async Task<List<SearchResultDto>> SearchAsync(string query, CancellationToken ct)
        {
            // USDB searches artist and title separately; a query matches either.
            async Task<List<SearchResultDto>> Search(string field)
            {
                var html = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Post, BaseUrl + "index.php?link=list")
                {
                    Content = new FormUrlEncodedContent(new Dictionary<string, string>
                    {
                        [field] = query, ["order"] = "rating", ["ud"] = "desc", ["limit"] = "50", ["details"] = "1", ["start"] = "0"
                    })
                }, ct);
                return ParseList(html);
            }

            var byArtist = await Search("interpret");
            var byTitle = await Search("title");
            return byArtist.Concat(byTitle).GroupBy(r => r.SourceId).Select(g => g.First()).ToList();
        }
        #endregion

        #region Download
        [GeneratedRegex(@"<textarea[^>]*>(.*?)</textarea>", RegexOptions.Singleline)]
        private static partial Regex _TextArea();

        /// <summary>Media hints of a song text file (#VIDEO tag in the usdb_syncer format).</summary>
        public sealed record CMediaHints(string? Video, string? Audio, string? Cover, string? Background);

        public static CMediaHints ParseVideoTag(string? value)
        {
            if (string.IsNullOrWhiteSpace(value) || !value.Contains('='))
            {
                return new CMediaHints(null, null, null, null);
            }

            var pairs = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
            foreach (var pair in value.Split(','))
            {
                var eq = pair.IndexOf('=');
                if (eq > 0)
                {
                    pairs[pair[..eq].Trim()] = pair[(eq + 1)..].Trim().Replace("%2C", ",");
                }
            }

            return new CMediaHints(pairs.GetValueOrDefault("v"), pairs.GetValueOrDefault("a"), pairs.GetValueOrDefault("co"), pairs.GetValueOrDefault("bg"));
        }

        /// <summary>URL of a cover/background hint: full URL, host/path, or a fanart.tv id.</summary>
        public static string ImageUrl(string source)
        {
            if (source.Contains("://"))
            {
                return source.Replace("images.fanart.tv", "assets.fanart.tv");
            }

            return source.Contains('/') ? "https://" + source : "https://assets.fanart.tv/fanart/" + source;
        }

        /// <summary>Splits a song text file into header tags (in order) and the rest (notes).</summary>
        public static (List<(string Tag, string Value)> Header, string Body) SplitSong(string text)
        {
            var header = new List<(string, string)>();
            var lines = text.Replace("\r\n", "\n").Split('\n');
            var i = 0;
            for (; i < lines.Length; i++)
            {
                var line = lines[i].Trim();
                if (line.Length == 0)
                {
                    continue;
                }

                if (line[0] != '#')
                {
                    break;
                }

                var colon = line.IndexOf(':');
                header.Add(colon > 0 ? (line[1..colon].Trim().ToUpperInvariant(), line[(colon + 1)..].Trim()) : (line[1..].Trim().ToUpperInvariant(), ""));
            }

            return (header, string.Join("\n", lines.Skip(i)));
        }

        /// <summary>The song text with the media tags replaced by the downloaded files (null = tag removed).</summary>
        public static string RewriteSong(string text, IReadOnlyDictionary<string, string?> media)
        {
            var (header, body) = SplitSong(text);
            var sb = new StringBuilder();
            foreach (var (tag, value) in header)
            {
                if (tag is "MP3" or "AUDIO" or "VIDEO" or "COVER" or "BACKGROUND" or "ENCODING")
                {
                    continue;
                }

                sb.Append('#').Append(tag).Append(':').Append(value).Append('\n');
            }

            foreach (var (tag, value) in media)
            {
                if (!string.IsNullOrEmpty(value))
                {
                    sb.Append('#').Append(tag).Append(':').Append(value).Append('\n');
                }
            }

            return sb.Append(body.TrimEnd('\n')).Append('\n').ToString();
        }

        public async Task DownloadAsync(string sourceId, string folder, IProgress<double> progress, CancellationToken ct)
        {
            if (!int.TryParse(sourceId, out var id))
            {
                throw new ArgumentException("Invalid USDB id " + sourceId);
            }

            if (!_YtDlp.Available)
            {
                throw new InvalidOperationException("yt-dlp is not installed, USDB songs can't be downloaded");
            }

            var page = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Post, BaseUrl + "index.php?link=gettxt&id=" + id)
            {
                Content = new FormUrlEncodedContent(new Dictionary<string, string> { ["wd"] = "1" })
            }, ct);
            var area = _TextArea().Match(page);
            if (!area.Success)
            {
                throw new InvalidOperationException("USDB returned no song text for #" + id);
            }

            var text = WebUtility.HtmlDecode(area.Groups[1].Value);
            var (header, _) = SplitSong(text);
            var hints = ParseVideoTag(header.FirstOrDefault(h => h.Tag == "VIDEO").Value);
            var artist = header.FirstOrDefault(h => h.Tag == "ARTIST").Value ?? "";
            var title = header.FirstOrDefault(h => h.Tag == "TITLE").Value ?? "";
            progress.Report(0.02);

            // Media: the #VIDEO hints, otherwise the YouTube links in the comments of the song page.
            var videoId = hints.Video != null ? CYtDlp.YouTubeId(hints.Video) : null;
            var audioId = hints.Audio != null ? CYtDlp.YouTubeId(hints.Audio) : null;
            if (videoId == null && audioId == null)
            {
                var detail = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Get, BaseUrl + "index.php?link=detail&id=" + id), ct);
                videoId = CYtDlp.FindYouTubeIds(detail).FirstOrDefault();
            }

            audioId ??= videoId;
            if (audioId == null)
            {
                throw new InvalidOperationException("No audio source known for this song (no YouTube link in the song or its comments)");
            }

            var media = new Dictionary<string, string?>();
            media["MP3"] = await _YtDlp.DownloadAsync("https://www.youtube.com/watch?v=" + audioId, folder, "audio",
                "bestaudio[ext=m4a]/bestaudio", new CScaledProgress(progress, 0.05, 0.45), ct);
            if (videoId != null)
            {
                try
                {
                    media["VIDEO"] = await _YtDlp.DownloadAsync("https://www.youtube.com/watch?v=" + videoId, folder, "video",
                        "bestvideo[height<=720][ext=mp4]/bestvideo[height<=720]/best[height<=720]", new CScaledProgress(progress, 0.45, 0.9), ct);
                }
                catch (Exception e) when (e is not OperationCanceledException)
                {
                    _Logger.LogWarning("USDB #{Id}: video download failed, the song is saved without video: {Message}", id, e.Message);
                }
            }

            media["COVER"] = await _DownloadImageAsync(hints.Cover != null ? ImageUrl(hints.Cover) : (videoId ?? audioId) is { } yt ? $"https://i.ytimg.com/vi/{yt}/hqdefault.jpg" : null,
                folder, "cover", ct);
            media["BACKGROUND"] = await _DownloadImageAsync(hints.Background != null ? ImageUrl(hints.Background) : null, folder, "background", ct);

            var name = CSourceHelper.SafeName(artist + " - " + title);
            await File.WriteAllTextAsync(Path.Combine(folder, name + ".txt"), RewriteSong(text, media), new UTF8Encoding(true), ct);
            progress.Report(1);
        }

        private async Task<string?> _DownloadImageAsync(string? url, string folder, string baseName, CancellationToken ct)
        {
            if (url == null)
            {
                return null;
            }

            try
            {
                using var response = await _Http.GetAsync(url, ct);
                response.EnsureSuccessStatusCode();
                var ext = response.Content.Headers.ContentType?.MediaType == "image/png" ? ".png" : ".jpg";
                await File.WriteAllBytesAsync(Path.Combine(folder, baseName + ext), await response.Content.ReadAsByteArrayAsync(ct), ct);
                return baseName + ext;
            }
            catch (Exception e) when (e is not OperationCanceledException)
            {
                _Logger.LogInformation("Image {Url} not downloaded: {Message}", url, e.Message);
                return null;
            }
        }
        #endregion
    }

    /// <summary>Maps the progress of a part of a job (0..1) to a range of the whole job.</summary>
    public sealed class CScaledProgress : IProgress<double>
    {
        private readonly IProgress<double> _Target;
        private readonly double _From;
        private readonly double _To;

        public CScaledProgress(IProgress<double> target, double from, double to)
        {
            _Target = target;
            _From = from;
            _To = to;
        }

        public void Report(double value)
        {
            _Target.Report(_From + (_To - _From) * Math.Clamp(value, 0, 1));
        }
    }
}