using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using VocaluxeSongServer.Contracts;
namespace VocaluxeSongServer.Sources
{
public sealed class CUsdbOptions
{
public string User { get; set; } = "";
public string Password { get; set; } = "";
}
///
/// USDB (usdb.animux.de), the largest UltraStar database. It has only the song text files and needs a (free)
/// account. Audio, video and images come from the sources the song references: the #VIDEO tag
/// ("v=<YouTube>,a=<YouTube>,co=<fanart.tv>,bg=...", the format of usdb_syncer) or YouTube links in the
/// comments. Media are downloaded with yt-dlp.
///
public sealed partial class CUsdbSongSource : ISongSource
{
public const string BaseUrl = "https://usdb.animux.de/";
private const string _NotLoggedIn = "You are not logged in";
private const string _LoginInvalid = "Login or Password invalid";
private readonly HttpClient _Http;
private readonly CUsdbOptions _Options;
private readonly CYtDlp _YtDlp;
private readonly ILogger _Logger;
private readonly SemaphoreSlim _LoginLock = new(1, 1);
private bool _LoggedIn;
/// A client with its own cookie container (the USDB session is a cookie).
public CUsdbSongSource(HttpClient http, CUsdbOptions options, CYtDlp ytDlp, ILogger logger)
{
_Http = http;
_Options = options;
_YtDlp = ytDlp;
_Logger = logger;
}
public string Id => "usdb";
public string Name => "USDB";
public string Description => "usdb.animux.de, the largest UltraStar database (notes and lyrics); audio and video from the YouTube links of the song";
public bool Enabled => _Options.User.Length > 0 && _Options.Password.Length > 0;
public string Hint => Enabled
? (_YtDlp.Available ? "" : "yt-dlp is not installed, songs can't be downloaded")
: "Needs a free account at usdb.animux.de (settings Usdb__User and Usdb__Password)";
#region HTTP
private async Task _LoginAsync(CancellationToken ct)
{
await _LoginLock.WaitAsync(ct);
try
{
if (_LoggedIn)
{
return;
}
var form = new FormUrlEncodedContent(new Dictionary { ["user"] = _Options.User, ["pass"] = _Options.Password, ["login"] = "Login" });
using var response = await _Http.PostAsync(BaseUrl, form, ct);
var html = await response.Content.ReadAsStringAsync(ct);
if (!response.IsSuccessStatusCode || html.Contains(_LoginInvalid))
{
throw new InvalidOperationException("USDB login failed (check user and password)");
}
_LoggedIn = true;
}
finally
{
_LoginLock.Release();
}
}
/// Sends a request (logs in first and again when the session expired) and returns the page.
private async Task _RequestAsync(Func createRequest, CancellationToken ct)
{
for (var attempt = 0; attempt < 2; attempt++)
{
await _LoginAsync(ct);
using var request = createRequest();
using var response = await _Http.SendAsync(request, ct);
response.EnsureSuccessStatusCode();
var bytes = await response.Content.ReadAsByteArrayAsync(ct);
var html = Encoding.UTF8.GetString(bytes);
if (!html.Contains(_NotLoggedIn))
{
return html;
}
_LoggedIn = false;
}
throw new InvalidOperationException("USDB: not logged in after login");
}
#endregion
#region Search
[GeneratedRegex(@"]*data-songid=""(\d+)""[^>]*>(.*?)
", RegexOptions.Singleline)]
private static partial Regex _Row();
[GeneratedRegex(@"]*>(.*?) | ", RegexOptions.Singleline)]
private static partial Regex _Cell();
[GeneratedRegex(@"<[^>]+>")]
private static partial Regex _Tag();
private static string _Text(string html)
{
return WebUtility.HtmlDecode(_Tag().Replace(html, " ")).Trim();
}
/// Parses the result list (link=list with details=1): id, artist, title and the year if there is one.
public static List ParseList(string html)
{
var results = new List();
foreach (Match row in _Row().Matches(html))
{
// Cells: sample, cover (no text), artist, title, genre, year, edition, golden notes, language, creator, ...
var cells = _Cell().Matches(row.Groups[2].Value).Select(c => _Text(c.Groups[1].Value)).Where(t => t.Length > 0).ToList();
if (cells.Count < 2)
{
continue;
}
var id = row.Groups[1].Value;
var year = cells.Skip(2).FirstOrDefault(c => Regex.IsMatch(c, @"^(19|20)\d\d$"));
results.Add(new SearchResultDto
{
Source = "usdb",
SourceId = id,
Artist = cells[0],
Title = cells[1],
Details = "USDB #" + id + (year != null ? " ยท " + year : ""),
CoverUrl = BaseUrl + "data/cover/" + id + ".jpg"
});
}
return results;
}
public async Task> SearchAsync(string query, CancellationToken ct)
{
// USDB searches artist and title separately; a query matches either.
async Task> Search(string field)
{
var html = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Post, BaseUrl + "index.php?link=list")
{
Content = new FormUrlEncodedContent(new Dictionary
{
[field] = query, ["order"] = "rating", ["ud"] = "desc", ["limit"] = "50", ["details"] = "1", ["start"] = "0"
})
}, ct);
return ParseList(html);
}
var byArtist = await Search("interpret");
var byTitle = await Search("title");
return byArtist.Concat(byTitle).GroupBy(r => r.SourceId).Select(g => g.First()).ToList();
}
#endregion
#region Download
[GeneratedRegex(@"", RegexOptions.Singleline)]
private static partial Regex _TextArea();
/// Media hints of a song text file (#VIDEO tag in the usdb_syncer format).
public sealed record CMediaHints(string? Video, string? Audio, string? Cover, string? Background);
public static CMediaHints ParseVideoTag(string? value)
{
if (string.IsNullOrWhiteSpace(value) || !value.Contains('='))
{
return new CMediaHints(null, null, null, null);
}
var pairs = new Dictionary(StringComparer.OrdinalIgnoreCase);
foreach (var pair in value.Split(','))
{
var eq = pair.IndexOf('=');
if (eq > 0)
{
pairs[pair[..eq].Trim()] = pair[(eq + 1)..].Trim().Replace("%2C", ",");
}
}
return new CMediaHints(pairs.GetValueOrDefault("v"), pairs.GetValueOrDefault("a"), pairs.GetValueOrDefault("co"), pairs.GetValueOrDefault("bg"));
}
/// URL of a cover/background hint: full URL, host/path, or a fanart.tv id.
public static string ImageUrl(string source)
{
if (source.Contains("://"))
{
return source.Replace("images.fanart.tv", "assets.fanart.tv");
}
return source.Contains('/') ? "https://" + source : "https://assets.fanart.tv/fanart/" + source;
}
/// Splits a song text file into header tags (in order) and the rest (notes).
public static (List<(string Tag, string Value)> Header, string Body) SplitSong(string text)
{
var header = new List<(string, string)>();
var lines = text.Replace("\r\n", "\n").Split('\n');
var i = 0;
for (; i < lines.Length; i++)
{
var line = lines[i].Trim();
if (line.Length == 0)
{
continue;
}
if (line[0] != '#')
{
break;
}
var colon = line.IndexOf(':');
header.Add(colon > 0 ? (line[1..colon].Trim().ToUpperInvariant(), line[(colon + 1)..].Trim()) : (line[1..].Trim().ToUpperInvariant(), ""));
}
return (header, string.Join("\n", lines.Skip(i)));
}
/// The song text with the media tags replaced by the downloaded files (null = tag removed).
public static string RewriteSong(string text, IReadOnlyDictionary media)
{
var (header, body) = SplitSong(text);
var sb = new StringBuilder();
foreach (var (tag, value) in header)
{
if (tag is "MP3" or "AUDIO" or "VIDEO" or "COVER" or "BACKGROUND" or "ENCODING")
{
continue;
}
sb.Append('#').Append(tag).Append(':').Append(value).Append('\n');
}
foreach (var (tag, value) in media)
{
if (!string.IsNullOrEmpty(value))
{
sb.Append('#').Append(tag).Append(':').Append(value).Append('\n');
}
}
return sb.Append(body.TrimEnd('\n')).Append('\n').ToString();
}
public async Task DownloadAsync(string sourceId, string folder, IProgress progress, CancellationToken ct)
{
if (!int.TryParse(sourceId, out var id))
{
throw new ArgumentException("Invalid USDB id " + sourceId);
}
if (!_YtDlp.Available)
{
throw new InvalidOperationException("yt-dlp is not installed, USDB songs can't be downloaded");
}
var page = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Post, BaseUrl + "index.php?link=gettxt&id=" + id)
{
Content = new FormUrlEncodedContent(new Dictionary { ["wd"] = "1" })
}, ct);
var area = _TextArea().Match(page);
if (!area.Success)
{
throw new InvalidOperationException("USDB returned no song text for #" + id);
}
var text = WebUtility.HtmlDecode(area.Groups[1].Value);
var (header, _) = SplitSong(text);
var hints = ParseVideoTag(header.FirstOrDefault(h => h.Tag == "VIDEO").Value);
var artist = header.FirstOrDefault(h => h.Tag == "ARTIST").Value ?? "";
var title = header.FirstOrDefault(h => h.Tag == "TITLE").Value ?? "";
progress.Report(0.02);
// Media: the #VIDEO hints, otherwise the YouTube links in the comments of the song page.
var videoId = hints.Video != null ? CYtDlp.YouTubeId(hints.Video) : null;
var audioId = hints.Audio != null ? CYtDlp.YouTubeId(hints.Audio) : null;
if (videoId == null && audioId == null)
{
var detail = await _RequestAsync(() => new HttpRequestMessage(HttpMethod.Get, BaseUrl + "index.php?link=detail&id=" + id), ct);
videoId = CYtDlp.FindYouTubeIds(detail).FirstOrDefault();
}
audioId ??= videoId;
if (audioId == null)
{
throw new InvalidOperationException("No audio source known for this song (no YouTube link in the song or its comments)");
}
var media = new Dictionary();
media["MP3"] = await _YtDlp.DownloadAsync("https://www.youtube.com/watch?v=" + audioId, folder, "audio",
"bestaudio[ext=m4a]/bestaudio", new CScaledProgress(progress, 0.05, 0.45), ct);
if (videoId != null)
{
try
{
media["VIDEO"] = await _YtDlp.DownloadAsync("https://www.youtube.com/watch?v=" + videoId, folder, "video",
"bestvideo[height<=720][ext=mp4]/bestvideo[height<=720]/best[height<=720]", new CScaledProgress(progress, 0.45, 0.9), ct);
}
catch (Exception e) when (e is not OperationCanceledException)
{
_Logger.LogWarning("USDB #{Id}: video download failed, the song is saved without video: {Message}", id, e.Message);
}
}
media["COVER"] = await _DownloadImageAsync(hints.Cover != null ? ImageUrl(hints.Cover) : (videoId ?? audioId) is { } yt ? $"https://i.ytimg.com/vi/{yt}/hqdefault.jpg" : null,
folder, "cover", ct);
media["BACKGROUND"] = await _DownloadImageAsync(hints.Background != null ? ImageUrl(hints.Background) : null, folder, "background", ct);
var name = CSourceHelper.SafeName(artist + " - " + title);
await File.WriteAllTextAsync(Path.Combine(folder, name + ".txt"), RewriteSong(text, media), new UTF8Encoding(true), ct);
progress.Report(1);
}
private async Task _DownloadImageAsync(string? url, string folder, string baseName, CancellationToken ct)
{
if (url == null)
{
return null;
}
try
{
using var response = await _Http.GetAsync(url, ct);
response.EnsureSuccessStatusCode();
var ext = response.Content.Headers.ContentType?.MediaType == "image/png" ? ".png" : ".jpg";
await File.WriteAllBytesAsync(Path.Combine(folder, baseName + ext), await response.Content.ReadAsByteArrayAsync(ct), ct);
return baseName + ext;
}
catch (Exception e) when (e is not OperationCanceledException)
{
_Logger.LogInformation("Image {Url} not downloaded: {Message}", url, e.Message);
return null;
}
}
#endregion
}
/// Maps the progress of a part of a job (0..1) to a range of the whole job.
public sealed class CScaledProgress : IProgress
{
private readonly IProgress _Target;
private readonly double _From;
private readonly double _To;
public CScaledProgress(IProgress target, double from, double to)
{
_Target = target;
_From = from;
_To = to;
}
public void Report(double value)
{
_Target.Report(_From + (_To - _From) * Math.Clamp(value, 0, 1));
}
}
}