using System.Text.Json; using VocaluxeSongServer.Contracts; namespace VocaluxeSongServer.Sources { /// /// Openly licensed songs with all media from a GitHub repository, by default UltraStar-Deluxe/songs /// (Creative Commons / public domain; one song per folder, license.txt in every folder). /// The repository tree is read with one API call and cached for a day (the API allows 60 calls/hour without token). /// public sealed class CGitHubSongSource : ISongSource { private sealed record CSong(string Folder, string Artist, string Title, string License, List<(string Path, long Size)> Files); private readonly HttpClient _Http; private readonly string _Repo; private readonly string _Branch; private readonly SemaphoreSlim _Lock = new(1, 1); private List? _Catalog; private DateTime _CatalogTime; public CGitHubSongSource(HttpClient http, string repo = "UltraStar-Deluxe/songs", string branch = "master") { _Http = http; _Repo = repo; _Branch = branch; } public string Id => "free"; public string Name => "Free songs"; public string Description => "Creative Commons and public domain songs with audio, from github.com/" + _Repo; public bool Enabled => true; public string Hint => ""; private async Task> _GetCatalogAsync(CancellationToken ct) { await _Lock.WaitAsync(ct); try { if (_Catalog != null && DateTime.UtcNow - _CatalogTime < TimeSpan.FromDays(1)) { return _Catalog; } using var request = new HttpRequestMessage(HttpMethod.Get, $"https://api.github.com/repos/{_Repo}/git/trees/{_Branch}?recursive=1"); request.Headers.UserAgent.ParseAdd("VocaluxeSongServer"); using var response = await _Http.SendAsync(request, ct); response.EnsureSuccessStatusCode(); using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync(ct)); var files = doc.RootElement.GetProperty("tree").EnumerateArray() .Where(e => e.GetProperty("type").GetString() == "blob") .Select(e => (Path: e.GetProperty("path").GetString()!, Size: e.TryGetProperty("size", out var s) ? s.GetInt64() : 0)) .ToList(); var songs = new List(); foreach (var folder in files.GroupBy(f => f.Path.Contains('/') ? f.Path[..f.Path.LastIndexOf('/')] : "")) { var hasSong = folder.Any(f => f.Path.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && !f.Path.EndsWith("/license.txt", StringComparison.OrdinalIgnoreCase)); if (!hasSong || folder.Key == "") { continue; } // Folder names are "Artist - Title"; the category is the parent folder ("Creative Commons", ...). var name = folder.Key[(folder.Key.LastIndexOf('/') + 1)..]; var category = folder.Key.Contains('/') ? folder.Key[..folder.Key.LastIndexOf('/')] : ""; var dash = name.IndexOf(" - ", StringComparison.Ordinal); var artist = dash > 0 ? name[..dash] : ""; var title = dash > 0 ? name[(dash + 3)..] : name; songs.Add(new CSong(folder.Key, artist, title, category, folder.ToList())); } _Catalog = songs; _CatalogTime = DateTime.UtcNow; return songs; } finally { _Lock.Release(); } } private string _RawUrl(string path) { return $"https://raw.githubusercontent.com/{_Repo}/{_Branch}/" + string.Join('/', path.Split('/').Select(Uri.EscapeDataString)); } public async Task> SearchAsync(string query, CancellationToken ct) { var normalized = CSourceHelper.Normalize(query); var catalog = await _GetCatalogAsync(ct); return catalog.Where(s => CSourceHelper.Matches(normalized, s.Artist + " " + s.Title)) .Select(s => { var cover = s.Files.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f.Path).Equals("cover", StringComparison.OrdinalIgnoreCase)).Path; return new SearchResultDto { Source = Id, SourceId = s.Folder, Artist = s.Artist, Title = s.Title, Details = $"{s.License} ยท {s.Files.Sum(f => f.Size) / 1_000_000.0:0.#} MB", CoverUrl = cover != null ? _RawUrl(cover) : "" }; }) .ToList(); } public async Task DownloadAsync(string sourceId, string folder, IProgress progress, CancellationToken ct) { var song = (await _GetCatalogAsync(ct)).FirstOrDefault(s => s.Folder == sourceId) ?? throw new InvalidOperationException("Song not found in " + _Repo); var total = Math.Max(1, song.Files.Sum(f => f.Size)); long done = 0; foreach (var file in song.Files) { var target = Path.Combine(folder, CSourceHelper.SafeName(Path.GetFileName(file.Path))); using var response = await _Http.GetAsync(_RawUrl(file.Path), HttpCompletionOption.ResponseHeadersRead, ct); response.EnsureSuccessStatusCode(); await using var source = await response.Content.ReadAsStreamAsync(ct); await using var stream = File.Create(target); await CSourceHelper.CopyAsync(source, stream, n => { done += n; progress.Report((double)done / total); }, ct); } } } }