Vocaluxe/VocaluxeSongServer/Sources/CGitHubSongSource.cs

using System.Text.Json;
using VocaluxeSongServer.Contracts;

namespace VocaluxeSongServer.Sources
{
    /// <summary>
    ///     Openly licensed songs with all media from a GitHub repository, by default UltraStar-Deluxe/songs
    ///     (Creative Commons / public domain; one song per folder, license.txt in every folder).
    ///     The repository tree is read with one API call and cached for a day (the API allows 60 calls/hour without token).
    /// </summary>
    public sealed class CGitHubSongSource : ISongSource
    {
        private sealed record CSong(string Folder, string Artist, string Title, string License, List<(string Path, long Size)> Files);

        private readonly HttpClient _Http;
        private readonly string _Repo;
        private readonly string _Branch;
        private readonly SemaphoreSlim _Lock = new(1, 1);
        private List<CSong>? _Catalog;
        private DateTime _CatalogTime;

        public CGitHubSongSource(HttpClient http, string repo = "UltraStar-Deluxe/songs", string branch = "master")
        {
            _Http = http;
            _Repo = repo;
            _Branch = branch;
        }

        public string Id => "free";
        public string Name => "Free songs";
        public string Description => "Creative Commons and public domain songs with audio, from github.com/" + _Repo;
        public bool Enabled => true;
        public string Hint => "";

        private async Task<List<CSong>> _GetCatalogAsync(CancellationToken ct)
        {
            await _Lock.WaitAsync(ct);
            try
            {
                if (_Catalog != null && DateTime.UtcNow - _CatalogTime < TimeSpan.FromDays(1))
                {
                    return _Catalog;
                }

                using var request = new HttpRequestMessage(HttpMethod.Get, $"https://api.github.com/repos/{_Repo}/git/trees/{_Branch}?recursive=1");
                request.Headers.UserAgent.ParseAdd("VocaluxeSongServer");
                using var response = await _Http.SendAsync(request, ct);
                response.EnsureSuccessStatusCode();
                using var doc = JsonDocument.Parse(await response.Content.ReadAsStringAsync(ct));

                var files = doc.RootElement.GetProperty("tree").EnumerateArray()
                               .Where(e => e.GetProperty("type").GetString() == "blob")
                               .Select(e => (Path: e.GetProperty("path").GetString()!, Size: e.TryGetProperty("size", out var s) ? s.GetInt64() : 0))
                               .ToList();

                var songs = new List<CSong>();
                foreach (var folder in files.GroupBy(f => f.Path.Contains('/') ? f.Path[..f.Path.LastIndexOf('/')] : ""))
                {
                    var hasSong = folder.Any(f => f.Path.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) &&
                                                  !f.Path.EndsWith("/license.txt", StringComparison.OrdinalIgnoreCase));
                    if (!hasSong || folder.Key == "")
                    {
                        continue;
                    }

                    // Folder names are "Artist - Title"; the category is the parent folder ("Creative Commons", ...).
                    var name = folder.Key[(folder.Key.LastIndexOf('/') + 1)..];
                    var category = folder.Key.Contains('/') ? folder.Key[..folder.Key.LastIndexOf('/')] : "";
                    var dash = name.IndexOf(" - ", StringComparison.Ordinal);
                    var artist = dash > 0 ? name[..dash] : "";
                    var title = dash > 0 ? name[(dash + 3)..] : name;
                    songs.Add(new CSong(folder.Key, artist, title, category, folder.ToList()));
                }

                _Catalog = songs;
                _CatalogTime = DateTime.UtcNow;
                return songs;
            }
            finally
            {
                _Lock.Release();
            }
        }

        private string _RawUrl(string path)
        {
            return $"https://raw.githubusercontent.com/{_Repo}/{_Branch}/" + string.Join('/', path.Split('/').Select(Uri.EscapeDataString));
        }

        public async Task<List<SearchResultDto>> SearchAsync(string query, CancellationToken ct)
        {
            var normalized = CSourceHelper.Normalize(query);
            var catalog = await _GetCatalogAsync(ct);
            return catalog.Where(s => CSourceHelper.Matches(normalized, s.Artist + " " + s.Title))
                          .Select(s =>
                          {
                              var cover = s.Files.FirstOrDefault(f => Path.GetFileNameWithoutExtension(f.Path).Equals("cover", StringComparison.OrdinalIgnoreCase)).Path;
                              return new SearchResultDto
                              {
                                  Source = Id,
                                  SourceId = s.Folder,
                                  Artist = s.Artist,
                                  Title = s.Title,
                                  Details = $"{s.License} · {s.Files.Sum(f => f.Size) / 1_000_000.0:0.#} MB",
                                  CoverUrl = cover != null ? _RawUrl(cover) : ""
                              };
                          })
                          .ToList();
        }

        public async Task DownloadAsync(string sourceId, string folder, IProgress<double> progress, CancellationToken ct)
        {
            var song = (await _GetCatalogAsync(ct)).FirstOrDefault(s => s.Folder == sourceId) ?? throw new InvalidOperationException("Song not found in " + _Repo);
            var total = Math.Max(1, song.Files.Sum(f => f.Size));
            long done = 0;
            foreach (var file in song.Files)
            {
                var target = Path.Combine(folder, CSourceHelper.SafeName(Path.GetFileName(file.Path)));
                using var response = await _Http.GetAsync(_RawUrl(file.Path), HttpCompletionOption.ResponseHeadersRead, ct);
                response.EnsureSuccessStatusCode();
                await using var source = await response.Content.ReadAsStreamAsync(ct);
                await using var stream = File.Create(target);
                await CSourceHelper.CopyAsync(source, stream, n =>
                {
                    done += n;
                    progress.Report((double)done / total);
                }, ct);
            }
        }
    }
}