From 96435ba60e22bf0a82d777fc271cb6e1e2edc3f5 Mon Sep 17 00:00:00 2001 From: ivarlovlie Date: Sun, 4 Dec 2022 14:16:57 +0900 Subject: fea: !WIP nrk radio --- src/Endpoints/CreateRadioIndex.cs | 18 +++++++ src/Endpoints/RadioSearchEndpoint.cs | 17 ------- src/Models/Database/RadioEpisode.cs | 13 +++++ src/Models/Database/RadioSeason.cs | 11 ++++ src/Models/Database/RadioSeries.cs | 11 ++++ src/Models/NrkLinks.cs | 14 ++++++ src/Models/NrkPlaybackManifest.cs | 19 +++++++ src/Models/NrkRadioCategorySearchResult.cs | 47 +++++++++++++++++ src/Models/NrkRadioSeries.cs | 79 +++++++++++++++++++++++++++++ src/Models/RadioCategorySearchResult.cs | 58 --------------------- src/Models/RadioSeries.cs | 18 ------- src/Program.cs | 6 +++ src/RadioIndexDb.cs | 81 ++++++++++++++++++++++++++---- src/Services/NrkRadioService.cs | 63 ++++++++++++++++++++--- 14 files changed, 343 insertions(+), 112 deletions(-) create mode 100644 src/Endpoints/CreateRadioIndex.cs delete mode 100644 src/Endpoints/RadioSearchEndpoint.cs create mode 100644 src/Models/Database/RadioEpisode.cs create mode 100644 src/Models/Database/RadioSeason.cs create mode 100644 src/Models/Database/RadioSeries.cs create mode 100644 src/Models/NrkLinks.cs create mode 100644 src/Models/NrkPlaybackManifest.cs create mode 100644 src/Models/NrkRadioCategorySearchResult.cs create mode 100644 src/Models/NrkRadioSeries.cs delete mode 100644 src/Models/RadioCategorySearchResult.cs delete mode 100644 src/Models/RadioSeries.cs diff --git a/src/Endpoints/CreateRadioIndex.cs b/src/Endpoints/CreateRadioIndex.cs new file mode 100644 index 0000000..1ef19ab --- /dev/null +++ b/src/Endpoints/CreateRadioIndex.cs @@ -0,0 +1,18 @@ +using Microsoft.AspNetCore.Mvc; + +namespace I2R.LightNews.Endpoints; + +public class RadioSearchEndpoint : EndpointBase +{ + private readonly NrkRadioService _radio; + + public RadioSearchEndpoint(NrkRadioService radio) { + _radio = radio; + } + + [HttpGet("~/create-radio-index")] + public async Task HandleASync() { + await _radio.CreateIndex(); + return Ok(); + } +} \ No newline at end of file diff --git a/src/Endpoints/RadioSearchEndpoint.cs b/src/Endpoints/RadioSearchEndpoint.cs deleted file mode 100644 index 79f24ce..0000000 --- a/src/Endpoints/RadioSearchEndpoint.cs +++ /dev/null @@ -1,17 +0,0 @@ -using Microsoft.AspNetCore.Mvc; - -namespace I2R.LightNews.Endpoints; - -public class RadioSearchEndpoint : EndpointBase -{ - private readonly NrkRadioService _radio; - - public RadioSearchEndpoint(NrkRadioService radio) { - _radio = radio; - } - - [HttpGet("~/radio-search")] - public async Task HandleASync(string q) { - - } -} \ No newline at end of file diff --git a/src/Models/Database/RadioEpisode.cs b/src/Models/Database/RadioEpisode.cs new file mode 100644 index 0000000..508177b --- /dev/null +++ b/src/Models/Database/RadioEpisode.cs @@ -0,0 +1,13 @@ +namespace I2R.LightNews.Models; + +public class RadioEpisode +{ + public int Id { get; set; } + public int SeriesId { get; set; } + public int SeasonId { get; set; } + public string NrkId { get; set; } + public string Title { get; set; } + public string Subtitle { get; set; } + public string SourceUrl { get; set; } + public string CanonicalUrl { get; set; } +} \ No newline at end of file diff --git a/src/Models/Database/RadioSeason.cs b/src/Models/Database/RadioSeason.cs new file mode 100644 index 0000000..5db925d --- /dev/null +++ b/src/Models/Database/RadioSeason.cs @@ -0,0 +1,11 @@ +namespace I2R.LightNews.Models; + +public class RadioSeason +{ + public int Id { get; set; } + public int SeriesId { get; set; } + public string Name { get; set; } + public string Description { get; set; } + public string CanonicalUrl { get; set; } + public string NrkId { get; set; } +} \ No newline at end of file diff --git a/src/Models/Database/RadioSeries.cs b/src/Models/Database/RadioSeries.cs new file mode 100644 index 0000000..946b6ce --- /dev/null +++ b/src/Models/Database/RadioSeries.cs @@ -0,0 +1,11 @@ +namespace I2R.LightNews.Models; + +public class RadioSeries +{ + public int Id { get; set; } + public string NrkId { get; set; } + public string Name { get; set; } + public string Description { get; set; } + public string Type { get; set; } + public string CanonicalUrl { get; set; } +} \ No newline at end of file diff --git a/src/Models/NrkLinks.cs b/src/Models/NrkLinks.cs new file mode 100644 index 0000000..69f5ae3 --- /dev/null +++ b/src/Models/NrkLinks.cs @@ -0,0 +1,14 @@ +namespace I2R.LightNews.Models; + +public class NrkLinks +{ + public LinkModel NextPage { get; set; } + public LinkModel LastPage { get; set; } + public LinkModel Share { get; set; } + public LinkModel Episodes { get; set; } + + public class LinkModel + { + public string Href { get; set; } + } +} \ No newline at end of file diff --git a/src/Models/NrkPlaybackManifest.cs b/src/Models/NrkPlaybackManifest.cs new file mode 100644 index 0000000..52f9503 --- /dev/null +++ b/src/Models/NrkPlaybackManifest.cs @@ -0,0 +1,19 @@ +namespace I2R.LightNews.Models; + +public class NrkPlaybackManifest +{ + public PlayableModel Playable { get; set; } + + public class PlayableModel + { + public List Assets { get; set; } + + public class Asset + { + public string Url { get; set; } + public string Format { get; set; } + public string MimeType { get; set; } + public bool Encrypted { get; set; } + } + } +} \ No newline at end of file diff --git a/src/Models/NrkRadioCategorySearchResult.cs b/src/Models/NrkRadioCategorySearchResult.cs new file mode 100644 index 0000000..6672105 --- /dev/null +++ b/src/Models/NrkRadioCategorySearchResult.cs @@ -0,0 +1,47 @@ +using System.Text.Json.Serialization; + +namespace I2R.LightNews.Models; + +public class RadioCategorySearchResult +{ + [JsonPropertyName("_links")] + public NrkLinks Links { get; set; } + + public List Letters { get; set; } + public string Title { get; set; } + public List Series { get; set; } + public long TotalCount { get; set; } + + public class SeriesModel + { + [JsonPropertyName("_links")] + public LinksModel Links { get; set; } + + public string Id { get; set; } + public string SeriesId { get; set; } + public string SeasonId { get; set; } + public string Title { get; set; } + public string Type { get; set; } + public string InitialCharacter { get; set; } + public List Images { get; set; } + + public class ImageModel + { + public string Uri { get; set; } + public int Width { get; set; } + } + + public class LinksModel + { + public NrkLinks.LinkModel CustomSeason { get; set; } + public NrkLinks.LinkModel Series { get; set; } + } + } + + public class LetterModel + { + public string Letter { get; set; } + public int Count { get; set; } + public string Link { get; set; } + } +} \ No newline at end of file diff --git a/src/Models/NrkRadioSeries.cs b/src/Models/NrkRadioSeries.cs new file mode 100644 index 0000000..3496f36 --- /dev/null +++ b/src/Models/NrkRadioSeries.cs @@ -0,0 +1,79 @@ +using System.Text.Json.Serialization; + +namespace I2R.LightNews.Models; + +public class NrkRadioSeries +{ + [JsonPropertyName("_links")] + public NrkLinks Links { get; set; } + + [JsonPropertyName("_embedded")] + public EmbeddedModel Embedded { get; set; } + + public class EmbeddedModel + { + public List Seasons { get; set; } + + public class SeasonModel + { + public List Titles { get; set; } + public List Episodes { get; set; } + public string Id { get; set; } + public bool HasAvailableEpisodes { get; set; } + public int EpisodeCount { get; set; } + + public class EpisodeModel + { + [JsonPropertyName("_embedded")] + public EmbeddedModel Embedded { get; set; } + + public class EmbeddedModel + { + public List Episodes { get; set; } + + public class EpisodeModel + { + [JsonPropertyName("_links")] + public LinksModel Links { get; set; } + + public string Id { get; set; } + public string EpisodeId { get; set; } + public List Titles { get; set; } + public DateTime Date { get; set; } + public int DurationInSeconds { get; set; } + public int ProductionYear { get; set; } + + public class TitlesModel + { + public string Title { get; set; } + public string Subtitle { get; set; } + } + + public class LinksModel + { + public NrkLinks.LinkModel Playback { get; set; } + public NrkLinks.LinkModel Share { get; set; } + } + } + } + } + + public class TitleModel + { + public string Title { get; set; } + } + } + } + + public class NrkRadioSeriesLinks : NrkLinks + { + public List Seasons { get; set; } + + public class Season + { + public string Name { get; set; } + public string Href { get; set; } + public string Title { get; set; } + } + } +} \ No newline at end of file diff --git a/src/Models/RadioCategorySearchResult.cs b/src/Models/RadioCategorySearchResult.cs deleted file mode 100644 index 7fd4c5d..0000000 --- a/src/Models/RadioCategorySearchResult.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Text.Json.Serialization; - -namespace I2R.LightNews.Models; - -public class RadioCategorySearchResult -{ - [JsonPropertyName("_links")] - public LinksModel Links { get; set; } - - public List Letters { get; set; } - public string Title { get; set; } - public List Series { get; set; } - public long TotalCount { get; set; } - - public class SeriesModel - { - [JsonPropertyName("_links")] - public LinksModel Links { get; set; } - - public Guid Id { get; set; } - public string SeriesId { get; set; } - public string SeasonId { get; set; } - public string Title { get; set; } - public string Type { get; set; } - public string InitialCharacter { get; set; } - public List Images { get; set; } - - public class ImageModel - { - public string Uri { get; set; } - public int Width { get; set; } - } - - public class LinksModel - { - public RadioCategorySearchResult.LinksModel.LinkModel CustomSeason { get; set; } - } - } - - public class LetterModel - { - public string Letter { get; set; } - public int Count { get; set; } - public string Link { get; set; } - } - - public class LinksModel - { - public LinkModel NextPage { get; set; } - public LinkModel LastPage { get; set; } - - - public class LinkModel - { - public string Href { get; set; } - } - } -} \ No newline at end of file diff --git a/src/Models/RadioSeries.cs b/src/Models/RadioSeries.cs deleted file mode 100644 index 6bd4efe..0000000 --- a/src/Models/RadioSeries.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace I2R.LightNews.Models; - -public class RadioSeries -{ - public string Name { get; set; } - public string Description { get; set; } - public string Type { get; set; } - public Uri CanonicalUri { get; set; } - public List Episodes { get; set; } - - public class Episode - { - public string Title { get; set; } - public string Subtitle { get; set; } - public Uri SourceUri { get; set; } - public Uri CanonicalUri { get; set; } - } -} \ No newline at end of file diff --git a/src/Program.cs b/src/Program.cs index ba011a6..7fd2903 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -1,7 +1,9 @@ global using I2R.LightNews.Services; global using I2R.LightNews.Models; global using IOL.Helpers; +using System.Text.Json.Serialization; using I2R.LightNews; +using Microsoft.AspNetCore.Http.Json; var builder = WebApplication.CreateBuilder(args); @@ -10,6 +12,10 @@ builder.Services.AddHttpClient(); builder.Services.AddMemoryCache(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.Configure(options => { + options.SerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.Never; + options.SerializerOptions.PropertyNameCaseInsensitive = true; +}); builder.Services.AddControllers(); builder.Services.AddRazorPages().AddRazorRuntimeCompilation(); diff --git a/src/RadioIndexDb.cs b/src/RadioIndexDb.cs index f96c4af..e69ca4b 100644 --- a/src/RadioIndexDb.cs +++ b/src/RadioIndexDb.cs @@ -9,27 +9,58 @@ public static class RadioIndexDb { private static readonly string ConnectionString = "data source=AppData/radio-index.db"; - public static void AddEntry(RadioSeries entry) { + public static int AddSeries(RadioSeries entry) { using var db = new SqliteConnection(ConnectionString); - if (!db.TableExists("series")) return; - var addOperation = db.Execute(@"insert series(name,description,canonical_url,episodes) values (@name,@description,@canonical_url,@episodes)", new { + if (!db.TableExists("series")) return -1; + return db.ExecuteScalar(@" +insert into series(name,description,canonical_url,nrk_id) values (@name,@description,@canonical_url,@nrk_id); +select last_insert_rowid();", new { name = entry.Name, description = entry.Description, - canonical_url = entry.CanonicalUri, - episodes = JsonSerializer.Serialize(entry.Episodes) + canonical_url = entry.CanonicalUrl, + nrk_id = entry.NrkId }); - if (addOperation == 0) Console.WriteLine("No rows were added"); } - - public static void DeleteEntry(int id) { } - public static RadioSeries GetEntry(int id) { + public static int AddSeason(RadioSeason entry) { + using var db = new SqliteConnection(ConnectionString); + if (!db.TableExists("seasons")) return -1; + return db.ExecuteScalar(@" +insert into seasons(name,description,canonical_url,nrk_id,series_id) values (@name,@description,@canonical_url,@nrk_id,@series_id); +select last_insert_rowid();", new { + name = entry.Name, + description = entry.Description, + canonical_url = entry.CanonicalUrl, + nrk_id = entry.NrkId, + series_id = entry.SeriesId + }); + } + + public static int AddEpisode(RadioEpisode entry) { + using var db = new SqliteConnection(ConnectionString); + if (!db.TableExists("episodes")) return -1; + return db.ExecuteScalar(@" +insert into episodes(title,subtitle,canonical_url,nrk_id,series_id,season_id,source_url) values (@title,@subtitle,@canonical_url,@nrk_id,@series_id,@season_id,@source_url); +select last_insert_rowid();", new { + title = entry.Title, + subtitle = entry.Subtitle, + canonical_url = entry.CanonicalUrl, + nrk_id = entry.NrkId, + series_id = entry.SeriesId, + season_id = entry.SeasonId, + source_url = entry.SourceUrl, + }); + } + + public static void DeleteSeries(int id) { } + + public static RadioSeries GetSeries(int id) { using var db = new SqliteConnection(ConnectionString); if (!db.TableExists("series")) return default; return db.QueryFirstOrDefault(@"select * from series where id=@id", new {id}); } - public static List GetEntries(string query, bool includeEpisodes = false) { + public static List GetSeries(string query, bool includeEpisodes = false) { using var db = new SqliteConnection(ConnectionString); if (!db.TableExists("series")) return default; var selectSet = includeEpisodes ? "*" : "id,name,description,type,canonical_url"; @@ -49,7 +80,35 @@ public static class RadioIndexDb description text, type text, canonical_url text, - episodes json + nrk_id text + ) + "); + } + + if (!db.TableExists("seasons")) { + db.Execute(@" + create table seasons( + id integer primary key autoincrement, + series_id integer, + name text, + description text, + canonical_url text, + nrk_id text + ) + "); + } + + if (!db.TableExists("episodes")) { + db.Execute(@" + create table episodes( + id integer primary key autoincrement, + series_id integer, + season_id integer, + name text, + description text, + canonical_url text, + source_url text, + nrk_id text ) "); } diff --git a/src/Services/NrkRadioService.cs b/src/Services/NrkRadioService.cs index a2889ce..d4e06a5 100644 --- a/src/Services/NrkRadioService.cs +++ b/src/Services/NrkRadioService.cs @@ -7,23 +7,70 @@ public class NrkRadioService private readonly IMemoryCache _cache; private readonly HttpClient _http; private const string CATEGORY_SEARCH_CACHE_KEY = "category_search"; + private readonly ILogger _logger; - public NrkRadioService(IMemoryCache cache, HttpClient http) { + public NrkRadioService(IMemoryCache cache, HttpClient http, ILogger logger) { _cache = cache; http.BaseAddress = new Uri("https://psapi.nrk.no"); _http = http; + _logger = logger; } - public async Task GetEverythingAsync() { - var path = "/radio/search/categories/alt-innhold"; - var everything = new List(); - while (path.HasValue()) { - var response = await _http.GetFromJsonAsync(path); - + public async Task CreateIndex() { + var letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ".ToCharArray(); + var skip = 0; + foreach (var letter in letters) { + var path = "/radio/search/categories/alt-innhold?letter=" + letter + "&skip=0&take=50"; + while (path.HasValue()) { + var response = await _http.GetFromJsonAsync(path); + if (response == default) break; + await Task.Delay(2000); + foreach (var series in response.Series) { + var dbSeries = new RadioSeries { + Name = series.Title, + NrkId = series.Id, + Type = series.Type, + }; + var seriesId = RadioIndexDb.AddSeries(dbSeries); + _logger.LogInformation("Added series {0} with id {1}, to the database", dbSeries.Name, seriesId); + if (!series.Links.Series.Href.HasValue()) continue; + var seriesMetadata = await _http.GetFromJsonAsync(series.Links.Series.Href); + if (seriesMetadata == default) continue; + await Task.Delay(1000); + foreach (var season in seriesMetadata.Embedded.Seasons) { + var dbSeason = new RadioSeason() { + Name = season.Titles.FirstOrDefault()?.Title, + NrkId = season.Id, + SeriesId = seriesId + }; + var seasonId = RadioIndexDb.AddSeason(dbSeason); + _logger.LogInformation("Added season {0} to series {1} with id {2}, to the database", dbSeason.Name, dbSeries.Name, seasonId); + foreach (var episode in season.Episodes) { + foreach (var actuallyEpisode in episode.Embedded.Episodes) { + var dbEpisode = new RadioEpisode { + CanonicalUrl = actuallyEpisode.Links.Share.Href, + Title = actuallyEpisode.Titles.FirstOrDefault()?.Title, + Subtitle = actuallyEpisode.Titles.FirstOrDefault()?.Subtitle, + NrkId = actuallyEpisode.EpisodeId, + SeasonId = seasonId, + SeriesId = dbSeason.SeriesId + }; + var playbackResponse = await _http.GetFromJsonAsync("/playback/manifest/program/" + dbEpisode.NrkId); + if (playbackResponse == default) continue; + dbEpisode.SourceUrl = playbackResponse.Playable.Assets.FirstOrDefault()?.Url; + var episodeId = RadioIndexDb.AddEpisode(dbEpisode); + _logger.LogInformation("Added episode {0} to series {1} season {2} with id {3}, to the database", dbEpisode.Title, dbSeries.Name, dbSeason.Name, episodeId); + } + } + } + } + + path = response.Links.NextPage.Href.HasValue() ? response.Links.NextPage.Href : ""; + } } } - public async Task SearchCategoriesAsync(string query, int take = 50, int skip = 50) { + public async Task SearchCategoriesAsync(string query, int take = 50, int skip = 0) { return await _http.GetFromJsonAsync( "/radio/search/categories/alt-innhold?q=" + query + "&take=" + take + "&skip=" + skip ); -- cgit v1.3