From 2ea053d4c436a79d77b205d3d74716244da91058 Mon Sep 17 00:00:00 2001 From: ivarlovlie Date: Sun, 4 Dec 2022 17:54:53 +0900 Subject: feat: Initial indexing of seasons and episode is working --- src/Models/Database/RadioEpisode.cs | 4 +-- src/Models/NrkRadioSeries.cs | 10 ++++-- src/Program.cs | 7 +--- src/RadioIndexDb.cs | 24 +++++++++---- src/Services/NrkRadioService.cs | 72 ++++++++++++++++++++++--------------- 5 files changed, 70 insertions(+), 47 deletions(-) (limited to 'src') diff --git a/src/Models/Database/RadioEpisode.cs b/src/Models/Database/RadioEpisode.cs index 508177b..1604c74 100644 --- a/src/Models/Database/RadioEpisode.cs +++ b/src/Models/Database/RadioEpisode.cs @@ -6,8 +6,8 @@ public class RadioEpisode public int SeriesId { get; set; } public int SeasonId { get; set; } public string NrkId { get; set; } - public string Title { get; set; } - public string Subtitle { get; set; } + public string Name { get; set; } + public string Description { get; set; } public string SourceUrl { get; set; } public string CanonicalUrl { get; set; } } \ No newline at end of file diff --git a/src/Models/NrkRadioSeries.cs b/src/Models/NrkRadioSeries.cs index 3496f36..c4ee110 100644 --- a/src/Models/NrkRadioSeries.cs +++ b/src/Models/NrkRadioSeries.cs @@ -13,11 +13,12 @@ public class NrkRadioSeries public class EmbeddedModel { public List Seasons { get; set; } + public SeasonModel.EpisodeModel Episodes { get; set; } public class SeasonModel { - public List Titles { get; set; } - public List Episodes { get; set; } + public TitleModel Titles { get; set; } + public EpisodeModel Episodes { get; set; } public string Id { get; set; } public bool HasAvailableEpisodes { get; set; } public int EpisodeCount { get; set; } @@ -38,9 +39,12 @@ public class NrkRadioSeries public string Id { get; set; } public string EpisodeId { get; set; } - public List Titles { get; set; } + public TitlesModel Titles { get; set; } public DateTime Date { get; set; } + public int DurationInSeconds { get; set; } + + // int 😬 public int ProductionYear { get; set; } public class TitlesModel diff --git a/src/Program.cs b/src/Program.cs index 7fd2903..0f54b87 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -1,9 +1,7 @@ global using I2R.LightNews.Services; global using I2R.LightNews.Models; global using IOL.Helpers; -using System.Text.Json.Serialization; using I2R.LightNews; -using Microsoft.AspNetCore.Http.Json; var builder = WebApplication.CreateBuilder(args); @@ -12,10 +10,6 @@ builder.Services.AddHttpClient(); builder.Services.AddMemoryCache(); builder.Services.AddScoped(); builder.Services.AddScoped(); -builder.Services.Configure(options => { - options.SerializerOptions.DefaultIgnoreCondition = JsonIgnoreCondition.Never; - options.SerializerOptions.PropertyNameCaseInsensitive = true; -}); builder.Services.AddControllers(); builder.Services.AddRazorPages().AddRazorRuntimeCompilation(); @@ -26,5 +20,6 @@ app.UseStatusCodePages(); app.UseRouting(); app.MapRazorPages(); app.MapControllers(); +Dapper.DefaultTypeMap.MatchNamesWithUnderscores = true; RadioIndexDb.CreateIfNotExists(); app.Run(); \ No newline at end of file diff --git a/src/RadioIndexDb.cs b/src/RadioIndexDb.cs index e69ca4b..6b629a7 100644 --- a/src/RadioIndexDb.cs +++ b/src/RadioIndexDb.cs @@ -40,10 +40,10 @@ select last_insert_rowid();", new { using var db = new SqliteConnection(ConnectionString); if (!db.TableExists("episodes")) return -1; return db.ExecuteScalar(@" -insert into episodes(title,subtitle,canonical_url,nrk_id,series_id,season_id,source_url) values (@title,@subtitle,@canonical_url,@nrk_id,@series_id,@season_id,@source_url); +insert into episodes(name,description,canonical_url,nrk_id,series_id,season_id,source_url) values (@name,@description,@canonical_url,@nrk_id,@series_id,@season_id,@source_url); select last_insert_rowid();", new { - title = entry.Title, - subtitle = entry.Subtitle, + name = entry.Name, + description = entry.Description, canonical_url = entry.CanonicalUrl, nrk_id = entry.NrkId, series_id = entry.SeriesId, @@ -52,12 +52,22 @@ select last_insert_rowid();", new { }); } - public static void DeleteSeries(int id) { } - - public static RadioSeries GetSeries(int id) { + public static RadioSeries GetSeriesByNrkId(string nrkId) { using var db = new SqliteConnection(ConnectionString); if (!db.TableExists("series")) return default; - return db.QueryFirstOrDefault(@"select * from series where id=@id", new {id}); + return db.QueryFirstOrDefault(@"select * from series where nrk_id=@nrkId", new {nrkId}); + } + + public static RadioSeason GetSeasonByNrkId(string nrkId) { + using var db = new SqliteConnection(ConnectionString); + if (!db.TableExists("seasons")) return default; + return db.QueryFirstOrDefault(@"select * from seasons where nrk_id=@nrkId", new {nrkId}); + } + + public static RadioEpisode GetEpisodeByNrkId(string nrkId) { + using var db = new SqliteConnection(ConnectionString); + if (!db.TableExists("episodes")) return default; + return db.QueryFirstOrDefault(@"select * from episodes where nrk_id=@nrkId", new {nrkId}); } public static List GetSeries(string query, bool includeEpisodes = false) { diff --git a/src/Services/NrkRadioService.cs b/src/Services/NrkRadioService.cs index d4e06a5..7077d16 100644 --- a/src/Services/NrkRadioService.cs +++ b/src/Services/NrkRadioService.cs @@ -17,51 +17,41 @@ public class NrkRadioService } public async Task CreateIndex() { - var letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ".ToCharArray(); + var letters = "#ABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ".ToCharArray(); var skip = 0; foreach (var letter in letters) { - var path = "/radio/search/categories/alt-innhold?letter=" + letter + "&skip=0&take=50"; + + // TODO: Support series._links.podcast in addition to series._links.seasons + var path = "/radio/search/categories/alt-innhold?letter=" + letter == "#" ? "%23" : letter + "&skip=0&take=50"; while (path.HasValue()) { var response = await _http.GetFromJsonAsync(path); if (response == default) break; await Task.Delay(2000); foreach (var series in response.Series) { - var dbSeries = new RadioSeries { + var dbSeries = RadioIndexDb.GetSeriesByNrkId(series.Id) ?? new RadioSeries { Name = series.Title, NrkId = series.Id, Type = series.Type, }; - var seriesId = RadioIndexDb.AddSeries(dbSeries); + var seriesId = dbSeries.Id > 0 ? dbSeries.Id : RadioIndexDb.AddSeries(dbSeries); _logger.LogInformation("Added series {0} with id {1}, to the database", dbSeries.Name, seriesId); - if (!series.Links.Series.Href.HasValue()) continue; + if (!series.Links?.Series?.Href?.HasValue() ?? true) continue; var seriesMetadata = await _http.GetFromJsonAsync(series.Links.Series.Href); if (seriesMetadata == default) continue; await Task.Delay(1000); - foreach (var season in seriesMetadata.Embedded.Seasons) { - var dbSeason = new RadioSeason() { - Name = season.Titles.FirstOrDefault()?.Title, - NrkId = season.Id, - SeriesId = seriesId - }; - var seasonId = RadioIndexDb.AddSeason(dbSeason); - _logger.LogInformation("Added season {0} to series {1} with id {2}, to the database", dbSeason.Name, dbSeries.Name, seasonId); - foreach (var episode in season.Episodes) { - foreach (var actuallyEpisode in episode.Embedded.Episodes) { - var dbEpisode = new RadioEpisode { - CanonicalUrl = actuallyEpisode.Links.Share.Href, - Title = actuallyEpisode.Titles.FirstOrDefault()?.Title, - Subtitle = actuallyEpisode.Titles.FirstOrDefault()?.Subtitle, - NrkId = actuallyEpisode.EpisodeId, - SeasonId = seasonId, - SeriesId = dbSeason.SeriesId - }; - var playbackResponse = await _http.GetFromJsonAsync("/playback/manifest/program/" + dbEpisode.NrkId); - if (playbackResponse == default) continue; - dbEpisode.SourceUrl = playbackResponse.Playable.Assets.FirstOrDefault()?.Url; - var episodeId = RadioIndexDb.AddEpisode(dbEpisode); - _logger.LogInformation("Added episode {0} to series {1} season {2} with id {3}, to the database", dbEpisode.Title, dbSeries.Name, dbSeason.Name, episodeId); - } + if (seriesMetadata.Embedded.Seasons?.Any() ?? false) { + foreach (var season in seriesMetadata.Embedded.Seasons) { + var dbSeason = RadioIndexDb.GetSeasonByNrkId(season.Id) ?? new RadioSeason() { + Name = season.Titles.Title, + NrkId = season.Id, + SeriesId = seriesId + }; + var seasonId = dbSeason.Id > 0 ? dbSeason.Id : RadioIndexDb.AddSeason(dbSeason); + _logger.LogInformation("Added season {0} to series {1} with id {2}, to the database", dbSeason.Name, dbSeries.Name, seasonId); + await AddEpisodesAsync(season.Episodes.Embedded.Episodes, dbSeries, dbSeason); } + } else if (seriesMetadata.Embedded.Episodes.Embedded.Episodes?.Any() ?? false) { + await AddEpisodesAsync(seriesMetadata.Embedded.Episodes.Embedded.Episodes, dbSeries); } } @@ -70,6 +60,30 @@ public class NrkRadioService } } + private async Task AddEpisodesAsync(List lol, RadioSeries dbSeries, RadioSeason dbSeason = default) { + foreach (var episode in lol) { + var dbEpisode = RadioIndexDb.GetEpisodeByNrkId(episode.EpisodeId) ?? new RadioEpisode { + CanonicalUrl = episode.Links.Share.Href, + Name = episode.Titles.Title, + Description = episode.Titles.Subtitle, + NrkId = episode.EpisodeId, + SeasonId = dbSeason?.Id ?? -1, + SeriesId = dbSeries.Id + }; + + if (dbEpisode.NrkId.IsNullOrWhiteSpace()) { + _logger.LogWarning("No nrk id was found for episode {0}", dbEpisode.Name); + continue; + } + + var playbackResponse = await _http.GetFromJsonAsync("/playback/manifest/program/" + dbEpisode.NrkId); + if (playbackResponse == default) continue; + dbEpisode.SourceUrl = playbackResponse.Playable.Assets.FirstOrDefault()?.Url; + var episodeId = dbEpisode.Id > 0 ? dbEpisode.Id : RadioIndexDb.AddEpisode(dbEpisode); + _logger.LogInformation("Added episode {0} to series {1} season {2} with id {3}, to the database", dbEpisode.Name, dbSeries.Name, dbSeason?.Name ?? "!!NO SEASON!!", episodeId); + } + } + public async Task SearchCategoriesAsync(string query, int take = 50, int skip = 0) { return await _http.GetFromJsonAsync( "/radio/search/categories/alt-innhold?q=" + query + "&take=" + take + "&skip=" + skip -- cgit v1.3