From 1bd30ee34323f150c63fc537e0d131dca29dc4ef Mon Sep 17 00:00:00 2001 From: ivarlovlie Date: Sun, 5 Jun 2022 00:19:10 +0200 Subject: refactor: Implement caching in VaultService and use VaultService instead of IOptions --- server/src/Data/Static/AppConfiguration.cs | 25 ++++++++++ server/src/Data/Static/AppConstants.cs | 1 + server/src/Data/Static/AppEnvironmentVariables.cs | 1 + server/src/Data/Static/AppHeaders.cs | 1 + .../Internal/Root/ReadConfigurationRoute.cs | 17 +++++++ .../Internal/Root/RefreshConfigurationRoute.cs | 15 ++++++ .../src/Endpoints/V1/ApiTokens/CreateTokenRoute.cs | 4 +- server/src/Program.cs | 34 +++----------- server/src/Services/ForgotPasswordService.cs | 14 +++--- server/src/Services/MailService.cs | 11 +++-- server/src/Services/VaultService.cs | 53 ++++++++++++++++++--- server/src/Utilities/BasicAuthenticationHandler.cs | 4 +- server/src/Utilities/ConfigurationExtensions.cs | 54 ++++++++++++++++++++-- 13 files changed, 179 insertions(+), 55 deletions(-) create mode 100644 server/src/Endpoints/Internal/Root/ReadConfigurationRoute.cs create mode 100644 server/src/Endpoints/Internal/Root/RefreshConfigurationRoute.cs (limited to 'server/src') diff --git a/server/src/Data/Static/AppConfiguration.cs b/server/src/Data/Static/AppConfiguration.cs index a6494ea..08fc716 100644 --- a/server/src/Data/Static/AppConfiguration.cs +++ b/server/src/Data/Static/AppConfiguration.cs @@ -24,4 +24,29 @@ public class AppConfiguration public string GITHUB_CLIENT_ID { get; set; } public string GITHUB_CLIENT_SECRET { get; set; } public string APP_AES_KEY { get; set; } + + public object GetPublicVersion() { + return new { + DB_HOST, + DB_PORT, + DB_USER, + DB_PASSWORD = DB_PASSWORD.Obfuscate() ?? "", + QUARTZ_DB_HOST, + QUARTZ_DB_PORT, + QUARTZ_DB_USER, + QUARTZ_DB_PASSWORD = QUARTZ_DB_PASSWORD.Obfuscate() ?? "", + SEQ_API_KEY = SEQ_API_KEY.Obfuscate() ?? "", + SEQ_API_URL, + SMTP_HOST, + SMTP_PORT, + SMTP_USER = SMTP_USER.Obfuscate() ?? "", + SMTP_PASSWORD = SMTP_PASSWORD.Obfuscate() ?? "", + EMAIL_FROM_ADDRESS, + EMAIL_FROM_DISPLAY_NAME, + PORTAL_URL, + GITHUB_CLIENT_ID = GITHUB_CLIENT_ID.Obfuscate() ?? "", + GITHUB_CLIENT_SECRET = GITHUB_CLIENT_SECRET.Obfuscate() ?? "", + APP_AES_KEY = APP_AES_KEY.Obfuscate() ?? "", + }; + } } diff --git a/server/src/Data/Static/AppConstants.cs b/server/src/Data/Static/AppConstants.cs index 61e5cd5..461317b 100644 --- a/server/src/Data/Static/AppConstants.cs +++ b/server/src/Data/Static/AppConstants.cs @@ -8,4 +8,5 @@ public static class AppConstants public const string TOKEN_ALLOW_CREATE = "TOKEN_ALLOW_CREATE"; public const string TOKEN_ALLOW_UPDATE = "TOKEN_ALLOW_UPDATE"; public const string TOKEN_ALLOW_DELETE = "TOKEN_ALLOW_DELETE"; + public const string VAULT_CACHE_KEY = "VAULT_CACHE_KEY"; } diff --git a/server/src/Data/Static/AppEnvironmentVariables.cs b/server/src/Data/Static/AppEnvironmentVariables.cs index 181eced..4555bfb 100644 --- a/server/src/Data/Static/AppEnvironmentVariables.cs +++ b/server/src/Data/Static/AppEnvironmentVariables.cs @@ -5,4 +5,5 @@ public static class AppEnvironmentVariables public const string VAULT_TOKEN = "VAULT_TOKEN"; public const string VAULT_URL = "VAULT_URL"; public const string MAIN_CONFIG_SHEET = "MAIN_CONFIG_SHEET"; + public const string VAULT_CACHE_TTL = "CONFIG_CACHE_TTL"; } diff --git a/server/src/Data/Static/AppHeaders.cs b/server/src/Data/Static/AppHeaders.cs index 41a3085..7912418 100644 --- a/server/src/Data/Static/AppHeaders.cs +++ b/server/src/Data/Static/AppHeaders.cs @@ -3,4 +3,5 @@ namespace IOL.GreatOffice.Api.Data.Static; public static class AppHeaders { public const string BROWSER_TIME_ZONE = "X-TimeZone"; + public const string VAULT_TOKEN = "X-Vault-Token"; } diff --git a/server/src/Endpoints/Internal/Root/ReadConfigurationRoute.cs b/server/src/Endpoints/Internal/Root/ReadConfigurationRoute.cs new file mode 100644 index 0000000..e0dcca3 --- /dev/null +++ b/server/src/Endpoints/Internal/Root/ReadConfigurationRoute.cs @@ -0,0 +1,17 @@ +namespace IOL.GreatOffice.Api.Endpoints.Internal.Root; + +public class ReadConfigurationRoute : RouteBaseSync.WithoutRequest.WithActionResult +{ + private readonly VaultService _vaultService; + + public ReadConfigurationRoute(VaultService vaultService) { + _vaultService = vaultService; + } + + [AllowAnonymous] + [HttpGet("~/_/configuration")] + public override ActionResult Handle() { + var config = _vaultService.GetCurrentAppConfiguration(); + return Content(JsonSerializer.Serialize(config.GetPublicVersion()), "application/json"); + } +} diff --git a/server/src/Endpoints/Internal/Root/RefreshConfigurationRoute.cs b/server/src/Endpoints/Internal/Root/RefreshConfigurationRoute.cs new file mode 100644 index 0000000..4b1beec --- /dev/null +++ b/server/src/Endpoints/Internal/Root/RefreshConfigurationRoute.cs @@ -0,0 +1,15 @@ +namespace IOL.GreatOffice.Api.Endpoints.Internal.Root; + +public class RefreshConfigurationRoute : RouteBaseSync.WithoutRequest.WithoutResult +{ + private readonly VaultService _vaultService; + + public RefreshConfigurationRoute(VaultService vaultService) { + _vaultService = vaultService; + } + + [HttpGet("~/_/refresh-configuration")] + public override void Handle() { + _vaultService.RefreshCurrentAppConfiguration(); + } +} diff --git a/server/src/Endpoints/V1/ApiTokens/CreateTokenRoute.cs b/server/src/Endpoints/V1/ApiTokens/CreateTokenRoute.cs index 3e19626..352ad18 100644 --- a/server/src/Endpoints/V1/ApiTokens/CreateTokenRoute.cs +++ b/server/src/Endpoints/V1/ApiTokens/CreateTokenRoute.cs @@ -9,9 +9,9 @@ public class CreateTokenRoute : RouteBaseSync.WithRequest _logger; - public CreateTokenRoute(AppDbContext context, IOptions configuration, ILogger logger) { + public CreateTokenRoute(AppDbContext context, VaultService vaultService, ILogger logger) { _context = context; - _configuration = configuration.Value; + _configuration = vaultService.GetCurrentAppConfiguration(); _logger = logger; } diff --git a/server/src/Program.cs b/server/src/Program.cs index e6e8369..4499b1a 100644 --- a/server/src/Program.cs +++ b/server/src/Program.cs @@ -42,7 +42,6 @@ using IOL.GreatOffice.Api.Endpoints.V1; using IOL.GreatOffice.Api.Jobs; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.AspNetCore.Mvc.Versioning; -using Microsoft.Extensions.Options; using Quartz; namespace IOL.GreatOffice.Api; @@ -53,20 +52,13 @@ public static class Program var builder = WebApplication.CreateBuilder(args); builder.Services.AddLogging(); builder.Services.AddHttpClient(); - + builder.Services.AddMemoryCache(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); - builder.Services.AddScoped(); + builder.Services.AddTransient(); var vaultService = builder.Services.BuildServiceProvider().GetRequiredService(); - var configurationResponse = vaultService.GetSecretAsync(builder.Configuration.GetValue(AppEnvironmentVariables.MAIN_CONFIG_SHEET)).Result; - builder.Services.AddOptions() - .Configure(o => { - foreach (var property in typeof(AppConfiguration).GetProperties().Where(p => p.CanWrite)) { - property.SetValue(o, property.GetValue(configurationResponse.Data.Data, null), null); - } - }); - var configuration = builder.Services.BuildServiceProvider().GetRequiredService>().Value; + var configuration = vaultService.GetCurrentAppConfiguration(); var logger = new LoggerConfiguration() .Enrich.FromLogContext() .ReadFrom.Configuration(builder.Configuration) @@ -78,20 +70,7 @@ public static class Program Log.Logger = logger.CreateLogger(); Log.Information("Starting web host, " - + JsonSerializer.Serialize(new { - DateTime.UtcNow, - PID = Environment.ProcessId, - configuration.DB_HOST, - configuration.DB_PORT, - configuration.DB_USER, - configuration.DB_NAME, - DB_PASS = configuration.DB_PASSWORD.Obfuscate() ?? "!!!Empty!!!", - configuration.QUARTZ_DB_HOST, - configuration.QUARTZ_DB_PORT, - configuration.QUARTZ_DB_USER, - configuration.QUARTZ_DB_NAME, - QUARTZ_DB_PASS = configuration.QUARTZ_DB_PASSWORD.Obfuscate() ?? "!!!Empty!!!", - }, + + JsonSerializer.Serialize(configuration.GetPublicVersion(), new JsonSerializerOptions() { WriteIndented = true })); @@ -115,7 +94,7 @@ public static class Program builder.Services.Configure(JsonSettings.Default); builder.Services.AddQuartz(options => { options.UsePersistentStore(o => { - o.UsePostgres(builder.Configuration.GetQuartzDatabaseConnectionString(configuration)); + o.UsePostgres(builder.Configuration.GetQuartzDatabaseConnectionString(vaultService.GetCurrentAppConfiguration)); o.UseSerializer(); }); options.UseMicrosoftDependencyInjectionJobFactory(); @@ -154,9 +133,8 @@ public static class Program }) .AddScheme(AppConstants.BASIC_AUTH_SCHEME, default); - builder.Services.AddDbContext(options => { - options.UseNpgsql(builder.Configuration.GetAppDatabaseConnectionString(configuration), + options.UseNpgsql(builder.Configuration.GetAppDatabaseConnectionString(vaultService.GetCurrentAppConfiguration), npgsqlDbContextOptionsBuilder => { npgsqlDbContextOptionsBuilder.UseQuerySplittingBehavior(QuerySplittingBehavior.SplitQuery); npgsqlDbContextOptionsBuilder.EnableRetryOnFailure(5, TimeSpan.FromSeconds(10), default); diff --git a/server/src/Services/ForgotPasswordService.cs b/server/src/Services/ForgotPasswordService.cs index e6b6acf..6874d37 100644 --- a/server/src/Services/ForgotPasswordService.cs +++ b/server/src/Services/ForgotPasswordService.cs @@ -1,23 +1,21 @@ -using Microsoft.Extensions.Options; - namespace IOL.GreatOffice.Api.Services; public class ForgotPasswordService { private readonly AppDbContext _context; private readonly MailService _mailService; - private readonly IOptions _configuration; + private readonly AppConfiguration _configuration; private readonly ILogger _logger; public ForgotPasswordService( AppDbContext context, - IOptions configuration, + VaultService vaultService, ILogger logger, MailService mailService ) { _context = context; - _configuration = configuration; + _configuration = vaultService.GetCurrentAppConfiguration(); _logger = logger; _mailService = mailService; } @@ -59,9 +57,9 @@ public class ForgotPasswordService var request = new ForgotPasswordRequest(user); _context.ForgotPasswordRequests.Add(request); await _context.SaveChangesAsync(cancellationToken); - var portalUrl = _configuration.Value.PORTAL_URL; - var emailFromAddress = _configuration.Value.EMAIL_FROM_ADDRESS; - var emailFromDisplayName = _configuration.Value.EMAIL_FROM_DISPLAY_NAME; + var portalUrl = _configuration.PORTAL_URL; + var emailFromAddress = _configuration.EMAIL_FROM_ADDRESS; + var emailFromDisplayName = _configuration.EMAIL_FROM_DISPLAY_NAME; var zonedExpirationDate = TimeZoneInfo.ConvertTimeBySystemTimeZoneId(request.ExpirationDate, requestTz.Id); var message = new MailMessage { From = new MailAddress(emailFromAddress, emailFromDisplayName), diff --git a/server/src/Services/MailService.cs b/server/src/Services/MailService.cs index d773303..eaff764 100644 --- a/server/src/Services/MailService.cs +++ b/server/src/Services/MailService.cs @@ -15,12 +15,13 @@ public class MailService /// /// /// - public MailService(IOptions configuration, ILogger logger) { + public MailService(VaultService vaultService, ILogger logger) { + var configuration = vaultService.GetCurrentAppConfiguration(); _logger = logger; - _emailHost = configuration.Value.SMTP_HOST; - _emailPort = Convert.ToInt32(configuration.Value.SMTP_PORT); - _emailUser = configuration.Value.SMTP_USER; - _emailPassword = configuration.Value.SMTP_PASSWORD; + _emailHost = configuration.SMTP_HOST; + _emailPort = Convert.ToInt32(configuration.SMTP_PORT); + _emailUser = configuration.SMTP_USER; + _emailPassword = configuration.SMTP_PASSWORD; } /// diff --git a/server/src/Services/VaultService.cs b/server/src/Services/VaultService.cs index 388f8d4..6034586 100644 --- a/server/src/Services/VaultService.cs +++ b/server/src/Services/VaultService.cs @@ -1,21 +1,52 @@ +using Microsoft.Extensions.Caching.Memory; + namespace IOL.GreatOffice.Api.Services; public class VaultService { private readonly HttpClient _client; + private readonly IMemoryCache _cache; + private readonly IConfiguration _configuration; + private int CACHE_TTL { get; set; } - public VaultService(HttpClient client, IConfiguration configuration) { - var token = configuration.GetValue("VAULT_TOKEN"); - var vaultUrl = configuration.GetValue("VAULT_URL"); + public VaultService(HttpClient client, IConfiguration configuration, IMemoryCache cache) { + var token = configuration.GetValue(AppEnvironmentVariables.VAULT_TOKEN); + var vaultUrl = configuration.GetValue(AppEnvironmentVariables.VAULT_URL); + CACHE_TTL = configuration.GetValue(AppEnvironmentVariables.VAULT_CACHE_TTL, 60 * 60 * 12); if (token.IsNullOrWhiteSpace()) throw new ApplicationException("VAULT_TOKEN is empty"); if (vaultUrl.IsNullOrWhiteSpace()) throw new ApplicationException("VAULT_URL is empty"); - client.DefaultRequestHeaders.Add("X-Vault-Token", token); + client.DefaultRequestHeaders.Add(AppHeaders.VAULT_TOKEN, token); client.BaseAddress = new Uri(vaultUrl); _client = client; + _cache = cache; + _configuration = configuration; + } + + public static object Data { get; set; } + + public T Get(string path) { + return _cache.GetOrCreate(AppConstants.VAULT_CACHE_KEY, + cacheEntry => { + cacheEntry.AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(CACHE_TTL); + var getSecretResponse = _client.GetFromJsonAsync>("/v1/kv/data/" + path).Result; + if (getSecretResponse != null) { + Log.Debug("Setting new Vault cache, " + + new { + PATH = path, + CACHE_TTL, + Data = JsonSerializer.Serialize(getSecretResponse.Data.Data) + }); + return getSecretResponse.Data.Data ?? default; + } + + return default; + }); } - public async Task> GetSecretAsync(string path) { - return await _client.GetFromJsonAsync>("/v1/kv/data/" + path); + public T Refresh(string path) { + _cache.Remove(AppConstants.VAULT_CACHE_KEY); + CACHE_TTL = _configuration.GetValue(AppEnvironmentVariables.VAULT_CACHE_TTL, 60 * 60 * 12); + return Get(path); } public async Task RenewTokenAsync(string token) { @@ -30,6 +61,16 @@ public class VaultService return default; } + public AppConfiguration GetCurrentAppConfiguration() { + var path = _configuration.GetValue(AppEnvironmentVariables.MAIN_CONFIG_SHEET); + return Get(path); + } + + public AppConfiguration RefreshCurrentAppConfiguration() { + var path = _configuration.GetValue(AppEnvironmentVariables.MAIN_CONFIG_SHEET); + return Refresh(path); + } + public class RenewTokenResponse { public Guid RequestId { get; set; } diff --git a/server/src/Utilities/BasicAuthenticationHandler.cs b/server/src/Utilities/BasicAuthenticationHandler.cs index 1793c95..6138193 100644 --- a/server/src/Utilities/BasicAuthenticationHandler.cs +++ b/server/src/Utilities/BasicAuthenticationHandler.cs @@ -17,11 +17,11 @@ public class BasicAuthenticationHandler : AuthenticationHandler configuration + VaultService vaultService ) : base(options, logger, encoder, clock) { _context = context; - _configuration = configuration.Value; + _configuration = vaultService.GetCurrentAppConfiguration(); _logger = logger.CreateLogger(); } diff --git a/server/src/Utilities/ConfigurationExtensions.cs b/server/src/Utilities/ConfigurationExtensions.cs index ff978f0..405c702 100644 --- a/server/src/Utilities/ConfigurationExtensions.cs +++ b/server/src/Utilities/ConfigurationExtensions.cs @@ -9,11 +9,34 @@ public static class ConfigurationExtensions var user = configuration.DB_USER; var password = configuration.DB_PASSWORD; + var res = ""; if (config.GetValue("ASPNETCORE_ENVIRONMENT") == "Development") { - return $"Server={host};Port={port};Database={database};User Id={user};Password={password};Include Error Detail=true"; + res = $"Server={host};Port={port};Database={database};User Id={user};Password={password};Include Error Detail=true"; + } else { + res = $"Server={host};Port={port};Database={database};User Id={user};Password={password}"; } - return $"Server={host};Port={port};Database={database};User Id={user};Password={password}"; + Log.Debug("Using app database connection string: " + res); + return res; + } + + public static string GetAppDatabaseConnectionString(this IConfiguration config, Func configuration) { + var _configuration = configuration(); + var host = _configuration.DB_HOST; + var port = _configuration.DB_PORT; + var database = _configuration.DB_NAME; + var user = _configuration.DB_USER; + var password = _configuration.DB_PASSWORD; + + var res = ""; + if (config.GetValue("ASPNETCORE_ENVIRONMENT") == "Development") { + res = $"Server={host};Port={port};Database={database};User Id={user};Password={password};Include Error Detail=true"; + } else { + res = $"Server={host};Port={port};Database={database};User Id={user};Password={password}"; + } + + Log.Debug("Using app database connection string: " + res); + return res; } public static string GetQuartzDatabaseConnectionString(this IConfiguration config, AppConfiguration configuration) { @@ -23,11 +46,34 @@ public static class ConfigurationExtensions var user = configuration.QUARTZ_DB_USER; var password = configuration.QUARTZ_DB_PASSWORD; + var res = ""; + if (config.GetValue("ASPNETCORE_ENVIRONMENT") == "Development") { + res = $"Server={host};Port={port};Database={database};User Id={user};Password={password};Include Error Detail=true"; + } else { + res = $"Server={host};Port={port};Database={database};User Id={user};Password={password}"; + } + + Log.Debug("Using quartz database connection string: " + res); + return res; + } + + public static string GetQuartzDatabaseConnectionString(this IConfiguration config, Func configuration) { + var _configuration = configuration(); + var host = _configuration.QUARTZ_DB_HOST; + var port = _configuration.QUARTZ_DB_PORT; + var database = _configuration.QUARTZ_DB_NAME; + var user = _configuration.QUARTZ_DB_USER; + var password = _configuration.QUARTZ_DB_PASSWORD; + + var res = ""; if (config.GetValue("ASPNETCORE_ENVIRONMENT") == "Development") { - return $"Server={host};Port={port};Database={database};User Id={user};Password={password};Include Error Detail=true"; + res = $"Server={host};Port={port};Database={database};User Id={user};Password={password};Include Error Detail=true"; + } else { + res = $"Server={host};Port={port};Database={database};User Id={user};Password={password}"; } - return $"Server={host};Port={port};Database={database};User Id={user};Password={password}"; + Log.Debug("Using quartz database connection string: " + res); + return res; } public static string GetVersion(this IConfiguration configuration) { -- cgit v1.3