Allowing multiple KV Paths and enable autorenewal for KV engine in vault

.github/workflows/Dylan.Convey.Secrets.Vault.yml

@@ -0,0 +1,39 @@
+name: .NET
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v2
+      with:
+        dotnet-version: 6.0.x
+    - name: Get current date
+      id: date
+      run: echo "::set-output name=date::$(date +'%Y.%m.%d%H')"
+    - name: Say current date
+      run: echo "${{ steps.date.outputs.date }}"
+    - name: Restore dependencies
+      run: dotnet restore
+      working-directory: src/Convey.Secrets.Vault/src/Convey.Secrets.Vault
+    - name: Build
+      run: dotnet build --no-restore
+      working-directory: src/Convey.Secrets.Vault/src/Convey.Secrets.Vault
+    - name: Test
+      run: dotnet test --no-build --verbosity normal
+      working-directory: src/Convey.Secrets.Vault/src/Convey.Secrets.Vault
+    - name: Pack
+      run: dotnet pack -p:PackageVersion=${{ steps.date.outputs.date }}
+      working-directory: src/Convey.Secrets.Vault/src/Convey.Secrets.Vault
+    - name: push
+      run:  dotnet nuget push **/*.nupkg -k ${{ secrets.NUGET_API_KEY }} -s https://api.nuget.org/v3/index.json
+      working-directory: src/Convey.Secrets.Vault/src/Convey.Secrets.Vault

samples/Conveyor.Services.Orders/Program.cs

@@ -31,6 +31,7 @@
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 
 namespace Conveyor.Services.Orders;
@@ -43,6 +44,12 @@ public static Task Main(string[] args)
     public static IHostBuilder CreateHostBuilder(string[] args)
         => Host.CreateDefaultBuilder(args).ConfigureWebHostDefaults(webBuilder =>
         {
+            var configuration = new ConfigurationBuilder()
+                .AddEnvironmentVariables()
+                .AddCommandLine(args)
+                .AddJsonFile("appsettings.json")
+                .Build();
+
             webBuilder.ConfigureServices(services => services
                     .AddConvey()
                     .AddErrorHandler<ExceptionToResponseMapper>()
@@ -87,6 +94,6 @@ public static IHostBuilder CreateHostBuilder(string[] args)
                     .UseRabbitMq()
                     .SubscribeEvent<DeliveryStarted>())
                 .UseLogging()
-                .UseVault();
+                .UseVault(configuration);
         });
 }

src/Convey.Secrets.Vault/src/Convey.Secrets.Vault/Convey.Secrets.Vault.csproj

@@ -2,17 +2,18 @@
 
   <PropertyGroup>
     <Description>Convey.Secrets.Vault</Description>
-    <Authors>DevMentors.io</Authors>
+	  <Authors>DevMentors.io</Authors>
     <TargetFramework>net6.0</TargetFramework>
     <LangVersion>Latest</LangVersion>
     <Title>Convey.Secrets.Vault</Title>
     <PackageId>Convey.Secrets.Vault</PackageId>
     <PackageTags>Convey.Secrets.Vault</PackageTags>
   </PropertyGroup>
 
-  <ItemGroup>
-    <ProjectReference Include="..\..\..\Convey\src\Convey\Convey.csproj" />
-  </ItemGroup>
+	<ItemGroup>
+		<ProjectReference Include="..\..\..\Convey\src\Convey\Convey.csproj" />
+	</ItemGroup>
+
 
   <ItemGroup>
     <PackageReference Include="VaultSharp" Version="1.7.0.4" />

src/Convey.Secrets.Vault/src/Convey.Secrets.Vault/Extensions.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using Convey;
 using Convey.Secrets.Vault.Internals;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Configuration;
@@ -23,9 +24,8 @@ public static class Extensions
     private static readonly ILeaseService LeaseService = new LeaseService();
     private static readonly ICertificatesService CertificatesService = new CertificatesService();
 
-    public static IHostBuilder UseVault(this IHostBuilder builder, string keyValuePath = null,
-        string sectionName = SectionName)
-        => builder.ConfigureServices(services => services.AddVault(sectionName))
+    public static IHostBuilder UseVault(this IHostBuilder builder,IConfiguration configuration, string sectionName = SectionName)
+        => builder.ConfigureServices(services => services.AddVault(configuration, sectionName))
             .ConfigureAppConfiguration((ctx, cfg) =>
             {
                 var options = cfg.Build().GetOptions<VaultOptions>(sectionName);
@@ -34,12 +34,11 @@ public static IHostBuilder UseVault(this IHostBuilder builder, string keyValuePa
                     return;
                 }
 
-                cfg.AddVaultAsync(options, keyValuePath).GetAwaiter().GetResult();
+                cfg.AddVaultAsync(options).GetAwaiter().GetResult();
             });
 
-    public static IWebHostBuilder UseVault(this IWebHostBuilder builder, string keyValuePath = null,
-        string sectionName = SectionName)
-        => builder.ConfigureServices(services => services.AddVault(sectionName))
+    public static IWebHostBuilder UseVault(this IWebHostBuilder builder,IConfiguration configuration, string sectionName = SectionName)
+        => builder.ConfigureServices(services => services.AddVault(configuration, sectionName))
             .ConfigureAppConfiguration((ctx, cfg) =>
             {
                 var options = cfg.Build().GetOptions<VaultOptions>(sectionName);
@@ -48,21 +47,17 @@ public static IWebHostBuilder UseVault(this IWebHostBuilder builder, string keyV
                     return;
                 }
 
-                cfg.AddVaultAsync(options, keyValuePath).GetAwaiter().GetResult();
+                cfg.AddVaultAsync(options).GetAwaiter().GetResult();
             });
 
-    private static IServiceCollection AddVault(this IServiceCollection services, string sectionName)
+    private static IServiceCollection AddVault(this IServiceCollection services,IConfiguration configuration, string sectionName)
     {
         if (string.IsNullOrWhiteSpace(sectionName))
         {
             sectionName = SectionName;
         }
 
-        IConfiguration configuration;
-        using (var serviceProvider = services.BuildServiceProvider())
-        {
-            configuration = serviceProvider.GetRequiredService<IConfiguration>();
-        }
+
 
         var options = configuration.GetOptions<VaultOptions>(sectionName);
         VerifyOptions(options);
@@ -95,7 +90,7 @@ private static void VerifyOptions(VaultOptions options)
                 options.Kv = new VaultOptions.KeyValueOptions
                 {
                     Enabled = options.Enabled,
-                    Path = options.Key
+                    Paths = new List<string> { options.Key }
                 };
             }
 
@@ -113,23 +108,11 @@ private static void VerifyOptions(VaultOptions options)
         }
     }
 
-    private static async Task AddVaultAsync(this IConfigurationBuilder builder, VaultOptions options,
-        string keyValuePath)
+    private static async Task AddVaultAsync(this IConfigurationBuilder builder, VaultOptions options)
     {
         VerifyOptions(options);
-        var kvPath = string.IsNullOrWhiteSpace(keyValuePath) ? options.Kv?.Path : keyValuePath;
         var (client, _) = GetClientAndSettings(options);
-        if (!string.IsNullOrWhiteSpace(kvPath) && options.Kv.Enabled)
-        {
-            Console.WriteLine($"Loading settings from Vault: '{options.Url}', KV path: '{kvPath}'.");
-            var keyValueSecrets = new KeyValueSecrets(client, options);
-            var secret = await keyValueSecrets.GetAsync(kvPath);
-            var parser = new JsonParser();
-            var json = JsonConvert.SerializeObject(secret);
-            var data = parser.Parse(json);
-            var source = new MemoryConfigurationSource {InitialData = data};
-            builder.Add(source);
-        }
+        builder.AddVaultKeyValueConfiguration(options, client);
 
         if (options.Pki is not null && options.Pki.Enabled)
         {

src/Convey.Secrets.Vault/src/Convey.Secrets.Vault/Internals/IVaultPeriodicalWatcher.cs

@@ -0,0 +1,10 @@
+using Microsoft.Extensions.Primitives;
+
+namespace Convey.Secrets.Vault.Internals
+{
+    internal interface IVaultPeriodicalWatcher
+    {
+        void Dispose();
+        IChangeToken Watch();
+    }
+}

src/Convey.Secrets.Vault/src/Convey.Secrets.Vault/Internals/VaultHostedService.cs

@@ -2,6 +2,7 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using VaultSharp;
@@ -17,9 +18,10 @@ internal sealed class VaultHostedService : BackgroundService
     private readonly VaultOptions _options;
     private readonly ILogger<VaultHostedService> _logger;
     private readonly int _interval;
+    private readonly IConfiguration _configuration;
 
     public VaultHostedService(IVaultClient client, ILeaseService leaseService, ICertificatesIssuer certificatesIssuer,
-        ICertificatesService certificatesService, VaultOptions options, ILogger<VaultHostedService> logger)
+        ICertificatesService certificatesService, VaultOptions options, ILogger<VaultHostedService> logger, IConfiguration configuration)
     {
         _client = client;
         _leaseService = leaseService;
@@ -28,6 +30,7 @@ public VaultHostedService(IVaultClient client, ILeaseService leaseService, ICert
         _options = options;
         _logger = logger;
         _interval = _options.RenewalsInterval <= 0 ? 10 : _options.RenewalsInterval;
+        _configuration = configuration;
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -37,7 +40,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             return;
         }
 
-        if ((_options.Pki is null || !_options.Pki.Enabled) &&
+        if ((_options.Kv is null || !_options.Kv.Enabled || !_options.Kv.AutoRenewal) &&
+            (_options.Pki is null || !_options.Pki.Enabled) &&
             (_options.Lease is null || _options.Lease.All(l => !l.Value.Enabled) ||
              !_options.Lease.Any(l => l.Value.AutoRenewal)))
         {
@@ -50,6 +54,12 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
             var now = DateTime.UtcNow;
             var nextIterationAt = now.AddSeconds(2 * _interval);
+            //if (_options.Kv is not null && _options.Kv.Enabled && _options.Kv.AutoRenewal)
+            //{
+            // _configuration.Pro   
+            //    //var manager = new VaultKeyValueConfigurationProvider(_client, _options);
+            //    //await manager.UpdateConfiguration();
+            //}
 
             if (_options.Pki is not null && _options.Pki.Enabled)
             {

src/Convey.Secrets.Vault/src/Convey.Secrets.Vault/Internals/VaultKeyValueConfigurationBuilderExtensions.cs

@@ -0,0 +1,31 @@
+using Microsoft.Extensions.Configuration;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using VaultSharp;
+
+namespace Convey.Secrets.Vault.Internals
+{
+    internal static  class VaultKeyValueConfigurationBuilderExtensions
+    {
+        public static IConfigurationBuilder AddVaultKeyValueConfiguration(this IConfigurationBuilder builder,
+           VaultOptions options,
+           IVaultClient client)
+        {
+            IVaultPeriodicalWatcher watcher = null;
+            if (options.Kv.AutoRenewal)
+            {
+                watcher = new VaultPeriodicalWatcher(TimeSpan.FromSeconds(options.RenewalsInterval));
+            }
+
+            return builder.Add(new VaultKeyValueConfigurationSource()
+            {
+                Options = options,
+                Client = client,
+                PeriodicalWatcher = watcher
+            });
+        }
+    }
+}

src/Convey.Secrets.Vault/src/Convey.Secrets.Vault/Internals/VaultKeyValueConfigurationProvider.cs

@@ -0,0 +1,63 @@
+
+using Convey.Secrets.Vault;
+using Convey.Secrets.Vault.Internals;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Primitives;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using VaultSharp;
+
+namespace Convey.Secrets.Vault.Internals
+{
+    internal class VaultKeyValueConfigurationProvider: ConfigurationProvider
+    {
+        private readonly VaultKeyValueConfigurationSource _source;
+        private readonly IVaultClient _client;
+        private readonly VaultOptions _options;
+        private readonly IDisposable _changeTokenRegistration;
+
+        public VaultKeyValueConfigurationProvider(VaultKeyValueConfigurationSource source)
+        {
+            _source = source;
+            _client = source.Client;
+            _options = source.Options;
+
+            if (_source.PeriodicalWatcher != null)
+            {
+                _changeTokenRegistration = ChangeToken.OnChange(
+                    () => _source.PeriodicalWatcher.Watch(),
+                    Load
+                );
+            }
+        }
+
+
+        public override void Load()
+        {
+            var kvPaths =  _options.Kv?.Paths;
+            if(kvPaths is null || kvPaths.Count() == 0)
+            {
+                kvPaths.Add(_options.Kv.Path);
+            }
+            JObject kvConfiguration = new JObject();
+            foreach (var kvPath in kvPaths)
+            {
+                if (!string.IsNullOrWhiteSpace(kvPath) && _options.Kv.Enabled)
+                {
+                    Console.WriteLine($"Loading settings from Vault: '{_options.Url}', KV path: '{kvPath}'.");
+                    var keyValueSecrets = new KeyValueSecrets(_client, _options);
+                    var secret =  keyValueSecrets.GetAsync(kvPath).GetAwaiter().GetResult();
+                    kvConfiguration.Merge(JObject.FromObject(secret));
+
+                }
+            }
+            Data = new JsonParser().Parse(kvConfiguration.ToString());
+        }
+    }
+}

src/Convey.Secrets.Vault/src/Convey.Secrets.Vault/Internals/VaultKeyValueConfigurationSource.cs

@@ -0,0 +1,24 @@
+using Convey.Secrets.Vault;
+using Convey.Secrets.Vault.Internals;
+using Microsoft.Extensions.Configuration;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using VaultSharp;
+
+namespace Convey.Secrets.Vault.Internals
+{
+    internal class VaultKeyValueConfigurationSource : IConfigurationSource
+    {
+        public VaultOptions Options { get; set; }
+        public IVaultClient Client { get; set; }
+        public IVaultPeriodicalWatcher PeriodicalWatcher { get; set; }
+        public IConfigurationProvider Build(IConfigurationBuilder builder)
+        {
+
+            return new VaultKeyValueConfigurationProvider(this);
+        }
+    }
+}