Add Blob AzureWebApp provider (#477)

* Add Blob AzureWebApp provider

Author: pakrym

src/Microsoft.Extensions.Logging.AzureWebAppDiagnostics/AzureWebAppDiagnosticsFactoryExtensions.cs

@@ -15,13 +15,22 @@ public static class AzureWebAppDiagnosticsFactoryExtensions
         /// Adds an Azure Web Apps diagnostics logger.
         /// </summary>
         /// <param name="factory">The extension method argument</param>
-        /// <param name="fileSizeLimitMb">A strictly positive value representing the maximum log size in megabytes. Once the log is full, no more message will be appended</param>
-        public static ILoggerFactory AddAzureWebAppDiagnostics(this ILoggerFactory factory, int fileSizeLimitMb = FileLoggerProvider.DefaultFileSizeLimitMb)
+        public static ILoggerFactory AddAzureWebAppDiagnostics(this ILoggerFactory factory)
+        {
+            return AddAzureWebAppDiagnostics(factory, new AzureWebAppDiagnosticsSettings());
+        }
+
+        /// <summary>
+        /// Adds an Azure Web Apps diagnostics logger.
+        /// </summary>
+        /// <param name="factory">The extension method argument</param>
+        /// <param name="settings">The setting object to configure loggers.</param>
+        public static ILoggerFactory AddAzureWebAppDiagnostics(this ILoggerFactory factory, AzureWebAppDiagnosticsSettings settings)
         {
             if (WebAppContext.Default.IsRunningInAzureWebApp)
             {
                 // Only add the provider if we're in Azure WebApp. That cannot change once the apps started
-                factory.AddProvider(new AzureWebAppDiagnosticsLoggerProvider(WebAppContext.Default, fileSizeLimitMb));
+                factory.AddProvider(new AzureWebAppDiagnosticsLoggerProvider(WebAppContext.Default, settings));
             }
             return factory;
         }

src/Microsoft.Extensions.Logging.AzureWebAppDiagnostics/AzureWebAppDiagnosticsLoggerProvider.cs

@@ -1,8 +1,11 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System;
 using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Logging.Abstractions.Internal;
 using Microsoft.Extensions.Logging.AzureWebAppDiagnostics.Internal;
+using Serilog;
 
 namespace Microsoft.Extensions.Logging.AzureWebAppDiagnostics
 {
@@ -13,45 +16,58 @@ public class AzureWebAppDiagnosticsLoggerProvider : ILoggerProvider
     {
         private readonly IWebAppLogConfigurationReader _configurationReader;
 
-        private readonly ILoggerProvider _innerLoggerProvider;
-        private readonly bool _runningInWebApp;
+        private readonly LoggerFactory _loggerFactory;
 
         /// <summary>
         /// Creates a new instance of the <see cref="AzureWebAppDiagnosticsLoggerProvider"/> class.
         /// </summary>
-        public AzureWebAppDiagnosticsLoggerProvider(WebAppContext context, int fileSizeLimitMb)
+        public AzureWebAppDiagnosticsLoggerProvider(WebAppContext context, AzureWebAppDiagnosticsSettings settings)
         {
+            if (settings == null)
+            {
+                throw new ArgumentNullException(nameof(settings));
+            }
+
             _configurationReader  = new WebAppLogConfigurationReader(context);
 
             var config = _configurationReader.Current;
-            _runningInWebApp = config.IsRunningInWebApp;
+            var runningInWebApp = config.IsRunningInWebApp;
 
-            if (!_runningInWebApp)
-            {
-                _innerLoggerProvider = NullLoggerProvider.Instance;
-            }
-            else
+            if (runningInWebApp)
             {
-                _innerLoggerProvider = new FileLoggerProvider(_configurationReader, fileSizeLimitMb);
+                _loggerFactory = new LoggerFactory();
+                var fileLoggerProvider = new FileLoggerProvider(
+                    settings.FileSizeLimit,
+                    settings.RetainedFileCountLimit,
+                    settings.BackgroundQueueSize,
+                    settings.OutputTemplate);
 
+                _loggerFactory.AddSerilog(fileLoggerProvider.ConfigureLogger(_configurationReader));
                 if (!string.IsNullOrEmpty(config.BlobContainerUrl))
                 {
-                    // TODO: Add the blob logger by creating a composite inner logger which calls
-                    // both loggers
+                    var blobLoggerProvider = new AzureBlobLoggerProvider(
+                        settings.OutputTemplate,
+                        context.SiteName,
+                        context.SiteInstanceId,
+                        settings.BlobName,
+                        settings.BlobBatchSize,
+                        settings.BackgroundQueueSize,
+                        settings.BlobCommitPeriod);
+                    _loggerFactory.AddSerilog(blobLoggerProvider.ConfigureLogger(_configurationReader));
                 }
             }
         }
 
         /// <inheritdoc />
         public ILogger CreateLogger(string categoryName)
         {
-            return _innerLoggerProvider.CreateLogger(categoryName);
+            return _loggerFactory?.CreateLogger(categoryName) ?? NullLogger.Instance;
         }
 
         /// <inheritdoc />
         public void Dispose()
         {
-            _innerLoggerProvider.Dispose();
+            _loggerFactory?.Dispose();
             _configurationReader.Dispose();
         }
     }

src/Microsoft.Extensions.Logging.AzureWebAppDiagnostics/AzureWebAppDiagnosticsSettings.cs

@@ -0,0 +1,46 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace Microsoft.Extensions.Logging.AzureWebAppDiagnostics
+{
+    /// <summary>
+    /// Settings for <see cref="AzureWebAppDiagnosticsLoggerProvider"/>.
+    /// </summary>
+    public class AzureWebAppDiagnosticsSettings
+    {
+        /// <summary>
+        /// Gets or sets a strictly positive value representing the maximum log size in bytes. Once the log is full, no more message will be appended.
+        /// </summary>
+        public int FileSizeLimit { get; set; } = 10 * 1024 * 1024;
+
+        /// <summary>
+        /// Gets or sets a strictly positive value representing the maximum retained file count.
+        /// </summary>
+        public int RetainedFileCountLimit { get; set; } = 2;
+
+        /// <summary>
+        /// Gets or sets a message template describing the output messages.
+        /// </summary>
+        public string OutputTemplate { get; set; } = "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level}] {Message}{NewLine}{Exception}";
+
+        /// <summary>
+        /// Gets or sets a maximum number of events to include in a single blob append batch.
+        /// </summary>
+        public int BlobBatchSize { get; set; } = 32;
+
+        /// <summary>
+        /// Gets or sets a time to wait between checking for blob log batches.
+        /// </summary>
+        public TimeSpan BlobCommitPeriod { get; set; } = TimeSpan.FromSeconds(5);
+
+        /// <summary>
+        /// Gets or sets the last section of log blob name.
+        /// </summary>
+        public string BlobName { get; set; } = "applicationLog.txt";
+
+        /// Gets of sets the maximum size of the background log message queue.
+        public int BackgroundQueueSize { get; set; }
+    }
+}

src/Microsoft.Extensions.Logging.AzureWebAppDiagnostics/Internal/AzureBlobLoggerProvider.cs

@@ -0,0 +1,99 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Serilog;
+using Serilog.Core;
+using Serilog.Formatting.Display;
+using Microsoft.WindowsAzure.Storage.Blob;
+
+namespace Microsoft.Extensions.Logging.AzureWebAppDiagnostics.Internal
+{
+    /// <summary>
+    /// The implemenation of logger provider that creates instances of <see cref="Serilog.Core.Logger"/>.
+    /// </summary>
+    public class AzureBlobLoggerProvider
+    {
+        private readonly string _outputTemplate;
+        private readonly string _appName;
+        private readonly string _instanceId;
+        private readonly string _fileName;
+        private readonly int _batchSize;
+        private readonly int _backgroundQueueSize;
+        private readonly TimeSpan _period;
+
+        /// <summary>
+        /// Creates a new instance of the <see cref="AzureBlobLoggerProvider"/> class.
+        /// </summary>
+        /// <param name="outputTemplate">A message template describing the output messages</param>
+        /// <param name="appName">The application name to use in blob name</param>
+        /// <param name="instanceId">The application instance id to use in blob name</param>
+        /// <param name="fileName">The last section in log blob name</param>
+        /// <param name="batchSize">A maximum number of events to include in a single blob append batch</param>
+        /// <param name="backgroundQueueSize">The maximum size of the background queue</param>
+        /// <param name="period">A time to wait between checking for blob log batches</param>
+        public AzureBlobLoggerProvider(string outputTemplate, string appName, string instanceId, string fileName, int batchSize, int backgroundQueueSize, TimeSpan period)
+        {
+            if (outputTemplate == null)
+            {
+                throw new ArgumentNullException(nameof(outputTemplate));
+            }
+            if (appName == null)
+            {
+                throw new ArgumentNullException(nameof(appName));
+            }
+            if (instanceId == null)
+            {
+                throw new ArgumentNullException(nameof(instanceId));
+            }
+            if (fileName == null)
+            {
+                throw new ArgumentNullException(nameof(fileName));
+            }
+            if (batchSize <= 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(batchSize), $"{nameof(batchSize)} should be a positive number.");
+            }
+            if (period <= TimeSpan.Zero)
+            {
+                throw new ArgumentOutOfRangeException(nameof(period), $"{nameof(period)} should be longer than zero.");
+            }
+
+            _outputTemplate = outputTemplate;
+            _appName = appName;
+            _instanceId = instanceId;
+            _fileName = fileName;
+            _batchSize = batchSize;
+            _backgroundQueueSize = backgroundQueueSize;
+            _period = period;
+        }
+
+        /// <inheritdoc />
+        public Logger ConfigureLogger(IWebAppLogConfigurationReader reader)
+        {
+            var messageFormatter = new MessageTemplateTextFormatter(_outputTemplate, null);
+            var container = new CloudBlobContainer(new Uri(reader.Current.BlobContainerUrl));
+            var fileName = _instanceId + "-" + _fileName;
+            var azureBlobSink = new AzureBlobSink(
+                name => new BlobAppendReferenceWrapper(container.GetAppendBlobReference(name)),
+                _appName,
+                fileName,
+                messageFormatter,
+                _batchSize,
+                _period);
+
+            var backgroundSink = new BackgroundSink(azureBlobSink, _backgroundQueueSize);
+            var loggerConfiguration = new LoggerConfiguration();
+
+            loggerConfiguration.WriteTo.Sink(backgroundSink);
+            loggerConfiguration.MinimumLevel.ControlledBy(new WebConfigurationReaderLevelSwitch(reader,
+                configuration =>
+                {
+                    return configuration.BlobLoggingEnabled ? configuration.BlobLoggingLevel : LogLevel.None;
+                }));
+
+            return loggerConfiguration.CreateLogger();
+        }
+
+    }
+}

src/Microsoft.Extensions.Logging.AzureWebAppDiagnostics/Internal/AzureBlobSink.cs

@@ -0,0 +1,111 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Storage;
+using Serilog.Core;
+using Serilog.Events;
+using Serilog.Formatting;
+using Serilog.Sinks.PeriodicBatching;
+
+namespace Microsoft.Extensions.Logging.AzureWebAppDiagnostics.Internal
+{
+    /// <summary>
+    /// The <see cref="ILogEventSink"/> implemenation that stores messages by appending them to Azure Blob in batches.
+    /// </summary>
+    public class AzureBlobSink : PeriodicBatchingSink
+    {
+        private readonly string _appName;
+        private readonly string _fileName;
+        private readonly ITextFormatter _formatter;
+        private readonly Func<string, ICloudAppendBlob> _blobReferenceFactory;
+
+        /// <summary>
+        /// Creates a new instance of <see cref="AzureBlobSink"/>
+        /// </summary>
+        /// <param name="blobReferenceFactory">The container to store logs to.</param>
+        /// <param name="appName">The application name to use in blob path generation.</param>
+        /// <param name="fileName">The last segment of blob name.</param>
+        /// <param name="formatter">The <see cref="ITextFormatter"/> for log messages.</param>
+        /// <param name="batchSizeLimit">The maximum number of events to include in a single batch.</param>
+        /// <param name="period">The time to wait between checking for event batches.</param>
+        public AzureBlobSink(Func<string, ICloudAppendBlob> blobReferenceFactory,
+            string appName,
+            string fileName,
+            ITextFormatter formatter,
+            int batchSizeLimit,
+            TimeSpan period) : base(batchSizeLimit, period)
+        {
+            if (appName == null)
+            {
+                throw new ArgumentNullException(nameof(appName));
+            }
+            if (fileName == null)
+            {
+                throw new ArgumentNullException(nameof(fileName));
+            }
+            if (formatter == null)
+            {
+                throw new ArgumentNullException(nameof(formatter));
+            }
+            if (batchSizeLimit <= 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(batchSizeLimit), $"{nameof(batchSizeLimit)} should be a positive number.");
+            }
+            if (period <= TimeSpan.Zero)
+            {
+                throw new ArgumentOutOfRangeException(nameof(period), $"{nameof(period)} should be longer than zero.");
+            }
+
+            _appName = appName;
+            _fileName = fileName;
+            _formatter = formatter;
+            _blobReferenceFactory = blobReferenceFactory;
+        }
+
+        /// <inheritdoc />
+        protected override async Task EmitBatchAsync(IEnumerable<LogEvent> events)
+        {
+            var eventGroups = events.GroupBy(GetBlobKey);
+            foreach (var eventGroup in eventGroups)
+            {
+                var key = eventGroup.Key;
+                var blobName = $"{_appName}/{key.Item1}/{key.Item2:00}/{key.Item3:00}/{key.Item4:00}/{_fileName}";
+
+                var blob = _blobReferenceFactory(blobName);
+
+                Stream stream;
+                try
+                {
+                    stream = await blob.OpenWriteAsync();
+                }
+                // Blob does not exist
+                catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 404)
+                {
+                    await blob.CreateAsync();
+                    stream = await blob.OpenWriteAsync();
+                }
+
+                using (var writer = new StreamWriter(stream))
+                {
+                    foreach (var logEvent in eventGroup)
+                    {
+                        _formatter.Format(logEvent, writer);
+                    }
+                }
+            }
+        }
+
+        private Tuple<int,int,int,int> GetBlobKey(LogEvent e)
+        {
+            return Tuple.Create(e.Timestamp.Year,
+                e.Timestamp.Month,
+                e.Timestamp.Day,
+                e.Timestamp.Hour);
+        }
+    }
+}