From c8a43194cdbd86be873399fc09e4b87761b1dbc4 Mon Sep 17 00:00:00 2001 From: Scott W Harden Date: Sun, 10 Jul 2022 14:06:52 -0400 Subject: [PATCH 1/6] Update README.md clarify that the audio file reader works for WAV and MP3 files --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3c160f0..aee7e88 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ _"I'm sorry Dave... I'm afraid I can't do that"_ * Source code for the WAV reading method is at the bottom of this page. ```cs -(double[] audio, int sampleRate) = ReadWavMono("hal.wav"); +(double[] audio, int sampleRate) = ReadMono("hal.wav"); var sg = new SpectrogramGenerator(sampleRate, fftSize: 4096, stepSize: 500, maxFreq: 3000); sg.Add(audio); sg.SaveImage("hal.png"); @@ -81,7 +81,7 @@ Review the source code of the demo application for additional details and consid This example demonstrates how to convert a MP3 file to a spectrogram image. A sample MP3 audio file in the [data folder](data) contains the audio track from Ken Barker's excellent piano performance of George Frideric Handel's Suite No. 5 in E major for harpsichord ([_The Harmonious Blacksmith_](https://en.wikipedia.org/wiki/The_Harmonious_Blacksmith)). This audio file is included [with permission](dev/Handel%20-%20Air%20and%20Variations.txt), and the [original video can be viewed on YouTube](https://www.youtube.com/watch?v=Mza-xqk770k). ```cs -(double[] audio, int sampleRate) = ReadWavMono("song.wav"); +(double[] audio, int sampleRate) = ReadMono("song.wav"); int fftSize = 16384; int targetWidthPx = 3000; @@ -117,7 +117,7 @@ Spectrogram (2993, 817) These examples demonstrate the identical spectrogram analyzed with a variety of different colormaps. Spectrogram colormaps can be changed by calling the `SetColormap()` method: ```cs -(double[] audio, int sampleRate) = ReadWavMono("hal.wav"); +(double[] audio, int sampleRate) = ReadMono("hal.wav"); var sg = new SpectrogramGenerator(sampleRate, fftSize: 8192, stepSize: 200, maxFreq: 3000); sg.Add(audio); sg.SetColormap(Colormap.Jet); @@ -141,7 +141,7 @@ Cropped Linear Scale (0-3kHz) | Mel Scale (0-22 kHz) Amplitude perception in humans, like frequency perception, is logarithmic. Therefore, Mel spectrograms typically display log-transformed spectral power and are presented using Decibel units. ```cs -(double[] audio, int sampleRate) = ReadWavMono("hal.wav"); +(double[] audio, int sampleRate) = ReadMono("hal.wav"); var sg = new SpectrogramGenerator(sampleRate, fftSize: 4096, stepSize: 500, maxFreq: 3000); sg.Add(audio); @@ -153,12 +153,12 @@ Bitmap bmp = sg.GetBitmapMel(melSizePoints: 250); bmp.Save("halMel.png", ImageFormat.Png); ``` -## Read data from a WAV File +## Read Data from an Audio File You should customize your file-reading method to suit your specific application. I frequently use the NAudio package to read data from WAV and MP3 files. This function reads audio data from a mono WAV file and will be used for the examples on this page. ```cs -(double[] audio, int sampleRate) ReadWavMono(string filePath, double multiplier = 16_000) +(double[] audio, int sampleRate) ReadMono(string filePath, double multiplier = 16_000) { using var afr = new NAudio.Wave.AudioFileReader(filePath); int sampleRate = afr.WaveFormat.SampleRate; From caa99e12aba729c47580c706ea394a2a41965c3d Mon Sep 17 00:00:00 2001 From: Scott W Harden Date: Sun, 10 Jul 2022 14:09:07 -0400 Subject: [PATCH 2/6] Tests: target .NET 6 --- src/Spectrogram.Tests/Spectrogram.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Spectrogram.Tests/Spectrogram.Tests.csproj b/src/Spectrogram.Tests/Spectrogram.Tests.csproj index 073a9df..3ec3d29 100644 --- a/src/Spectrogram.Tests/Spectrogram.Tests.csproj +++ b/src/Spectrogram.Tests/Spectrogram.Tests.csproj @@ -1,7 +1,7 @@ - net5.0 + net6.0 false From 4130489618472a0464898cb002983d27fadd0b15 Mon Sep 17 00:00:00 2001 From: Scott W Harden Date: Sun, 10 Jul 2022 14:09:14 -0400 Subject: [PATCH 3/6] Demo: target .NET 6 --- src/Spectrogram.MicrophoneDemo/Spectrogram.Demo.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Spectrogram.MicrophoneDemo/Spectrogram.Demo.csproj b/src/Spectrogram.MicrophoneDemo/Spectrogram.Demo.csproj index 0aee820..607dc15 100644 --- a/src/Spectrogram.MicrophoneDemo/Spectrogram.Demo.csproj +++ b/src/Spectrogram.MicrophoneDemo/Spectrogram.Demo.csproj @@ -1,6 +1,6 @@  - net5.0-windows + net6.0-windows WinExe false true From 1880e71a2ef547eadc2d65101c9bdbc9e7cbf187 Mon Sep 17 00:00:00 2001 From: Scott W Harden Date: Sun, 10 Jul 2022 14:10:27 -0400 Subject: [PATCH 4/6] CI: target .NET 6 --- .github/workflows/ci.yaml | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c1dd401..2966d92 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,16 +19,10 @@ jobs: - name: 🛒 Checkout uses: actions/checkout@v2 - - name: ✨ Setup .NET 5 - uses: actions/setup-dotnet@v1 - with: - dotnet-version: "5.0.x" - - name: ✨ Setup .NET 6 uses: actions/setup-dotnet@v1 with: dotnet-version: "6.0.x" - include-prerelease: true - name: 🚚 Restore run: dotnet restore src @@ -42,22 +36,12 @@ jobs: - name: 📦 Pack run: dotnet pack src --configuration Release --no-build - - name: 💾 Store Release Package - if: github.event_name == 'release' - uses: actions/upload-artifact@v2 - with: - name: Packages - retention-days: 1 - path: | - src/Spectrogram/bin/Release/*.nupkg - src/Spectrogram/bin/Release/*.snupkg - - - name: 🔑 Configure NuGet Secrets + - name: 🔑 Configure Secrets if: github.event_name == 'release' uses: nuget/setup-nuget@v1 with: nuget-api-key: ${{ secrets.NUGET_API_KEY }} - - name: 🚀 Deploy Release Package + - name: 🚀 Deploy Package if: github.event_name == 'release' run: nuget push "src\Spectrogram\bin\Release\*.nupkg" -SkipDuplicate -Source https://api.nuget.org/v3/index.json From bc4ad217e80df9df665d78f13b89041a5dcfd2f0 Mon Sep 17 00:00:00 2001 From: Scott W Harden Date: Sun, 10 Jul 2022 14:43:24 -0400 Subject: [PATCH 5/6] GetBitmap: add rotate argument resolves #47 --- src/Spectrogram.Tests/ImageTests.cs | 21 +++++ src/Spectrogram/Image.cs | 59 +++----------- src/Spectrogram/ImageMaker.cs | 101 ++++++++++++++++++++++++ src/Spectrogram/Spectrogram.csproj | 1 + src/Spectrogram/SpectrogramGenerator.cs | 5 +- 5 files changed, 139 insertions(+), 48 deletions(-) create mode 100644 src/Spectrogram.Tests/ImageTests.cs create mode 100644 src/Spectrogram/ImageMaker.cs diff --git a/src/Spectrogram.Tests/ImageTests.cs b/src/Spectrogram.Tests/ImageTests.cs new file mode 100644 index 0000000..d02a218 --- /dev/null +++ b/src/Spectrogram.Tests/ImageTests.cs @@ -0,0 +1,21 @@ +using NUnit.Framework; + +namespace Spectrogram.Tests; + +internal class ImageTests +{ + [Test] + public void Test_Image_Rotations() + { + string filePath = $"../../../../../data/cant-do-that-44100.wav"; + (double[] audio, int sampleRate) = AudioFile.ReadWAV(filePath); + SpectrogramGenerator sg = new(sampleRate, 4096, 500, maxFreq: 3000); + sg.Add(audio); + + System.Drawing.Bitmap bmp1 = sg.GetBitmap(rotate: false); + bmp1.Save("test-image-original.png"); + + System.Drawing.Bitmap bmp2 = sg.GetBitmap(rotate: true); + bmp2.Save("test-image-rotated.png"); + } +} diff --git a/src/Spectrogram/Image.cs b/src/Spectrogram/Image.cs index 7b4866a..283543b 100644 --- a/src/Spectrogram/Image.cs +++ b/src/Spectrogram/Image.cs @@ -10,55 +10,22 @@ namespace Spectrogram { public static class Image { - public static Bitmap GetBitmap( - List ffts, - Colormap cmap, - double intensity = 1, - bool dB = false, - double dBScale = 1, - bool roll = false, - int rollOffset = 0) + public static Bitmap GetBitmap(List ffts, Colormap cmap, double intensity = 1, + bool dB = false, double dBScale = 1, bool roll = false, int rollOffset = 0, bool rotate = false) { - if (ffts.Count == 0) - throw new ArgumentException("Not enough data in FFTs to generate an image yet."); - int Width = ffts.Count; - int Height = ffts[0].Length; - - Bitmap bmp = new Bitmap(Width, Height, PixelFormat.Format8bppIndexed); - cmap.Apply(bmp); - - var lockRect = new Rectangle(0, 0, Width, Height); - BitmapData bitmapData = bmp.LockBits(lockRect, ImageLockMode.ReadOnly, bmp.PixelFormat); - int stride = bitmapData.Stride; - - byte[] bytes = new byte[bitmapData.Stride * bmp.Height]; - Parallel.For(0, Width, col => + ImageMaker maker = new() { - int sourceCol = col; - if (roll) - { - sourceCol += Width - rollOffset % Width; - if (sourceCol >= Width) - sourceCol -= Width; - } - - for (int row = 0; row < Height; row++) - { - double value = ffts[sourceCol][row]; - if (dB) - value = 20 * Math.Log10(value * dBScale + 1); - value *= intensity; - value = Math.Min(value, 255); - int bytePosition = (Height - 1 - row) * stride + col; - bytes[bytePosition] = (byte)value; - } - }); - - Marshal.Copy(bytes, 0, bitmapData.Scan0, bytes.Length); - bmp.UnlockBits(bitmapData); - - return bmp; + Colormap = cmap, + Intensity = intensity, + IsDecibel = dB, + DecibelScaleFactor = dBScale, + IsRoll = roll, + RollOffset = rollOffset, + IsRotated = rotate, + }; + + return maker.GetBitmap(ffts); } } } diff --git a/src/Spectrogram/ImageMaker.cs b/src/Spectrogram/ImageMaker.cs new file mode 100644 index 0000000..5d70013 --- /dev/null +++ b/src/Spectrogram/ImageMaker.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Drawing.Imaging; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; + +namespace Spectrogram +{ + /// + /// This class converts a collection of FFTs to a colormapped spectrogram image + /// + public class ImageMaker + { + /// + /// Colormap used to translate intensity to pixel color + /// + public Colormap Colormap; + + /// + /// Intensity is multiplied by this number before converting it to the pixel color according to the colormap + /// + public double Intensity = 1; + + /// + /// If True, intensity will be log-scaled to represent Decibels + /// + public bool IsDecibel = false; + + /// + /// If is enabled, intensity will be scaled by this value prior to log transformation + /// + public double DecibelScaleFactor = 1; + + /// + /// If False, the spectrogram will proceed in time from left to right across the whole image. + /// If True, the image will be broken and the newest FFTs will appear on the left and oldest on the right. + /// + public bool IsRoll = false; + + /// + /// If is enabled, this value indicates the pixel position of the break point. + /// + public int RollOffset = 0; + + /// + /// If True, the spectrogram will flow top-down (oldest to newest) rather than left-right. + /// + public bool IsRotated = false; + + public ImageMaker() + { + + } + + public Bitmap GetBitmap(List ffts) + { + if (ffts.Count == 0) + throw new ArgumentException("Not enough data in FFTs to generate an image yet."); + + int Width = IsRotated ? ffts[0].Length : ffts.Count; + int Height = IsRotated ? ffts.Count : ffts[0].Length; + + Bitmap bmp = new(Width, Height, PixelFormat.Format8bppIndexed); + Colormap.Apply(bmp); + + Rectangle lockRect = new(0, 0, Width, Height); + BitmapData bitmapData = bmp.LockBits(lockRect, ImageLockMode.ReadOnly, bmp.PixelFormat); + int stride = bitmapData.Stride; + + byte[] bytes = new byte[bitmapData.Stride * bmp.Height]; + Parallel.For(0, Width, col => + { + int sourceCol = col; + if (IsRoll) + { + sourceCol += Width - RollOffset % Width; + if (sourceCol >= Width) + sourceCol -= Width; + } + + for (int row = 0; row < Height; row++) + { + double value = IsRotated ? ffts[row][sourceCol] : ffts[sourceCol][row]; + if (IsDecibel) + value = 20 * Math.Log10(value * DecibelScaleFactor + 1); + value *= Intensity; + value = Math.Min(value, 255); + int bytePosition = (Height - 1 - row) * stride + col; + bytes[bytePosition] = (byte)value; + } + }); + + Marshal.Copy(bytes, 0, bitmapData.Scan0, bytes.Length); + bmp.UnlockBits(bitmapData); + + return bmp; + } + } +} diff --git a/src/Spectrogram/Spectrogram.csproj b/src/Spectrogram/Spectrogram.csproj index 028ff85..7d543ea 100644 --- a/src/Spectrogram/Spectrogram.csproj +++ b/src/Spectrogram/Spectrogram.csproj @@ -21,6 +21,7 @@ true true true + latest diff --git a/src/Spectrogram/SpectrogramGenerator.cs b/src/Spectrogram/SpectrogramGenerator.cs index 367d91d..bf4cac9 100644 --- a/src/Spectrogram/SpectrogramGenerator.cs +++ b/src/Spectrogram/SpectrogramGenerator.cs @@ -267,15 +267,16 @@ public List GetMelFFTs(int melBinCount) /// If true, output will be log-transformed. /// If dB scaling is in use, this multiplier will be applied before log transformation. /// Behavior of the spectrogram when it is full of data. + /// If True, the image will be rotated so time flows from top to bottom (rather than left to right). /// Roll (true) adds new columns on the left overwriting the oldest ones. /// Scroll (false) slides the whole image to the left and adds new columns to the right. - public Bitmap GetBitmap(double intensity = 1, bool dB = false, double dBScale = 1, bool roll = false) + public Bitmap GetBitmap(double intensity = 1, bool dB = false, double dBScale = 1, bool roll = false, bool rotate = false) { if (FFTs.Count == 0) throw new InvalidOperationException("Not enough data to create an image. " + $"Ensure {nameof(Width)} is >0 before calling {nameof(GetBitmap)}()."); - return Image.GetBitmap(FFTs, Colormap, intensity, dB, dBScale, roll, NextColumnIndex); + return Image.GetBitmap(FFTs, Colormap, intensity, dB, dBScale, roll, NextColumnIndex, rotate); } /// From 99b0c71ceb4b00ce07a1d9e669f6491ef48c450d Mon Sep 17 00:00:00 2001 From: Scott W Harden Date: Sun, 10 Jul 2022 14:44:56 -0400 Subject: [PATCH 6/6] Spectrogram 1.6 --- src/Spectrogram/Spectrogram.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Spectrogram/Spectrogram.csproj b/src/Spectrogram/Spectrogram.csproj index 7d543ea..bd74147 100644 --- a/src/Spectrogram/Spectrogram.csproj +++ b/src/Spectrogram/Spectrogram.csproj @@ -1,8 +1,8 @@ - + netstandard2.0 - 1.5.0 + 1.6.0 A .NET Standard library for creating spectrograms Scott Harden Harden Technologies, LLC