diff --git a/Amuencha/Amuencha.hpp b/Amuencha/Amuencha.hpp index 8602aeb..c95d5f6 100644 --- a/Amuencha/Amuencha.hpp +++ b/Amuencha/Amuencha.hpp @@ -1,4 +1,4 @@ #pragma once #include -#include \ No newline at end of file +#include diff --git a/Amuencha/AmuenchaModel.cpp b/Amuencha/AmuenchaModel.cpp index df09fe4..85ff0e6 100644 --- a/Amuencha/AmuenchaModel.cpp +++ b/Amuencha/AmuenchaModel.cpp @@ -2,12 +2,12 @@ namespace Amuencha { -void Model::prepare(halp::setup info) +void Model::prepare(setup info) { sampling_rate = info.rate; } -void Model::operator()(halp::tick t) +void Model::operator()(tick t) { analyzer.new_data(inputs.audio[0], t.frames); } diff --git a/Amuencha/AmuenchaModel.hpp b/Amuencha/AmuenchaModel.hpp index b19dfb0..624a9e8 100644 --- a/Amuencha/AmuenchaModel.hpp +++ b/Amuencha/AmuenchaModel.hpp @@ -17,11 +17,8 @@ public: halp_meta(c_name, "amuencha") halp_meta(uuid, "b37351b4-7b8d-4150-9e1c-708eec9182b2") - // This one will be memcpy'd as it is a trivial type struct processor_to_ui { - int min; - int max; std::vector reassigned_frequencies; std::vector power_spectrum; }; @@ -33,22 +30,9 @@ public: struct ins { halp::fixed_audio_bus<"Input", float, 1> audio; - struct : halp::hslider_i32<"Min", halp::range{.min = 0, .max = 127, .init = 24}> - { - void update(Model& self) - { - self.send_message({.min = this->value, .max = self.inputs.max}); - } - } min; - struct : halp::hslider_i32<"Min", halp::range{.min = 0, .max = 127, .init = 72}> - { - void update(Model& self) - { - self.send_message({.min = self.inputs.min, .max = this->value}); - } - } max; - halp::spinbox_i32<"Periods", - halp::range{.min = 0, .max = 99, .init = 30}> periods; + halp::hslider_i32<"Min", halp::range{.min = 0, .max = 127, .init = 24}> min; + halp::hslider_i32<"Max", halp::range{.min = 0, .max = 127, .init = 72}> max; + halp::spinbox_i32<"Periods", halp::range{.min = 0, .max = 99, .init = 30}> periods; } inputs; void process_message(const std::vector& frequencies) @@ -58,14 +42,13 @@ public: [&] (const std::vector& r_f, const std::vector& p_s) { - this->send_message({.min = inputs.min, - .max = inputs.max, - .reassigned_frequencies = r_f, + this->send_message({.reassigned_frequencies = r_f, .power_spectrum = p_s}); }, inputs.periods); - analyzer.start(QThread::NormalPriority); + if (!analyzer.isRunning()) + analyzer.start(QThread::NormalPriority); } struct outs @@ -74,13 +57,12 @@ public: } outputs; using setup = halp::setup; - void prepare(halp::setup info); + void prepare(setup info); // Do our processing for N samples using tick = halp::tick; - // Defined in the .cpp - void operator()(halp::tick t); + void operator()(tick t); // UI is defined in another file to keep things clear. struct ui; diff --git a/Amuencha/AmuenchaUi.hpp b/Amuencha/AmuenchaUi.hpp index dd72f2e..355f1f8 100644 --- a/Amuencha/AmuenchaUi.hpp +++ b/Amuencha/AmuenchaUi.hpp @@ -4,6 +4,7 @@ #include #include "SpiralDisplay.hpp" +#include "CustomSlider.hpp" namespace Amuencha { @@ -19,14 +20,13 @@ struct Model::ui struct { halp_meta(layout, vbox) - halp::item<&ins::min> min; - halp::item<&ins::max> max; + halp::custom_control min; + halp::custom_control max; halp::item<&ins::periods> periods; } controls; halp::custom_actions_item spiral{.x = 0, .y = 0}; - // Define the communication between UI and processor. struct bus { @@ -38,6 +38,15 @@ struct Model::ui self.spiral.set_frequencies_callback( [&] { this->send_message(self.spiral.get_frequencies()); } ); + + self.controls.min.on_changed = [&] (int min) + { + // self.spiral.set_min_max_notes(min, self.controls.max.value); + }; + self.controls.max.on_changed = [&] (int max) + { + // self.spiral.set_min_max_notes(self.controls.min.value, max); + }; } // Receive a message on the UI thread from the processing thread @@ -49,8 +58,6 @@ struct Model::ui first_msg = true; } - // self.spiral.set_min_max_notes(msg.min, msg.max); - if (msg.power_spectrum.empty() || msg.reassigned_frequencies.empty()) return; diff --git a/Amuencha/CustomSlider.cpp b/Amuencha/CustomSlider.cpp new file mode 100644 index 0000000..596546c --- /dev/null +++ b/Amuencha/CustomSlider.cpp @@ -0,0 +1,2 @@ +#include "CustomSlider.hpp" + diff --git a/Amuencha/CustomSlider.hpp b/Amuencha/CustomSlider.hpp new file mode 100644 index 0000000..c0ee94d --- /dev/null +++ b/Amuencha/CustomSlider.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include +#include +#include + +struct CustomSlider +{ + // Same as above + static constexpr double width() { return 100.; } + static constexpr double height() { return 20.; } + + // Needed for changing the ui. It's the type above - it's already defined as-is + // in the helpers library. + halp::transaction transaction; + + // Called when the value changes from the host software. + void set_value(const auto& control, int value) + { + this->value = avnd::map_control_to_01(control, value / 127.f); + } + + // When transaction.update() is called, this converts the value in the slider + // into one fit for the control definition passed as argument. + static auto value_to_control(auto& control, int value) + { + return avnd::map_control_from_01(control, value / 127.f); + } + + // Paint method: same as above + void paint(avnd::painter auto ctx) + { + ctx.set_stroke_color({200, 200, 200, 255}); + ctx.set_stroke_width(2.); + ctx.set_fill_color({120, 120, 120, 255}); + ctx.begin_path(); + ctx.draw_rect(0., 0., width(), height()); + ctx.fill(); + ctx.stroke(); + + ctx.begin_path(); + ctx.set_fill_color({90, 90, 90, 255}); + ctx.draw_rect(2., 2., (width() - 4) * (value / 127.f), (height() - 4)); + ctx.fill(); + } + + // Return true to handle the event. x, y, are the positions of the item in local coordinates. + bool mouse_press(double x, double y) + { + transaction.start(); + mouse_move(x, y); + return true; + } + + // Obvious :-) + void mouse_move(double x, double y) + { + const int res{static_cast(std::clamp(x / width(), 0., 1.) * 127)}; + transaction.update(res); + on_changed(value); + } + + // Same + void mouse_release(double x, double y) + { + mouse_move(x, y); + transaction.commit(); + } + + int value{}; + + std::function on_changed{[](int){}}; +}; + diff --git a/Amuencha/FrequencyAnalyzer.cpp b/Amuencha/FrequencyAnalyzer.cpp new file mode 100644 index 0000000..904d838 --- /dev/null +++ b/Amuencha/FrequencyAnalyzer.cpp @@ -0,0 +1,348 @@ +/* + Analyseur de MUsique et ENtraînement au CHAnt + + This file is released under either of the two licenses below, your choice: + - LGPL v2.1 or later, https://www.gnu.org + The GNU Lesser General Public Licence, version 2.1 or, + at your option, any later version. + - CeCILL-C, http://www.cecill.info + The CeCILL-C license is more adapted to the French laws, + but can be converted to the GNU LGPL. + + You can use, modify and/or redistribute the software under the terms of any + of these licences, which should have been provided to you together with this + sofware. If that is not the case, you can find a copy of the licences on + the indicated web sites. + + By Nicolas . Brodu @ Inria . fr + + See http://nicolas.brodu.net/programmation/amuencha/ for more information +*/ + +#include +#include + +#include +#include + +#include +#include + +#include + +#include "sse_mathfun.h" + +#include "FrequencyAnalyzer.hpp" + +using namespace std; +using namespace boost::math::float_constants; + +Amuencha::FrequencyAnalyzer::FrequencyAnalyzer(QObject *parent) : QThread(parent) +{ + status = NO_DATA; +} + +Amuencha::FrequencyAnalyzer::~FrequencyAnalyzer() +{ + mutex.lock(); + status = QUIT_NOW; + // Do not even wait the end of a cycle, quit + condition.wakeOne(); + mutex.unlock(); + wait(CYCLE_PERIOD*2); + terminate(); + wait(); // until run terminates +} + +void Amuencha::FrequencyAnalyzer::new_data(float* chunk, int size) +{ + // producer, called from another thread. + // Must not wake the main thread uselessly to notify that new data is available + // The idea is that the new_data may be called very fast from a RT audio thread + // but that frequencies are computed at most 10 times/second (or whatever the cyclic period is set) + // => decouple thread frequencies + // And when no data is here (recording off, no song...), the main thread is fully passive (no CPU hog) + + mutex.lock(); + + // Store a pointer here and not a full data copy, which is much faster + // The real data is stored in the AudioRecording object + // Kind of duplicates the AudioRecording list, however: + // - This way, there is no need for mutex/lock in the AudioRecording structure + // - The position of the last unprocessed chunk within that structure need not be stored there + chunks.emplace_back(make_pair(chunk, size)); + + // set the flag that data has arrived + status = HAS_NEW_DATA; + + // IF AND ONLY IF the thread was blocked forever, then wake it up + if (waiting_time != CYCLE_PERIOD) { + // and now resume the cyclic scheduling + waiting_time = CYCLE_PERIOD; + condition.wakeOne(); + } + + // Otherwise, do NOT wake the other thread, keep the low-freq cycle to decrease load + mutex.unlock(); + +} + +void Amuencha::FrequencyAnalyzer::run() +{ + // Solution with a wait condition + time + // other possible solution = timer, but that would need to be stopped + + mutex.lock(); + + // waiting_time is a mutex-protected info + waiting_time = CYCLE_PERIOD; + + // loop starts with mutex locked + while (true) { + condition.wait(&mutex, waiting_time); + + if (status==QUIT_NOW) break; + + if (status==HAS_NEW_DATA) { + // Swap the chunks to a local variable, so: + // - the class chunks becomes empty + // - the audio thread can feed it more data while computing frequencies + vector> chunks; + chunks.swap(this->chunks); + status = NO_DATA; // will be updated if new data indeed arrives + mutex.unlock(); + + // Now, we can take the time to do the frequency computations + data_mutex.lock(); + + int new_data_size = 0; + for (auto c: chunks) new_data_size += c.second; + // shift the old data to make room for the new + int new_data_pos = big_buffer.size()-new_data_size; +//cout << "data " << new_data_size << " " << big_buffer.size() << endl; + // std::copy can cope with overlapping regions in this copy direction + if (new_data_pos>0) copy(big_buffer.begin()+new_data_size, big_buffer.end(), big_buffer.begin()); + // now copy each chunk at its position in the big buffer + for (auto c: chunks) { + if (new_data_pos<0) { + // discard too old chunks + if (c.second <= -new_data_pos) { + new_data_pos += c.second; + continue; + } + // partial copy of chunks that fall on the edge + copy(c.first+new_data_pos, c.first+c.second, &big_buffer[0]); + new_data_pos = c.second+new_data_pos; + continue; + } + copy(c.first, c.first+c.second, &big_buffer[new_data_pos]); + new_data_pos += c.second; + } + + // Apply the filter bank + float *bbend = &big_buffer[0] + big_buffer.size(); + for (int idx=0; idx0) { + reassign -= (acc[0] * acc[3] - acc[1] * acc[2]) * samplerate_div_2pi / norm; + } + reassigned_frequencies[idx] = reassign; + power_spectrum[idx] = norm * power_normalization_factors[idx]; + } + + // Notify our listener that new power/frequency content has arrived + power_handler(reassigned_frequencies, power_spectrum); + + // setup can now lock and change data structures if needed + data_mutex.unlock(); + + // relock for the condition wait + mutex.lock(); + continue; + } + + // No more data ? Force waiting until data arrives + waiting_time = ULONG_MAX; + // keep the lock for next loop + } + mutex.unlock(); +} + +void Amuencha::FrequencyAnalyzer::setup(float sampling_rate, const std::vector& frequencies, PowerHandler&& handler, float periods, float max_buffer_duration) +{ + // Block data processing while changing the data structures + data_mutex.lock(); + + this->samplerate_div_2pi = sampling_rate/two_pi; + this->frequencies = frequencies; + + this->reassigned_frequencies = frequencies; + this->power_spectrum.resize(frequencies.size()); + + power_handler = std::move(handler); + + // Prepare the windows + //std::vector> windowed_sines; + windowed_sines.resize(frequencies.size()); + power_normalization_factors.resize(frequencies.size()); + + int big_buffer_size = 0; + + for (int idx{0}; idx < frequencies.size(); ++idx) + { + // for each freq, span at least 20 periods for more precise measurements + // This still gives reasonable latencies, e.g. 50ms at 400Hz, 100ms at 200Hz, 400ms at 50Hz... + // Could also span more for even better measurements, with larger + // computation cost and latency + float f = frequencies[idx]; + int window_size = (int)(min(periods / f, max_buffer_duration * 0.001f) * sampling_rate); + vector window(window_size); + vector window_deriv(window_size); + if (!read_from_cache(window, window_deriv)) { + initialize_window(window); + initialize_window_deriv(window_deriv); + write_to_cache(window, window_deriv); + } + windowed_sines[idx].resize(window_size); + float wsum = 0; + for (int i=0; i& window) { + // Kaiser window with a parameter of alpha=3 that nullifies the window on edges + int size = window.size(); + const float two_over_N = 2./size; + const float alpha = 3.; + const float alpha_pi = alpha * pi; + const float inv_denom = 1. /boost::math::cyl_bessel_i(0., alpha_pi); + + for (int i{0}; i < size; ++i) + { + float p = i * two_over_N - 1.; + window[i] = boost::math::cyl_bessel_i(0., alpha_pi * sqrt(1. - p * p)) * inv_denom; + } +} + +void Amuencha::FrequencyAnalyzer::initialize_window_deriv(std::vector& window) { + // Derivative of the Kaiser window with a parameter of alpha=3 that nullifies the window on edges + int size = window.size(); + const float two_over_N = 2./size; + const float alpha = 3.; + const float alpha_pi = alpha * pi; + const float inv_denom = 1./boost::math::cyl_bessel_i(0., alpha_pi); + + for (int i{1}; i0 = 1/2 + window[0] = 0.5 * inv_denom * alpha_pi * alpha_pi * two_over_N; +} + +bool Amuencha::FrequencyAnalyzer::read_from_cache(std::vector& window, std::vector& window_deriv) +{ + auto it = mem_win_cache.find(window.size()); + if (it != mem_win_cache.end()) + { + window = it->second; + auto itd = mem_winderiv_cache.find(window.size()); + if (itd != mem_winderiv_cache.end()) + { + window_deriv = itd->second; + return true; + } + // else, load from disk + } + + // TODO: make the cache location parametrizable (and an option to not use it) + ifstream file(".amuencha_cache/w" + to_string(window.size()) + ".bin", ios::binary); + if (file.is_open()) + { + file.read(reinterpret_cast(&window[0]), window.size() * sizeof(float)); + file.read(reinterpret_cast(&window_deriv[0]), window_deriv.size() * sizeof(float)); + if (file.tellg() != (window.size() + window_deriv.size()) * sizeof(float)) + { + cerr << "Error: invalid cache .amuencha_cache/w" + to_string(window.size()) + ".bin \n"; + return false; + } + return true; + } + return false; +} + +void Amuencha::FrequencyAnalyzer::write_to_cache(std::vector& window, std::vector& window_deriv) +{ +#if defined(_WIN32) || defined(_WIN64) + mkdir(".amuencha_cache"); +#else + mkdir(".amuencha_cache", 0755); +#endif + ofstream file(".amuencha_cache/w" + to_string(window.size()) + ".bin", ios::binary | ios::trunc); + file.write(reinterpret_cast(&window[0]), window.size() * sizeof(float)); + file.write(reinterpret_cast(&window_deriv[0]), window_deriv.size() * sizeof(float)); + mem_win_cache[window.size()] = window; + mem_winderiv_cache[window.size()] = window_deriv; +} diff --git a/Amuencha/FrequencyAnalyzer.hpp b/Amuencha/FrequencyAnalyzer.hpp new file mode 100644 index 0000000..3055a30 --- /dev/null +++ b/Amuencha/FrequencyAnalyzer.hpp @@ -0,0 +1,113 @@ +/* + Analyseur de MUsique et ENtraînement au CHAnt + + This file is released under either of the two licenses below, your choice: + - LGPL v2.1 or later, https://www.gnu.org + The GNU Lesser General Public Licence, version 2.1 or, + at your option, any later version. + - CeCILL-C, http://www.cecill.info + The CeCILL-C license is more adapted to the French laws, + but can be converted to the GNU LGPL. + + You can use, modify and/or redistribute the software under the terms of any + of these licences, which should have been provided to you together with this + sofware. If that is not the case, you can find a copy of the licences on + the indicated web sites. + + By Nicolas . Brodu @ Inria . fr + + See http://nicolas.brodu.net/programmation/amuencha/ for more information +*/ + +#ifndef AUDIOINPUTTHREAD_H +#define AUDIOINPUTTHREAD_H + +#include +#include +#include + + +#include +#include +#include +#include + +namespace Amuencha +{ +class FrequencyAnalyzer : public QThread +{ +public: + FrequencyAnalyzer(QObject *parent = 0); + ~FrequencyAnalyzer(); + + // called by the RT audio thread to feed new data + void new_data(float *chunk, int size); + + // Arguments are: frequency bins [f,f+1), and power in each bin + // hence the first vector size is 1 more than the second + // TODO if useful: make a proper listener API with id, etc + typedef std::function&,const std::vector&)> PowerHandler; + PowerHandler power_handler; + + // sampling rate in Hz + // freqs in Hz + // PowerHandler for the callback + // max_buffer_duration in milliseconds, specifies the largest buffer for computing the frequency content + // At lower frequencies, long buffers are needed for accurate frequency separation. + // When that max buffer duration is reached, then it is capped and the frequency resolution decreases + // Too low buffers also limit the min_freq, duration must be >= period + void setup(float sampling_rate, const std::vector& frequencies, PowerHandler&& handler, float periods = 20, float max_buffer_duration = 500); + + // call to remove all existing chunk references + // this may cause signal loss, but this is usually called precisely when the signal is lost... + void invalidate_samples(); + +protected: + void run() override; + + // Multi-threading related variables + static const unsigned long CYCLE_PERIOD = 20; // in milliseconds + QMutex mutex, data_mutex; + QWaitCondition condition; + unsigned long waiting_time = CYCLE_PERIOD; + enum Status : std::uint8_t + { + NO_DATA = 0, + HAS_NEW_DATA = 1, + QUIT_NOW = 2 + }; + Status status; + + // new data chunks arrived since the last periodic processing + std::vector> chunks; + + // The window is the usual Kaiser with alpha=3 + static void initialize_window(std::vector& window); + static void initialize_window_deriv(std::vector& window); + + // The filter bank. One filter per frequency + // The 4 entries in the v4sf are the real, imaginary parts of the windowed + // sine wavelet, and the real, imaginary parts of the derived windowed sine + // used for reassigning the power spectrum. + // Hopefully, with SIMD, computing all 4 of them is the same price as just one + // TODO: v8sf and compute 2 freqs at the same time + typedef float v4sf __attribute__ ((vector_size (16))); + std::vector> windowed_sines; + std::vector frequencies; + std::vector power_normalization_factors; + float samplerate_div_2pi; + std::vector big_buffer; + std::vector reassigned_frequencies; + std::vector power_spectrum; + + // caching computations for faster init + // on disk for persistence between executions, + // in memory for avoiding reloading from disk when changing the spiral size + bool read_from_cache(std::vector& window, std::vector& window_deriv); + void write_to_cache(std::vector& window, std::vector& window_deriv); + std::map> mem_win_cache, mem_winderiv_cache; +}; + +} // namespace Amuencha + +#endif // AUDIOINPUTTHREAD_H diff --git a/Amuencha/SpiralDisplay.hpp b/Amuencha/SpiralDisplay.hpp index 492c6c7..5403ace 100644 --- a/Amuencha/SpiralDisplay.hpp +++ b/Amuencha/SpiralDisplay.hpp @@ -30,6 +30,8 @@ struct SpiralDisplay if (display_bins.empty()) compute_frequencies(); + // Draw note axis and names + // TODO : replace with CSV background for (int i{0}; i < 12; i++) { ctx.move_to(half, half); diff --git a/CMakeLists.txt b/CMakeLists.txt index ac32af9..1fdfcdd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,11 +20,13 @@ avnd_score_plugin_add( Amuencha/Amuencha.hpp Amuencha/AmuenchaModel.hpp Amuencha/AmuenchaModel.cpp + Amuencha/FrequencyAnalyzer.hpp + Amuencha/FrequencyAnalyzer.cpp Amuencha/AmuenchaUi.hpp Amuencha/SpiralDisplay.hpp Amuencha/SpiralDisplay.cpp - Amuencha/FrequencyAnalyzer.hpp - Amuencha/FrequencyAnalyzer.cpp + Amuencha/CustomSlider.hpp + Amuencha/CustomSlider.cpp TARGET amuencha MAIN_CLASS Model NAMESPACE Amuencha