From ac4ee321dc187cbabb21d727d92e2f4357f82f41 Mon Sep 17 00:00:00 2001 From: Cyberflow Date: Thu, 23 Jul 2026 20:25:31 +0300 Subject: [PATCH 01/13] feat(transcription): extract minimal NeuralNote engine --- .../Native/Engine/BasicPitch.cpp | 134 + .../Transcription/Native/Engine/BasicPitch.h | 83 + .../Native/Engine/BasicPitchCNN.cpp | 147 + .../Native/Engine/BasicPitchCNN.h | 122 + .../Native/Engine/BasicPitchConstants.h | 27 + .../Transcription/Native/Engine/Features.cpp | 45 + .../Transcription/Native/Engine/Features.h | 54 + JammLab/Transcription/Native/Engine/Notes.cpp | 284 + JammLab/Transcription/Native/Engine/Notes.h | 272 + JammLab/Transcription/Native/Engine/Utils.h | 18 + .../Native/ThirdParty/RTNeural/CMakeLists.txt | 57 + .../Native/ThirdParty/RTNeural/Layer.h | 39 + .../Native/ThirdParty/RTNeural/Model.h | 118 + .../Native/ThirdParty/RTNeural/ModelT.h | 579 + .../Native/ThirdParty/RTNeural/RTNeural.cpp | 7 + .../Native/ThirdParty/RTNeural/RTNeural.h | 14 + .../RTNeural/activation/activation.h | 432 + .../RTNeural/activation/activation_eigen.h | 469 + .../RTNeural/activation/activation_xsimd.h | 436 + .../ThirdParty/RTNeural/batchnorm/batchnorm.h | 146 + .../RTNeural/batchnorm/batchnorm.tpp | 113 + .../RTNeural/batchnorm/batchnorm2d.h | 164 + .../RTNeural/batchnorm/batchnorm2d.tpp | 116 + .../RTNeural/batchnorm/batchnorm2d_eigen.h | 162 + .../RTNeural/batchnorm/batchnorm2d_eigen.tpp | 111 + .../RTNeural/batchnorm/batchnorm2d_xsimd.h | 170 + .../RTNeural/batchnorm/batchnorm2d_xsimd.tpp | 112 + .../RTNeural/batchnorm/batchnorm_eigen.h | 141 + .../RTNeural/batchnorm/batchnorm_eigen.tpp | 109 + .../RTNeural/batchnorm/batchnorm_xsimd.h | 149 + .../RTNeural/batchnorm/batchnorm_xsimd.tpp | 110 + .../Native/ThirdParty/RTNeural/common.h | 259 + .../Native/ThirdParty/RTNeural/config.h | 37 + .../ThirdParty/RTNeural/conv1d/conv1d.h | 324 + .../ThirdParty/RTNeural/conv1d/conv1d.tpp | 172 + .../ThirdParty/RTNeural/conv1d/conv1d_eigen.h | 274 + .../RTNeural/conv1d/conv1d_eigen.tpp | 111 + .../ThirdParty/RTNeural/conv1d/conv1d_xsimd.h | 402 + .../RTNeural/conv1d/conv1d_xsimd.tpp | 132 + .../conv1d_stateless/conv1d_stateless.h | 310 + .../conv1d_stateless/conv1d_stateless.tpp | 81 + .../conv1d_stateless/conv1d_stateless_eigen.h | 251 + .../conv1d_stateless_eigen.tpp | 75 + .../conv1d_stateless/conv1d_stateless_xsimd.h | 350 + .../conv1d_stateless_xsimd.tpp | 74 + .../ThirdParty/RTNeural/conv2d/conv2d.h | 254 + .../ThirdParty/RTNeural/conv2d/conv2d.tpp | 92 + .../ThirdParty/RTNeural/conv2d/conv2d_eigen.h | 247 + .../RTNeural/conv2d/conv2d_eigen.tpp | 91 + .../ThirdParty/RTNeural/conv2d/conv2d_xsimd.h | 252 + .../RTNeural/conv2d/conv2d_xsimd.tpp | 88 + .../Native/ThirdParty/RTNeural/dense/dense.h | 254 + .../ThirdParty/RTNeural/dense/dense_eigen.h | 210 + .../ThirdParty/RTNeural/dense/dense_xsimd.h | 389 + .../Native/ThirdParty/RTNeural/gru/gru.h | 357 + .../Native/ThirdParty/RTNeural/gru/gru.tpp | 364 + .../ThirdParty/RTNeural/gru/gru_eigen.h | 346 + .../ThirdParty/RTNeural/gru/gru_eigen.tpp | 240 + .../ThirdParty/RTNeural/gru/gru_xsimd.h | 396 + .../ThirdParty/RTNeural/gru/gru_xsimd.tpp | 337 + .../Native/ThirdParty/RTNeural/lstm/lstm.h | 363 + .../Native/ThirdParty/RTNeural/lstm/lstm.tpp | 287 + .../ThirdParty/RTNeural/lstm/lstm_eigen.h | 291 + .../ThirdParty/RTNeural/lstm/lstm_eigen.tpp | 208 + .../ThirdParty/RTNeural/lstm/lstm_xsimd.h | 408 + .../ThirdParty/RTNeural/lstm/lstm_xsimd.tpp | 259 + .../ThirdParty/RTNeural/maths/maths_eigen.h | 28 + .../ThirdParty/RTNeural/maths/maths_stl.h | 27 + .../ThirdParty/RTNeural/maths/maths_xsimd.h | 33 + .../Native/ThirdParty/RTNeural/model_loader.h | 728 + .../ThirdParty/RTNeural/torch_helpers.h | 151 + .../RTNeural/xsimd-legacy/README.md | 7 + .../xsimd-legacy/algorithms/algorithms.hpp | 201 + .../Native/ThirdParty/nlohmann/json.hpp | 24441 ++++++++++++++++ JammLab/Transcription/UPSTREAM.md | 37 + 75 files changed, 39178 insertions(+) create mode 100644 JammLab/Transcription/Native/Engine/BasicPitch.cpp create mode 100644 JammLab/Transcription/Native/Engine/BasicPitch.h create mode 100644 JammLab/Transcription/Native/Engine/BasicPitchCNN.cpp create mode 100644 JammLab/Transcription/Native/Engine/BasicPitchCNN.h create mode 100644 JammLab/Transcription/Native/Engine/BasicPitchConstants.h create mode 100644 JammLab/Transcription/Native/Engine/Features.cpp create mode 100644 JammLab/Transcription/Native/Engine/Features.h create mode 100644 JammLab/Transcription/Native/Engine/Notes.cpp create mode 100644 JammLab/Transcription/Native/Engine/Notes.h create mode 100644 JammLab/Transcription/Native/Engine/Utils.h create mode 100755 JammLab/Transcription/Native/ThirdParty/RTNeural/CMakeLists.txt create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/Layer.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/Model.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/ModelT.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/RTNeural.cpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/RTNeural.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation_eigen.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation_xsimd.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_eigen.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_eigen.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_xsimd.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_xsimd.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_eigen.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_eigen.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_xsimd.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_xsimd.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/common.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/config.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_eigen.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_eigen.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_xsimd.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_xsimd.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_eigen.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_eigen.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_xsimd.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_xsimd.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_eigen.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_eigen.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_xsimd.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_xsimd.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense_eigen.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense_xsimd.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_eigen.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_eigen.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_xsimd.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_xsimd.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_eigen.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_eigen.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_xsimd.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_xsimd.tpp create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_eigen.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_stl.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_xsimd.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/model_loader.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/torch_helpers.h create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/xsimd-legacy/README.md create mode 100644 JammLab/Transcription/Native/ThirdParty/RTNeural/xsimd-legacy/algorithms/algorithms.hpp create mode 100644 JammLab/Transcription/Native/ThirdParty/nlohmann/json.hpp create mode 100644 JammLab/Transcription/UPSTREAM.md diff --git a/JammLab/Transcription/Native/Engine/BasicPitch.cpp b/JammLab/Transcription/Native/Engine/BasicPitch.cpp new file mode 100644 index 0000000..9b17fbb --- /dev/null +++ b/JammLab/Transcription/Native/Engine/BasicPitch.cpp @@ -0,0 +1,134 @@ +// +// Created by Damien Ronssin on 10.03.23. +// + +#include "BasicPitch.h" + +#include +#include + +BasicPitch::BasicPitch(const std::filesystem::path& modelDirectory) + : mFeaturesCalculator(modelDirectory / "features_model.ort") + , mBasicPitchCNN(modelDirectory) +{ +} + +void BasicPitch::reset() +{ + mBasicPitchCNN.reset(); + mNotesCreator.clear(); + + mContoursPG.clear(); + mContoursPG.shrink_to_fit(); + mNotesPG.clear(); + mNotesPG.shrink_to_fit(); + mOnsetsPG.clear(); + mOnsetsPG.shrink_to_fit(); + mNoteEvents.clear(); + mNoteEvents.shrink_to_fit(); + + mNumFrames = 0; +} + +void BasicPitch::setParameters( + float inNoteSensitivity, + float inSplitSensitivity, + float inMinNoteDurationMs, + bool includePitchBends +) +{ + mParams.frameThreshold = 1.0f - inNoteSensitivity; + mParams.onsetThreshold = 1.0f - inSplitSensitivity; + + mParams.minNoteLength = + static_cast(std::round(inMinNoteDurationMs / 1000.0f / (FFT_HOP / BASIC_PITCH_SAMPLE_RATE))); + + mParams.pitchBend = includePitchBends ? MultiPitchBend : NoPitchBend; + mParams.melodiaTrick = true; + mParams.inferOnsets = true; +} + +void BasicPitch::transcribeToMIDI( + float* inAudio, + int inNumSamples, + const ProgressCallback& progressCallback +) +{ + if (inAudio == nullptr || inNumSamples <= 0) { + throw std::invalid_argument("Basic Pitch audio input is empty"); + } + if (progressCallback && !progressCallback(0)) { + throw std::runtime_error("transcription_cancelled"); + } + + const float* stacked_cqt = mFeaturesCalculator.computeFeatures(inAudio, inNumSamples, mNumFrames); + if (mNumFrames == 0) { + mNoteEvents.clear(); + return; + } + if (progressCallback && !progressCallback(0.15)) { + throw std::runtime_error("transcription_cancelled"); + } + + mOnsetsPG.resize(mNumFrames, std::vector(static_cast(NUM_FREQ_OUT), 0.0f)); + mNotesPG.resize(mNumFrames, std::vector(static_cast(NUM_FREQ_OUT), 0.0f)); + mContoursPG.resize(mNumFrames, std::vector(static_cast(NUM_FREQ_IN), 0.0f)); + + mOnsetsPG.shrink_to_fit(); + mNotesPG.shrink_to_fit(); + mContoursPG.shrink_to_fit(); + + mBasicPitchCNN.reset(); + + const size_t num_lh_frames = BasicPitchCNN::getNumFramesLookahead(); + + std::vector zero_stacked_cqt(NUM_HARMONICS * NUM_FREQ_IN, 0.0f); + + // Run the CNN with 0 input and discard output (only for num_lh_frames) + for (int i = 0; i < num_lh_frames; i++) { + mBasicPitchCNN.frameInference(zero_stacked_cqt.data(), mContoursPG[0], mNotesPG[0], mOnsetsPG[0]); + } + + // Run the CNN with real inputs and discard outputs (only for num_lh_frames) + for (size_t frame_idx = 0; frame_idx < num_lh_frames; frame_idx++) { + mBasicPitchCNN.frameInference( + stacked_cqt + frame_idx * NUM_HARMONICS * NUM_FREQ_IN, mContoursPG[0], mNotesPG[0], mOnsetsPG[0]); + } + + // Run the CNN with real inputs and correct outputs + for (size_t frame_idx = num_lh_frames; frame_idx < mNumFrames; frame_idx++) { + mBasicPitchCNN.frameInference(stacked_cqt + frame_idx * NUM_HARMONICS * NUM_FREQ_IN, + mContoursPG[frame_idx - num_lh_frames], + mNotesPG[frame_idx - num_lh_frames], + mOnsetsPG[frame_idx - num_lh_frames]); + if (progressCallback && (frame_idx % 32 == 0)) { + const auto progress = 0.15 + 0.75 * static_cast(frame_idx) / static_cast(mNumFrames); + if (!progressCallback(progress)) { + throw std::runtime_error("transcription_cancelled"); + } + } + } + + // Run end with zeroes as input and last frames as output + for (size_t frame_idx = mNumFrames; frame_idx < mNumFrames + num_lh_frames; frame_idx++) { + mBasicPitchCNN.frameInference(zero_stacked_cqt.data(), + mContoursPG[frame_idx - num_lh_frames], + mNotesPG[frame_idx - num_lh_frames], + mOnsetsPG[frame_idx - num_lh_frames]); + } + + mNoteEvents = mNotesCreator.convert(mNotesPG, mOnsetsPG, mContoursPG, mParams, true); + if (progressCallback && !progressCallback(1)) { + throw std::runtime_error("transcription_cancelled"); + } +} + +void BasicPitch::updateMIDI() +{ + mNoteEvents = mNotesCreator.convert(mNotesPG, mOnsetsPG, mContoursPG, mParams, false); +} + +const std::vector& BasicPitch::getNoteEvents() const +{ + return mNoteEvents; +} diff --git a/JammLab/Transcription/Native/Engine/BasicPitch.h b/JammLab/Transcription/Native/Engine/BasicPitch.h new file mode 100644 index 0000000..27a5f68 --- /dev/null +++ b/JammLab/Transcription/Native/Engine/BasicPitch.h @@ -0,0 +1,83 @@ +// +// Created by Damien Ronssin on 10.03.23. +// + +#ifndef BasicPitch_h +#define BasicPitch_h + +#include +#include + +#include "BasicPitchCNN.h" +#include "BasicPitchConstants.h" +#include "Features.h" +#include "Notes.h" + +/** + * Class to get midi transcription from raw audio. + */ +class BasicPitch +{ +public: + using ProgressCallback = std::function; + + explicit BasicPitch(const std::filesystem::path& modelDirectory); + + /** + * Resets all states of model, clear the posteriorgrams vector computed by the CNN and the note event vector. + */ + void reset(); + + /** + * Set parameters for next transcription or midi update. + * @param inNoteSensitivity Note sensitivity threshold (0.05, 0.95). Higher gives more notes. + * @param inSplitSensitivity Split sensitivity threshold (0.05, 0.95). Higher will split note more, lower will merge close notes with same pitch + * @param inMinNoteDurationMs Minimum note duration to keep in ms. + */ + void setParameters( + float inNoteSensitivity, + float inSplitSensitivity, + float inMinNoteDurationMs, + bool includePitchBends + ); + + /** + * Transcribe the input audio. The note event vector can be obtained after this with getNoteEvents + * @param inAudio Pointer to raw audio (must be at 22050 Hz) + * @param inNumSamples Number of input samples available. + */ + void transcribeToMIDI( + float* inAudio, + int inNumSamples, + const ProgressCallback& progressCallback = {} + ); + + /** + * Function to call to update the midi transcription with new parameters. + * The whole Features + CNN is not rerun for this. Only Notes::Convert is. + */ + void updateMIDI(); + + /** + * @return Note event vector. + */ + const std::vector& getNoteEvents() const; + +private: + // Posteriorgrams vector + std::vector> mContoursPG; + std::vector> mNotesPG; + std::vector> mOnsetsPG; + + std::vector mNoteEvents; + + Notes::ConvertParams mParams; + + size_t mNumFrames = 0; + + Features mFeaturesCalculator; + BasicPitchCNN mBasicPitchCNN; + Notes mNotesCreator; +}; + +#endif // BasicPitch_h diff --git a/JammLab/Transcription/Native/Engine/BasicPitchCNN.cpp b/JammLab/Transcription/Native/Engine/BasicPitchCNN.cpp new file mode 100644 index 0000000..d9bbe8d --- /dev/null +++ b/JammLab/Transcription/Native/Engine/BasicPitchCNN.cpp @@ -0,0 +1,147 @@ +// +// Created by Damien Ronssin on 03.03.23. +// + +#include "BasicPitchCNN.h" + +#include +#include + +using json = nlohmann::json; + +namespace +{ +json loadModelJson(const std::filesystem::path& modelDirectory, const char* filename) +{ + const auto path = modelDirectory / filename; + std::ifstream stream(path); + if (!stream.is_open()) { + throw std::runtime_error(std::string("Basic Pitch model resource is missing: ") + filename); + } + return json::parse(stream); +} +} // namespace + +BasicPitchCNN::BasicPitchCNN(const std::filesystem::path& modelDirectory) +{ + const auto json_cnn_contour = loadModelJson(modelDirectory, "cnn_contour_model.json"); + mCNNContour.parseJson(json_cnn_contour); + + const auto json_cnn_note = loadModelJson(modelDirectory, "cnn_note_model.json"); + mCNNNote.parseJson(json_cnn_note); + + const auto json_cnn_onset_input = loadModelJson(modelDirectory, "cnn_onset_1_model.json"); + mCNNOnsetInput.parseJson(json_cnn_onset_input); + + const auto json_cnn_onset_output = loadModelJson(modelDirectory, "cnn_onset_2_model.json"); + mCNNOnsetOutput.parseJson(json_cnn_onset_output); +} + +void BasicPitchCNN::reset() +{ + for (auto& array: mContoursCircularBuffer) { + array.fill(0.0f); + } + + for (auto& array: mNotesCircularBuffer) { + array.fill(0.0f); + } + + for (auto& array: mConcat2CircularBuffer) { + array.fill(0.0f); + } + + mCNNContour.reset(); + mCNNNote.reset(); + mCNNOnsetInput.reset(); + mCNNOnsetOutput.reset(); + + mNoteIdx = 0; + mContourIdx = 0; + mConcat2Idx = 0; + + mInputArray.fill(0.0f); +} + +int BasicPitchCNN::getNumFramesLookahead() +{ + return mTotalLookahead; +} + +void BasicPitchCNN::frameInference(const float* inData, + std::vector& outContours, + std::vector& outNotes, + std::vector& outOnsets) +{ + // Checks on parameters + assert(outContours.size() == NUM_FREQ_IN); + assert(outNotes.size() == NUM_FREQ_OUT); + assert(outOnsets.size() == NUM_FREQ_OUT); + + // Copy data in aligned input array for inference + std::copy(inData, inData + NUM_HARMONICS * NUM_FREQ_IN, mInputArray.begin()); + + _runModels(); + + // Fill output vectors + std::copy(mCNNOnsetOutput.getOutputs(), mCNNOnsetOutput.getOutputs() + NUM_FREQ_OUT, outOnsets.begin()); + + std::copy(mNotesCircularBuffer[(size_t) _wrapIndex(mNoteIdx + 1, mNumNoteStored)].begin(), + mNotesCircularBuffer[(size_t) _wrapIndex(mNoteIdx + 1, mNumNoteStored)].end(), + outNotes.begin()); + + std::copy(mContoursCircularBuffer[(size_t) _wrapIndex(mContourIdx + 1, mNumContourStored)].begin(), + mContoursCircularBuffer[(size_t) _wrapIndex(mContourIdx + 1, mNumContourStored)].end(), + outContours.begin()); + + // Increment index for different circular buffers + mContourIdx = (mContourIdx == mNumContourStored - 1) ? 0 : mContourIdx + 1; + mNoteIdx = (mNoteIdx == mNumNoteStored - 1) ? 0 : mNoteIdx + 1; + mConcat2Idx = (mConcat2Idx == mNumConcat2Stored - 1) ? 0 : mConcat2Idx + 1; +} + +void BasicPitchCNN::_runModels() +{ + // Run models and push results in appropriate circular buffer + mCNNOnsetInput.forward(mInputArray.data()); + std::copy(mCNNOnsetInput.getOutputs(), + mCNNOnsetInput.getOutputs() + 32 * NUM_FREQ_OUT, + mConcat2CircularBuffer[(size_t) mConcat2Idx].begin()); + + mCNNContour.forward(mInputArray.data()); + std::copy(mCNNContour.getOutputs(), + mCNNContour.getOutputs() + NUM_FREQ_IN, + mContoursCircularBuffer[(size_t) mContourIdx].begin()); + + mCNNNote.forward(mCNNContour.getOutputs()); + std::copy( + mCNNNote.getOutputs(), mCNNNote.getOutputs() + NUM_FREQ_OUT, mNotesCircularBuffer[(size_t) mNoteIdx].begin()); + + // Concat operation with correct frame shift + _concat(); + + mCNNOnsetOutput.forward(mConcatArray.data()); +} + +constexpr int BasicPitchCNN::_wrapIndex(int inIndex, int inSize) +{ + int wrapped_index = inIndex % inSize; + + if (wrapped_index < 0) { + wrapped_index += inSize; + } + + return wrapped_index; +} + +void BasicPitchCNN::_concat() +{ + auto concat2_index = (size_t) _wrapIndex(mConcat2Idx + 1, mNumConcat2Stored); + + for (size_t i = 0; i < NUM_FREQ_OUT; i++) { + mConcatArray[i * 33] = mCNNNote.getOutputs()[i]; + std::copy(mConcat2CircularBuffer[concat2_index].begin() + i * 32, + mConcat2CircularBuffer[concat2_index].begin() + (i + 1) * 32, + mConcatArray.begin() + i * 33 + 1); + } +} diff --git a/JammLab/Transcription/Native/Engine/BasicPitchCNN.h b/JammLab/Transcription/Native/Engine/BasicPitchCNN.h new file mode 100644 index 0000000..6d02b52 --- /dev/null +++ b/JammLab/Transcription/Native/Engine/BasicPitchCNN.h @@ -0,0 +1,122 @@ +// +// Created by Damien Ronssin on 03.03.23. +// + +#ifndef BasicPitchCNN_h +#define BasicPitchCNN_h + +#include + +#include "RTNeural/RTNeural.h" + +#include "BasicPitchConstants.h" + +/** + * Class to run basic pitch CNN with RTNeural + */ +class BasicPitchCNN +{ +public: + explicit BasicPitchCNN(const std::filesystem::path& modelDirectory); + + ~BasicPitchCNN() = default; + + /** + * Resets the internal state of the CNN. + */ + void reset(); + + /** + * @return The number of future lookahead of basic pitch cnn. + * It corresponds to the number of padded frames done left and right (in tensorflow for example) + * in order to have aligned input and output when running with valid padding. + */ + static int getNumFramesLookahead(); + + /** + * Run inference for a single frame. inData should have 8 * 264 elements + * @param inData input features (CQT harmonically stacked). + * @param outContours output vector for contour posteriorgrams. Size should be 264 + * @param outNotes output vector for note posteriorgrams. Size should be 88 + * @param outOnsets output vector for onset posteriorgrams. Size should be 88 + */ + void frameInference(const float* inData, + std::vector& outContours, + std::vector& outNotes, + std::vector& outOnsets); + +private: + /** + * Run different sequential models with correct time offset ... + */ + void _runModels(); + + /** + * Perform concat operation with correct time offset + */ + void _concat(); + + /** + * Return in-range index for given size as if periodic. + * @param inIndex maybe out of range index + * @param inSize Size of container + * @return Wrapped index (in-range) + */ + static constexpr int _wrapIndex(int inIndex, int inSize); + + alignas(RTNEURAL_DEFAULT_ALIGNMENT) std::array mInputArray {}; + + alignas(RTNEURAL_DEFAULT_ALIGNMENT) std::array mConcatArray {}; + + static constexpr int mLookaheadCNNContour = 3; + static constexpr int mLookaheadCNNNote = 6; + static constexpr int mLookaheadCNNOnsetInput = 2; + static constexpr int mLookaheadCNNOnsetOutput = 1; + static constexpr int mTotalLookahead = mLookaheadCNNContour + mLookaheadCNNNote + mLookaheadCNNOnsetOutput; + + static constexpr int mNumContourStored = mTotalLookahead - mLookaheadCNNContour + 1; + static constexpr int mNumNoteStored = mTotalLookahead - (mLookaheadCNNContour + mLookaheadCNNNote) + 1; + static constexpr int mNumConcat2Stored = mLookaheadCNNContour + mLookaheadCNNNote - mLookaheadCNNOnsetInput + 1; + + std::array, mNumContourStored> mContoursCircularBuffer {}; + std::array, mNumNoteStored> mNotesCircularBuffer {}; // Also concat 1 + std::array, mNumConcat2Stored> mConcat2CircularBuffer {}; + + int mContourIdx = 0; + int mNoteIdx = 0; + int mConcat2Idx = 0; + + RTNeural::ModelT, + RTNeural::ReLuActivationT, + RTNeural::Conv2DT, + RTNeural::SigmoidActivationT> + mCNNContour; + + RTNeural::ModelT, + RTNeural::ReLuActivationT, + RTNeural::Conv2DT, + RTNeural::SigmoidActivationT> + mCNNNote; + + RTNeural::ModelT, + RTNeural::ReLuActivationT> + mCNNOnsetInput; + + RTNeural::ModelT, + RTNeural::SigmoidActivationT> + mCNNOnsetOutput; +}; + +#endif // BasicPitchCNN_h diff --git a/JammLab/Transcription/Native/Engine/BasicPitchConstants.h b/JammLab/Transcription/Native/Engine/BasicPitchConstants.h new file mode 100644 index 0000000..03806b0 --- /dev/null +++ b/JammLab/Transcription/Native/Engine/BasicPitchConstants.h @@ -0,0 +1,27 @@ +// +// Created by Damien Ronssin on 04.03.23. +// + +#ifndef NN_CONSTANTS_H +#define NN_CONSTANTS_H + +static constexpr int NUM_HARMONICS = 8; +static constexpr int NUM_FREQ_IN = 264; +static constexpr int NUM_FREQ_OUT = 88; +static constexpr double BASIC_PITCH_SAMPLE_RATE = 22050.0; + +static constexpr int MIDI_OFFSET = 21; +static constexpr int FFT_HOP = 256; +static constexpr int AUDIO_SAMPLE_RATE = 22050; +static constexpr int MAX_NOTE_IDX = 87; +// duration in seconds of training examples - original 1 +static constexpr int AUDIO_WINDOW_LENGTH = 2; + +// lowest key on a piano +static constexpr float ANNOTATIONS_BASE_FREQUENCY = 27.5; +static constexpr int CONTOURS_BINS_PER_SEMITONE = 3; + +static constexpr int MIN_MIDI_NOTE = 21; +static constexpr int MAX_MIDI_NOTE = 108; + +#endif //NN_CONSTANTS_H diff --git a/JammLab/Transcription/Native/Engine/Features.cpp b/JammLab/Transcription/Native/Engine/Features.cpp new file mode 100644 index 0000000..cbd5d37 --- /dev/null +++ b/JammLab/Transcription/Native/Engine/Features.cpp @@ -0,0 +1,45 @@ +// +// Created by Damien Ronssin on 04.03.23. +// + +#include "Features.h" + +#include + +Features::Features(const std::filesystem::path& modelPath) + : mMemoryInfo(nullptr) + , mSession(nullptr) +{ + if (!std::filesystem::is_regular_file(modelPath)) { + throw std::runtime_error("Basic Pitch feature model is missing"); + } + + mMemoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + + mSessionOptions.SetInterOpNumThreads(1); + mSessionOptions.SetIntraOpNumThreads(1); + + mSession = Ort::Session(mEnv, modelPath.c_str(), mSessionOptions); +} + +const float* Features::computeFeatures(float* inAudio, size_t inNumSamples, size_t& outNumFrames) +{ + mInputShape[0] = 1; + mInputShape[1] = static_cast(inNumSamples); + mInputShape[2] = 1; + + mInput.clear(); + mInput.push_back( + Ort::Value::CreateTensor(mMemoryInfo, inAudio, inNumSamples, mInputShape.data(), mInputShape.size())); + + mOutput = mSession.Run(mRunOptions, mInputNames, mInput.data(), 1, mOutputNames, 1); + + auto out_shape = mOutput[0].GetTensorTypeAndShapeInfo().GetShape(); + assert(out_shape[0] == 1 && out_shape[2] == NUM_FREQ_IN && out_shape[3] == NUM_HARMONICS); + + outNumFrames = static_cast(out_shape[1]); + + mInput.clear(); + + return mOutput[0].GetTensorData(); +} diff --git a/JammLab/Transcription/Native/Engine/Features.h b/JammLab/Transcription/Native/Engine/Features.h new file mode 100644 index 0000000..42d793a --- /dev/null +++ b/JammLab/Transcription/Native/Engine/Features.h @@ -0,0 +1,54 @@ +// +// Created by Damien Ronssin on 04.03.23. +// + +#ifndef Features_h +#define Features_h + +#include +#include +#include +#include +#include + +#include "BasicPitchConstants.h" + +/** + * Class to compute the CQT and harmonically stack those. Output of this can be given as input to Basic Pitch cnn. + */ +class Features +{ +public: + explicit Features(const std::filesystem::path& modelPath); + + ~Features() = default; + + /** + * Compute features for full audio signal + * @param inAudio Input audio. Should contain inNumSamples + * @param inNumSamples Number of samples in inAudio + * @param outNumFrames Number of frames that have been computed. + * @return Pointer to features. + */ + const float* computeFeatures(float* inAudio, size_t inNumSamples, size_t& outNumFrames); + +private: + // ONNX Runtime Data + std::vector mInput; + std::vector mOutput; + + std::array mInputShape; + + // Input and output names of model + const char* mInputNames[1] = {"input_1"}; + const char* mOutputNames[1] = {"harmonic_stacking"}; + + // ONNX Runtime + Ort::MemoryInfo mMemoryInfo; + Ort::SessionOptions mSessionOptions; + Ort::Env mEnv; + Ort::Session mSession; + Ort::RunOptions mRunOptions; +}; + +#endif // Features_h diff --git a/JammLab/Transcription/Native/Engine/Notes.cpp b/JammLab/Transcription/Native/Engine/Notes.cpp new file mode 100644 index 0000000..a815c32 --- /dev/null +++ b/JammLab/Transcription/Native/Engine/Notes.cpp @@ -0,0 +1,284 @@ +// +// Created by Tibor Vass on 04.03.23. +// + +#include "Notes.h" + +bool Notes::Event::operator==(const Notes::Event& other) const +{ + return this->startTime == other.startTime && this->endTime == other.endTime && this->startFrame == other.startFrame + && this->endFrame == other.endFrame && this->pitch == other.pitch && this->amplitude == other.amplitude + && this->bends == other.bends; +} + +std::vector Notes::convert(const std::vector>& inNotesPG, + const std::vector>& inOnsetsPG, + const std::vector>& inContoursPG, + const ConvertParams& inParams, + bool inNewAudio) +{ + std::vector events; + events.reserve(1024); + + const auto n_frames = static_cast(inNotesPG.size()); + if (n_frames == 0) { + return events; + } + + const auto n_notes = static_cast(inNotesPG[0].size()); + assert(n_frames == inOnsetsPG.size()); + assert(n_frames == inContoursPG.size()); + assert(n_notes == inOnsetsPG[0].size()); + assert(n_notes == NUM_FREQ_OUT); + + std::vector> inferred_onsets; + auto onsets_ptr = &inOnsetsPG; + if (inParams.inferOnsets) { + inferred_onsets = _inferredOnsets(inOnsetsPG, inNotesPG); + onsets_ptr = &inferred_onsets; + } + auto& onsets = *onsets_ptr; + + if (inNewAudio) { + mRemainingEnergy = inNotesPG; + } else { + // Copy without changing the location of the original data + assert(mRemainingEnergy.size() == n_frames); + for (size_t f = 0; f < n_frames; f++) { + assert(inNotesPG[f].size() == NUM_FREQ_OUT); + assert(mRemainingEnergy[f].size() == NUM_FREQ_OUT); + + std::copy(inNotesPG[f].begin(), inNotesPG[f].end(), mRemainingEnergy[f].begin()); + } + } + + if (inParams.melodiaTrick) { + if (inNewAudio) { + // Fill mRemainingEnergyIndex + mRemainingEnergyIndex.clear(); + mRemainingEnergyIndex.reserve(static_cast(n_frames) * static_cast(NUM_FREQ_OUT)); + + for (int frame_idx = 0; frame_idx < n_frames; frame_idx++) { + for (int freq_idx = 0; freq_idx < NUM_FREQ_OUT; freq_idx++) { + mRemainingEnergyIndex.push_back( + {&mRemainingEnergy[static_cast(frame_idx)][static_cast(freq_idx)], + frame_idx, + freq_idx}); + } + } + + mRemainingEnergyIndex.shrink_to_fit(); + } + } + + const auto frame_threshold = inParams.frameThreshold; + // TODO: infer frame_threshold if < 0, can be merged with inferredOnsets. + + // constrain frequencies + const auto max_note_idx = + inParams.maxFrequency < 0 ? n_notes - 1 : NoteUtils::hzToMidi(inParams.maxFrequency) - MIDI_OFFSET; + const auto min_note_idx = inParams.minFrequency < 0 ? 0 : NoteUtils::hzToMidi(inParams.minFrequency) - MIDI_OFFSET; + + // stop 1 frame early to prevent edge case + // as per https://github.com/spotify/basic-pitch/blob/f85a8e9ade1f297b8adb39b155c483e2312e1aca/basic_pitch/note_creation.py#L399 + const int last_frame = n_frames - 1; + + // Go backwards in time + for (int frame_idx = last_frame - 1; frame_idx >= 0; frame_idx--) { + for (int note_idx = max_note_idx; note_idx >= min_note_idx; note_idx--) { + auto onset = onsets[frame_idx][note_idx]; + + // equivalent to argrelmax logic + auto prev = frame_idx <= 0 ? onset : onsets[frame_idx - 1][note_idx]; + auto next = frame_idx >= last_frame ? onset : onsets[frame_idx + 1][note_idx]; + + if (onset < inParams.onsetThreshold || onset < prev || onset < next) { + continue; + } + + // find time index at this frequency band where the frames drop below an energy threshold + int i = frame_idx + 1; + int k = 0; // number of frames since energy dropped below threshold + while (i < last_frame && k < inParams.energyThreshold) { + if (mRemainingEnergy[i][note_idx] < frame_threshold) { + k++; + } else { + k = 0; + } + i++; + } + + i -= k; // go back to frame above threshold + + // if the note is too short, skip it + if (i - frame_idx <= inParams.minNoteLength) { + continue; + } + + double amplitude = 0.0; + for (int f = frame_idx; f < i; f++) { + amplitude += mRemainingEnergy[f][note_idx]; + mRemainingEnergy[f][note_idx] = 0; + + if (note_idx < MAX_NOTE_IDX) { + mRemainingEnergy[f][note_idx + 1] = 0; + } + if (note_idx > 0) { + mRemainingEnergy[f][note_idx - 1] = 0; + } + } + + amplitude /= (i - frame_idx); + + events.push_back(Event { + _modelFrameToTime(frame_idx) /* startTime */, + _modelFrameToTime(i) /* endTime */, + frame_idx /* startFrame */, + i /* endFrame */, + note_idx + MIDI_OFFSET /* pitch */, + amplitude /* amplitude */, + }); + } + } + + if (inParams.melodiaTrick) { + std::sort(mRemainingEnergyIndex.begin(), + mRemainingEnergyIndex.end(), + [](const _pg_index& a, const _pg_index& b) { return *a.value > *b.value; }); + + // loop through each remaining note probability in descending order + // until reaching frame_threshold. + for (auto& [energy_ptr, frame_idx, note_idx]: mRemainingEnergyIndex) { + auto& energy = *energy_ptr; + + // skip those that have already been zeroed + if (energy == 0.0f) { + continue; + } + + if (energy <= frame_threshold) { + break; + } + energy = 0; + + // this inhibit function zeroes out neighbor notes and keeps track (with k) + // on how many consecutive frames were below frame_threshold. + auto inhibit = [frame_threshold](std::vector>& pg, int frame_i, int note_i, int k) { + if (pg[frame_i][note_i] < frame_threshold) { + k++; + } else { + k = 0; + } + + pg[frame_i][note_i] = 0; + if (note_i < MAX_NOTE_IDX) { + pg[frame_i][note_i + 1] = 0; + } + if (note_i > 0) { + pg[frame_i][note_i - 1] = 0; + } + return k; + }; + + // forward pass + int i = frame_idx + 1; + int k = 0; + while (i < last_frame && k < inParams.energyThreshold) { + k = inhibit(mRemainingEnergy, i, note_idx, k); + i++; + } + + const auto i_end = i - 1 - k; + + // backward pass + i = frame_idx - 1; + k = 0; + while (i > 0 && k < inParams.energyThreshold) { + k = inhibit(mRemainingEnergy, i, note_idx, k); + i--; + } + + const auto i_start = i + 1 + k; + + // if the note is too short, skip it + if (i_end - i_start <= inParams.minNoteLength) { + continue; + } + + double amplitude = 0.0; + for (i = i_start; i < i_end; i++) { + amplitude += inNotesPG[i][note_idx]; + } + amplitude /= (i_end - i_start); + + events.push_back(Event { + _modelFrameToTime(i_start /* startTime */), + _modelFrameToTime(i_end) /* endTime */, + i_start /* startFrame */, + i_end /* endFrame */, + note_idx + MIDI_OFFSET /* pitch */, + amplitude /* amplitude */, + }); + } + } + + sortEvents(events); + + if (inParams.pitchBend != NoPitchBend) { + _addPitchBends(events, inContoursPG); + if (inParams.pitchBend == SinglePitchBend) { + dropOverlappingPitchBends(events); + } + } + + return events; +} + +void Notes::clear() +{ + mRemainingEnergy.clear(); + mRemainingEnergy.shrink_to_fit(); + + mRemainingEnergyIndex.clear(); + mRemainingEnergyIndex.shrink_to_fit(); +} + +void Notes::_addPitchBends(std::vector& inOutEvents, + const std::vector>& inContoursPG, + int inNumBinsTolerance) +{ + for (auto& event: inOutEvents) { + // midi_pitch_to_contour_bin + int note_idx = + CONTOURS_BINS_PER_SEMITONE + * (event.pitch - 69 + 12 * static_cast(std::round(std::log2(440.0f / ANNOTATIONS_BASE_FREQUENCY)))); + + static constexpr int N_FREQ_BINS_CONTOURS = NUM_FREQ_OUT * CONTOURS_BINS_PER_SEMITONE; + int note_start_idx = std::max(note_idx - inNumBinsTolerance, 0); + int note_end_idx = std::min(N_FREQ_BINS_CONTOURS, note_idx + inNumBinsTolerance + 1); + + const auto gauss_start = static_cast(std::max(0, inNumBinsTolerance - note_idx)); + const auto pb_shift = inNumBinsTolerance - std::max(0, inNumBinsTolerance - note_idx); + + for (int i = event.startFrame; i < event.endFrame; i++) { + int bend = 0; + float max = 0; + for (int j = note_start_idx; j < note_end_idx; j++) { + int k = j - note_start_idx; + float x = gauss_start + static_cast(k); + float n = x - static_cast(inNumBinsTolerance); + + static constexpr float std = 5.0f; + + // Gaussian + float w = std::exp(-(n * n) / (2.0f * std * std)) * inContoursPG[i][j]; + + if (w > max) { + bend = k; + max = w; + } + } + event.bends.emplace_back(bend - pb_shift); + } + } +} \ No newline at end of file diff --git a/JammLab/Transcription/Native/Engine/Notes.h b/JammLab/Transcription/Native/Engine/Notes.h new file mode 100644 index 0000000..977091f --- /dev/null +++ b/JammLab/Transcription/Native/Engine/Notes.h @@ -0,0 +1,272 @@ +// +// Created by Tibor Vass on 04.03.23. +// + +#ifndef Notes_h +#define Notes_h + +#include +#include +#include + +#include "BasicPitchConstants.h" + +namespace NoteUtils +{ +static inline int hzToMidi(float hz) +{ + return static_cast(std::round(12.0f * std::log2(hz / 440.0f) + 69.0f)); +} +} // namespace NoteUtils + +enum PitchBendModes { NoPitchBend = 0, SinglePitchBend, MultiPitchBend }; + +/** + * Class to extract note events from posteriorgrams (outputs of basic pitch cnn). + */ +class Notes +{ +public: + typedef struct Event { + double startTime; + double endTime; + int startFrame; + int endFrame; + int pitch; // pitch is not in Hz, but in "MIDI note number" + double amplitude; + std::vector bends; // One vale of pitch bend per frame. Units is 1/3 of semitones. + + bool operator==(const struct Event&) const; + } Event; + + typedef struct ConvertParams { + /* Note segmentation (0.05 - 0.95, Split-Merge Notes) */ + float onsetThreshold = 0.3; + /* Confidence threshold (0.05 to 0.95, More-Less notes) */ + float frameThreshold = 0.5; + /* Minimum note length in number of frames */ + int minNoteLength = 11; + bool inferOnsets = true; + float maxFrequency = -1; // in Hz, -1 means unset + float minFrequency = -1; // in Hz, -1 means unset + bool melodiaTrick = true; + PitchBendModes pitchBend = NoPitchBend; + int energyThreshold = 11; + } ConvertParams; + + /** + * Create note events based on postegriorgram inputs + * @param inNotesPG Note posteriorgrams + * @param inOnsetsPG Onset posteriorgrams + * @param inContoursPG Contour posteriorgrams + * @param inParams input parameters + * @param inNewAudio True: first time calling this function with this audio (these inNotesPG, inOnsetsPG, inContoursPG). + * False if same audio as last time with updated parameters. + * @return + */ + std::vector convert(const std::vector>& inNotesPG, + const std::vector>& inOnsetsPG, + const std::vector>& inContoursPG, + const ConvertParams& inParams, + bool inNewAudio); + + /** + * Release any memory allocated by the class. + */ + void clear(); + + /** + * Inplace sort of note events. + * @param inOutEvents + */ + static inline void sortEvents(std::vector& inOutEvents) + { + std::sort(inOutEvents.begin(), inOutEvents.end(), [](const Event& a, const Event& b) { + return a.startFrame < b.startFrame || (a.startFrame == b.startFrame && a.endFrame < b.endFrame); + }); + } + + /** + * dropOverlappingPitchBends sets bends to an empty array to all the note events that are overlapping in time. + * inOutEvents is expected to be sorted. + * @param inOutEvents + */ + static void dropOverlappingPitchBends(std::vector& inOutEvents) + { + for (int i = 0; i < int(inOutEvents.size()) - 1; i++) { + auto& event = inOutEvents[i]; + // if there is an overlap between events, remove pitch bends + for (int j = i + 1; j < inOutEvents.size(); j++) { + auto& event2 = inOutEvents[j]; + if (event2.startFrame >= event.endFrame) { + break; + } + event.bends = std::vector(); + event2.bends = std::vector(); + } + } + } + + /** + * mergeOverlappingNotes merges note events of same pitch that are overlapping in time. + * inOutEvents is expected to be sorted. + * @param inOutEvents + */ + static void mergeOverlappingNotesWithSamePitch(std::vector& inOutEvents) + { + sortEvents(inOutEvents); + for (int i = 0; i < int(inOutEvents.size()) - 1; i++) { + auto& event = inOutEvents[i]; + for (auto j = i + 1; j < inOutEvents.size(); j++) { + auto& event2 = inOutEvents[j]; + + // If notes don't overlap, break + if (event2.startFrame >= event.endFrame) { + break; + } + + // If notes overlap and have the same pitch: merge them + if (event.pitch == event2.pitch) { + event.endTime = event2.endTime; + event.endFrame = event2.endFrame; + inOutEvents.erase(inOutEvents.begin() + j); + } + } + } + } + +private: + /** + * Add pitch bend vector to note events. + * @param inOutEvents event vector (input and output) + * @param inContoursPG Contour posteriorgram matrix + * @param inNumBinsTolerance + */ + static void _addPitchBends(std::vector& inOutEvents, + const std::vector>& inContoursPG, + int inNumBinsTolerance = 25); + + /** + * Get time in seconds given frame index. + * Different behaviour in test because of weirdness in basic-pitch code + * @param frame Index of frame. + * @return Corresponding time in seconds. + */ + static inline double _modelFrameToTime(int frame) + { + // The following are compile-time computed consts only used here. + // If they need to be used elsewhere, please move to Constants.h + + static constexpr int ANNOTATIONS_FPS = AUDIO_SAMPLE_RATE / FFT_HOP; + // number of frames in the time-frequency representations we compute + static constexpr int ANNOT_N_FRAMES = ANNOTATIONS_FPS * AUDIO_WINDOW_LENGTH; + // number of samples in the (clipped) audio that we use as input to the models + static constexpr int AUDIO_N_SAMPLES = AUDIO_SAMPLE_RATE * AUDIO_WINDOW_LENGTH - FFT_HOP; + // magic from Basic Pitch + static constexpr double WINDOW_OFFSET = + (double) FFT_HOP / AUDIO_SAMPLE_RATE * (ANNOT_N_FRAMES - AUDIO_N_SAMPLES / (double) FFT_HOP) + 0.0018; + + // Weird stuff from Basic Pitch. Use only in test so they can pass. +#if USE_TEST_NOTE_FRAME_TO_TIME + return (frame * FFT_HOP) / (double) (AUDIO_SAMPLE_RATE) -WINDOW_OFFSET * (frame / ANNOT_N_FRAMES); +#else + return (frame * FFT_HOP) / (double) (AUDIO_SAMPLE_RATE); +#endif + } + + /** + * Returns a version of inOnsetsPG augmented by detecting differences in note posteriorgrams + * across frames separated by varying offsets (up to inNumDiffs). + * @tparam T + * @param inOnsetsPG Onset posteriorgrams + * @param inNotesPG Note posteriorgrams + * @param inNumDiffs max varying offset. + * @return + */ + // TODO: change to float + template + static std::vector> _inferredOnsets(const std::vector>& inOnsetsPG, + const std::vector>& inNotesPG, + int inNumDiffs = 2) + { + auto n_frames = inNotesPG.size(); + auto n_notes = inNotesPG[0].size(); + + // The algorithm starts by calculating a diff of note posteriorgrams, hence the name notes_diff. + // This same variable will later morph into the inferred onsets output + // notes_diff needs to be initialized to all 1 to not interfere with minima + // calculations, assuming all values in inNotesPG are probabilities < 1. + auto notes_diff = std::vector>(n_frames, std::vector(n_notes, 1)); + + // max of minima of notes_diff + T max_min_notes_diff = 0; + // max of onsets + T max_onset = 0; + + // for each frame offset + for (int n = 0; n < inNumDiffs; n++) { + auto offset = n + 1; + // for each frame + for (int i = 0; i < n_frames; i++) { + // frame index slided back by offset + auto i_behind = i - offset; + // for each note + for (int j = 0; j < n_notes; j++) { + // calculate the difference in note probabilities between frame i and + // frame i_behind (the frame behind by offset). + auto diff = inNotesPG[i][j] - ((i_behind >= 0) ? inNotesPG[i_behind][j] : 0); + + // Basic Pitch calculates the minimum amongst positive and negative + // diffs instead of ignoring negative diffs (which mean "end of note") + // while we are only looking for "start of note" (aka onset). + // TODO: the zeroing of negative diff should probably happen before + // searching for minimum + auto& min = notes_diff[i][j]; + if (diff < min) { + diff = (diff < 0) ? 0 : diff; + // https://github.com/spotify/basic-pitch/blob/86fc60dab06e3115758eb670c92ead3b62a89b47/basic_pitch/note_creation.py#L298 + min = (i >= inNumDiffs) ? diff : 0; + } + + // if last diff, max_min_notes_diff can be computed + if (offset == inNumDiffs) { + auto onset = inOnsetsPG[i][j]; + if (onset > max_onset) { + max_onset = onset; + } + if (min > max_min_notes_diff) { + max_min_notes_diff = min; + } + } + } + } + } + + // Rescale notes_diff in-place to match scale of original onsets + // and choose the element-wise max between it and the original onsets. + // This is where notes_diff morphs truly into the inferred onsets. + for (int i = 0; i < n_frames; i++) { + for (int j = 0; j < n_notes; j++) { + auto& inferred = notes_diff[i][j]; + inferred = max_onset * inferred / max_min_notes_diff; + auto orig = inOnsetsPG[i][j]; + if (orig > inferred) { + inferred = orig; + } + } + } + + return notes_diff; + } + + struct _pg_index { + float* value; + int frameIdx; + int noteIdx; + }; + + std::vector> mRemainingEnergy; + std::vector<_pg_index> mRemainingEnergyIndex; +}; + +#endif // Notes_h diff --git a/JammLab/Transcription/Native/Engine/Utils.h b/JammLab/Transcription/Native/Engine/Utils.h new file mode 100644 index 0000000..f31eb29 --- /dev/null +++ b/JammLab/Transcription/Native/Engine/Utils.h @@ -0,0 +1,18 @@ +// +// Created by Tibor Vass on 07.03.23. +// + +#ifndef Utils_h +#define Utils_h + +#include +#include + +static int safe_divide(int a, int b) +{ + auto res = std::div(a, b); + assert(res.rem == 0); + return res.quot; +} + +#endif // Utils_h diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/CMakeLists.txt b/JammLab/Transcription/Native/ThirdParty/RTNeural/CMakeLists.txt new file mode 100755 index 0000000..81c15df --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/CMakeLists.txt @@ -0,0 +1,57 @@ +add_library(RTNeural STATIC + activation/activation.h + activation/activation_eigen.h + activation/activation_xsimd.h + Model.h + Layer.h + conv1d/conv1d.h + conv1d/conv1d.tpp + conv1d_stateless/conv1d_stateless.h + conv1d_stateless/conv1d_stateless.tpp + conv1d_stateless/conv1d_stateless_eigen.h + conv1d_stateless/conv1d_stateless_eigen.h + conv2d/conv2d.h + conv2d/conv2d.tpp + conv2d/conv2d_eigen.h + conv2d/conv2d_eigen.tpp + dense/dense.h + dense/dense_eigen.h + dense/dense_xsimd.h + gru/gru.h + gru/gru.tpp + gru/gru_eigen.h + gru/gru_eigen.tpp + gru/gru_xsimd.h + gru/gru_xsimd.tpp + lstm/lstm.h + lstm/lstm.tpp + lstm/lstm_eigen.h + lstm/lstm_eigen.tpp + lstm/lstm_xsimd.h + lstm/lstm_xsimd.tpp + batchnorm/batchnorm2d.h + batchnorm/batchnorm2d.tpp + batchnorm/batchnorm2d_eigen.h + batchnorm/batchnorm2d_eigen.tpp + model_loader.h + RTNeural.h + RTNeural.cpp +) + +set_property(TARGET RTNeural PROPERTY POSITION_INDEPENDENT_CODE ON) +set_target_properties(RTNeural PROPERTIES LINKER_LANGUAGE CXX) +target_include_directories(RTNeural + PUBLIC + ../modules/json + INTERFACE + .. +) +set(RTNEURAL_NAMESPACE "RTNeural" CACHE STRING "Namespace to use for RTNeural code") +target_compile_definitions(RTNeural + PUBLIC + RTNEURAL_NAMESPACE=${RTNEURAL_NAMESPACE} +) + +if(RTNEURAL_ENABLE_RADSAN) + rtneural_radsan_configure(RTNeural) +endif() diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/Layer.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/Layer.h new file mode 100644 index 0000000..d5dcb74 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/Layer.h @@ -0,0 +1,39 @@ +#ifndef LAYER_H_INCLUDED +#define LAYER_H_INCLUDED + +#include +#include + +namespace RTNEURAL_NAMESPACE +{ + +/** Virtual base class for a generic neural network layer. */ +template +class Layer +{ +public: + /** Constructs a layer with given input and output size. */ + Layer(int in_size, int out_size) + : in_size(in_size) + , out_size(out_size) + { + } + + virtual ~Layer() = default; + + /** Returns the name of this layer. */ + virtual std::string getName() const noexcept { return ""; } + + /** Resets the state of this layer. */ + virtual void reset() { } + + /** Implements the forward propagation step for this layer. */ + virtual void forward(const T* input, T* out) noexcept = 0; + + const int in_size; + const int out_size; +}; + +} // namespace RTNEURAL_NAMESPACE + +#endif // LAYER_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/Model.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/Model.h new file mode 100644 index 0000000..d2a60a3 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/Model.h @@ -0,0 +1,118 @@ +#ifndef MODEL_H_INCLUDED +#define MODEL_H_INCLUDED + +#include + +#include "Layer.h" +#include "activation/activation.h" +#include "batchnorm/batchnorm.h" +#include "batchnorm/batchnorm.tpp" +#include "batchnorm/batchnorm2d.h" +#include "batchnorm/batchnorm2d.tpp" +#include "config.h" +#include "conv1d/conv1d.h" +#include "conv1d/conv1d.tpp" +#include "conv2d/conv2d.h" +#include "conv2d/conv2d.tpp" +#include "dense/dense.h" +#include "gru/gru.h" +#include "gru/gru.tpp" +#include "lstm/lstm.h" +#include "lstm/lstm.tpp" + +namespace RTNEURAL_NAMESPACE +{ + +/** + * A dynamic sequential neural network model. + * + * Instances of this class should typically be created + * `json_parser::parseJson`. + */ +template +class Model +{ +public: + /** Constructs a sequential model for a given input size. */ + explicit Model(int in_size) + : in_size(in_size) + { + } + + /** Destructor. */ + ~Model() + { + for(auto l : layers) + delete l; + layers.clear(); + + outs.clear(); + } + + /** Returns the model's input size */ + int getInSize() const { return layers.front()->in_size; } + + /** Returns the model's output size */ + int getOutSize() const { return layers.back()->out_size; } + + /** Returns the required input size for the next layer being added to the network. */ + int getNextInSize() const + { + if(layers.empty()) + return in_size; + + return layers.back()->out_size; + } + + /** Adds a new layer to the sequential model. */ + void addLayer(Layer* layer) + { + layers.push_back(layer); + outs.push_back(vec_type(layer->out_size, (T)0)); + } + + /** Resets the state of the network layers. */ + RTNEURAL_REALTIME void reset() + { + for(auto* l : layers) + l->reset(); + } + + /** Performs forward propagation for this model. */ + RTNEURAL_REALTIME inline T forward(const T* input) + { + layers[0]->forward(input, outs[0].data()); + + for(int i = 1; i < (int)layers.size(); ++i) + { + layers[i]->forward(outs[i - 1].data(), outs[i].data()); + } + + return outs.back()[0]; + } + + /** Returns a pointer to the output of the final layer in the network. */ + RTNEURAL_REALTIME inline const T* getOutputs() const noexcept + { + return outs.back().data(); + } + + /** A vector storing the network layers in sequential order. */ + std::vector*> layers; + +private: +#if RTNEURAL_USE_XSIMD + using vec_type = std::vector>; +#elif RTNEURAL_USE_EIGEN + using vec_type = std::vector>; +#else + using vec_type = std::vector; +#endif + + const int in_size; + std::vector outs; +}; + +} // namespace RTNEURAL_NAMESPACE + +#endif // MODEL_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/ModelT.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/ModelT.h new file mode 100644 index 0000000..745be60 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/ModelT.h @@ -0,0 +1,579 @@ +#pragma once + +#include "model_loader.h" + +namespace RTNEURAL_NAMESPACE +{ + +#ifndef DOXYGEN +/** + * Some utilities for constructing and working + * with variadic templates of layers. + * + * Note that this API may change at any time, + * so probably don't use any of this directly. + */ +namespace modelt_detail +{ + /** utils for making offset index sequences */ + template + struct offset_sequence; + + template + struct offset_sequence> + { + using type = std::index_sequence; + }; + template + using offset_sequence_t = typename offset_sequence::type; + + /** Functions to do a function for each element in the tuple */ + template + constexpr void forEachInTuple(Fn&& fn, Tuple&& tuple, std::index_sequence) noexcept(noexcept(std::initializer_list { (fn(std::get(tuple), std::integral_constant()), 0)... })) + { + (void)std::initializer_list { ((void)fn(std::get(tuple), std::integral_constant()), 0)... }; + } + + template + using TupleIndexSequence = std::make_index_sequence>>::value>; + + template + constexpr void forEachInTuple(Fn&& fn, Tuple&& tuple) noexcept(noexcept(forEachInTuple(std::forward(fn), std::forward(tuple), TupleIndexSequence {}))) + { + forEachInTuple(std::forward(fn), std::forward(tuple), TupleIndexSequence {}); + } + + template + using TupleIndexSequenceRange = offset_sequence_t>; + + template + constexpr void forEachInTupleRange(Fn&& fn, Tuple&& tuple) noexcept(noexcept(forEachInTuple(std::forward(fn), std::forward(tuple), TupleIndexSequenceRange {}))) + { + forEachInTuple(std::forward(fn), std::forward(tuple), TupleIndexSequenceRange {}); + } + + // unrolled loop for forward inferencing + template + struct forward_unroll + { + template + static void call(T& t) + { + std::get(t).forward(std::get(t).outs); + forward_unroll::call(t); + } + }; + + template + struct forward_unroll + { + template + static void call(T&) { } + }; + + template + void loadLayer(LayerType&, int&, const nlohmann::json&, const std::string&, int, bool debug) + { + json_parser::debug_print("Loading a no-op layer!", debug); + } + + template + void loadLayer(DenseT& dense, int& json_stream_idx, const nlohmann::json& l, + const std::string& type, int layerDims, bool debug) + { + using namespace json_parser; + + debug_print("Layer: " + type, debug); + debug_print(" Dims: " + std::to_string(layerDims), debug); + const auto& weights = l["weights"]; + + if(checkDense(dense, type, layerDims, debug)) + loadDense(dense, weights); + + if(!l.contains("activation")) + { + json_stream_idx++; + } + else + { + const auto activationType = l["activation"].get(); + if(activationType.empty()) + json_stream_idx++; + } + } + + template + void loadLayer(Conv1DT& conv, int& json_stream_idx, const nlohmann::json& l, + const std::string& type, int layerDims, bool debug) + { + using namespace json_parser; + + debug_print("Layer: " + type, debug); + debug_print(" Dims: " + std::to_string(layerDims), debug); + const auto& l_weights = l["weights"]; + const auto l_kernel = l["kernel_size"].back().get(); + const auto l_dilation = l["dilation"].back().get(); + const auto l_groups = l.value("groups", 1); + + if(checkConv1D(conv, type, layerDims, l_kernel, l_dilation, l_groups, debug)) + loadConv1D(conv, l_kernel, l_dilation, l_weights); + + if(!l.contains("activation")) + { + json_stream_idx++; + } + else + { + const auto activationType = l["activation"].get(); + if(activationType.empty()) + json_stream_idx++; + } + } + template + void loadLayer(Conv2DT& conv, + int& json_stream_idx, const nlohmann::json& l, + const std::string& type, int layerDims, bool debug) + { + using namespace json_parser; + + debug_print("Layer: " + type, debug); + debug_print(" Dims: " + std::to_string(layerDims), debug); + const auto& weights = l["weights"]; + const auto kernel_time = l["kernel_size_time"].back().get(); + const auto kernel_feature = l["kernel_size_feature"].back().get(); + + const auto dilation = l["dilation"].back().get(); + const auto strides = l["strides"].back().get(); + const bool valid_pad = l["padding"].get() == "valid"; + + if(checkConv2D(conv, type, layerDims, kernel_time, kernel_feature, dilation, strides, valid_pad, debug)) + loadConv2D(conv, weights); + + if(!l.contains("activation")) + { + json_stream_idx++; + } + else + { + const auto activationType = l["activation"].get(); + if(activationType.empty()) + json_stream_idx++; + } + } + + template + void loadLayer(GRULayerT& gru, int& json_stream_idx, const nlohmann::json& l, + const std::string& type, int layerDims, bool debug) + { + using namespace json_parser; + + debug_print("Layer: " + type, debug); + debug_print(" Dims: " + std::to_string(layerDims), debug); + const auto& weights = l["weights"]; + + if(checkGRU(gru, type, layerDims, debug)) + loadGRU(gru, weights); + + json_stream_idx++; + } + + template + void loadLayer(LSTMLayerT& lstm, int& json_stream_idx, const nlohmann::json& l, + const std::string& type, int layerDims, bool debug) + { + using namespace json_parser; + + debug_print("Layer: " + type, debug); + debug_print(" Dims: " + std::to_string(layerDims), debug); + const auto& weights = l["weights"]; + + if(checkLSTM(lstm, type, layerDims, debug)) + loadLSTM(lstm, weights); + + json_stream_idx++; + } + + template + void loadLayer(PReLUActivationT& prelu, int& json_stream_idx, const nlohmann::json& l, + const std::string& type, int layerDims, bool debug) + { + using namespace json_parser; + + debug_print("Layer: " + type, debug); + debug_print(" Dims: " + std::to_string(layerDims), debug); + const auto& weights = l["weights"]; + + if(checkPReLU(prelu, type, layerDims, debug)) + loadPReLU(prelu, weights); + + json_stream_idx++; + } + + template + void loadLayer(BatchNorm1DT& batch_norm, int& json_stream_idx, const nlohmann::json& l, + const std::string& type, int layerDims, bool debug) + { + using namespace json_parser; + + debug_print("Layer: " + type, debug); + debug_print(" Dims: " + std::to_string(layerDims), debug); + const auto& weights = l["weights"]; + + if(checkBatchNorm(batch_norm, type, layerDims, weights, debug)) + { + loadBatchNorm(batch_norm, weights); + batch_norm.setEpsilon(l["epsilon"].get()); + } + + json_stream_idx++; + } + + template + void loadLayer(BatchNorm2DT& batch_norm, int& json_stream_idx, const nlohmann::json& l, + const std::string& type, int layerDims, bool debug) + { + using namespace json_parser; + + debug_print("Layer: " + type, debug); + debug_print(" Dims: " + std::to_string(layerDims), debug); + const auto& weights = l["weights"]; + + if(checkBatchNorm2D(batch_norm, type, layerDims, weights, debug)) + { + loadBatchNorm(batch_norm, weights); + batch_norm.setEpsilon(l["epsilon"].get()); + } + + json_stream_idx++; + } + + template + void parseJson(const nlohmann::json& parent, std::tuple& layers, const bool debug = false, std::initializer_list custom_layers = {}) + { + using namespace json_parser; + + auto shape = parent["in_shape"]; + auto json_layers = parent["layers"]; + + if(!shape.is_array() || !json_layers.is_array()) + return; + + // If 4D: nDims is num_features * num_channels + const int nDims = shape.size() == 4 ? shape[2].get() * shape[3].get() : shape.back().get(); + + debug_print("# dimensions: " + std::to_string(nDims), debug); + + if(nDims != in_size) + { + debug_print("Incorrect input size!", debug); + return; + } + + int json_stream_idx = 0; + modelt_detail::forEachInTuple([&](auto& layer, size_t) + { + if(json_stream_idx >= (int)json_layers.size()) + { + debug_print("Too many layers!", debug); + return; + } + + const auto l = json_layers.at(json_stream_idx); + const auto type = l["type"].get(); + const auto layerShape = l["shape"]; + + // If 4D: layerDims is num_features * num_channels + const int layerDims = layerShape.size() == 4 ? layerShape[2].get() * layerShape[3].get() : layerShape.back().get(); + + if(layer.isActivation()) // activation layers don't need initialisation + { + if(!l.contains("activation")) + { + debug_print("No activation layer expected!", debug); + return; + } + + const auto activationType = l["activation"].get(); + if(!activationType.empty()) + { + debug_print(" activation: " + activationType, debug); + checkActivation(layer, activationType, layerDims, debug); + } + + json_stream_idx++; + return; + } + + if(std::find(custom_layers.begin(), custom_layers.end(), type) != custom_layers.end()) + { + debug_print ("Skipping loading weights for custom layer: " + type, debug); + json_stream_idx++; + return; + } + + modelt_detail::loadLayer(layer, json_stream_idx, l, type, layerDims, debug); }, + layers); + } +} // namespace modelt_detail +#endif // DOXYGEN + +/** + * A static sequential neural network model. + * + * To use this class, you must define the layers at compile-time: + * ``` + * ModelT, + * TanhActivationT, + * DenseT + * > model; + * ``` + */ +template +class ModelT +{ +public: + static constexpr auto input_size = in_size; + static constexpr auto output_size = out_size; + + ModelT() + { +#if RTNEURAL_USE_XSIMD + for(int i = 0; i < v_in_size; ++i) + v_ins[i] = v_type((T)0); +#elif RTNEURAL_USE_EIGEN + auto& layer_outs = get().outs; + new(&layer_outs) Eigen::Map, RTNeuralEigenAlignment>(outs); +#endif + } + + /** Get a reference to the layer at index `Index`. */ + template + RTNEURAL_REALTIME auto& get() noexcept + { + return std::get(layers); + } + + /** Get a reference to the layer at index `Index`. */ + template + RTNEURAL_REALTIME const auto& get() const noexcept + { + return std::get(layers); + } + + /** Resets the state of the network layers. */ + RTNEURAL_REALTIME void reset() + { + modelt_detail::forEachInTuple([&](auto& layer, size_t) + { layer.reset(); }, + layers); + } + + /** Performs forward propagation for this model. */ + template + RTNEURAL_REALTIME inline typename std::enable_if<(N > 1), T>::type + forward(const T* input) + { +#if RTNEURAL_USE_XSIMD + for(int i = 0; i < v_in_size; ++i) + v_ins[i] = xsimd::load_aligned(input + i * v_size); +#elif RTNEURAL_USE_EIGEN + auto v_ins = Eigen::Map(input); +#else // RTNEURAL_USE_STL + std::copy(input, input + in_size, v_ins); +#endif + std::get<0>(layers).forward(v_ins); + modelt_detail::forward_unroll<1, n_layers - 1>::call(layers); + +#if RTNEURAL_USE_XSIMD + for(int i = 0; i < v_out_size; ++i) + xsimd::store_aligned(outs + i * v_size, get().outs[i]); +#elif RTNEURAL_USE_EIGEN +#else // RTNEURAL_USE_STL + auto& layer_outs = get().outs; + std::copy(layer_outs, layer_outs + out_size, outs); +#endif + return outs[0]; + } + + /** Performs forward propagation for this model. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const T* input) + { +#if RTNEURAL_USE_XSIMD + v_ins[0] = (v_type)input[0]; +#elif RTNEURAL_USE_EIGEN + const auto v_ins = vec_type::Constant(input[0]); +#else // RTNEURAL_USE_STL + v_ins[0] = input[0]; +#endif + + std::get<0>(layers).forward(v_ins); + modelt_detail::forward_unroll<1, n_layers - 1>::call(layers); + +#if RTNEURAL_USE_XSIMD + for(int i = 0; i < v_out_size; ++i) + xsimd::store_aligned(outs + i * v_size, get().outs[i]); +#elif RTNEURAL_USE_EIGEN +#else // RTNEURAL_USE_STL + auto& layer_outs = get().outs; + std::copy(layer_outs, layer_outs + out_size, outs); +#endif + return outs[0]; + } + + /** Returns a pointer to the output of the final layer in the network. */ + RTNEURAL_REALTIME inline const T* getOutputs() const noexcept + { + return outs; + } + + /** Loads neural network model weights from a json stream. */ + void parseJson(const nlohmann::json& parent, const bool debug = false, std::initializer_list custom_layers = {}) + { + modelt_detail::parseJson(parent, layers, debug, custom_layers); + } + + /** Loads neural network model weights from a json stream. */ + void parseJson(std::ifstream& jsonStream, const bool debug = false, std::initializer_list custom_layers = {}) + { + nlohmann::json parent; + jsonStream >> parent; + return parseJson(parent, debug, custom_layers); + } + +private: +#if RTNEURAL_USE_XSIMD + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_in_size = ceil_div(in_size, v_size); + static constexpr auto v_out_size = ceil_div(out_size, v_size); + v_type v_ins[v_in_size]; +#elif RTNEURAL_USE_EIGEN + using vec_type = Eigen::Matrix; +#else // RTNEURAL_USE_STL + T v_ins alignas(RTNEURAL_DEFAULT_ALIGNMENT)[in_size]; +#endif + +#if RTNEURAL_USE_XSIMD + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[v_out_size * v_size]; +#else + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; +#endif + + std::tuple layers; + static constexpr size_t n_layers = sizeof...(Layers); +}; + +#if RTNEURAL_USE_EIGEN || !RTNEURAL_USE_XSIMD +/** A static sequential 2D neural network model. */ +template +using ModelT2D = ModelT; +#else +/** + * A static sequential 2D neural network model. + * + * This API is still somewhat unstable, so maybe hold off on using it for now, + * unless you're in the mood to help with debugging. + */ +template +class ModelT2D +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_num_filters_in = ceil_div(num_filters_in, v_size); + static constexpr auto v_num_filters_out = ceil_div(num_filters_out, v_size); + static constexpr auto v_in_size = v_num_filters_in * num_features_in; + static constexpr auto v_out_size = v_num_filters_out * num_features_out; + +public: + static constexpr auto input_size = num_filters_in * num_features_in; + static constexpr auto output_size = num_filters_out * num_features_out; + static constexpr auto input_size_padded = v_in_size * v_size; + static constexpr auto output_size_padded = v_out_size * v_size; + + ModelT2D() + { + for(int i = 0; i < v_in_size; ++i) + v_ins[i] = v_type((T)0); + } + + /** Get a reference to the layer at index `Index`. */ + template + auto& get() noexcept + { + return std::get(layers); + } + + /** Get a reference to the layer at index `Index`. */ + template + const auto& get() const noexcept + { + return std::get(layers); + } + + /** Resets the state of the network layers. */ + void reset() + { + modelt_detail::forEachInTuple([&](auto& layer, size_t) + { layer.reset(); }, + layers); + } + + /** Performs forward propagation for this model. */ + inline T forward(const T* input) + { + for(int feature_index = 0; feature_index < num_features_in; ++feature_index) + { + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T load_arr[v_size * v_num_filters_in] {}; + std::copy(input + feature_index * num_filters_in, input + feature_index * num_filters_in + num_filters_in, std::begin(load_arr)); + for(int i = 0; i < v_num_filters_in; ++i) + v_ins[feature_index * num_filters_in + i] = xsimd::load_aligned(load_arr + i * v_size); + } + std::get<0>(layers).forward(v_ins); + modelt_detail::forward_unroll<1, n_layers - 1>::call(layers); + + for(int feature_index = 0; feature_index < num_features_out; ++feature_index) + { + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T store_arr[v_size * v_num_filters_out] {}; + for(int i = 0; i < v_num_filters_out; ++i) + xsimd::store_aligned(store_arr + i * v_size, get().outs[feature_index * num_filters_out + i]); + std::copy(std::begin(store_arr), std::begin(store_arr) + num_filters_out, outs + feature_index * num_filters_out); + } + + return outs[0]; + } + + /** Returns a pointer to the output of the final layer in the network. */ + inline const T* getOutputs() const noexcept + { + return outs; + } + + /** Loads neural network model weights from a json stream. */ + void parseJson(const nlohmann::json& parent, const bool debug = false, std::initializer_list custom_layers = {}) + { + modelt_detail::parseJson(parent, layers, debug, custom_layers); + } + + /** Loads neural network model weights from a json stream. */ + void parseJson(std::ifstream& jsonStream, const bool debug = false, std::initializer_list custom_layers = {}) + { + nlohmann::json parent; + jsonStream >> parent; + return parseJson(parent, debug, custom_layers); + } + +private: + v_type v_ins[v_in_size] {}; + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[output_size] {}; + + std::tuple layers; + static constexpr size_t n_layers = sizeof...(Layers); +}; +#endif // RTNEURAL_USE_XSIMD +} // namespace RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/RTNeural.cpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/RTNeural.cpp new file mode 100644 index 0000000..1ce85cd --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/RTNeural.cpp @@ -0,0 +1,7 @@ +#include "RTNeural.h" + +// forward declare some template classes +template class RTNEURAL_NAMESPACE::Model; +template class RTNEURAL_NAMESPACE::Model; +template class RTNEURAL_NAMESPACE::Layer; +template class RTNEURAL_NAMESPACE::Layer; diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/RTNeural.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/RTNeural.h new file mode 100644 index 0000000..48926ea --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/RTNeural.h @@ -0,0 +1,14 @@ +#pragma once + +// global include file for the RTNeural library! + +// C++ STL includes +#include + +// RTNeural includes: +#include "config.h" + +#include "Model.h" +#include "ModelT.h" +#include "model_loader.h" +#include "torch_helpers.h" diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation.h new file mode 100644 index 0000000..6f80bbb --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation.h @@ -0,0 +1,432 @@ +#ifndef ACTIVATION_H_INCLUDED +#define ACTIVATION_H_INCLUDED + +#include "../Layer.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ + +/** Base class for activation layers. */ +template +class Activation : public Layer +{ +public: + /** Constructs an activation layers for a given size and function. */ + Activation(int size, std::function func, const std::string& name) + : Layer(size, size) + , name(name) + , func(func) + { + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return name; } + + /** Implements the forward propagation step for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + for(int i = 0; i < Layer::out_size; ++i) + out[i] = func(input[i]); + } + +private: + const std::string name; + const std::function func; +}; + +} // namespace RTNEURAL_NAMESPACE + +#if RTNEURAL_USE_EIGEN +#include "activation_eigen.h" + +#elif RTNEURAL_USE_XSIMD +#include "activation_xsimd.h" + +#else +#include "../common.h" +#include "../maths/maths_stl.h" +#include + +namespace RTNEURAL_NAMESPACE +{ + +/** Dynamic implementation of a tanh activation layer. */ +template +class TanhActivation final : public Activation +{ +public: + /** Constructs a tanh activation layer for a given size. */ + explicit TanhActivation(int size) + : Activation( + size, [](T x) + { return MathsProvider::tanh(x); }, + "tanh") + { + } + + TanhActivation(std::initializer_list sizes) + : TanhActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for tanh activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + for(int i = 0; i < Layer::out_size; ++i) + out[i] = MathsProvider::tanh(input[i]); + } +}; + +/** Static implementation of a tanh activation layer. */ +template +class TanhActivationT +{ +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + TanhActivationT() = default; + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "tanh"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for tanh activation. */ + RTNEURAL_REALTIME inline void forward(const T (&ins)[size]) noexcept + { + for(int i = 0; i < size; ++i) + outs[i] = MathsProvider::tanh(ins[i]); + } + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size]; +}; + +/** Dynamic implementation of a ReLU activation layer. */ +template +class ReLuActivation final : public Activation +{ +public: + /** Constructs a ReLU activation layer for a given size. */ + explicit ReLuActivation(int size) + : Activation( + size, [](T x) + { return std::max((T)0, x); }, + "relu") + { + } + + ReLuActivation(std::initializer_list sizes) + : ReLuActivation(*sizes.begin()) + { + } +}; + +/** Static implementation of a ReLU activation layer. */ +template +class ReLuActivationT +{ +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + ReLuActivationT() = default; + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "relu"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for ReLU activation. */ + RTNEURAL_REALTIME inline void forward(const T (&ins)[size]) noexcept + { + for(int i = 0; i < size; ++i) + outs[i] = std::max((T)0, ins[i]); + } + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size]; +}; + +/** Dynamic implementation of a sigmoid activation layer. */ +template +class SigmoidActivation final : public Activation +{ +public: + /** Constructs a sigmoid activation layer for a given size. */ + explicit SigmoidActivation(int size) + : Activation( + size, [](T x) + { return MathsProvider::sigmoid(x); }, + "sigmoid") + { + } + + SigmoidActivation(std::initializer_list sizes) + : SigmoidActivation(*sizes.begin()) + { + } +}; + +/** Static implementation of a sigmoid activation layer. */ +template +class SigmoidActivationT +{ +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + SigmoidActivationT() = default; + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "sigmoid"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for sigmoid activation. */ + RTNEURAL_REALTIME inline void forward(const T (&ins)[size]) noexcept + { + for(int i = 0; i < size; ++i) + outs[i] = MathsProvider::sigmoid(ins[i]); + } + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size]; +}; + +/** Dynamic implementation of a softmax activation layer. */ +template +class SoftmaxActivation final : public Activation +{ +public: + /** Constructs a softmax activation layer for a given size. */ + explicit SoftmaxActivation(int size) + : Activation( + size, [](T x) + { return (T)0; }, + "softmax") + { + } + + SoftmaxActivation(std::initializer_list sizes) + : SoftmaxActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for softmax activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + T exp_sum = 0; + for(int i = 0; i < Layer::out_size; ++i) + { + out[i] = MathsProvider::exp(input[i]); + exp_sum += out[i]; + } + + const auto exp_sum_recip = (T)1 / exp_sum; + for(int i = 0; i < Layer::out_size; ++i) + { + out[i] *= exp_sum_recip; + } + } +}; + +/** Static implementation of a softmax activation layer. */ +template +class SoftmaxActivationT +{ +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + SoftmaxActivationT() = default; + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "softmax"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for softmax activation. */ + RTNEURAL_REALTIME inline void forward(const T (&ins)[size]) noexcept + { + T exp_sum = 0; + for(int i = 0; i < size; ++i) + { + outs[i] = MathsProvider::exp(ins[i]); + exp_sum += outs[i]; + } + + const auto exp_sum_recip = (T)1 / exp_sum; + for(int i = 0; i < size; ++i) + { + outs[i] *= exp_sum_recip; + } + } + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size]; +}; + +/** Dynamic implementation of a elu activation layer. */ +template +class ELuActivation final : public Activation +{ +public: + /** Constructs a softmax activation layer for a given size. */ + explicit ELuActivation(int size) + : Activation( + size, [this](T x) + { return x > (T)0 ? x : (alpha * (MathsProvider::exp(x) - (T)1)); }, + "elu") + { + } + + ELuActivation(std::initializer_list sizes) + : ELuActivation(*sizes.begin()) + { + } + + /** Sets a custom value for the layer's "alpha" parameter. */ + RTNEURAL_REALTIME void set_alpha(T newAlpha) { alpha = newAlpha; } + +private: + T alpha = (T)1; +}; + +/** Static implementation of a elu activation layer. */ +template +class ELuActivationT +{ +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + ELuActivationT() = default; + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "elu"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for elu activation. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const T (&ins)[size]) noexcept + { + for(int i = 0; i < size; ++i) + outs[i] = ins[i] > (T)0 ? ins[i] : (MathsProvider::exp(ins[i]) - (T)1); + } + + /** Performs forward propagation for elu activation (with custom alpha parameter). */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const T (&ins)[size]) noexcept + { + static constexpr T alpha = (T)AlphaNumerator / (T)AlphaDenominator; + for(int i = 0; i < size; ++i) + outs[i] = ins[i] > (T)0 ? ins[i] : (alpha * (MathsProvider::exp(ins[i]) - (T)1)); + } + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[size]; +}; + +/** Dynamic implementation of a PReLU activation layer. */ +template +class PReLUActivation final : public Activation +{ +public: + explicit PReLUActivation(int size) + : Activation(size, {}, "prelu") + , alpha(size, {}) + { + } + + /** Performs forward propagation for prelu activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + for(auto i = 0; i < Layer::in_size; ++i) + out[i] = input[i] >= (T)0 ? input[i] : (input[i] * alpha[i]); + } + + RTNEURAL_REALTIME void setAlphaVals(const std::vector& alphaVals) + { + if(alphaVals.size() == 1) + { + std::fill(alpha.begin(), alpha.end(), alphaVals[0]); + } + else + { + std::copy(alphaVals.begin(), alphaVals.end(), alpha.begin()); + } + } + + std::vector alpha; +}; + +/** Static implementation of a PReLU activation layer. */ +template +class PReLUActivationT +{ +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + PReLUActivationT() + { + for(int i = 0; i < size; ++i) + { + outs[i] = (T)0; + alpha[i] = (T)0; + } + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "prelu"; } + + /** Returns false since this layer has weights even though it is an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for prelu activation. */ + RTNEURAL_REALTIME inline void forward(const T (&ins)[size]) noexcept + { + for(auto i = 0; i < size; ++i) + outs[i] = ins[i] >= (T)0 ? ins[i] : (ins[i] * alpha[i]); + } + + RTNEURAL_REALTIME void setAlphaVals(const std::vector& alphaVals) + { + if(alphaVals.size() == 1) + { + std::fill(std::begin(alpha), std::end(alpha), alphaVals[0]); + } + else + { + for(size_t i = 0; i < size; i += alphaVals.size()) + std::copy(alphaVals.begin(), alphaVals.end(), std::begin(alpha) + i); + } + } + + T outs[size]; + T alpha[size]; +}; +} // namespace RTNEURAL_NAMESPACE + +#endif // RTNEURAL_USE_EIGEN + +#endif // ACTIVATION_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation_eigen.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation_eigen.h new file mode 100644 index 0000000..cdfecb0 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation_eigen.h @@ -0,0 +1,469 @@ +#ifndef ACTIVATIONEIGEN_H_INCLUDED +#define ACTIVATIONEIGEN_H_INCLUDED + +#include "../common.h" +#include "../config.h" +#include "../maths/maths_eigen.h" + +namespace RTNEURAL_NAMESPACE +{ + +/** Dynamic implementation of a tanh activation layer. */ +template +class TanhActivation : public Activation +{ +public: + /** Constructs a tanh activation layer for a given size. */ + explicit TanhActivation(int size) + : Activation(size, {}, "tanh") + { + inVec = Eigen::Matrix::Zero(size, 1); + outVec = Eigen::Matrix::Zero(size, 1); + } + + TanhActivation(std::initializer_list sizes) + : TanhActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for tanh activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + inVec = Eigen::Map, RTNeuralEigenAlignment>( + input, Layer::in_size, 1); + outVec = MathsProvider::tanh(inVec); + + std::copy(outVec.data(), outVec.data() + Layer::in_size, out); + } + + Eigen::Matrix inVec; + Eigen::Matrix outVec; +}; + +/** Static implementation of a tanh activation layer. */ +template +class TanhActivationT +{ + using v_type = Eigen::Matrix; + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + TanhActivationT() + : outs(outs_internal) + { + outs = v_type::Zero(); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "tanh"; } + + /** Returns true if this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for tanh activation. */ + RTNEURAL_REALTIME inline void forward(const v_type& ins) noexcept + { + outs = MathsProvider::tanh(ins); + } + + Eigen::Map outs; + +private: + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; +}; + +/** Dynamic implementation of a ReLU activation layer. */ +template +class ReLuActivation : public Activation +{ +public: + /** Constructs a ReLU activation layer for a given size. */ + explicit ReLuActivation(int size) + : Activation(size, {}, "relu") + { + inVec = Eigen::Matrix::Zero(size, 1); + outVec = Eigen::Matrix::Zero(size, 1); + } + + ReLuActivation(std::initializer_list sizes) + : ReLuActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for ReLU activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + inVec = Eigen::Map, RTNeuralEigenAlignment>( + input, Layer::in_size, 1); + outVec = inVec.array().max((T)0); + + std::copy(outVec.data(), outVec.data() + Layer::in_size, out); + } + + Eigen::Matrix inVec; + Eigen::Matrix outVec; +}; + +/** Static implementation of a ReLU activation layer. */ +template +class ReLuActivationT +{ + using v_type = Eigen::Matrix; + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + ReLuActivationT() + : outs(outs_internal) + { + outs = v_type::Zero(); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "relu"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for ReLU activation. */ + RTNEURAL_REALTIME inline void forward(const v_type& ins) noexcept + { + outs = ins.array().max((T)0); + } + + Eigen::Map outs; + +private: + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; +}; + +/** Dynamic implementation of a sigmoid activation layer. */ + +template +class SigmoidActivation : public Activation +{ +public: + /** Constructs a sigmoid activation layer for a given size. */ + explicit SigmoidActivation(int size) + : Activation(size, {}, "sigmoid") + { + inVec = Eigen::Matrix::Zero(size, 1); + outVec = Eigen::Matrix::Zero(size, 1); + } + + SigmoidActivation(std::initializer_list sizes) + : SigmoidActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for sigmoid activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + inVec = Eigen::Map, RTNeuralEigenAlignment>( + input, Layer::in_size, 1); + outVec = MathsProvider::sigmoid(inVec); + + std::copy(outVec.data(), outVec.data() + Layer::in_size, out); + } + + Eigen::Matrix inVec; + Eigen::Matrix outVec; +}; + +/** Static implementation of a sigmoid activation layer. */ +template +class SigmoidActivationT +{ + using v_type = Eigen::Matrix; + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + SigmoidActivationT() + : outs(outs_internal) + { + outs = v_type::Zero(); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "sigmoid"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for sigmoid activation. */ + RTNEURAL_REALTIME inline void forward(const v_type& ins) noexcept + { + outs = MathsProvider::sigmoid(ins); + } + + Eigen::Map outs; + +private: + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; +}; + +/** Dynamic implementation of a softmax activation layer. */ +template +class SoftmaxActivation : public Activation +{ +public: + /** Constructs a softmax activation layer for a given size. */ + explicit SoftmaxActivation(int size) + : Activation(size, {}, "softmax") + { + inVec = Eigen::Matrix::Zero(size, 1); + outVec = Eigen::Matrix::Zero(size, 1); + } + + SoftmaxActivation(std::initializer_list sizes) + : SoftmaxActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for softmax activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + inVec = Eigen::Map, RTNeuralEigenAlignment>( + input, Layer::in_size, 1); + outVec = MathsProvider::exp(inVec); + outVec = outVec / outVec.sum(); + + std::copy(outVec.data(), outVec.data() + Layer::in_size, out); + } + + Eigen::Matrix inVec; + Eigen::Matrix outVec; +}; + +/** Static implementation of a softmax activation layer. */ +template +class SoftmaxActivationT +{ + using v_type = Eigen::Matrix; + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + SoftmaxActivationT() + : outs(outs_internal) + { + outs = v_type::Zero(); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "softmax"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for softmax activation. */ + RTNEURAL_REALTIME inline void forward(const v_type& ins) noexcept + { + outs = MathsProvider::exp(ins); + outs = outs / outs.sum(); + } + + Eigen::Map outs; + +private: + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; +}; + +/** Dynamic implementation of a elu activation layer. */ +template +class ELuActivation : public Activation +{ +public: + /** Constructs a elu activation layer for a given size. */ + explicit ELuActivation(int size) + : Activation(size, {}, "elu") + , ones(Eigen::Matrix::Ones(size, 1)) + { + inVec = Eigen::Matrix::Zero(size, 1); + outVec = Eigen::Matrix::Zero(size, 1); + } + + ELuActivation(std::initializer_list sizes) + : ELuActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for softmax activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + inVec = Eigen::Map, RTNeuralEigenAlignment>( + input, Layer::in_size, 1); + + outVec = (inVec.array() > (T)0).select(inVec, alpha * (MathsProvider::exp(inVec) - ones.array())); + std::copy(outVec.data(), outVec.data() + Layer::in_size, out); + } + + Eigen::Matrix inVec; + Eigen::Matrix outVec; + + /** Sets a custom value for the layer's "alpha" parameter. */ + RTNEURAL_REALTIME void set_alpha(T newAlpha) { alpha = newAlpha; } + +private: + const Eigen::Matrix ones; + T alpha = (T)1; +}; + +/** Static implementation of a elu activation layer. */ +template +class ELuActivationT +{ + using v_type = Eigen::Matrix; + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + ELuActivationT() + : outs(outs_internal) + { + outs = v_type::Zero(); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "elu"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for elu activation. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type& ins) noexcept + { + outs = (ins.array() > (T)0).select(ins, MathsProvider::exp(ins) - ones.array()); + } + + /** Performs forward propagation for elu activation (with custom alpha parameter). */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type& ins) noexcept + { + static constexpr T alpha = (T)AlphaNumerator / (T)AlphaDenominator; + outs = (ins.array() > (T)0).select(ins, alpha * (MathsProvider::exp(ins) - ones.array())); + } + + Eigen::Map outs; + +private: + const v_type ones = v_type::Ones(); + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; +}; + +/** Dynamic implementation of a PReLU activation layer. */ +template +class PReLUActivation final : public Activation +{ +public: + explicit PReLUActivation(int size) + : Activation(size, {}, "prelu") + { + alpha = Eigen::Matrix::Zero(size, 1); + inVec = Eigen::Matrix::Zero(size, 1); + outVec = Eigen::Matrix::Zero(size, 1); + } + + /** Performs forward propagation for prelu activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + inVec = Eigen::Map, RTNeuralEigenAlignment>( + input, Layer::in_size, 1); + + outVec = (inVec.array() >= (T)0).select(inVec, alpha.cwiseProduct(inVec)); + std::copy(outVec.data(), outVec.data() + Layer::in_size, out); + } + + RTNEURAL_REALTIME void setAlphaVals(const std::vector& alphaVals) + { + if(alphaVals.size() == 1) + { + std::fill(alpha.begin(), alpha.end(), alphaVals[0]); + } + else + { + std::copy(alphaVals.begin(), alphaVals.end(), alpha.begin()); + } + } + + Eigen::Matrix inVec; + Eigen::Matrix outVec; + +private: + Eigen::Matrix alpha; +}; + +/** Static implementation of a PReLU activation layer. */ +template +class PReLUActivationT +{ + using v_type = Eigen::Matrix; + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + PReLUActivationT() + : outs(outs_internal) + { + outs = v_type::Zero(); + alpha = v_type::Zero(); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "prelu"; } + + /** Returns false since this layer has weights even though it is an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for prelu activation. */ + RTNEURAL_REALTIME inline void forward(const v_type& ins) noexcept + { + outs = (ins.array() >= (T)0).select(ins, alpha.cwiseProduct(ins)); + } + + RTNEURAL_REALTIME void setAlphaVals(const std::vector& alphaVals) + { + if(alphaVals.size() == 1) + { + std::fill(std::begin(alpha), std::end(alpha), alphaVals[0]); + } + else + { + for(size_t i = 0; i < (size_t)alpha.size(); i += alphaVals.size()) + std::copy(alphaVals.begin(), alphaVals.end(), std::begin(alpha) + i); + } + } + + Eigen::Map outs; + +private: + const v_type ones = v_type::Ones(); + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + v_type alpha; +}; +} // namespace RTNEURAL_NAMESPACE + +#endif // ACTIVATIONEIGEN_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation_xsimd.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation_xsimd.h new file mode 100644 index 0000000..c903f9b --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/activation/activation_xsimd.h @@ -0,0 +1,436 @@ +#ifndef ACTIVATIONXSIMD_H_INCLUDED +#define ACTIVATIONXSIMD_H_INCLUDED + +#include "../common.h" +#include "../config.h" +#include "../maths/maths_xsimd.h" + +namespace RTNEURAL_NAMESPACE +{ + +/** Dynamic implementation of a tanh activation layer. */ +template +class TanhActivation : public Activation +{ +public: + /** Constructs a tanh activation layer for a given size. */ + explicit TanhActivation(int size) + : Activation(size, {}, "tanh") + { + } + + TanhActivation(std::initializer_list sizes) + : TanhActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for tanh activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + tanh(input, out, Layer::in_size); + } +}; + +/** Static implementation of a tanh activation layer. */ +template +class TanhActivationT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_io_size = ceil_div(size, v_size); + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + TanhActivationT() + { + for(int i = 0; i < v_io_size; ++i) + outs[i] = v_type((T)0); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "tanh"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for tanh activation. */ + RTNEURAL_REALTIME inline void forward(const v_type (&ins)[v_io_size]) noexcept + { + for(int i = 0; i < v_io_size; ++i) + outs[i] = MathsProvider::tanh(ins[i]); + } + + v_type outs[v_io_size]; +}; + +/** Dynamic implementation of a ReLU activation layer. */ +template +class ReLuActivation : public Activation +{ +public: + /** Constructs a ReLU activation layer for a given size. */ + explicit ReLuActivation(int size) + : Activation(size, {}, "relu") + { + zeros.resize(size, (T)0); + } + + ReLuActivation(std::initializer_list sizes) + : ReLuActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for ReLU activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + xsimd::transform( + input, &input[Layer::in_size], zeros.begin(), out, + [](auto const& a, auto const& b) + { return xsimd::max(a, b); }); + } + + std::vector> zeros; +}; + +/** Static implementation of a ReLU activation layer. */ +template +class ReLuActivationT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_io_size = ceil_div(size, v_size); + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + ReLuActivationT() + { + for(int i = 0; i < v_io_size; ++i) + outs[i] = v_type((T)0); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "relu"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for ReLU activation. */ + RTNEURAL_REALTIME inline void forward(const v_type (&ins)[v_io_size]) noexcept + { + for(int i = 0; i < v_io_size; ++i) + outs[i] = xsimd::max(ins[i], v_type((T)0)); + } + + v_type outs[v_io_size]; +}; + +/** Dynamic implementation of a sigmoid activation layer. */ +template +class SigmoidActivation : public Activation +{ +public: + /** Constructs a sigmoid activation layer for a given size. */ + explicit SigmoidActivation(int size) + : Activation(size, {}, "sigmoid") + { + } + + SigmoidActivation(std::initializer_list sizes) + : SigmoidActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for sigmoid activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + sigmoid(input, out, Layer::in_size); + } +}; + +/** Static implementation of a sigmoid activation layer. */ +template +class SigmoidActivationT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_io_size = ceil_div(size, v_size); + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + SigmoidActivationT() + { + for(int i = 0; i < v_io_size; ++i) + outs[i] = v_type((T)0); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "sigmoid"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for sigmoid activation. */ + RTNEURAL_REALTIME inline void forward(const v_type (&ins)[v_io_size]) noexcept + { + for(int i = 0; i < v_io_size; ++i) + outs[i] = MathsProvider::sigmoid(ins[i]); + } + + v_type outs[v_io_size]; +}; + +/** Dynamic implementation of a softmax activation layer. */ +template +class SoftmaxActivation : public Activation +{ +public: + /** Constructs a softmax activation layer for a given size. */ + explicit SoftmaxActivation(int size) + : Activation(size, {}, "softmax") + { + } + + SoftmaxActivation(std::initializer_list sizes) + : SoftmaxActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for softmax activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + softmax(input, out, Layer::in_size); + } +}; + +/** Static implementation of a softmax activation layer. */ +template +class SoftmaxActivationT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_io_size = ceil_div(size, v_size); + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + SoftmaxActivationT() + { + for(int i = 0; i < v_io_size; ++i) + outs[i] = v_type((T)0); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "softmax"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for softmax activation. */ + RTNEURAL_REALTIME inline void forward(const v_type (&ins)[v_io_size]) noexcept + { + v_type exp_sum {}; + for(int i = 0; i < v_io_size; ++i) + { + outs[i] = MathsProvider::exp(ins[i]); + exp_sum += outs[i]; + } + + const auto exp_sum_recip = v_type((T)1 / xsimd::reduce_add(exp_sum)); + for(int i = 0; i < v_io_size; ++i) + outs[i] *= exp_sum_recip; + } + + v_type outs[v_io_size]; +}; + +/** Dynamic implementation of a elu activation layer. */ +template +class ELuActivation final : public Activation +{ +public: + /** Constructs a elu activation layer for a given size. */ + explicit ELuActivation(int size) + : Activation(size, {}, "elu") + { + } + + ELuActivation(std::initializer_list sizes) + : ELuActivation(*sizes.begin()) + { + } + + /** Performs forward propagation for softmax activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + elu(input, out, Layer::in_size, alpha); + } + + /** Sets a custom value for the layer's "alpha" parameter. */ + RTNEURAL_REALTIME void set_alpha(T newAlpha) { alpha = newAlpha; } + +private: + T alpha = (T)1; +}; + +/** Static implementation of a elu activation layer. */ +template +class ELuActivationT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_io_size = ceil_div(size, v_size); + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + ELuActivationT() = default; + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "elu"; } + + /** Returns true since this layer is an activation layer. */ + constexpr bool isActivation() const noexcept { return true; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for elu activation. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type (&ins)[v_io_size]) noexcept + { + for(int i = 0; i < v_io_size; ++i) + outs[i] = xsimd::select(ins[i] > (T)0, ins[i], MathsProvider::exp(ins[i]) - (T)1); + } + + /** Performs forward propagation for elu activation (with custom alpha parameter). */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type (&ins)[v_io_size]) noexcept + { + static constexpr T alpha = (T)AlphaNumerator / (T)AlphaDenominator; + for(int i = 0; i < v_io_size; ++i) + outs[i] = xsimd::select(ins[i] > (T)0, ins[i], alpha * (MathsProvider::exp(ins[i]) - (T)1)); + } + + v_type outs[v_io_size]; +}; + +/** Dynamic implementation of a PReLU activation layer. */ +template +class PReLUActivation final : public Activation +{ +public: + explicit PReLUActivation(int size) + : Activation(size, {}, "prelu") + , alpha(size, {}) + { + } + + /** Performs forward propagation for prelu activation. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + using b_type = xsimd::simd_type; + constexpr auto inc = (int)b_type::size; + + // size for which the vectorization is possible + auto vec_size = Layer::in_size - Layer::in_size % inc; + for(int i = 0; i < vec_size; i += inc) + { + b_type x_vec = xsimd::load_aligned(&input[i]); + b_type a_vec = xsimd::load_aligned(&alpha[i]); + b_type y_vec = xsimd::select(x_vec >= (T)0, x_vec, x_vec * a_vec); + xsimd::store_aligned(&out[i], y_vec); + } + + // Remaining part that cannot be vectorized + for(auto i = vec_size; i < Layer::in_size; ++i) + out[i] = input[i] >= (T)0 ? input[i] : (input[i] * alpha[i]); + } + + RTNEURAL_REALTIME void setAlphaVals(const std::vector& alphaVals) + { + if(alphaVals.size() == 1) + { + std::fill(alpha.begin(), alpha.end(), alphaVals[0]); + } + else + { + std::copy(alphaVals.begin(), alphaVals.end(), alpha.begin()); + } + } + + std::vector> alpha; +}; + +/** Static implementation of a PReLU activation layer. */ +template +class PReLUActivationT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_io_size = ceil_div(size, v_size); + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + + PReLUActivationT() + { + for(int i = 0; i < v_io_size; ++i) + { + outs[i] = v_type((T)0); + alpha[i] = v_type((T)0); + } + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "prelu"; } + + /** Returns false since this layer has weights even though it is an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for prelu activation. */ + RTNEURAL_REALTIME inline void forward(const v_type (&ins)[v_io_size]) noexcept + { + for(int i = 0; i < v_io_size; ++i) + outs[i] = xsimd::select(ins[i] >= (T)0, ins[i], ins[i] * alpha[i]); + } + + RTNEURAL_REALTIME void setAlphaVals(const std::vector& alphaVals) + { + if(alphaVals.size() == 1) + { + for(int i = 0; i < v_io_size; ++i) + alpha[i] = alphaVals[0]; + } + else + { + for(int i = 0; i < out_size; ++i) + alpha[i / v_size] = set_value(alpha[i / v_size], i % v_size, alphaVals[i]); + } + } + + v_type outs[v_io_size]; + v_type alpha[v_io_size]; +}; +} // namespace RTNEURAL_NAMESPACE + +#endif // ACTIVATIONXSIMD_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm.h new file mode 100644 index 0000000..3cb1a94 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm.h @@ -0,0 +1,146 @@ +#ifndef BATCHNORM_H_INCLUDED +#define BATCHNORM_H_INCLUDED + +#if RTNEURAL_USE_EIGEN +#include "batchnorm_eigen.h" +#include "batchnorm_eigen.tpp" +#elif RTNEURAL_USE_XSIMD +#include "batchnorm_xsimd.h" +#include "batchnorm_xsimd.tpp" +#else +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ +/** Dynamic batch normalization layer. */ +template +class BatchNorm1DLayer final : public Layer +{ +public: + explicit BatchNorm1DLayer(int size); + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "batchnorm"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + for(int i = 0; i < Layer::out_size; ++i) + out[i] = multiplier[i] * (input[i] - running_mean[i]) + beta[i]; + } + + /** Sets the layer "gamma" values. */ + RTNEURAL_REALTIME void setGamma(const std::vector& gammaVals); + + /** Sets the layer "beta" values. */ + RTNEURAL_REALTIME void setBeta(const std::vector& betaVals); + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + +private: + void updateMultiplier(); + + std::vector gamma; + std::vector beta; + + std::vector running_mean; + std::vector running_var; + + std::vector multiplier; + + T epsilon = (T)0; +}; + +/** Static batch normalization layer. */ +template +class BatchNorm1DT +{ +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + static constexpr bool is_affine = affine; + + BatchNorm1DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "batchnorm"; } + + /** Returns false since batch-norm is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const T (&ins)[in_size]) noexcept + { + for(int i = 0; i < size; ++i) + outs[i] = multiplier[i] * (ins[i] - running_mean[i]) + beta[i]; + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const T (&ins)[in_size]) noexcept + { + for(int i = 0; i < size; ++i) + outs[i] = multiplier[i] * (ins[i] - running_mean[i]); + } + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector& gammaVals); + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector&) { } + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector& betaVals); + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector&) { } + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + +private: + void updateMultiplier(); + + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T gamma[out_size]; + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T beta[out_size]; + + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T running_mean[out_size]; + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T running_var[out_size]; + + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T multiplier[out_size]; + + T epsilon = (T)0; +}; +} + +#endif // RTNEURAL_STL + +#endif // BATCHNORM_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm.tpp new file mode 100644 index 0000000..7382849 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm.tpp @@ -0,0 +1,113 @@ +#include "batchnorm.h" + +namespace RTNEURAL_NAMESPACE +{ +#if !RTNEURAL_USE_EIGEN && !RTNEURAL_USE_XSIMD + +template +BatchNorm1DLayer::BatchNorm1DLayer(int size) + : Layer(size, size) + , gamma(size, (T)1) + , beta(size, (T)0) + , running_mean(size, (T)0) + , running_var(size, (T)1) + , multiplier(size, (T)1) +{ +} + +template +void BatchNorm1DLayer::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), gamma.begin()); + updateMultiplier(); +} + +template +void BatchNorm1DLayer::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), beta.begin()); +} + +template +void BatchNorm1DLayer::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), running_mean.begin()); +} + +template +void BatchNorm1DLayer::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), running_var.begin()); + updateMultiplier(); +} + +template +void BatchNorm1DLayer::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm1DLayer::updateMultiplier() +{ + for(int i = 0; i < Layer::out_size; ++i) + multiplier[i] = gamma[i] / std::sqrt(running_var[i] + epsilon); +} + +//============================================================ +template +BatchNorm1DT::BatchNorm1DT() +{ + std::fill(std::begin(outs), std::end(outs), (T)0); + + std::fill(std::begin(gamma), std::end(gamma), (T)1); + std::fill(std::begin(beta), std::end(beta), (T)0); + std::fill(std::begin(running_mean), std::end(running_mean), (T)0); + std::fill(std::begin(running_var), std::end(running_var), (T)1); + std::fill(std::begin(multiplier), std::end(multiplier), (T)1); +} + +template +template +typename std::enable_if::type BatchNorm1DT::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), std::begin(gamma)); + updateMultiplier(); +} + +template +template +typename std::enable_if::type BatchNorm1DT::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), std::begin(beta)); +} + +template +void BatchNorm1DT::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), std::begin(running_mean)); +} + +template +void BatchNorm1DT::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), std::begin(running_var)); + updateMultiplier(); +} + +template +void BatchNorm1DT::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm1DT::updateMultiplier() +{ + for(int i = 0; i < out_size; ++i) + multiplier[i] = gamma[i] / std::sqrt(running_var[i] + epsilon); +} +#endif +} \ No newline at end of file diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d.h new file mode 100644 index 0000000..c9e8db7 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d.h @@ -0,0 +1,164 @@ +#ifndef BATCHNORM2D_H_INCLUDED +#define BATCHNORM2D_H_INCLUDED + +#if RTNEURAL_USE_EIGEN +#include "batchnorm2d_eigen.h" +#include "batchnorm2d_eigen.tpp" +#elif RTNEURAL_USE_XSIMD +#include "batchnorm2d_xsimd.h" +#include "batchnorm2d_xsimd.tpp" +#else +#include "../Layer.h" +#include "../config.h" + +namespace RTNEURAL_NAMESPACE +{ +/** Dynamic batch normalization layer. */ +template +class BatchNorm2DLayer final : public Layer +{ +public: + BatchNorm2DLayer(int num_filters, int num_features); + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "batchnorm2d"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + for(int i = 0; i < num_features; i++) + { + for(int j = 0; j < num_filters; ++j) + { + out[i * num_filters + j] = (input[i * num_filters + j] - running_mean[j]) * multiplier[j] + beta[j]; + } + } + } + + /** Sets the layer "gamma" values. */ + RTNEURAL_REALTIME void setGamma(const std::vector& gammaVals); + + /** Sets the layer "beta" values. */ + RTNEURAL_REALTIME void setBeta(const std::vector& betaVals); + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + +private: + void updateMultiplier(); + + const int num_filters; + const int num_features; + + std::vector gamma; + std::vector beta; + + std::vector running_mean; + std::vector running_var; + + std::vector multiplier; + + T epsilon = (T)0; +}; + +/** Static batch normalization layer. */ +template +class BatchNorm2DT +{ +public: + static constexpr auto in_size = num_filters_t * num_features_t; + static constexpr auto out_size = num_filters_t * num_features_t; + static constexpr auto num_filters = num_filters_t; + static constexpr auto num_features = num_features_t; + static constexpr bool is_affine = affine; + + BatchNorm2DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "batchnorm2d"; } + + /** Returns false since batch-norm is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const T (&ins)[in_size]) noexcept + { + for(int i = 0; i < num_features; i++) + { + for(int j = 0; j < num_filters; ++j) + { + outs[i * num_filters + j] = (ins[i * num_filters + j] - running_mean[j]) * multiplier[j] + beta[j]; + } + } + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const T (&ins)[in_size]) noexcept + { + for(int i = 0; i < num_features; i++) + { + for(int j = 0; j < num_filters; ++j) + { + outs[i * num_filters + j] = (ins[i * num_filters + j] - running_mean[j]) * multiplier[j]; + } + } + } + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector& gammaVals); + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector&) { } + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector& betaVals); + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector&) { } + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + +private: + void updateMultiplier(); + + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T gamma[num_filters_t]; + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T beta[num_filters_t]; + + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T running_mean[num_filters_t]; + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T running_var[num_filters_t]; + + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T multiplier[num_filters_t]; + + T epsilon = (T)0; +}; +} + +#endif // RTNEURAL_USE_STL + +#endif // BATCHNORM2D_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d.tpp new file mode 100644 index 0000000..12699e3 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d.tpp @@ -0,0 +1,116 @@ +#include "batchnorm2d.h" + +#if !RTNEURAL_USE_EIGEN && !RTNEURAL_USE_XSIMD + +namespace RTNEURAL_NAMESPACE +{ +template +BatchNorm2DLayer::BatchNorm2DLayer(int in_num_filters, int in_num_features) + : Layer(in_num_filters * in_num_features, in_num_filters * in_num_features) + , num_filters(in_num_filters) + , num_features(in_num_features) + , gamma(num_filters, (T)1) + , beta(num_filters, (T)0) + , running_mean(num_filters, (T)0) + , running_var(num_filters, (T)1) + , multiplier(num_filters, (T)1) +{ +} + +template +void BatchNorm2DLayer::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), gamma.begin()); + updateMultiplier(); +} + +template +void BatchNorm2DLayer::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), beta.begin()); +} + +template +void BatchNorm2DLayer::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), running_mean.begin()); +} + +template +void BatchNorm2DLayer::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), running_var.begin()); + updateMultiplier(); +} + +template +void BatchNorm2DLayer::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm2DLayer::updateMultiplier() +{ + for(int i = 0; i < num_filters; ++i) + multiplier[i] = gamma[i] / std::sqrt(running_var[i] + epsilon); +} + +//============================================================ +template +BatchNorm2DT::BatchNorm2DT() +{ + std::fill(std::begin(outs), std::end(outs), (T)0); + + std::fill(std::begin(gamma), std::end(gamma), (T)1); + std::fill(std::begin(beta), std::end(beta), (T)0); + std::fill(std::begin(running_mean), std::end(running_mean), (T)0); + std::fill(std::begin(running_var), std::end(running_var), (T)1); + std::fill(std::begin(multiplier), std::end(multiplier), (T)1); +} + +template +template +typename std::enable_if::type BatchNorm2DT::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), std::begin(gamma)); + updateMultiplier(); +} + +template +template +typename std::enable_if::type BatchNorm2DT::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), std::begin(beta)); +} + +template +void BatchNorm2DT::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), std::begin(running_mean)); +} + +template +void BatchNorm2DT::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), std::begin(running_var)); + updateMultiplier(); +} + +template +void BatchNorm2DT::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm2DT::updateMultiplier() +{ + for(int i = 0; i < num_filters_t; ++i) + multiplier[i] = gamma[i] / std::sqrt(running_var[i] + epsilon); +} +} + +#endif // RTNEURAL_USE_STL diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_eigen.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_eigen.h new file mode 100644 index 0000000..6d17723 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_eigen.h @@ -0,0 +1,162 @@ +#ifndef BATCHNORM2DEIGEN_H_INCLUDED +#define BATCHNORM2DEIGEN_H_INCLUDED + +#include "../Layer.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ +/** Dynamic batch normalization layer. */ +template +class BatchNorm2DLayer final : public Layer +{ +public: + BatchNorm2DLayer(int num_filters, int num_features); + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "batchnorm2d"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + auto inMat = Eigen::Map, RTNeuralEigenAlignment>( + input, num_filters, num_features); + + auto outMat = Eigen::Map, RTNeuralEigenAlignment>( + out, num_filters, num_features); + + // TODO: Should be possible to do it in one line with .colwise() but did not manage to do it yet. + for(int i = 0; i < num_features; i++) + { + outMat.col(i) = (inMat.col(i) - running_mean).cwiseProduct(multiplier) + beta; + } + } + + /** Sets the layer "gamma" values. */ + RTNEURAL_REALTIME void setGamma(const std::vector& gammaVals); + + /** Sets the layer "beta" values. */ + RTNEURAL_REALTIME void setBeta(const std::vector& betaVals); + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + +private: + void updateMultiplier(); + + const int num_filters; + const int num_features; + + Eigen::Vector gamma; + Eigen::Vector beta; + + Eigen::Vector running_mean; + Eigen::Vector running_var; + + Eigen::Vector multiplier; + + T epsilon = (T)0; +}; + +/** Static batch normalization layer. */ +template +class BatchNorm2DT +{ +public: + static constexpr auto in_size = num_filters_t * num_features_t; + static constexpr auto out_size = num_filters_t * num_features_t; + static constexpr auto num_filters = num_filters_t; + static constexpr auto num_features = num_features_t; + static constexpr bool is_affine = affine; + + BatchNorm2DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "batchnorm2d"; } + + /** Returns false since batch-norm is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const Eigen::Vector& ins) noexcept + { + auto inMat = Eigen::Map, RTNeuralEigenAlignment>(ins.data()); + auto outMat = Eigen::Map, RTNeuralEigenAlignment>(outs.data()); + + for(int i = 0; i < num_features; i++) + { + outMat.col(i) = (inMat.col(i) - running_mean).cwiseProduct(multiplier) + beta; + } + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const Eigen::Vector& ins) noexcept + { + auto inMat = Eigen::Map, RTNeuralEigenAlignment>(ins.data()); + auto outMat = Eigen::Map, RTNeuralEigenAlignment>(outs.data()); + + for(int i = 0; i < num_features; i++) + { + outMat.col(i) = (inMat.col(i) - running_mean).cwiseProduct(multiplier); + } + } + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector& gammaVals); + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector&) { } + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector& betaVals); + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector&) { } + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + + Eigen::Map, RTNeuralEigenAlignment> outs; + +private: + void updateMultiplier(); + + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + Eigen::Vector gamma; + Eigen::Vector beta; + + Eigen::Vector running_mean; + Eigen::Vector running_var; + + Eigen::Vector multiplier; + + T epsilon = (T)0; +}; +} + +#endif // BATCHNORM2DEIGEN_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_eigen.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_eigen.tpp new file mode 100644 index 0000000..2e41a4a --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_eigen.tpp @@ -0,0 +1,111 @@ +#include "batchnorm2d_eigen.h" + +namespace RTNEURAL_NAMESPACE +{ +template +BatchNorm2DLayer::BatchNorm2DLayer(int in_num_filters, int in_num_features) + : Layer(in_num_filters * in_num_features, in_num_filters * in_num_features) + , num_filters(in_num_filters) + , num_features(in_num_features) +{ + gamma = Eigen::Vector::Ones(num_filters); + beta = Eigen::Vector::Zero(num_filters); + running_mean = Eigen::Vector::Zero(num_filters); + running_var = Eigen::Vector::Ones(num_filters); + multiplier = Eigen::Vector::Ones(num_filters); +} + +template +void BatchNorm2DLayer::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), gamma.begin()); + updateMultiplier(); +} + +template +void BatchNorm2DLayer::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), beta.begin()); +} + +template +void BatchNorm2DLayer::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), running_mean.begin()); +} + +template +void BatchNorm2DLayer::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), running_var.begin()); + updateMultiplier(); +} + +template +void BatchNorm2DLayer::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm2DLayer::updateMultiplier() +{ + for(int i = 0; i < num_filters; ++i) + multiplier[i] = gamma[i] / std::sqrt(running_var[i] + epsilon); +} + +//============================================================ +template +BatchNorm2DT::BatchNorm2DT() + : outs(outs_internal) +{ + gamma = Eigen::Vector::Ones(num_filters_t); + beta = Eigen::Vector::Zero(num_filters_t); + running_mean = Eigen::Vector::Zero(num_filters_t); + running_var = Eigen::Vector::Ones(num_filters_t); + multiplier = Eigen::Vector::Ones(num_filters_t); +} + +template +template +typename std::enable_if::type BatchNorm2DT::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), std::begin(gamma)); + updateMultiplier(); +} + +template +template +typename std::enable_if::type BatchNorm2DT::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), std::begin(beta)); +} + +template +void BatchNorm2DT::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), std::begin(running_mean)); +} + +template +void BatchNorm2DT::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), std::begin(running_var)); + updateMultiplier(); +} + +template +void BatchNorm2DT::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm2DT::updateMultiplier() +{ + for(int i = 0; i < num_filters_t; ++i) + multiplier[i] = gamma[i] / std::sqrt(running_var[i] + epsilon); +} +} diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_xsimd.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_xsimd.h new file mode 100644 index 0000000..5613e71 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_xsimd.h @@ -0,0 +1,170 @@ +#ifndef RTNEURAL_BATCHNORM2D_XSIMD_H +#define RTNEURAL_BATCHNORM2D_XSIMD_H + +#include "../Layer.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ +/** Dynamic batch normalization layer. */ +template +class BatchNorm2DLayer final : public Layer +{ +public: + BatchNorm2DLayer(int num_filters, int num_features); + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "batchnorm2d"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + for(int i = 0; i < num_features; i++) + { + const auto* inCol = input + i * num_filters; + auto* outCol = out + i * num_filters; + xsimd::transform(inCol, inCol + num_filters, running_mean.begin(), outCol, + [](auto const& a, auto const& b) + { return a - b; }); + xsimd::transform(outCol, outCol + num_filters, multiplier.begin(), outCol, + [](auto const& a, auto const& b) + { return a * b; }); + xsimd::transform(outCol, outCol + num_filters, beta.begin(), outCol, + [](auto const& a, auto const& b) + { return a + b; }); + } + } + + /** Sets the layer "gamma" values. */ + RTNEURAL_REALTIME void setGamma(const std::vector& gammaVals); + + /** Sets the layer "beta" values. */ + RTNEURAL_REALTIME void setBeta(const std::vector& betaVals); + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + +private: + void updateMultiplier(); + + const int num_filters; + const int num_features; + + using vec_type = std::vector>; + + vec_type gamma; + vec_type beta; + + vec_type running_mean; + vec_type running_var; + + vec_type multiplier; + + T epsilon = (T)0; +}; + +/** Static batch normalization layer. */ +template +class BatchNorm2DT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_num_filters = ceil_div(num_filters_t, v_size); + static constexpr auto v_io_size = v_num_filters * num_features_t; + +public: + static constexpr auto in_size = num_filters_t * num_features_t; + static constexpr auto out_size = num_filters_t * num_features_t; + static constexpr auto num_filters = num_filters_t; + static constexpr auto num_features = num_features_t; + static constexpr bool is_affine = affine; + + BatchNorm2DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "batchnorm2d"; } + + /** Returns false since batch-norm is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type (&ins)[v_io_size]) noexcept + { + for(int i = 0; i < num_features; i++) + { + for(int j = 0; j < v_num_filters; ++j) + { + outs[i * v_num_filters + j] = (ins[i * v_num_filters + j] - running_mean[j]) * multiplier[j] + beta[j]; + } + } + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type (&ins)[v_io_size]) noexcept + { + for(int i = 0; i < num_features; i++) + { + for(int j = 0; j < v_num_filters; ++j) + { + outs[i * v_num_filters + j] = (ins[i * v_num_filters + j] - running_mean[j]) * multiplier[j]; + } + } + } + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector& gammaVals); + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector&) { } + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector& betaVals); + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector&) { } + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + + v_type outs[v_io_size]; + +private: + void updateMultiplier(); + + v_type gamma[v_num_filters]; + v_type beta[v_num_filters]; + + v_type running_mean[v_num_filters]; + v_type running_var[v_num_filters]; + + v_type multiplier[v_num_filters]; + + T epsilon = (T)0; +}; +} + +#endif // RTNEURAL_BATCHNORM2D_XSIMD_H diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_xsimd.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_xsimd.tpp new file mode 100644 index 0000000..ca17b5f --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm2d_xsimd.tpp @@ -0,0 +1,112 @@ +#include "batchnorm2d_xsimd.h" + +namespace RTNEURAL_NAMESPACE +{ +template +BatchNorm2DLayer::BatchNorm2DLayer(int in_num_filters, int in_num_features) + : Layer(in_num_filters * in_num_features, in_num_filters * in_num_features) + , num_filters(in_num_filters) + , num_features(in_num_features) + , gamma(num_filters, (T)1) + , beta(num_filters, (T)0) + , running_mean(num_filters, (T)0) + , running_var(num_filters, (T)1) + , multiplier(num_filters, (T)1) +{ +} + +template +void BatchNorm2DLayer::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), gamma.begin()); + updateMultiplier(); +} + +template +void BatchNorm2DLayer::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), beta.begin()); +} + +template +void BatchNorm2DLayer::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), running_mean.begin()); +} + +template +void BatchNorm2DLayer::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), running_var.begin()); + updateMultiplier(); +} + +template +void BatchNorm2DLayer::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm2DLayer::updateMultiplier() +{ + for(int i = 0; i < num_filters; ++i) + multiplier[i] = gamma[i] / std::sqrt(running_var[i] + epsilon); +} + +//============================================================ +template +BatchNorm2DT::BatchNorm2DT() +{ + std::fill(std::begin(outs), std::end(outs), (T)0); + + std::fill(std::begin(gamma), std::end(gamma), (T)1); + std::fill(std::begin(beta), std::end(beta), (T)0); + std::fill(std::begin(running_mean), std::end(running_mean), (T)0); + std::fill(std::begin(running_var), std::end(running_var), (T)1); + std::fill(std::begin(multiplier), std::end(multiplier), (T)1); +} + +template +template +typename std::enable_if::type BatchNorm2DT::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), reinterpret_cast(std::begin(gamma))); + updateMultiplier(); +} + +template +template +typename std::enable_if::type BatchNorm2DT::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), reinterpret_cast(std::begin(beta))); +} + +template +void BatchNorm2DT::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), reinterpret_cast(std::begin(running_mean))); +} + +template +void BatchNorm2DT::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), reinterpret_cast(std::begin(running_var))); + updateMultiplier(); +} + +template +void BatchNorm2DT::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm2DT::updateMultiplier() +{ + for(int i = 0; i < v_num_filters; ++i) + multiplier[i] = gamma[i] / xsimd::sqrt(running_var[i] + epsilon); +} +} diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_eigen.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_eigen.h new file mode 100644 index 0000000..a8d42ec --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_eigen.h @@ -0,0 +1,141 @@ +#ifndef BATCHNORMEIGEN_H_INCLUDED +#define BATCHNORMEIGEN_H_INCLUDED + +#include "../Layer.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ +/** Dynamic batch normalization layer. */ +template +class BatchNorm1DLayer final : public Layer +{ +public: + explicit BatchNorm1DLayer(int size); + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "batchnorm"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + auto inVec = Eigen::Map, RTNeuralEigenAlignment>( + input, Layer::in_size, 1); + + auto outVec = Eigen::Map, RTNeuralEigenAlignment>( + out, Layer::in_size, 1); + + outVec = multiplier.cwiseProduct(inVec - running_mean) + beta; + } + + /** Sets the layer "gamma" values. */ + RTNEURAL_REALTIME void setGamma(const std::vector& gammaVals); + + /** Sets the layer "beta" values. */ + RTNEURAL_REALTIME void setBeta(const std::vector& betaVals); + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + +private: + void updateMultiplier(); + + Eigen::Vector gamma; + Eigen::Vector beta; + + Eigen::Vector running_mean; + Eigen::Vector running_var; + + Eigen::Vector multiplier; + + T epsilon = (T)0; +}; + +/** Static batch normalization layer. */ +template +class BatchNorm1DT +{ +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + static constexpr bool is_affine = affine; + + BatchNorm1DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "batchnorm"; } + + /** Returns false since batch-norm is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const Eigen::Matrix& ins) noexcept + { + outs = multiplier.cwiseProduct(ins - running_mean) + beta; + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const Eigen::Matrix& ins) noexcept + { + outs = multiplier.cwiseProduct(ins - running_mean); + } + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector& gammaVals); + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector&) { } + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector& betaVals); + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector&) { } + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + + Eigen::Map, RTNeuralEigenAlignment> outs; + +private: + void updateMultiplier(); + + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + Eigen::Vector gamma; + Eigen::Vector beta; + + Eigen::Vector running_mean; + Eigen::Vector running_var; + + Eigen::Vector multiplier; + + T epsilon = (T)0; +}; +} + +#endif // BATCHNORMEIGEN_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_eigen.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_eigen.tpp new file mode 100644 index 0000000..b2657f6 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_eigen.tpp @@ -0,0 +1,109 @@ +#include "batchnorm_eigen.h" + +namespace RTNEURAL_NAMESPACE +{ +template +BatchNorm1DLayer::BatchNorm1DLayer(int size) + : Layer(size, size) +{ + gamma = Eigen::Vector::Ones(size); + beta = Eigen::Vector::Zero(size); + running_mean = Eigen::Vector::Zero(size); + running_var = Eigen::Vector::Ones(size); + multiplier = Eigen::Vector::Ones(size); +} + +template +void BatchNorm1DLayer::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), gamma.begin()); + updateMultiplier(); +} + +template +void BatchNorm1DLayer::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), beta.begin()); +} + +template +void BatchNorm1DLayer::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), running_mean.begin()); +} + +template +void BatchNorm1DLayer::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), running_var.begin()); + updateMultiplier(); +} + +template +void BatchNorm1DLayer::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm1DLayer::updateMultiplier() +{ + for(int i = 0; i < Layer::out_size; ++i) + multiplier[i] = gamma[i] / std::sqrt(running_var[i] + epsilon); +} + +//============================================================ +template +BatchNorm1DT::BatchNorm1DT() + : outs(outs_internal) +{ + gamma = Eigen::Vector::Ones(size); + beta = Eigen::Vector::Zero(size); + running_mean = Eigen::Vector::Zero(size); + running_var = Eigen::Vector::Ones(size); + multiplier = Eigen::Vector::Ones(size); +} + +template +template +typename std::enable_if::type BatchNorm1DT::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), std::begin(gamma)); + updateMultiplier(); +} + +template +template +typename std::enable_if::type BatchNorm1DT::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), std::begin(beta)); +} + +template +void BatchNorm1DT::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), std::begin(running_mean)); +} + +template +void BatchNorm1DT::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), std::begin(running_var)); + updateMultiplier(); +} + +template +void BatchNorm1DT::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm1DT::updateMultiplier() +{ + for(int i = 0; i < out_size; ++i) + multiplier[i] = gamma[i] / std::sqrt(running_var[i] + epsilon); +} +} diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_xsimd.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_xsimd.h new file mode 100644 index 0000000..cd20737 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_xsimd.h @@ -0,0 +1,149 @@ +#ifndef BATCHNORMXSIMD_H_INCLUDED +#define BATCHNORMXSIMD_H_INCLUDED + +#include "../Layer.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ +/** Dynamic batch normalization layer. */ +template +class BatchNorm1DLayer final : public Layer +{ +public: + explicit BatchNorm1DLayer(int size); + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "batchnorm"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + xsimd::transform(input, input + Layer::in_size, running_mean.begin(), out, + [](auto const& a, auto const& b) + { return a - b; }); + xsimd::transform(out, out + Layer::in_size, multiplier.begin(), out, + [](auto const& a, auto const& b) + { return a * b; }); + xsimd::transform(out, out + Layer::in_size, beta.begin(), out, + [](auto const& a, auto const& b) + { return a + b; }); + } + + /** Sets the layer "gamma" values. */ + RTNEURAL_REALTIME void setGamma(const std::vector& gammaVals); + + /** Sets the layer "beta" values. */ + RTNEURAL_REALTIME void setBeta(const std::vector& betaVals); + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + +private: + void updateMultiplier(); + + using vec_type = std::vector>; + + vec_type gamma; + vec_type beta; + + vec_type running_mean; + vec_type running_var; + + vec_type multiplier; + + T epsilon = (T)0; +}; + +/** Static batch normalization layer. */ +template +class BatchNorm1DT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_out_size = ceil_div(size, v_size); + +public: + static constexpr auto in_size = size; + static constexpr auto out_size = size; + static constexpr bool is_affine = affine; + + BatchNorm1DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "batchnorm"; } + + /** Returns false since batch-norm is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type (&ins)[v_out_size]) noexcept + { + for(int k = 0; k < v_out_size; ++k) + outs[k] = multiplier[k] * (ins[k] - running_mean[k]) + beta[k]; + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type (&ins)[v_out_size]) noexcept + { + for(int k = 0; k < v_out_size; ++k) + outs[k] = multiplier[k] * (ins[k] - running_mean[k]); + } + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector& gammaVals); + + /** Sets the layer "gamma" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setGamma(const std::vector&) { } + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector& betaVals); + + /** Sets the layer "beta" values. */ + template + RTNEURAL_REALTIME typename std::enable_if::type setBeta(const std::vector&) { } + + /** Sets the layer's trained running mean. */ + RTNEURAL_REALTIME void setRunningMean(const std::vector& runningMean); + + /** Set's the layer's trained running variance. */ + RTNEURAL_REALTIME void setRunningVariance(const std::vector& runningVar); + + /** Set's the layer "epsilon" value. */ + RTNEURAL_REALTIME void setEpsilon(T epsilon); + + v_type outs[v_out_size]; + +private: + void updateMultiplier(); + + v_type gamma[v_out_size]; + v_type beta[v_out_size]; + + v_type running_mean[v_out_size]; + v_type running_var[v_out_size]; + + v_type multiplier[v_out_size]; + + T epsilon = (T)0; +}; +} + +#endif // BATCHNORMXSIMD_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_xsimd.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_xsimd.tpp new file mode 100644 index 0000000..3946af3 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/batchnorm/batchnorm_xsimd.tpp @@ -0,0 +1,110 @@ +#include "batchnorm_xsimd.h" + +namespace RTNEURAL_NAMESPACE +{ +template +BatchNorm1DLayer::BatchNorm1DLayer(int size) + : Layer(size, size) + , gamma(size, (T)1) + , beta(size, (T)0) + , running_mean(size, (T)0) + , running_var(size, (T)1) + , multiplier(size, (T)1) +{ +} + +template +void BatchNorm1DLayer::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), gamma.begin()); + updateMultiplier(); +} + +template +void BatchNorm1DLayer::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), beta.begin()); +} + +template +void BatchNorm1DLayer::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), running_mean.begin()); +} + +template +void BatchNorm1DLayer::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), running_var.begin()); + updateMultiplier(); +} + +template +void BatchNorm1DLayer::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm1DLayer::updateMultiplier() +{ + for(int i = 0; i < Layer::out_size; ++i) + multiplier[i] = gamma[i] / std::sqrt(running_var[i] + epsilon); +} + +//============================================================ +template +BatchNorm1DT::BatchNorm1DT() +{ + std::fill(std::begin(outs), std::end(outs), (T)0); + + std::fill(std::begin(gamma), std::end(gamma), (T)1); + std::fill(std::begin(beta), std::end(beta), (T)0); + std::fill(std::begin(running_mean), std::end(running_mean), (T)0); + std::fill(std::begin(running_var), std::end(running_var), (T)1); + std::fill(std::begin(multiplier), std::end(multiplier), (T)1); +} + +template +template +typename std::enable_if::type BatchNorm1DT::setGamma(const std::vector& gammaVals) +{ + std::copy(gammaVals.begin(), gammaVals.end(), reinterpret_cast(std::begin(gamma))); + updateMultiplier(); +} + +template +template +typename std::enable_if::type BatchNorm1DT::setBeta(const std::vector& betaVals) +{ + std::copy(betaVals.begin(), betaVals.end(), reinterpret_cast(std::begin(beta))); +} + +template +void BatchNorm1DT::setRunningMean(const std::vector& runningMean) +{ + std::copy(runningMean.begin(), runningMean.end(), reinterpret_cast(std::begin(running_mean))); +} + +template +void BatchNorm1DT::setRunningVariance(const std::vector& runningVar) +{ + std::copy(runningVar.begin(), runningVar.end(), reinterpret_cast(std::begin(running_var))); + updateMultiplier(); +} + +template +void BatchNorm1DT::setEpsilon(T newEpsilon) +{ + epsilon = newEpsilon; + updateMultiplier(); +} + +template +void BatchNorm1DT::updateMultiplier() +{ + for(int i = 0; i < v_out_size; ++i) + multiplier[i] = gamma[i] / xsimd::sqrt(running_var[i] + epsilon); +} +} diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/common.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/common.h new file mode 100644 index 0000000..29750e9 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/common.h @@ -0,0 +1,259 @@ +#pragma once + +namespace RTNEURAL_NAMESPACE +{ + +/** + * For templated recurrent layers (e.g. LSTMLayerT, GRULayerT), + * this class can be used as a template argument to allow the + * recurrent layer to perform real-time sample-rate correction. + * + * For example, if you have a GRU network that was trained at 48 kHz + * and want to process data at 96 kHz, you could enable sample-rate + * correction for that layer, and prepare it to use a 2-sample delay, + * instead of the standard 1-sample delay (since the target sample rate + * is 2x the training sample rate). Note that sample-rate correction + * does not support delay lengths less than 1-sample, so the target sample + * rate must always be greater than or equal to the training sample rate. + */ +enum class SampleRateCorrectionMode +{ + None, // no sample rate correction + NoInterp, // sample rate correction with no interpolation (only appropriate for integer delay lengths) + LinInterp, // sample rate correction with linear interpolation (can be used with non-integer delay lengths) +}; + +/** Divides two numbers and rounds up if there is a remainder. */ +template +constexpr T ceil_div(T num, T den) +{ + return (num + den - 1) / den; +} +} // namespace RTNEURAL_NAMESPACE + +#if RTNEURAL_USE_EIGEN +#include + +namespace RTNEURAL_NAMESPACE +{ +#if RTNEURAL_DEFAULT_ALIGNMENT == 32 +constexpr auto RTNeuralEigenAlignment = Eigen::Aligned32; +#elif RTNEURAL_DEFAULT_ALIGNMENT == 16 +constexpr auto RTNeuralEigenAlignment = Eigen::Aligned16; +#elif RTNEURAL_DEFAULT_ALIGNMENT == 8 +constexpr auto RTNeuralEigenAlignment = Eigen::Aligned8; +#else +#error "Unsupported alignment" +#endif +} // namespace RTNEURAL_NAMESPACE + +#elif RTNEURAL_USE_XSIMD +#include + +#include "xsimd-legacy/algorithms/algorithms.hpp" + +namespace RTNEURAL_NAMESPACE +{ + +template +static inline xsimd::simd_type set_value(xsimd::simd_type x, int idx, T value) noexcept +{ + union UnionType + { + xsimd::simd_type v; + T s[xsimd::simd_type::size]; + }; + UnionType u { x }; + + u.s[idx] = value; + return u.v; +} + +template +static inline T get_value(xsimd::simd_type x, int idx) noexcept +{ + union UnionType + { + xsimd::simd_type v; + T s[xsimd::simd_type::size]; + }; + UnionType u { x }; + + return u.s[idx]; +} + +template +static inline T vMult(const T* arg1, const T* arg2, T* prod, + int dim) noexcept +{ + xsimd::transform(arg1, &arg1[dim], arg2, prod, + [](auto const& a, auto const& b) + { return a * b; }); + + return xsimd::reduce(prod, &prod[dim], (T)0); +} + +template +static inline void vAdd(const T* in1, const T* in2, T* out, + int dim) noexcept +{ + xsimd::transform(in1, &in1[dim], in2, out, + [](auto const& a, auto const& b) + { return a + b; }); +} + +template +static inline void vSub(const T* in1, const T* in2, T* out, + int dim) noexcept +{ + xsimd::transform(in1, &in1[dim], in2, out, + [](auto const& a, auto const& b) + { return a - b; }); +} + +template +static inline void vProd(const T* in1, const T* in2, T* out, + int dim) noexcept +{ + xsimd::transform(in1, &in1[dim], in2, out, + [](auto const& a, auto const& b) + { return a * b; }); +} + +template +static inline void vCopy(const T* in, T* out, int dim) noexcept +{ + using b_type = xsimd::simd_type; + constexpr auto inc = (int)b_type::size; + + // size for which the vectorization is possible + auto vec_size = dim - dim % inc; + for(int i = 0; i < vec_size; i += inc) + { + b_type vec = xsimd::load_aligned(&in[i]); + xsimd::store_aligned(&out[i], vec); + } + + // Remaining part that cannot be vectorize + for(auto i = vec_size; i < dim; ++i) + out[i] = in[i]; +} + +template +static inline void sigmoid(const T* in, T* out, int dim) noexcept +{ + using b_type = xsimd::simd_type; + constexpr auto inc = (int)b_type::size; + + // size for which the vectorization is possible + auto vec_size = dim - dim % inc; + for(int i = 0; i < vec_size; i += inc) + { + b_type x_vec = xsimd::load_aligned(&in[i]); + b_type y_vec = MathsProvider::sigmoid(x_vec); + xsimd::store_aligned(&out[i], y_vec); + } + + // Remaining part that cannot be vectorize + for(auto i = vec_size; i < dim; ++i) + out[i] = MathsProvider::sigmoid(in[i]); +} + +template +static inline void softmax(const T* in, T* out, int dim) noexcept +{ + using b_type = xsimd::simd_type; + constexpr auto inc = (int)b_type::size; + + b_type exp_sum_vec {}; + + // size for which the vectorization is possible + auto vec_size = dim - dim % inc; + for(int i = 0; i < vec_size; i += inc) + { + b_type x_vec = xsimd::load_aligned(&in[i]); + b_type y_vec = MathsProvider::exp(x_vec); + exp_sum_vec += y_vec; + xsimd::store_aligned(&out[i], y_vec); + } + + T exp_sum = xsimd::reduce_add(exp_sum_vec); + + // Remaining part that cannot be vectorize + for(auto i = vec_size; i < dim; ++i) + { + out[i] = MathsProvider::exp(in[i]); + exp_sum += out[i]; + } + + const auto exp_sum_recip = (T)1 / exp_sum; + for(int i = 0; i < vec_size; i += inc) + { + b_type x_vec = xsimd::load_aligned(&out[i]); + b_type y_vec = x_vec * exp_sum_recip; + xsimd::store_aligned(&out[i], y_vec); + } + + // Remaining part that cannot be vectorize + for(auto i = vec_size; i < dim; ++i) + { + out[i] *= exp_sum_recip; + } +} + +template +static inline void tanh(const T* in, T* out, int dim) noexcept +{ + using b_type = xsimd::simd_type; + constexpr auto inc = (int)b_type::size; + + // size for which the vectorization is possible + auto vec_size = dim - dim % inc; + for(int i = 0; i < vec_size; i += inc) + { + b_type x_vec = xsimd::load_aligned(&in[i]); + b_type y_vec = MathsProvider::tanh(x_vec); + xsimd::store_aligned(&out[i], y_vec); + } + + // Remaining part that cannot be vectorize + for(auto i = vec_size; i < dim; ++i) + out[i] = MathsProvider::tanh(in[i]); +} + +template +static inline void elu(const T* in, T* out, int dim, T alpha) noexcept +{ + using b_type = xsimd::simd_type; + constexpr auto inc = (int)b_type::size; + + // size for which the vectorization is possible + auto vec_size = dim - dim % inc; + for(int i = 0; i < vec_size; i += inc) + { + b_type x_vec = xsimd::load_aligned(&in[i]); + b_type y_vec = xsimd::select(x_vec > (T)0, x_vec, alpha * (MathsProvider::exp(x_vec) - (T)1)); + xsimd::store_aligned(&out[i], y_vec); + } + + // Remaining part that cannot be vectorized + for(auto i = vec_size; i < dim; ++i) + out[i] = in[i] > (T)0 ? in[i] : (alpha * (MathsProvider::exp(in[i]) - (T)1)); +} +} // namespace RTNEURAL_NAMESPACE + +#else // STL backend +#include +#include +#include + +namespace RTNEURAL_NAMESPACE +{ +template +static inline T vMult(const T* arg1, const T* arg2, int dim) noexcept +{ + return std::inner_product(arg1, arg1 + dim, arg2, (T)0); +} +} // namespace RTNEURAL_NAMESPACE + +#endif diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/config.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/config.h new file mode 100644 index 0000000..ac85f37 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/config.h @@ -0,0 +1,37 @@ +#pragma once + +#ifndef RTNEURAL_NAMESPACE +#define RTNEURAL_NAMESPACE RTNeural +#endif + +#ifndef RTNEURAL_DEFAULT_ALIGNMENT +#if _MSC_VER +#pragma message("RTNEURAL_DEFAULT_ALIGNMENT was not defined! Using default alignment = 16.") +#else +#warning "RTNEURAL_DEFAULT_ALIGNMENT was not defined! Using default alignment = 16." +#endif +#define RTNEURAL_DEFAULT_ALIGNMENT 16 +#endif + +/** + Facilitate testing real-time safety with RealtimeSanitizer (RADSan) + + For more information, see https://github.com/realtime-sanitizer/radsan. + The `[[clang::realtime]]` attribute is unique to a RADSan-modified + version of clang, and its appearance will result in an error for other + compilers. Here, we make its presence configurable. RealtimeSanitizer is + very early stage, and this configuration may change. + + This real-time safety checking is designed to function mostly in CI. If you + wish to test it locally on your dev machine, you'll need to either: + + i) use the RADSan clang Docker image (recommended), or + ii) get the RADSan clang compiler, + + for which instructions may be found in the RADSan repository above. +*/ +#ifdef RTNEURAL_RADSAN_ENABLED +#define RTNEURAL_REALTIME [[clang::realtime]] +#else +#define RTNEURAL_REALTIME +#endif diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d.h new file mode 100644 index 0000000..e161c31 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d.h @@ -0,0 +1,324 @@ +#ifndef CONV1D_H_INCLUDED +#define CONV1D_H_INCLUDED + +#if RTNEURAL_USE_EIGEN +#include "conv1d_eigen.h" +#include "conv1d_eigen.tpp" +#elif RTNEURAL_USE_XSIMD +#include "conv1d_xsimd.h" +#include "conv1d_xsimd.tpp" +#else +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ + +/** + * Dynamic implementation of a 1-dimensional convolution layer + * with no activation. + * + * This implementation was designed to be used for "temporal + * convolution", so the layer has a "state" made up of past inputs + * to the layer. To ensure that the state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + */ +template +class Conv1D final : public Layer +{ +public: + /** + * Constructs a convolution layer for the given dimensions. + * + * @param in_size: the input size for the layer + * @param out_size: the output size for the layer + * @param kernel_size: the size of the convolution kernel + * @param dilation: the dilation rate to use for dilated convolution + */ + Conv1D(int in_size, int out_size, int kernel_size, int dilation, int groups = 1); + Conv1D(std::initializer_list sizes); + Conv1D(const Conv1D& other); + Conv1D& operator=(const Conv1D& other); + virtual ~Conv1D(); + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset() override; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "conv1d"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override + { + // insert input into a circular buffer + std::copy(input, input + Layer::in_size, state[state_ptr]); + + // set state pointers to particular columns of the buffer + setStatePointers(); + + if(groups == 1) + { + // copy selected columns to a helper variable + for(int k = 0; k < kernel_size; ++k) + { + const auto& col = state[state_ptrs[k]]; + std::copy(col, col + Layer::in_size, state_cols[k]); + } + + // perform multi-channel convolution + for(int i = 0; i < Layer::out_size; ++i) + { + h[i] = bias[i]; + for(int k = 0; k < kernel_size; ++k) + h[i] = std::inner_product( + weights[i][k], + weights[i][k] + filters_per_group, + state_cols[k], + h[i]); + } + } + else + { + // perform multi-channel convolution + for(int i = 0; i < Layer::out_size; ++i) + { + h[i] = bias[i]; + const auto ii = ((i / channels_per_group) * filters_per_group); + for(int k = 0; k < kernel_size; ++k) + { + // copy selected columns to a helper variable + const auto& column = state[state_ptrs[k]]; + + const auto column_begin = column + ii; + const auto column_end = column_begin + filters_per_group; + std::copy(column_begin, column_end, state_cols[k]); + + h[i] = std::inner_product( + weights[i][k], + weights[i][k] + filters_per_group, + state_cols[k], + h[i]); + } + } + } + + state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[out_size][in_size][kernel_size * dilation] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& weights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const std::vector& biasVals); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size; } + + /** Returns the convolution dilation rate. */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate; } + + /** Returns the number of "groups" in the convolution. */ + int getGroups() const noexcept { return groups; } + +private: + const int dilation_rate; + const int kernel_size; + const int state_size; + const int groups; + const int filters_per_group; + const int channels_per_group; + + T*** weights; + T* bias; + + T** state; + T** state_cols; + + int* state_ptrs; + int state_ptr = 0; + + /** Sets pointers to state array columns. */ + inline void setStatePointers() + { + for(int k = 0; k < kernel_size; ++k) + state_ptrs[k] = (state_ptr + state_size - k * dilation_rate) % state_size; + } +}; + +//==================================================== +/** + * Static implementation of a 1-dimensional convolution layer + * with no activation. + * + * This implementation was designed to be used for "temporal + * convolution", so the layer has a "state" made up of past inputs + * to the layer. To ensure that the state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * @param in_sizet: the input size for the layer + * @param out_sizet: the output size for the layer + * @param kernel_size: the size of the convolution kernel + * @param dilation_rate: the dilation rate to use for dilated convolution + * @param dynamic_state: use dynamically allocated layer state + * @param groups: controls connections between inputs and outputs + */ +template +class Conv1DT +{ + static_assert((in_sizet % groups == 0) && (out_sizet % groups == 0), "in_size and out_size must be divisible by groups!"); + + static constexpr auto state_size = (kernel_size - 1) * dilation_rate + 1; + +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + static constexpr auto filters_per_group = in_size / groups; + static constexpr auto channels_per_group = out_size / groups; + + Conv1DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "conv1d"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset(); + + template = true> + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T (&ins)[in_size]) noexcept + { + // insert input into a circular buffer + std::copy(std::begin(ins), std::end(ins), state[state_ptr].begin()); + + // set state pointers to particular columns of the buffer + setStatePointers(); + + // copy selected columns to a helper variable + for(int k = 0; k < kernel_size; ++k) + { + const auto& col = state[state_ptrs[k]]; + std::copy(col.begin(), col.end(), state_cols[k].begin()); + } + + // perform multi-channel convolution + for(int i = 0; i < out_size; ++i) + { + outs[i] = bias[i]; + for(int k = 0; k < kernel_size; ++k) + outs[i] = std::inner_product( + weights[i][k].begin(), + weights[i][k].end(), + state_cols[k].begin(), + outs[i]); + } + + state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards + } + + template = true> + /** Performs forward propagation for this layer. */ + inline void forward(const T (&ins)[in_size]) noexcept + { + // insert input into a circular buffer + std::copy(std::begin(ins), std::end(ins), state[state_ptr].begin()); + + // set state pointers to particular columns of the buffer + setStatePointers(); + + // perform multi-channel convolution + for(int i = 0; i < out_size; ++i) + { + outs[i] = bias[i]; + + const auto ii = ((i / channels_per_group) * filters_per_group); + for(int k = 0; k < kernel_size; ++k) + { + // copy selected columns to a helper variable + const auto& column = state[state_ptrs[k]]; + const auto column_begin = column.begin() + ii; + const auto column_end = column_begin + filters_per_group; + std::copy(column_begin, column_end, state_cols[k].begin()); + + outs[i] = std::inner_product( + weights[i][k].begin(), + weights[i][k].end(), + state_cols[k].begin(), + outs[i]); + } + } + + state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[out_size][group_count][kernel_size * dilation] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& weights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const std::vector& biasVals); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size; } + + /** Returns the convolution dilation rate. */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate; } + + /** Returns the number of "groups" in the convolution. */ + int getGroups() const noexcept { return groups; } + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + +private: + template + typename std::enable_if::type resize_state() + { + state.resize(state_size, {}); + } + + template + typename std::enable_if::type resize_state() { } + + using state_type = typename std::conditional>, std::array, state_size>>::type; + using weights_type = std::array, kernel_size>; + + alignas(RTNEURAL_DEFAULT_ALIGNMENT) state_type state; + alignas(RTNEURAL_DEFAULT_ALIGNMENT) weights_type state_cols; + + int state_ptr = 0; + std::array state_ptrs; + + alignas(RTNEURAL_DEFAULT_ALIGNMENT) weights_type weights[out_size]; + alignas(RTNEURAL_DEFAULT_ALIGNMENT) std::array bias; + + /** Sets pointers to state array columns. */ + inline void setStatePointers() + { + for(int k = 0; k < kernel_size; ++k) + state_ptrs[k] = (state_ptr + state_size - k * dilation_rate) % state_size; + } +}; +} // namespace RTNEURAL_NAMESPACE +#endif +#endif // CONV1D_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d.tpp new file mode 100644 index 0000000..a454d3b --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d.tpp @@ -0,0 +1,172 @@ +#include "conv1d.h" + +namespace RTNEURAL_NAMESPACE +{ + +#if !RTNEURAL_USE_EIGEN && !RTNEURAL_USE_XSIMD + +template +Conv1D::Conv1D(int in_size, int out_size, int kernel_size, int dilation, int num_groups) + : Layer(in_size, out_size) + , dilation_rate(dilation) + , kernel_size(kernel_size) + , state_size((kernel_size - 1) * dilation + 1) + , groups(num_groups) + , filters_per_group(in_size / groups) + , channels_per_group(out_size / groups) +{ + weights = new T**[out_size]; + for(int i = 0; i < out_size; ++i) + { + weights[i] = new T*[kernel_size]; + for(int k = 0; k < kernel_size; ++k) + { + weights[i][k] = new T[filters_per_group]; + std::fill(weights[i][k], weights[i][k] + filters_per_group, (T)0); + } + } + + bias = new T[out_size]; + + state = new T*[state_size]; + for(int k = 0; k < state_size; ++k) + state[k] = new T[in_size]; + + state_cols = new T*[kernel_size]; + for(int k = 0; k < kernel_size; ++k) + state_cols[k] = new T[filters_per_group]; + + state_ptrs = new int[kernel_size]; +} + +template +Conv1D::Conv1D(std::initializer_list sizes) + : Conv1D(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3)) +{ +} + +template +Conv1D::Conv1D(const Conv1D& other) + : Conv1D(other.in_size, other.out_size, other.kernel_size, other.dilation_rate) +{ +} + +template +Conv1D& Conv1D::operator=(const Conv1D& other) +{ + if(&other != this) + *this = Conv1D(other); + + return *this; +} + +template +Conv1D::~Conv1D() +{ + for(int i = 0; i < Layer::out_size; ++i) + { + for(int k = 0; k < kernel_size; ++k) + delete[] weights[i][k]; + + delete[] weights[i]; + } + + delete[] weights; + delete[] bias; + + for(int k = 0; k < state_size; ++k) + delete[] state[k]; + delete[] state; + + for(int k = 0; k < kernel_size; ++k) + delete[] state_cols[k]; + delete[] state_cols; + + delete[] state_ptrs; +} + +template +void Conv1D::reset() +{ + for(int k = 0; k < state_size; ++k) + std::fill(state[k], state[k] + Layer::in_size, (T)0); + + for(int k = 0; k < kernel_size; ++k) + std::fill(state_cols[k], state_cols[k] + filters_per_group, (T)0); + + for(int k = 0; k < kernel_size; ++k) + state_ptrs[k] = 0; + + state_ptr = 0; +} + +template +void Conv1D::setWeights(const std::vector>>& ws) +{ + for(int i = 0; i < Layer::out_size; ++i) + for(int k = 0; k < filters_per_group; ++k) + for(int j = 0; j < kernel_size; ++j) + weights[i][j][k] = ws[i][k][j]; +} + +template +void Conv1D::setBias(const std::vector& biasVals) +{ + for(int i = 0; i < Layer::out_size; ++i) + bias[i] = biasVals[i]; +} + +//==================================================== +template +Conv1DT::Conv1DT() +{ + for(int i = 0; i < out_size; ++i) + for(int j = 0; j < kernel_size; ++j) + for(int k = 0; k < filters_per_group; ++k) + weights[i][j][k] = (T)0.0; + + for(int i = 0; i < out_size; ++i) + bias[i] = (T)0.0; + + for(int i = 0; i < out_size; ++i) + outs[i] = (T)0.0; + + resize_state(); + reset(); +} + +template +void Conv1DT::reset() +{ + for(int i = 0; i < state_size; ++i) + for(int k = 0; k < in_size; ++k) + state[i][k] = (T)0.0; + + for(int i = 0; i < kernel_size; ++i) + for(int k = 0; k < filters_per_group; ++k) + state_cols[i][k] = (T)0.0; + + state_ptr = 0; + for(int i = 0; i < kernel_size; ++i) + state_ptrs[i] = 0; +} + +template +void Conv1DT::setWeights(const std::vector>>& ws) +{ + for(int i = 0; i < out_size; ++i) + for(int k = 0; k < filters_per_group; ++k) + for(int j = 0; j < kernel_size; ++j) + weights[i][j][k] = ws[i][k][j]; +} + +template +void Conv1DT::setBias(const std::vector& biasVals) +{ + for(int i = 0; i < out_size; ++i) + bias[i] = biasVals[i]; +} + +#endif + +} // namespace RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_eigen.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_eigen.h new file mode 100644 index 0000000..582d819 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_eigen.h @@ -0,0 +1,274 @@ +#ifndef CONV1DEIGEN_H_INCLUDED +#define CONV1DEIGEN_H_INCLUDED + +#include "../Layer.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ + +/** + * Dynamic implementation of a 1-dimensional convolution layer + * with no activation. + * + * This implementation was designed to be used for "temporal + * convolution", so the layer has a "state" made up of past inputs + * to the layer. To ensure that the state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + */ +template +class Conv1D : public Layer +{ +public: + /** + * Constructs a convolution layer for the given dimensions. + * + * @param in_size: the input size for the layer + * @param out_size: the output size for the layer + * @param kernel_size: the size of the convolution kernel + * @param dilation: the dilation rate to use for dilated convolution + */ + Conv1D(int in_size, int out_size, int kernel_size, int dilation, int groups = 1); + Conv1D(std::initializer_list sizes); + Conv1D(const Conv1D& other); + Conv1D& operator=(const Conv1D& other); + virtual ~Conv1D(); + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset() override; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "conv1d"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override + { + // insert input into a circular buffer + state.col(state_ptr) = Eigen::Map, + RTNeuralEigenAlignment>(input, Layer::in_size); + + // set state pointers to the particular columns of the buffer + setStatePointers(); + + if(groups == 1) + { + // copy selected columns to a helper variable + for(int k = 0; k < kernel_size; ++k) + state_cols.col(k) = state.col(state_ptrs(k)); + + // perform a multichannel convolution + for(int i = 0; i < Layer::out_size; ++i) + h[i] = state_cols.cwiseProduct(kernelWeights[i]).sum() + bias(i); + } + else + { + // perform a multichannel convolution + for(int i = 0; i < Layer::out_size; ++i) + { + // copy selected columns to a helper variable + const auto ii = ((i / channels_per_group) * filters_per_group); + for(int k = 0; k < kernel_size; ++k) + state_cols.col(k) = state.col(state_ptrs(k))(Eigen::seqN(ii, filters_per_group)); + + h[i] = state_cols.cwiseProduct(kernelWeights[i]).sum() + bias(i); + } + } + + state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[out_size][in_size][kernel_size * dilation] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& weights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const std::vector& biasVals); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size; } + + /** Returns the convolution dilation rate. */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate; } + + /** Returns the number of "groups" in the convolution. */ + int getGroups() const noexcept { return groups; } + +private: + const int dilation_rate; + const int kernel_size; + const int state_size; + const int groups; + const int filters_per_group; + const int channels_per_group; + + std::vector> kernelWeights; + Eigen::Vector bias; + + Eigen::Matrix state; + Eigen::Matrix state_cols; + Eigen::VectorXi state_ptrs; + int state_ptr = 0; + + /** Sets pointers to state array columns. */ + inline void setStatePointers() + { + for(int k = 0; k < kernel_size; ++k) + state_ptrs[k] = (state_ptr + state_size - k * dilation_rate) % state_size; + } +}; + +//==================================================== +/** + * Static implementation of a 1-dimensional convolution layer + * with no activation. + * + * This implementation was designed to be used for "temporal + * convolution", so the layer has a "state" made up of past inputs + * to the layer. To ensure that the state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * @param in_sizet: the input size for the layer + * @param out_sizet: the output size for the layer + * @param kernel_size: the size of the convolution kernel + * @param dilation_rate: the dilation rate to use for dilated convolution + * @param dynamic_state: use dynamically allocated layer state + */ +template +class Conv1DT +{ + using vec_type = Eigen::Vector; + + static_assert((in_sizet % groups == 0) && (out_sizet % groups == 0), "in_sizet and out_sizet must be divisible by groups!"); + +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + static constexpr auto dilation = dilation_rate; + static constexpr auto kernel_length = kernel_size; + static constexpr auto filters_per_group = in_size / groups; + static constexpr auto channels_per_group = out_size / groups; + + static constexpr auto state_size = (kernel_size - 1) * dilation_rate + 1; + using state_type = Eigen::Matrix; + using weights_type = Eigen::Matrix; + using state_ptrs_type = Eigen::Vector; + + Conv1DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "conv1d"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset(); + + /** Performs forward propagation for this layer. */ + template = true> + RTNEURAL_REALTIME inline void forward(const Eigen::Matrix& ins) noexcept + { + // insert input into a circular buffer + state.col(state_ptr) = ins; + + // set state pointers to particular columns of the buffer + setStatePointers(); + + // copy selected columns to a helper variable + for(int k = 0; k < kernel_length; ++k) + state_cols.col(k) = state.col(state_ptrs(k)); + + // perform a multichannel convolution + for(int i = 0; i < out_size; ++i) + outs(i) = state_cols.cwiseProduct(weights[i]).sum() + bias(i); + + state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards + } + + /** Performs forward propagation for this layer (groups > 1). */ + template = true> + RTNEURAL_REALTIME inline void forward(const Eigen::Matrix& ins) noexcept + { + // insert input into a circular buffer + state.col(state_ptr) = ins; + + // set state pointers to particular columns of the buffer + setStatePointers(); + + // perform a multichannel convolution + for(int i = 0; i < out_size; ++i) + { + // copy selected columns to a helper variable + const auto ii = ((i / channels_per_group) * filters_per_group); + for(int k = 0; k < kernel_length; ++k) + state_cols.col(k) = state.col(state_ptrs(k))(Eigen::seqN(ii, filters_per_group)); + + outs(i) = state_cols.cwiseProduct(weights[i]).sum() + bias(i); + } + + state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[out_size][in_size][kernel_size * dilation] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& weights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const std::vector& biasVals); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size; } + + /** Returns the convolution dilation rate. */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate; } + + /** Returns the number of "groups" in the convolution. */ + int getGroups() const noexcept { return groups; } + + Eigen::Map outs; + +private: + void resize_state() + { + state.resize(in_sizet, state_size); + } + + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + state_type state; + weights_type state_cols; + + int state_ptr = 0; + state_ptrs_type state_ptrs; + + weights_type weights[out_size]; + vec_type bias; + + /** Sets pointers to state array columns. */ + inline void setStatePointers() + { + for(int k = 0; k < kernel_size; ++k) + state_ptrs[k] = (state_ptr + state_size - k * dilation_rate) % state_size; + } +}; + +} // RTNEURAL_NAMESPACE + +#endif // CONV1DEIGEN_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_eigen.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_eigen.tpp new file mode 100644 index 0000000..fc9073a --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_eigen.tpp @@ -0,0 +1,111 @@ +#include "conv1d_eigen.h" + +namespace RTNEURAL_NAMESPACE +{ + +template +Conv1D::Conv1D(int in_size, int out_size, int kernel_size, int dilation, int num_groups) + : Layer(in_size, out_size) + , dilation_rate(dilation) + , kernel_size(kernel_size) + , state_size((kernel_size - 1) * dilation + 1) + , groups(num_groups) + , filters_per_group(in_size / groups) + , channels_per_group(out_size / groups) +{ + kernelWeights.resize(out_size); + for(int i = 0; i < out_size; ++i) + kernelWeights[i] = Eigen::Matrix::Zero(filters_per_group, kernel_size); + + bias = Eigen::Vector::Zero(out_size); + state = Eigen::Matrix::Zero(in_size, state_size); + state_cols = Eigen::Matrix::Zero(filters_per_group, kernel_size); + state_ptrs = Eigen::Vector::Zero(kernel_size); +} + +template +Conv1D::Conv1D(std::initializer_list sizes) + : Conv1D(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3)) +{ +} + +template +Conv1D::Conv1D(const Conv1D& other) + : Conv1D(other.in_size, other.out_size, other.kernel_size, other.dilation_rate) +{ +} + +template +Conv1D& Conv1D::operator=(const Conv1D& other) +{ + return *this = Conv1D(other); +} + +template +Conv1D::~Conv1D() = default; + +template +void Conv1D::reset() +{ + state_ptr = 0; + state_ptrs.setZero(); + state_cols.setZero(); + state.setZero(); +} + +template +void Conv1D::setWeights(const std::vector>>& weights) +{ + for(int i = 0; i < Layer::out_size; ++i) + for(int k = 0; k < filters_per_group; ++k) + for(int j = 0; j < kernel_size; ++j) + kernelWeights[i](k, j) = weights[i][k][j]; +} + +template +void Conv1D::setBias(const std::vector& biasVals) +{ + for(int i = 0; i < Layer::out_size; ++i) + bias(i) = biasVals[i]; +} + +//==================================================== +template +Conv1DT::Conv1DT() + : outs(outs_internal) +{ + for(int k = 0; k < out_size; ++k) + weights[k] = weights_type::Zero(); + + bias = vec_type::Zero(); + + resize_state(); + reset(); +} + +template +void Conv1DT::reset() +{ + state.setZero(); + state_cols = weights_type::Zero(); + state_ptrs = state_ptrs_type::Zero(); + state_ptr = 0; +} + +template +void Conv1DT::setWeights(const std::vector>>& ws) +{ + for(int i = 0; i < out_size; ++i) + for(int k = 0; k < filters_per_group; ++k) + for(int j = 0; j < kernel_size; ++j) + weights[i](k, j) = ws[i][k][j]; +} + +template +void Conv1DT::setBias(const std::vector& biasVals) +{ + for(int i = 0; i < out_size; ++i) + bias(i) = biasVals[i]; +} + +} // namespace RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_xsimd.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_xsimd.h new file mode 100644 index 0000000..780f666 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_xsimd.h @@ -0,0 +1,402 @@ +#ifndef CONV1DXSIMD_H_INCLUDED +#define CONV1DXSIMD_H_INCLUDED + +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include +#include +#include + +namespace RTNEURAL_NAMESPACE +{ + +/** + * Dynamic implementation of a 1-dimensional convolution layer + * with no activation. + * + * This implementation was designed to be used for "temporal + * convolution", so the layer has a "state" made up of past inputs + * to the layer. To ensure that the state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + */ +template +class Conv1D : public Layer +{ +public: + /** + * Constructs a convolution layer for the given dimensions. + * + * @param in_size: the input size for the layer + * @param out_size: the output size for the layer + * @param kernel_size: the size of the convolution kernel + * @param dilation: the dilation rate to use for dilated convolution + */ + Conv1D(int in_size, int out_size, int kernel_size, int dilation, int groups = 1); + Conv1D(std::initializer_list sizes); + Conv1D(const Conv1D& other); + Conv1D& operator=(const Conv1D& other); + virtual ~Conv1D(); + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset() override; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "conv1d"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override + { + // insert input into a circular buffer + vCopy(input, state[state_ptr].data(), Layer::in_size); + + // set state pointers to particular columns of the buffer + setStatePointers(); + + if(groups == 1) + { + // copy selected columns to a helper variable + for(int k = 0; k < kernel_size; ++k) + { + const auto& col = state[state_ptrs[k]]; + vCopy(col.data(), state_cols[k].data(), Layer::in_size); + } + + // perform multi-channel convolution + vCopy(bias.data(), h, Layer::out_size); + for(int i = 0; i < Layer::out_size; ++i) + { + for(int k = 0; k < kernel_size; ++k) + h[i] += vMult(weights[i][k].data(), state_cols[k].data(), prod_state.data(), Layer::in_size); + } + } + else + { + // perform multi-channel convolution + vCopy(bias.data(), h, Layer::out_size); + for(int i = 0; i < Layer::out_size; ++i) + { + const auto ii = ((i / channels_per_group) * filters_per_group); + for(int k = 0; k < kernel_size; ++k) + { + // copy selected columns to a helper variable + const auto& column = state[state_ptrs[k]]; + const auto column_begin = column.begin() + ii; + const auto column_end = column_begin + filters_per_group; + std::copy(column_begin, column_end, state_cols[k].begin()); + + h[i] += vMult(weights[i][k].data(), state_cols[k].data(), prod_state.data(), filters_per_group); + } + } + } + + state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[out_size][in_size][kernel_size * dilation] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& weights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const std::vector& biasVals); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size; } + + /** Returns the convolution dilation rate. */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate; } + + /** Returns the number of "groups" in the convolution. */ + int getGroups() const noexcept { return groups; } + +private: + using vec_type = std::vector>; + using vec2_type = std::vector; + using vec3_type = std::vector; + + const int dilation_rate; + const int kernel_size; + const int state_size; + const int groups; + const int filters_per_group; + const int channels_per_group; + + vec3_type weights; + vec_type bias; + + vec2_type state; + vec2_type state_cols; + + int state_ptr = 0; + std::vector state_ptrs; + + vec_type prod_state; + + /** Sets pointers to state array columns. */ + inline void setStatePointers() + { + for(int k = 0; k < kernel_size; ++k) + state_ptrs[k] = (state_ptr + state_size - k * dilation_rate) % state_size; + } +}; + +//==================================================== +/** + * Static implementation of a 1-dimensional convolution layer + * with no activation. + * + * This implementation was designed to be used for "temporal + * convolution", so the layer has a "state" made up of past inputs + * to the layer. To ensure that the state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * @param in_sizet: the input size for the layer + * @param out_sizet: the output size for the layer + * @param kernel_size: the size of the convolution kernel + * @param dilation_rate: the dilation rate to use for dilated convolution + * @param dynamic_state: use dynamically allocated layer state + */ +template +class Conv1DT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto state_size = (kernel_size - 1) * dilation_rate + 1; + static constexpr auto v_in_size = ceil_div(in_sizet, v_size); + static constexpr auto v_out_size = ceil_div(out_sizet, v_size); + + static_assert((in_sizet % groups == 0) && (out_sizet % groups == 0), "in_size and out_size must be divisible by groups!"); + +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + static constexpr auto filters_per_group = in_size / groups; + static constexpr auto channels_per_group = out_size / groups; + static constexpr auto v_filters_per_group = ceil_div(filters_per_group, v_size); + + Conv1DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "conv1d"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset(); + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if<(G > 1), void>::type + forward(const v_type (&ins)[v_in_size]) noexcept + { + // insert input into a circular buffer + std::copy(std::begin(ins), std::end(ins), state[state_ptr].begin()); + + // set state pointers to particular columns of the buffer + setStatePointers(); + + // perform multi-channel convolution + for(int i = 0; i < v_out_size; ++i) + { + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T out_sum[v_size] {}; + for(int k = 0; k < v_size && (i * v_size + k) < out_size; ++k) + { + assert(i * v_size + k < out_size); + const auto& subWeights = weights[i * v_size + k]; + v_type accum {}; + + const auto ii = (((i * v_size + k) / channels_per_group) * filters_per_group); + for(int j = 0; j < kernel_size; ++j) + { + // copy selected columns to a helper variable + // @TODO: I'm not sure the reinterpret_casts are 100% safe here, but they seem to work in testing! + const auto& column = reinterpret_cast&>(state[state_ptrs[j]]); + const auto column_begin = column.begin() + ii; + const auto column_end = column_begin + filters_per_group; + std::copy(column_begin, column_end, reinterpret_cast&>(state_cols[j]).begin()); + + accum += std::inner_product( + subWeights[j].begin(), + subWeights[j].end(), + state_cols[j].begin(), + v_type {}); + } + out_sum[k] = xsimd::reduce_add(accum); + } + + outs[i] = xsimd::load_aligned(out_sum) + bias[i]; + } + + state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if<(DR > 1 && G == 1), void>::type + forward(const v_type (&ins)[v_in_size]) noexcept + { + // insert input into a circular buffer + std::copy(std::begin(ins), std::end(ins), state[state_ptr].begin()); + + // set state pointers to particular columns of the buffer + setStatePointers(); + + // copy selected columns to a helper variable + for(int k = 0; k < kernel_size; ++k) + { + const auto& col = state[state_ptrs[k]]; + std::copy(col.begin(), col.end(), state_cols[k].begin()); + } + + // perform multi-channel convolution + for(int i = 0; i < v_out_size; ++i) + { + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T out_sum[v_size] {}; + for(int k = 0; k < v_size && (i * v_size + k) < out_size; ++k) + { + assert(i * v_size + k < out_size); + const auto& subWeights = weights[i * v_size + k]; + v_type accum {}; + for(int j = 0; j < kernel_size; ++j) + { + accum += std::inner_product( + subWeights[j].begin(), + subWeights[j].end(), + state_cols[j].begin(), + v_type {}); + } + out_sum[k] = xsimd::reduce_add(accum); + } + + outs[i] = xsimd::load_aligned(out_sum) + bias[i]; + } + + state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if<(DR == 1 && KS > 1 && G == 1), void>::type + forward(const v_type (&ins)[v_in_size]) noexcept + { + // insert input into a circular buffer + std::copy(std::begin(ins), std::end(ins), state[state_ptr].begin()); + + // set state pointers to particular columns of the buffer + setStatePointers(); + + // perform multi-channel convolution + for(int i = 0; i < v_out_size; ++i) + { + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T out_sum[v_size] {}; + for(int k = 0; k < v_size && (i * v_size + k) < out_size; ++k) + { + const auto& subWeights = weights[i * v_size + k]; + v_type accum {}; + for(int j = 0; j < kernel_size; ++j) + { + accum += std::inner_product( + subWeights[j].begin(), + subWeights[j].end(), + state[(state_ptr + state_size - j) % state_size].begin(), + v_type {}); + } + out_sum[k] = xsimd::reduce_add(accum); + } + + outs[i] = xsimd::load_aligned(out_sum) + bias[i]; + } + + state_ptr = (state_ptr == state_size - 1 ? 0 : state_ptr + 1); // iterate state pointer forwards + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type (&ins)[v_in_size]) noexcept + { + for(int i = 0; i < v_out_size; ++i) + { + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T out_sum[v_size] {}; + for(int k = 0; k < v_size && (i * v_size + k) < out_size; ++k) + { + const auto& subWeights = weights[i * v_size + k][0]; + + v_type accum {}; + for(int j = 0; j < v_in_size; ++j) + accum += subWeights[j] * ins[j]; + out_sum[k] = xsimd::reduce_add(accum); + } + + outs[i] = xsimd::load_aligned(out_sum) + bias[i]; + } + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[out_size][in_size][kernel_size * dilation] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& weights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const std::vector& biasVals); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size; } + + /** Returns the convolution dilation rate. */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate; } + + /** Returns the number of "groups" in the convolution. */ + int getGroups() const noexcept { return groups; } + + v_type outs[v_out_size]; + +private: + template + typename std::enable_if::type resize_state() + { + state.resize(state_size, {}); + } + + template + typename std::enable_if::type resize_state() { } + + using state_col_type = std::array; + using state_type = typename std::conditional>, std::array>::type; + using weights_type = std::array, kernel_size>; + + state_type state {}; + weights_type state_cols {}; + + int state_ptr = 0; + std::array state_ptrs {}; + + weights_type weights[out_size] {}; + v_type bias[v_out_size] {}; + + /** Sets pointers to state array columns. */ + inline void setStatePointers() + { + for(int k = 0; k < kernel_size; ++k) + state_ptrs[k] = (state_ptr + state_size - k * dilation_rate) % state_size; + } +}; +} // namespace RTNEURAL_NAMESPACE + +#endif // CONV1DXSIMD_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_xsimd.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_xsimd.tpp new file mode 100644 index 0000000..4ec335e --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d/conv1d_xsimd.tpp @@ -0,0 +1,132 @@ +#include "conv1d_xsimd.h" + +namespace RTNEURAL_NAMESPACE +{ + +template +Conv1D::Conv1D(int in_size, int out_size, int kernel_size, int dilation, int num_groups) + : Layer(in_size, out_size) + , dilation_rate(dilation) + , kernel_size(kernel_size) + , state_size((kernel_size - 1) * dilation + 1) + , groups(num_groups) + , filters_per_group(in_size / groups) + , channels_per_group(out_size / groups) +{ + weights = vec3_type(out_size, vec2_type(kernel_size, vec_type(filters_per_group, (T)0))); + bias.resize(out_size, (T)0); + state = vec2_type(state_size, vec_type(in_size, (T)0)); + state_cols = vec2_type(kernel_size, vec_type(filters_per_group, (T)0)); + state_ptrs.resize(kernel_size); + prod_state.resize(filters_per_group); +} + +template +Conv1D::Conv1D(std::initializer_list sizes) + : Conv1D(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3)) +{ +} + +template +Conv1D::Conv1D(const Conv1D& other) + : Conv1D(other.in_size, other.out_size, other.kernel_size, other.dilation_rate) +{ +} + +template +Conv1D& Conv1D::operator=(const Conv1D& other) +{ + return *this = Conv1D(other); +} + +template +Conv1D::~Conv1D() = default; + +template +void Conv1D::reset() +{ + for(int k = 0; k < state_size; ++k) + std::fill(state[k].begin(), state[k].end(), (T)0); + + for(int k = 0; k < kernel_size; ++k) + std::fill(state_cols[k].begin(), state_cols[k].end(), (T)0); + + std::fill(state_ptrs.begin(), state_ptrs.end(), 0); + state_ptr = 0; +} + +template +void Conv1D::setWeights(const std::vector>>& ws) +{ + for(int i = 0; i < Layer::out_size; ++i) + for(int k = 0; k < filters_per_group; ++k) + for(int j = 0; j < kernel_size; ++j) + weights[i][j][k] = ws[i][k][j]; +} + +template +void Conv1D::setBias(const std::vector& biasVals) +{ + for(int i = 0; i < Layer::out_size; ++i) + bias[i] = biasVals[i]; +} + +//==================================================== +template +Conv1DT::Conv1DT() +{ + for(int i = 0; i < out_size; ++i) + for(int j = 0; j < kernel_size; ++j) + for(int k = 0; k < v_filters_per_group; ++k) + weights[i][j][k] = v_type((T)0.0); + + for(int i = 0; i < v_out_size; ++i) + bias[i] = v_type((T)0.0); + + for(int i = 0; i < v_out_size; ++i) + outs[i] = v_type((T)0.0); + + resize_state(); + reset(); +} + +template +void Conv1DT::reset() +{ + for(int i = 0; i < state_size; ++i) + for(int k = 0; k < v_in_size; ++k) + state[i][k] = v_type((T)0.0); + + for(int i = 0; i < kernel_size; ++i) + for(int k = 0; k < v_filters_per_group; ++k) + state_cols[i][k] = v_type((T)0.0); + + state_ptr = 0; + for(int i = 0; i < kernel_size; ++i) + state_ptrs[i] = 0; +} + +template +void Conv1DT::setWeights(const std::vector>>& ws) +{ + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < filters_per_group; ++k) + { + for(int j = 0; j < kernel_size; ++j) + { + auto& w = weights[i][j][k / v_size]; + w = set_value(w, k % v_size, ws[i][k][j]); + } + } + } +} + +template +void Conv1DT::setBias(const std::vector& biasVals) +{ + for(int i = 0; i < out_size; ++i) + bias[i / v_size] = set_value(bias[i / v_size], i % v_size, biasVals[i]); +} + +} // namespace RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless.h new file mode 100644 index 0000000..9252852 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless.h @@ -0,0 +1,310 @@ +#ifndef CONV1D_STATELESS_H_INCLUDED +#define CONV1D_STATELESS_H_INCLUDED + +#if RTNEURAL_USE_EIGEN +#include "conv1d_stateless_eigen.h" +#include "conv1d_stateless_eigen.tpp" +#elif RTNEURAL_USE_XSIMD +#include "conv1d_stateless_xsimd.h" +#include "conv1d_stateless_xsimd.tpp" +#else +#include "../Layer.h" +#include "../config.h" + +namespace RTNEURAL_NAMESPACE +{ +/** + * Dynamic implementation of a 1-dimensional stateless convolution layer with no activation. + * This implementation was designed to be used for a single frame of features, fully available at each forward call. + * So the layer has a NO internal "state" + * + * @tparam T Type of the layer (float, double, int ...) + */ +template +class Conv1DStateless : public Layer +{ +public: + Conv1DStateless(int in_num_filters_in, int in_num_features_in, int in_num_filters_out, int in_kernel_size, int in_stride, bool in_valid_pad); + Conv1DStateless(std::initializer_list sizes); + Conv1DStateless(const Conv1DStateless& other); + Conv1DStateless& operator=(const Conv1DStateless& other); + virtual ~Conv1DStateless() = default; + + static constexpr int computeNumFeaturesOut(int num_features_in, int kernel_size, int stride, int valid_pad) + { + // Based on tensorflow docs: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2 + // Custom implementation of ceil since std::ceil is not constexpr. + + if(valid_pad) + { + float f = static_cast(num_features_in - kernel_size + 1) / static_cast(stride); + int i = static_cast(f); + return f > static_cast(i) ? i + 1 : i; + } + + float f = static_cast(num_features_in) / static_cast(stride); + int i = static_cast(f); + return f > static_cast(i) ? i + 1 : i; + } + + static constexpr int computePadLeft(int num_features_in, int kernel_size, int stride, bool valid_pad) + { + // Based on tensorflow: Based on tensorflow: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2ow: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2 + return valid_pad ? 0 : std::max(num_features_in % stride == 0 ? kernel_size - stride : kernel_size - num_features_in % stride, 0) / 2; + } + + static constexpr int computePadRight(int num_features_in, int kernel_size, int stride, bool valid_pad) + { + // Based on tensorflow: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2 + if(!valid_pad) + { + int total_pad = std::max(num_features_in % stride == 0 ? kernel_size - stride : kernel_size - num_features_in % stride, 0); + return total_pad - total_pad / 2; + } + + return 0; + } + + /** Resets the layer state. */ + void reset() override {}; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "conv1d_stateless"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* output) noexcept override + { + if(valid_pad) + { + for(int out_row_idx = 0; out_row_idx < num_filters_out; ++out_row_idx) + { + for(int out_col_idx = 0; out_col_idx < num_features_out; ++out_col_idx) + { + T sum {}; + for(int in_col_idx = out_col_idx * stride; in_col_idx < out_col_idx * stride + kernel_size; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - out_col_idx * stride; + for(int in_row_idx = 0; in_row_idx < num_filters_in; ++in_row_idx) + sum += kernelWeights[out_row_idx][in_row_idx][kernel_col_idx] * (input[in_col_idx * num_filters_in + in_row_idx]); + } + output[out_col_idx * num_filters_out + out_row_idx] += sum; + } + } + } + else + { + for(int out_row_idx = 0; out_row_idx < num_filters_out; ++out_row_idx) + { + int out_col_idx = 0; + + for(; out_col_idx * stride < pad_left; ++out_col_idx) + { + T sum {}; + const int eff_kernel_size = kernel_size - pad_left + out_col_idx * stride; + for(int in_col_idx = 0; in_col_idx < eff_kernel_size; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx + (kernel_size - eff_kernel_size); + for(int in_row_idx = 0; in_row_idx < num_filters_in; ++in_row_idx) + sum += kernelWeights[out_row_idx][in_row_idx][kernel_col_idx] * (input[in_col_idx * num_filters_in + in_row_idx]); + } + output[out_col_idx * num_filters_out + out_row_idx] += sum; + } + + for(; out_col_idx * stride - pad_left + kernel_size < num_features_in; ++out_col_idx) + { + T sum {}; + for(int in_col_idx = out_col_idx * stride - pad_left; in_col_idx < out_col_idx * stride - pad_left + kernel_size; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - (out_col_idx * stride - pad_left); + for(int in_row_idx = 0; in_row_idx < num_filters_in; ++in_row_idx) + sum += kernelWeights[out_row_idx][in_row_idx][kernel_col_idx] * (input[in_col_idx * num_filters_in + in_row_idx]); + } + output[out_col_idx * num_filters_out + out_row_idx] += sum; + } + + for(; out_col_idx * stride - pad_left + kernel_size <= num_features_in + pad_right; ++out_col_idx) + { + T sum {}; + const int eff_kernel_size = num_features_in - (out_col_idx * stride - pad_left); + for(int in_col_idx = (num_features_in - eff_kernel_size); in_col_idx < num_features_in; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - (num_features_in - eff_kernel_size); + for(int in_row_idx = 0; in_row_idx < num_filters_in; ++in_row_idx) + sum += kernelWeights[out_row_idx][in_row_idx][kernel_col_idx] * (input[in_col_idx * num_filters_in + in_row_idx]); + } + output[out_col_idx * num_filters_out + out_row_idx] += sum; + } + } + } + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[num_filters_out][num_filters_in][kernel_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& inWeights); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size; } + + /** Returns the stride. */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride; } + +private: + const int num_filters_in; + const int num_features_in; + const int num_filters_out; + const int kernel_size; + const int stride; + const int num_features_out; + const bool valid_pad; + const int pad_left; + const int pad_right; + + using Matrix = std::vector>; + std::vector kernelWeights; +}; + +//==================================================== + +/** + * Static implementation of a 1-dimensional stateless convolution layer with no activation. + * This implementation was designed to be used for a single frame of features, fully available at each forward call. + * So the layer has a NO internal "state" + * + * @tparam T Type of the layer (float, double, int ...) + * @tparam num_filters_in_t number of input filters + * @tparam num_features_in_t number of input features + * @tparam num_filters_out_t number of output filters + * @tparam kernel_size_t size of the convolution kernel + * @tparam stride_t convolution stride + * @tparam valid_pad_t if true: pad is "valid", if false: pad is "same" + */ +template +class Conv1DStatelessT +{ + static constexpr int num_features_out = Conv1DStateless::computeNumFeaturesOut(num_features_in_t, kernel_size_t, stride_t, valid_pad_t); + static constexpr int pad_left = Conv1DStateless::computePadLeft(num_features_in_t, kernel_size_t, stride_t, valid_pad_t); + static constexpr int pad_right = Conv1DStateless::computePadRight(num_features_in_t, kernel_size_t, stride_t, valid_pad_t); + + using weights_type = std::array, num_filters_in_t>; + +public: + Conv1DStatelessT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "conv1d_stateless"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Empty function, this layer has no state */ + RTNEURAL_REALTIME void reset() {}; + + /** Performs forward propagation for this layer if pad is "valid". */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const T (&inMatrix)[num_features_in_t * num_filters_in_t]) noexcept + { + for(int out_row_idx = 0; out_row_idx < num_filters_out_t; ++out_row_idx) + { + for(int out_col_idx = 0; out_col_idx < num_features_out; ++out_col_idx) + { + T sum {}; + for(int in_col_idx = out_col_idx * stride_t; in_col_idx < out_col_idx * stride_t + kernel_size_t; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - out_col_idx * stride_t; + for(int in_row_idx = 0; in_row_idx < num_filters_in_t; ++in_row_idx) + sum += kernelWeights[out_row_idx][in_row_idx][kernel_col_idx] * (inMatrix[in_col_idx * num_filters_in_t + in_row_idx]); + } + outs[out_col_idx * num_filters_out_t + out_row_idx] += sum; + } + } + } + + /** Performs forward propagation for this layer if pad is "same" */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const T (&inMatrix)[num_features_in_t * num_filters_in_t]) noexcept + { + for(int out_row_idx = 0; out_row_idx < num_filters_out_t; ++out_row_idx) + { + int out_col_idx = 0; + + for(; out_col_idx * stride_t < pad_left; ++out_col_idx) + { + T sum {}; + const int eff_kernel_size = kernel_size_t - pad_left + out_col_idx * stride_t; + for(int in_col_idx = 0; in_col_idx < eff_kernel_size; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx + (kernel_size_t - eff_kernel_size); + for(int in_row_idx = 0; in_row_idx < num_filters_in_t; ++in_row_idx) + sum += kernelWeights[out_row_idx][in_row_idx][kernel_col_idx] * (inMatrix[in_col_idx * num_filters_in_t + in_row_idx]); + } + outs[out_col_idx * num_filters_out_t + out_row_idx] = sum; + } + + for(; out_col_idx * stride_t - pad_left + kernel_size_t < num_features_in_t; ++out_col_idx) + { + T sum {}; + for(int in_col_idx = out_col_idx * stride_t - pad_left; in_col_idx < out_col_idx * stride_t - pad_left + kernel_size_t; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - (out_col_idx * stride_t - pad_left); + for(int in_row_idx = 0; in_row_idx < num_filters_in_t; ++in_row_idx) + sum += kernelWeights[out_row_idx][in_row_idx][kernel_col_idx] * (inMatrix[in_col_idx * num_filters_in_t + in_row_idx]); + } + outs[out_col_idx * num_filters_out_t + out_row_idx] = sum; + } + + for(; out_col_idx * stride_t - pad_left + kernel_size_t <= num_features_in_t + pad_right; ++out_col_idx) + { + T sum {}; + const int eff_kernel_size = num_features_in_t - (out_col_idx * stride_t - pad_left); + for(int in_col_idx = (num_features_in_t - eff_kernel_size); in_col_idx < num_features_in_t; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - (num_features_in_t - eff_kernel_size); + for(int in_row_idx = 0; in_row_idx < num_filters_in_t; ++in_row_idx) + sum += kernelWeights[out_row_idx][in_row_idx][kernel_col_idx] * (inMatrix[in_col_idx * num_filters_in_t + in_row_idx]); + } + outs[out_col_idx * num_filters_out_t + out_row_idx] = sum; + } + } + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[num_filters_out][num_filters_in][kernel_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& inWeights); + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[kernel_size][num_filters_in][num_filters_out] + */ + RTNEURAL_REALTIME void setWeightsTransposed(const std::vector>>& inWeights); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size_t; } + + /** Returns the convolution dilation rate. */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride_t; } + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[num_filters_out_t * num_features_out] {}; + +private: + weights_type kernelWeights[num_filters_out_t]; +}; + +} // RTNEURAL + +#include "conv1d_stateless.tpp" + +#endif // RTNEURAL_USE_STL +#endif // CONV1D_STATELESS_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless.tpp new file mode 100644 index 0000000..2431143 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless.tpp @@ -0,0 +1,81 @@ +#include "conv1d_stateless.h" + +namespace RTNEURAL_NAMESPACE +{ +template +Conv1DStateless::Conv1DStateless(int in_num_filters_in, int in_num_features_in, int in_num_filters_out, int in_kernel_size, int in_stride, bool in_valid_pad) + : num_filters_in(in_num_filters_in) + , num_features_in(in_num_features_in) + , num_filters_out(in_num_filters_out) + , kernel_size(in_kernel_size) + , stride(in_stride) + , valid_pad(in_valid_pad) + , num_features_out(computeNumFeaturesOut(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) + , pad_left(computePadLeft(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) + , pad_right(computePadRight(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) + , Layer(in_num_filters_in * in_num_features_in, in_num_filters_out * computeNumFeaturesOut(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) +{ + kernelWeights.resize(num_filters_out); + for(auto& kw : kernelWeights) + { + kw.resize(num_filters_in); + for(auto& col : kw) + col.resize(kernel_size, (T)0); + } +} + +template +Conv1DStateless::Conv1DStateless(std::initializer_list sizes) + : Conv1DStateless(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3), *(sizes.begin() + 4), *(sizes.begin() + 5)) +{ +} + +template +Conv1DStateless::Conv1DStateless(const Conv1DStateless& other) + : Conv1DStateless(other.num_filters_in, other.num_features_in, other.num_filters_out, other.kernel_size, other.stride, other.valid_pad) +{ +} + +template +Conv1DStateless& Conv1DStateless::operator=(const Conv1DStateless& other) +{ + return *this = Conv1DStateless(other); +} + +template +void Conv1DStateless::setWeights(const std::vector>>& inWeights) +{ + for(int i = 0; i < num_filters_out; ++i) + for(int k = 0; k < num_filters_in; ++k) + for(int j = 0; j < kernel_size; ++j) + kernelWeights[i][k][j] = inWeights.at(i).at(k).at(j); +} + +//==================================================== + +template +Conv1DStatelessT::Conv1DStatelessT() +{ + for(int i = 0; i < num_filters_out_t; ++i) + for(int k = 0; k < num_filters_in_t; ++k) + std::fill(std::begin(kernelWeights[i][k]), std::end(kernelWeights[i][k]), (T)0); +} + +template +void Conv1DStatelessT::setWeights(const std::vector>>& inWeights) +{ + for(int i = 0; i < num_filters_out_t; ++i) + for(int k = 0; k < num_filters_in_t; ++k) + for(int j = 0; j < kernel_size_t; ++j) + kernelWeights[i][k][j] = inWeights.at(i).at(k).at(j); +} + +template +void Conv1DStatelessT::setWeightsTransposed(const std::vector>>& inWeights) +{ + for(int i = 0; i < num_filters_out_t; ++i) + for(int k = 0; k < num_filters_in_t; ++k) + for(int j = 0; j < kernel_size_t; ++j) + kernelWeights[i][k][j] = inWeights.at(j).at(k).at(i); +} +} // RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_eigen.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_eigen.h new file mode 100644 index 0000000..db2929f --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_eigen.h @@ -0,0 +1,251 @@ +#ifndef CONV1D_STATELESS_EIGEN_H_INCLUDED +#define CONV1D_STATELESS_EIGEN_H_INCLUDED + +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ +/** + * Dynamic implementation of a 1-dimensional stateless convolution layer with no activation. + * This implementation was designed to be used for a single frame of features, fully available at each forward call. + * So the layer has a NO internal "state" + * + * @tparam T Type of the layer (float, double, int ...) + */ +template +class Conv1DStateless : public Layer +{ +public: + Conv1DStateless(int in_num_filters_in, int in_num_features_in, int in_num_filters_out, int in_kernel_size, int in_stride, bool in_valid_pad); + Conv1DStateless(std::initializer_list sizes); + Conv1DStateless(const Conv1DStateless& other); + Conv1DStateless& operator=(const Conv1DStateless& other); + virtual ~Conv1DStateless() = default; + + static constexpr int computeNumFeaturesOut(int num_features_in, int kernel_size, int stride, int valid_pad) + { + // Based on tensorflow docs: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2 + // Custom implementation of ceil since std::ceil is not constexpr. + + if(valid_pad) + { + float f = static_cast(num_features_in - kernel_size + 1) / static_cast(stride); + int i = static_cast(f); + return f > static_cast(i) ? i + 1 : i; + } + + float f = static_cast(num_features_in) / static_cast(stride); + int i = static_cast(f); + return f > static_cast(i) ? i + 1 : i; + } + + static constexpr int computePadLeft(int num_features_in, int kernel_size, int stride, bool valid_pad) + { + // Based on tensorflow: Based on tensorflow: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2ow: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2 + return valid_pad ? 0 : std::max(num_features_in % stride == 0 ? kernel_size - stride : kernel_size - num_features_in % stride, 0) / 2; + } + + static constexpr int computePadRight(int num_features_in, int kernel_size, int stride, bool valid_pad) + { + // Based on tensorflow: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2 + if(!valid_pad) + { + int total_pad = std::max(num_features_in % stride == 0 ? kernel_size - stride : kernel_size - num_features_in % stride, 0); + return total_pad - total_pad / 2; + } + + return 0; + } + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset() override {}; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "conv1d_stateless"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* output) noexcept override + { + auto inMatrix = Eigen::Map, + RTNeuralEigenAlignment>(input, num_filters_in, num_features_in); + + auto outMatrix = Eigen::Map, + RTNeuralEigenAlignment>(output, num_filters_out, num_features_out); + + if(valid_pad) + { + for(int i = 0; i < num_filters_out; i++) + for(int j = 0; j < num_features_out; j++) + outMatrix(i, j) += kernelWeights[i].cwiseProduct(inMatrix.middleCols(j * stride, kernel_size)).sum(); + } + else + { + for(int i = 0; i < num_filters_out; i++) + { + int j = 0; + + for(; j * stride < pad_left; j++) + { + const int eff_kernel_size = kernel_size - pad_left + j * stride; + outMatrix(i, j) += kernelWeights[i].rightCols(eff_kernel_size).cwiseProduct(inMatrix.leftCols(eff_kernel_size)).sum(); + } + + for(; j * stride - pad_left + kernel_size < num_features_in; j++) + outMatrix(i, j) += kernelWeights[i].cwiseProduct(inMatrix.middleCols(j * stride - pad_left, kernel_size)).sum(); + + for(; j * stride - pad_left + kernel_size <= num_features_in + pad_right; j++) + { + const int eff_kernel_size = num_features_in - (j * stride - pad_left); + outMatrix(i, j) += kernelWeights[i].leftCols(eff_kernel_size).cwiseProduct(inMatrix.rightCols(eff_kernel_size)).sum(); + } + } + } + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[num_filters_out][num_filters_in][kernel_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& inWeights); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size; } + + /** Returns the stride. */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride; } + +private: + const int num_filters_in; + const int num_features_in; + const int num_filters_out; + const int kernel_size; + const int stride; + const int num_features_out; + const bool valid_pad; + const int pad_left; + const int pad_right; + + std::vector> kernelWeights; +}; + +//==================================================== + +/** + * Static implementation of a 1-dimensional stateless convolution layer with no activation. + * This implementation was designed to be used for a single frame of features, fully available at each forward call. + * So the layer has a NO internal "state" + * + * @tparam T Type of the layer (float, double, int ...) + * @tparam num_filters_in_t number of input filters + * @tparam num_features_in_t number of input features + * @tparam num_filters_out_t number of output filters + * @tparam kernel_size_t size of the convolution kernel + * @tparam stride_t convolution stride + * @tparam valid_pad_t if true: pad is "valid", if false: pad is "same" + */ +template +class Conv1DStatelessT +{ + static constexpr int num_features_out = Conv1DStateless::computeNumFeaturesOut(num_features_in_t, kernel_size_t, stride_t, valid_pad_t); + static constexpr int pad_left = Conv1DStateless::computePadLeft(num_features_in_t, kernel_size_t, stride_t, valid_pad_t); + static constexpr int pad_right = Conv1DStateless::computePadRight(num_features_in_t, kernel_size_t, stride_t, valid_pad_t); + + using weights_type = Eigen::Matrix; + using input_type = Eigen::Matrix; + using output_type = Eigen::Matrix; + +public: + Conv1DStatelessT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "conv1d_stateless"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Empty function, this layer has no state */ + RTNEURAL_REALTIME void reset() {}; + + /** Performs forward propagation for this layer if pad is "valid". */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const input_type& inMatrix) noexcept + { + // perform a multichannel convolution + for(int i = 0; i < num_filters_out_t; i++) + { + for(int j = 0; j < num_features_out; j++) + { + // TODO: manage to use middleCols(j*stride) + outs(i, j) = kernelWeights[i].cwiseProduct(inMatrix.middleCols(j * stride_t, kernel_size_t)).sum(); + } + } + } + + /** Performs forward propagation for this layer if pad is "same" */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const input_type& inMatrix) noexcept + { + // perform a multichannel convolution + for(int i = 0; i < num_filters_out_t; i++) + { + int j = 0; + + for(; j * stride_t < pad_left; j++) + { + const int eff_kernel_size = kernel_size_t - pad_left + j * stride_t; + outs(i, j) = kernelWeights[i].rightCols(eff_kernel_size).cwiseProduct(inMatrix.leftCols(eff_kernel_size)).sum(); + } + + for(; j * stride_t - pad_left + kernel_size_t < num_features_in_t; j++) + // TODO: manage to use middleCols(j*stride) + outs(i, j) = kernelWeights[i].cwiseProduct(inMatrix.middleCols(j * stride_t - pad_left, kernel_size_t)).sum(); + + for(; j * stride_t - pad_left + kernel_size_t <= num_features_in_t + pad_right; j++) + { + const int eff_kernel_size = num_features_in_t - (j * stride_t - pad_left); + outs(i, j) = kernelWeights[i].leftCols(eff_kernel_size).cwiseProduct(inMatrix.rightCols(eff_kernel_size)).sum(); + } + } + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[num_filters_out][num_filters_in][kernel_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& inWeights); + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[kernel_size][num_filters_in][num_filters_out] + */ + RTNEURAL_REALTIME void setWeightsTransposed(const std::vector>>& inWeights); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size_t; } + + /** Returns the convolution dilation rate. */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride_t; } + + Eigen::Map outs; + +private: + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[num_filters_out_t * num_features_out]; + + weights_type kernelWeights[num_filters_out_t]; +}; + +} // RTNEURAL + +#endif // CONV1D_STATELESS_EIGEN_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_eigen.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_eigen.tpp new file mode 100644 index 0000000..f86ca00 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_eigen.tpp @@ -0,0 +1,75 @@ +#include "conv1d_stateless_eigen.h" + +namespace RTNEURAL_NAMESPACE +{ +template +Conv1DStateless::Conv1DStateless(int in_num_filters_in, int in_num_features_in, int in_num_filters_out, int in_kernel_size, int in_stride, bool in_valid_pad) + : num_filters_in(in_num_filters_in) + , num_features_in(in_num_features_in) + , num_filters_out(in_num_filters_out) + , kernel_size(in_kernel_size) + , stride(in_stride) + , valid_pad(in_valid_pad) + , num_features_out(computeNumFeaturesOut(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) + , pad_left(computePadLeft(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) + , pad_right(computePadRight(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) + , Layer(in_num_filters_in * in_num_features_in, in_num_filters_out * computeNumFeaturesOut(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) +{ + kernelWeights.resize(num_filters_out, Eigen::Matrix::Zero(num_filters_in, kernel_size)); +} + +template +Conv1DStateless::Conv1DStateless(std::initializer_list sizes) + : Conv1DStateless(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3), *(sizes.begin() + 4), *(sizes.begin() + 5)) +{ +} + +template +Conv1DStateless::Conv1DStateless(const Conv1DStateless& other) + : Conv1DStateless(other.num_filters_in, other.num_features_in, other.num_filters_out, other.kernel_size, other.stride, other.valid_pad) +{ +} + +template +Conv1DStateless& Conv1DStateless::operator=(const Conv1DStateless& other) +{ + return *this = Conv1DStateless(other); +} + +template +void Conv1DStateless::setWeights(const std::vector>>& inWeights) +{ + for(int i = 0; i < num_filters_out; ++i) + for(int k = 0; k < num_filters_in; ++k) + for(int j = 0; j < kernel_size; ++j) + kernelWeights[i](k, j) = inWeights.at(i).at(k).at(j); +} + +//==================================================== + +template +Conv1DStatelessT::Conv1DStatelessT() + : outs(outs_internal) +{ + for(int k = 0; k < num_filters_out_t; ++k) + kernelWeights[k] = weights_type::Zero(); +} + +template +void Conv1DStatelessT::setWeights(const std::vector>>& inWeights) +{ + for(int i = 0; i < num_filters_out_t; ++i) + for(int k = 0; k < num_filters_in_t; ++k) + for(int j = 0; j < kernel_size_t; ++j) + kernelWeights[i](k, j) = inWeights.at(i).at(k).at(j); +} + +template +void Conv1DStatelessT::setWeightsTransposed(const std::vector>>& inWeights) +{ + for(int i = 0; i < num_filters_out_t; ++i) + for(int k = 0; k < num_filters_in_t; ++k) + for(int j = 0; j < kernel_size_t; ++j) + kernelWeights[i](k, j) = inWeights.at(j).at(k).at(i); +} +} // RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_xsimd.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_xsimd.h new file mode 100644 index 0000000..d43ab7a --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_xsimd.h @@ -0,0 +1,350 @@ +#ifndef RTNEURAL_CONV1D_STATELESS_XSIMD_H +#define RTNEURAL_CONV1D_STATELESS_XSIMD_H + +#include "../Layer.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ +/** + * Dynamic implementation of a 1-dimensional stateless convolution layer with no activation. + * This implementation was designed to be used for a single frame of features, fully available at each forward call. + * So the layer has a NO internal "state" + * + * @tparam T Type of the layer (float, double, int ...) + */ +template +class Conv1DStateless : public Layer +{ +public: + Conv1DStateless(int in_num_filters_in, int in_num_features_in, int in_num_filters_out, int in_kernel_size, int in_stride, bool in_valid_pad); + Conv1DStateless(std::initializer_list sizes); + Conv1DStateless(const Conv1DStateless& other); + Conv1DStateless& operator=(const Conv1DStateless& other); + virtual ~Conv1DStateless() = default; + + static constexpr int computeNumFeaturesOut(int num_features_in, int kernel_size, int stride, int valid_pad) + { + // Based on tensorflow docs: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2 + // Custom implementation of ceil since std::ceil is not constexpr. + + if(valid_pad) + { + float f = static_cast(num_features_in - kernel_size + 1) / static_cast(stride); + int i = static_cast(f); + return f > static_cast(i) ? i + 1 : i; + } + + float f = static_cast(num_features_in) / static_cast(stride); + int i = static_cast(f); + return f > static_cast(i) ? i + 1 : i; + } + + static constexpr int computePadLeft(int num_features_in, int kernel_size, int stride, bool valid_pad) + { + // Based on tensorflow: Based on tensorflow: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2ow: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2 + return valid_pad ? 0 : std::max(num_features_in % stride == 0 ? kernel_size - stride : kernel_size - num_features_in % stride, 0) / 2; + } + + static constexpr int computePadRight(int num_features_in, int kernel_size, int stride, bool valid_pad) + { + // Based on tensorflow: https://www.tensorflow.org/api_docs/python/tf/nn#notes_on_padding_2 + if(!valid_pad) + { + int total_pad = std::max(num_features_in % stride == 0 ? kernel_size - stride : kernel_size - num_features_in % stride, 0); + return total_pad - total_pad / 2; + } + + return 0; + } + + /** Resets the layer state. */ + RTNEURAL_REALTIME void reset() override { } + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "conv1d_stateless"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* output) noexcept override + { + if(valid_pad) + { + for(int out_row_idx = 0; out_row_idx < num_filters_out; ++out_row_idx) + { + for(int out_col_idx = 0; out_col_idx < num_features_out; ++out_col_idx) + { + T sum {}; + for(int in_col_idx = out_col_idx * stride; in_col_idx < out_col_idx * stride + kernel_size; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - out_col_idx * stride; + xsimd::transform(kernelWeights[out_row_idx][kernel_col_idx].begin(), + kernelWeights[out_row_idx][kernel_col_idx].end(), + input + in_col_idx * num_filters_in, + scratch.begin(), + [](auto a, auto b) + { return a * b; }); + sum += xsimd::reduce(scratch.begin(), scratch.end(), T {}); + } + output[out_col_idx * num_filters_out + out_row_idx] += sum; + } + } + } + else + { + for(int out_row_idx = 0; out_row_idx < num_filters_out; ++out_row_idx) + { + int out_col_idx = 0; + + for(; out_col_idx * stride < pad_left; ++out_col_idx) + { + T sum {}; + const int eff_kernel_size = kernel_size - pad_left + out_col_idx * stride; + for(int in_col_idx = 0; in_col_idx < eff_kernel_size; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx + (kernel_size - eff_kernel_size); + xsimd::transform(kernelWeights[out_row_idx][kernel_col_idx].begin(), + kernelWeights[out_row_idx][kernel_col_idx].end(), + input + in_col_idx * num_filters_in, + scratch.begin(), + [](auto a, auto b) + { return a * b; }); + sum += xsimd::reduce(scratch.begin(), scratch.end(), T {}); + } + output[out_col_idx * num_filters_out + out_row_idx] += sum; + } + + for(; out_col_idx * stride - pad_left + kernel_size < num_features_in; ++out_col_idx) + { + T sum {}; + for(int in_col_idx = out_col_idx * stride - pad_left; in_col_idx < out_col_idx * stride - pad_left + kernel_size; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - (out_col_idx * stride - pad_left); + xsimd::transform(kernelWeights[out_row_idx][kernel_col_idx].begin(), + kernelWeights[out_row_idx][kernel_col_idx].end(), + input + in_col_idx * num_filters_in, + scratch.begin(), + [](auto a, auto b) + { return a * b; }); + sum += xsimd::reduce(scratch.begin(), scratch.end(), T {}); + } + output[out_col_idx * num_filters_out + out_row_idx] += sum; + } + + for(; out_col_idx * stride - pad_left + kernel_size <= num_features_in + pad_right; ++out_col_idx) + { + T sum {}; + const int eff_kernel_size = num_features_in - (out_col_idx * stride - pad_left); + for(int in_col_idx = (num_features_in - eff_kernel_size); in_col_idx < num_features_in; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - (num_features_in - eff_kernel_size); + xsimd::transform(kernelWeights[out_row_idx][kernel_col_idx].begin(), + kernelWeights[out_row_idx][kernel_col_idx].end(), + input + in_col_idx * num_filters_in, + scratch.begin(), + [](auto a, auto b) + { return a * b; }); + sum += xsimd::reduce(scratch.begin(), scratch.end(), T {}); + } + output[out_col_idx * num_filters_out + out_row_idx] += sum; + } + } + } + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[num_filters_out][num_filters_in][kernel_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& inWeights); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size; } + + /** Returns the stride. */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride; } + +private: + const int num_filters_in; + const int num_features_in; + const int num_filters_out; + const int kernel_size; + const int stride; + const int num_features_out; + const bool valid_pad; + const int pad_left; + const int pad_right; + + using Matrix = std::vector>>; + std::vector kernelWeights; + + std::vector> scratch; +}; + +//==================================================== + +/** + * Static implementation of a 1-dimensional stateless convolution layer with no activation. + * This implementation was designed to be used for a single frame of features, fully available at each forward call. + * So the layer has a NO internal "state" + * + * @tparam T Type of the layer (float, double, int ...) + * @tparam num_filters_in_t number of input filters + * @tparam num_features_in_t number of input features + * @tparam num_filters_out_t number of output filters + * @tparam kernel_size_t size of the convolution kernel + * @tparam stride_t convolution stride + * @tparam valid_pad_t if true: pad is "valid", if false: pad is "same" + */ +template +class Conv1DStatelessT +{ + static constexpr int num_features_out = Conv1DStateless::computeNumFeaturesOut(num_features_in_t, kernel_size_t, stride_t, valid_pad_t); + static constexpr int pad_left = Conv1DStateless::computePadLeft(num_features_in_t, kernel_size_t, stride_t, valid_pad_t); + static constexpr int pad_right = Conv1DStateless::computePadRight(num_features_in_t, kernel_size_t, stride_t, valid_pad_t); + + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_num_filters_in = ceil_div(num_filters_in_t, v_size); + static constexpr auto v_num_filters_out = ceil_div(num_filters_out_t, v_size); + static constexpr auto v_in_size = v_num_filters_in * num_features_in_t; + static constexpr auto v_out_size = v_num_filters_out * num_features_out; + + using weights_type = std::array, kernel_size_t>; + +public: + Conv1DStatelessT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "conv1d_stateless"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Empty function, this layer has no state */ + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for this layer if pad is "valid". */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type (&inMatrix)[v_in_size]) noexcept + { + // @TODO: can we vectorize in the other direction if num_filters == 1? + for(int out_col_idx = 0; out_col_idx < num_features_out; ++out_col_idx) + { + for(int out_row_idx = 0; out_row_idx < v_num_filters_out; ++out_row_idx) + { + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T out_temp[v_size] {}; + for(int i = 0; i < v_size; ++i) + { + v_type sum {}; + for(int in_col_idx = out_col_idx * stride_t; in_col_idx < out_col_idx * stride_t + kernel_size_t; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - out_col_idx * stride_t; + for(int in_row_idx = 0; in_row_idx < v_num_filters_in; ++in_row_idx) + sum += kernelWeights[out_row_idx * v_size + i][kernel_col_idx][in_row_idx] * (inMatrix[in_col_idx * v_num_filters_in + in_row_idx]); + } + out_temp[i] = xsimd::reduce_add(sum); + } + outs[out_col_idx * v_num_filters_out + out_row_idx] += xsimd::load_aligned(out_temp); + } + } + } + + /** Performs forward propagation for this layer if pad is "same" */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type (&inMatrix)[v_in_size]) noexcept + { + int out_col_idx = 0; + + for(; out_col_idx * stride_t < pad_left; ++out_col_idx) + { + for(int out_row_idx = 0; out_row_idx < v_num_filters_out; ++out_row_idx) + { + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T out_temp[v_size] {}; + for(int i = 0; i < v_size; ++i) + { + v_type sum {}; + const int eff_kernel_size = kernel_size_t - pad_left + out_col_idx * stride_t; + for(int in_col_idx = 0; in_col_idx < eff_kernel_size; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx + (kernel_size_t - eff_kernel_size); + for(int in_row_idx = 0; in_row_idx < v_num_filters_in; ++in_row_idx) + sum += kernelWeights[out_row_idx * v_size + i][kernel_col_idx][in_row_idx] * (inMatrix[in_col_idx * v_num_filters_in + in_row_idx]); + } + out_temp[i] = xsimd::reduce_add(sum); + } + outs[out_col_idx * v_num_filters_out + out_row_idx] += xsimd::load_aligned(out_temp); + } + } + + for(; out_col_idx * stride_t - pad_left + kernel_size_t < num_features_in_t; ++out_col_idx) + { + for(int out_row_idx = 0; out_row_idx < v_num_filters_out; ++out_row_idx) + { + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T out_temp[v_size] {}; + for(int i = 0; i < v_size; ++i) + { + v_type sum {}; + for(int in_col_idx = out_col_idx * stride_t - pad_left; in_col_idx < out_col_idx * stride_t - pad_left + kernel_size_t; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - (out_col_idx * stride_t - pad_left); + for(int in_row_idx = 0; in_row_idx < v_num_filters_in; ++in_row_idx) + sum += kernelWeights[out_row_idx * v_size + i][kernel_col_idx][in_row_idx] * (inMatrix[in_col_idx * v_num_filters_in + in_row_idx]); + } + out_temp[i] = xsimd::reduce_add(sum); + } + outs[out_col_idx * v_num_filters_out + out_row_idx] += xsimd::load_aligned(out_temp); + } + } + + for(; out_col_idx * stride_t - pad_left + kernel_size_t <= num_features_in_t + pad_right; ++out_col_idx) + { + for(int out_row_idx = 0; out_row_idx < v_num_filters_out; ++out_row_idx) + { + alignas(RTNEURAL_DEFAULT_ALIGNMENT) T out_temp[v_size] {}; + for(int i = 0; i < v_size; ++i) + { + v_type sum {}; + const int eff_kernel_size = num_features_in_t - (out_col_idx * stride_t - pad_left); + for(int in_col_idx = (num_features_in_t - eff_kernel_size); in_col_idx < num_features_in_t; ++in_col_idx) + { + const auto kernel_col_idx = in_col_idx - (num_features_in_t - eff_kernel_size); + for(int in_row_idx = 0; in_row_idx < v_num_filters_in; ++in_row_idx) + sum += kernelWeights[out_row_idx * v_size + i][kernel_col_idx][in_row_idx] * (inMatrix[in_col_idx * v_num_filters_in + in_row_idx]); + } + out_temp[i] = xsimd::reduce_add(sum); + } + outs[out_col_idx * v_num_filters_out + out_row_idx] += xsimd::load_aligned(out_temp); + } + } + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[num_filters_out][num_filters_in][kernel_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>& inWeights); + + /** Returns the size of the convolution kernel. */ + RTNEURAL_REALTIME int getKernelSize() const noexcept { return kernel_size_t; } + + /** Returns the convolution dilation rate. */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride_t; } + + v_type outs[v_out_size]; + +private: + weights_type kernelWeights[num_filters_out_t]; +}; + +} // RTNEURAL + +#endif // RTNEURAL_CONV1D_STATELESS_XSIMD_H diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_xsimd.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_xsimd.tpp new file mode 100644 index 0000000..42cccff --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv1d_stateless/conv1d_stateless_xsimd.tpp @@ -0,0 +1,74 @@ +#include "conv1d_stateless_xsimd.h" + +namespace RTNEURAL_NAMESPACE +{ +template +Conv1DStateless::Conv1DStateless(int in_num_filters_in, int in_num_features_in, int in_num_filters_out, int in_kernel_size, int in_stride, bool in_valid_pad) + : num_filters_in(in_num_filters_in) + , num_features_in(in_num_features_in) + , num_filters_out(in_num_filters_out) + , kernel_size(in_kernel_size) + , stride(in_stride) + , valid_pad(in_valid_pad) + , num_features_out(computeNumFeaturesOut(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) + , pad_left(computePadLeft(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) + , pad_right(computePadRight(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) + , Layer(in_num_filters_in * in_num_features_in, in_num_filters_out * computeNumFeaturesOut(in_num_features_in, in_kernel_size, in_stride, in_valid_pad)) +{ + kernelWeights.resize(num_filters_out); + for(auto& kw : kernelWeights) + { + kw.resize(kernel_size); + for(auto& col : kw) + col.resize(num_filters_in, (T)0); + } + + scratch.resize(num_filters_in, (T)0); +} + +template +Conv1DStateless::Conv1DStateless(std::initializer_list sizes) + : Conv1DStateless(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3), *(sizes.begin() + 4), *(sizes.begin() + 5)) +{ +} + +template +Conv1DStateless::Conv1DStateless(const Conv1DStateless& other) + : Conv1DStateless(other.num_filters_in, other.num_features_in, other.num_filters_out, other.kernel_size, other.stride, other.valid_pad) +{ +} + +template +Conv1DStateless& Conv1DStateless::operator=(const Conv1DStateless& other) +{ + return *this = Conv1DStateless(other); +} + +template +void Conv1DStateless::setWeights(const std::vector>>& inWeights) +{ + for(int i = 0; i < num_filters_out; ++i) + for(int k = 0; k < num_filters_in; ++k) + for(int j = 0; j < kernel_size; ++j) + kernelWeights[i][j][k] = inWeights.at(i).at(k).at(j); +} + +//==================================================== + +template +Conv1DStatelessT::Conv1DStatelessT() +{ + for(int i = 0; i < num_filters_out_t; ++i) + for(int k = 0; k < num_filters_in_t; ++k) + std::fill(std::begin(kernelWeights[i][k]), std::end(kernelWeights[i][k]), (T)0); +} + +template +void Conv1DStatelessT::setWeights(const std::vector>>& inWeights) +{ + for(int i = 0; i < num_filters_out_t; ++i) + for(int k = 0; k < num_filters_in_t; ++k) + for(int j = 0; j < kernel_size_t; ++j) + kernelWeights[i][j][k / v_size] = set_value(kernelWeights[i][j][k / v_size], k % v_size, inWeights.at(i).at(k).at(j)); +} +} // RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d.h new file mode 100644 index 0000000..1752a5e --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d.h @@ -0,0 +1,254 @@ +#ifndef CONV2D_H_INCLUDED +#define CONV2D_H_INCLUDED + +#if RTNEURAL_USE_EIGEN +#include "conv2d_eigen.h" +#include "conv2d_eigen.tpp" +#elif RTNEURAL_USE_XSIMD +#include "conv2d_xsimd.h" +#include "conv2d_xsimd.tpp" +#else +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include "../conv1d_stateless/conv1d_stateless.h" + +namespace RTNEURAL_NAMESPACE +{ +/** + * Dynamic implementation of a 2-dimensional convolution layer with no activation. + * + * @tparam T Type of the layer (float, double, int ...) + */ +template +class Conv2D : public Layer +{ +public: + /** + * @param in_num_filters_in number of input filters (channels) + * @param in_num_filters_out number of output filters (channels) + * @param in_num_features_in number of input features + * @param in_kernel_size_time size of the convolution kernel (time axis) + * @param in_kernel_size_feature size of the convolution kernel (feature axis) + * @param in_dilation_rate dilation_rate (time axis) + * @param in_stride convolution stride (feature axis) + * @param in_valid_pad whether the padding is "valid" or not ("same" otherwise) + */ + Conv2D(int in_num_filters_in, int in_num_filters_out, int in_num_features_in, int in_kernel_size_time, int in_kernel_size_feature, int in_dilation_rate, int in_stride, bool in_valid_pad); + Conv2D(std::initializer_list sizes); + Conv2D(const Conv2D& other); + Conv2D& operator=(const Conv2D& other); + virtual ~Conv2D() = default; + + /** Reset the layer's state */ + RTNEURAL_REALTIME void reset() override + { + state_index = 0; + + for(int i = 0; i < receptive_field; i++) + { + std::fill(state[i].begin(), state[i].end(), (T)0); + } + }; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "conv2d"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* output) noexcept override + { + for(int i = 0; i < kernel_size_time; ++i) + { + int state_idx_to_use = (state_index + (receptive_field - 1) - i * dilation_rate) % receptive_field; + + conv1dLayers[i].forward(input, state[state_idx_to_use].data()); + } + + for(int i = 0; i < num_features_out; ++i) + { + for(int j = 0; j < num_filters_out; ++j) + { + output[i * num_filters_out + j] = state[state_index][i * num_filters_out + j] + bias[j]; + } + } + + std::fill(state[state_index].begin(), state[state_index].end(), (T)0); + state_index = state_index == receptive_field - 1 ? 0 : state_index + 1; + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[num_filters_out][num_filters_in][kernel_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>>& inWeights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[num_filters_out] + */ + RTNEURAL_REALTIME void setBias(const std::vector& inBias); + + /** Returns the size of the convolution kernel (time axis). */ + RTNEURAL_REALTIME int getKernelSizeTime() const noexcept { return kernel_size_time; } + + /** Returns the size of the convolution kernel (feature axis). */ + RTNEURAL_REALTIME int getKernelSizeFeature() const noexcept { return kernel_size_feature; } + + /** Returns the convolution stride (feature axis) */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride; } + + /** Returns the convolution dilation rate (time axis) */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate; } + + const int num_filters_in; + const int num_features_in; + const int num_filters_out; + const int kernel_size_time; + const int kernel_size_feature; + const int dilation_rate; + const int stride; + const int num_features_out; + const int receptive_field; + const bool valid_pad; + +private: + std::vector> conv1dLayers; + + std::vector> state; + + int state_index = 0; + + std::vector bias; +}; + +//==================================================== + +/** + * Static implementation of a 2-dimensional convolution layer with no activation. + * + * @tparam T Type of the layer (float, double, int ...) + * @tparam num_filters_in_t number of input filters (channels) + * @tparam num_filters_out_t number of output filters (channels) + * @tparam num_features_in_t number of input features + * @tparam kernel_size_time_t size of the convolution kernel (time axis) + * @tparam kernel_size_feature_t size of the convolution kernel (feature axis) + * @tparam dilation_rate_t dilation_rate (time axis) + * @tparam stride_t convolution stride (feature axis) + * @tparam valid_pad_t if true: pad is "valid". if false: pad is "same" + */ +template +class Conv2DT +{ +public: + static constexpr int num_features_out = Conv1DStateless::computeNumFeaturesOut(num_features_in_t, kernel_size_feature_t, stride_t, valid_pad_t); + static constexpr auto in_size = num_filters_in_t * num_features_in_t; + static constexpr auto out_size = num_filters_out_t * num_features_out; + + using bias_type = std::array; + using output_type = std::array; + + static constexpr int receptive_field = 1 + (kernel_size_time_t - 1) * dilation_rate_t; + static constexpr int num_filters_in = num_filters_in_t; + static constexpr int num_features_in = num_features_in_t; + static constexpr int num_filters_out = num_filters_out_t; + static constexpr int kernel_size_time = kernel_size_time_t; + static constexpr int kernel_size_feature = kernel_size_feature_t; + static constexpr int dilation_rate = dilation_rate_t; + static constexpr int stride = stride_t; + static constexpr bool valid_pad = valid_pad_t; + + Conv2DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "conv2d"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Reset the layer's state */ + RTNEURAL_REALTIME void reset() + { + state_index = 0; + + for(int i = 0; i < receptive_field; i++) + { + std::fill(state[i].begin(), state[i].end(), (T)0); + } + }; + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T (&ins)[in_size]) noexcept + { + for(int i = 0; i < kernel_size_time; ++i) + { + int state_idx_to_use = (state_index + (receptive_field - 1) - i * dilation_rate) % receptive_field; + + std::fill(std::begin(conv1dLayers[i].outs), std::end(conv1dLayers[i].outs), (T)0); + conv1dLayers[i].forward(ins); + + for(int j = 0; j < state[state_idx_to_use].size(); ++j) + state[state_idx_to_use][j] += conv1dLayers[i].outs[j]; + } + + for(int i = 0; i < num_features_out; ++i) + { + for(int j = 0; j < num_filters_out; ++j) + { + outs[i * num_filters_out + j] = state[state_index][i * num_filters_out + j] + bias[j]; + } + } + + std::fill(state[state_index].begin(), state[state_index].end(), (T)0); + state_index = state_index == receptive_field - 1 ? 0 : state_index + 1; + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights [kernel_size_time][num_filters_out][num_filters_in][kernel_size_feature] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>>& inWeights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[num_filters_out] + */ + RTNEURAL_REALTIME void setBias(const std::vector& inBias); + + /** Returns the size of the convolution kernel (time axis). */ + RTNEURAL_REALTIME int getKernelSizeTime() const noexcept { return kernel_size_time_t; } + + /** Returns the size of the convolution kernel (feature axis). */ + RTNEURAL_REALTIME int getKernelSizeFeature() const noexcept { return kernel_size_feature_t; } + + /** Returns the convolution stride */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride_t; } + + /** Returns the convolution dilation rate */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate_t; } + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[num_filters_out_t * num_features_out]; + +private: + std::array, + kernel_size_time_t> + conv1dLayers; + + std::array state; + + int state_index = 0; + + alignas(RTNEURAL_DEFAULT_ALIGNMENT) bias_type bias; +}; + +} // RTNEURAL + +#endif // RTNEURAL_USE_STL +#endif // CONV2D_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d.tpp new file mode 100644 index 0000000..47304c3 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d.tpp @@ -0,0 +1,92 @@ +#include "conv2d.h" + +#if !RTNEURAL_USE_EIGEN && !RTNEURAL_USE_XSIMD + +namespace RTNEURAL_NAMESPACE +{ +template +Conv2D::Conv2D(int in_num_filters_in, int in_num_filters_out, int in_num_features_in, int in_kernel_size_time, int in_kernel_size_feature, + int in_dilation_rate, int in_stride, bool in_valid_pad) + : num_filters_in(in_num_filters_in) + , num_filters_out(in_num_filters_out) + , num_features_in(in_num_features_in) + , kernel_size_time(in_kernel_size_time) + , kernel_size_feature(in_kernel_size_feature) + , dilation_rate(in_dilation_rate) + , stride(in_stride) + , num_features_out(Conv1DStateless::computeNumFeaturesOut(in_num_features_in, in_kernel_size_feature, in_stride, in_valid_pad)) + , receptive_field(1 + (in_kernel_size_time - 1) * in_dilation_rate) // See "Dilated (atrous) convolution" note here: https://distill.pub/2019/computing-receptive-fields/ + , valid_pad(in_valid_pad) + , Layer(in_num_features_in * in_num_filters_in, Conv1DStateless::computeNumFeaturesOut(in_num_features_in, in_kernel_size_feature, in_stride, in_valid_pad) * in_num_filters_out) +{ + conv1dLayers.resize(kernel_size_time, Conv1DStateless(num_filters_in, num_features_in, num_filters_out, kernel_size_feature, stride, valid_pad)); + bias.resize(num_filters_out, (T)0); + + state.resize(receptive_field); + for(auto& stateMat : state) + { + stateMat.resize(num_filters_out * num_features_out, (T)0); + } +} + +template +Conv2D::Conv2D(std::initializer_list sizes) + : Conv2D(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3), *(sizes.begin() + 4), + *(sizes.begin() + 5), *(sizes.begin() + 6), *(sizes.begin() + 7)) +{ +} + +template +Conv2D::Conv2D(const Conv2D& other) + : Conv2D(other.num_filters_in, other.num_filters_out, other.num_features_in, other.kernel_size_time, other.kernel_size_feature, + other.dilation_rate, other.stride, other.valid_pad) +{ +} + +template +Conv2D& Conv2D::operator=(const Conv2D& other) +{ + return *this = Conv2D(other); +} + +template +void Conv2D::setWeights(const std::vector>>>& inWeights) +{ + for(int i = 0; i < kernel_size_time; i++) + { + conv1dLayers[i].setWeights(inWeights[i]); + } +} + +template +void Conv2D::setBias(const std::vector& inBias) +{ + std::copy(inBias.begin(), inBias.end(), bias.begin()); +} + +template +Conv2DT::Conv2DT() +{ +} + +template +void Conv2DT::setWeights(const std::vector>>>& inWeights) +{ + for(int i = 0; i < kernel_size_time_t; i++) + { + conv1dLayers[i].setWeights(inWeights[i]); + } +} + +template +void Conv2DT::setBias(const std::vector& inBias) +{ + std::copy(inBias.begin(), inBias.end(), bias.begin()); +} +} // RTNEURAL_NAMESPACE + +#endif // RTNEURAL_USE_STL diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_eigen.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_eigen.h new file mode 100644 index 0000000..9dd7ed7 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_eigen.h @@ -0,0 +1,247 @@ +#ifndef CONV2D_EIGEN_H_INCLUDED +#define CONV2D_EIGEN_H_INCLUDED + +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include "../conv1d_stateless/conv1d_stateless.h" +#include + +namespace RTNEURAL_NAMESPACE +{ +/** + * Dynamic implementation of a 2-dimensional convolution layer with no activation. + * + * @tparam T Type of the layer (float, double, int ...) + */ +template +class Conv2D : public Layer +{ +public: + /** + * @param in_num_filters_in number of input filters (channels) + * @param in_num_filters_out number of output filters (channels) + * @param in_num_features_in number of input features + * @param in_kernel_size_time size of the convolution kernel (time axis) + * @param in_kernel_size_feature size of the convolution kernel (feature axis) + * @param in_dilation_rate dilation_rate (time axis) + * @param in_stride convolution stride (feature axis) + * @param in_valid_pad whether the padding is "valid" or not ("same" otherwise) + */ + Conv2D(int in_num_filters_in, int in_num_filters_out, int in_num_features_in, int in_kernel_size_time, int in_kernel_size_feature, int in_dilation_rate, int in_stride, bool in_valid_pad); + Conv2D(std::initializer_list sizes); + Conv2D(const Conv2D& other); + Conv2D& operator=(const Conv2D& other); + virtual ~Conv2D() = default; + + /** Reset the layer's state */ + RTNEURAL_REALTIME void reset() override + { + state_index = 0; + + for(int i = 0; i < receptive_field; i++) + { + state[i].setZero(); + } + }; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "conv2d"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* output) noexcept override + { + auto inMatrix = Eigen::Map, + RTNeuralEigenAlignment>(input, num_filters_in, num_features_in); + + auto outMatrix = Eigen::Map, + RTNeuralEigenAlignment>(output, num_filters_out, num_features_out); + + for(int i = 0; i < kernel_size_time; i++) + { + int state_idx_to_use = (state_index + (receptive_field - 1) - i * dilation_rate) % receptive_field; + + conv1dLayers[i].forward(inMatrix.data(), state[state_idx_to_use].data()); + } + + outMatrix = state[state_index].colwise() + bias; + + state[state_index].setZero(); + state_index = state_index == receptive_field - 1 ? 0 : state_index + 1; + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[num_filters_out][num_filters_in][kernel_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>>& inWeights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[num_filters_out] + */ + RTNEURAL_REALTIME void setBias(const std::vector& inBias); + + /** Returns the size of the convolution kernel (time axis). */ + RTNEURAL_REALTIME int getKernelSizeTime() const noexcept { return kernel_size_time; } + + /** Returns the size of the convolution kernel (feature axis). */ + RTNEURAL_REALTIME int getKernelSizeFeature() const noexcept { return kernel_size_feature; } + + /** Returns the convolution stride (feature axis) */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride; } + + /** Returns the convolution dilation rate (time axis) */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate; } + + const int num_filters_in; + const int num_features_in; + const int num_filters_out; + const int kernel_size_time; + const int kernel_size_feature; + const int dilation_rate; + const int stride; + const int num_features_out; + const int receptive_field; + const bool valid_pad; + +private: + std::vector> conv1dLayers; + + std::vector> state; + + int state_index = 0; + + Eigen::Vector bias; +}; + +//==================================================== + +/** + * Static implementation of a 2-dimensional convolution layer with no activation. + * + * @tparam T Type of the layer (float, double, int ...) + * @tparam num_filters_in_t number of input filters (channels) + * @tparam num_filters_out_t number of output filters (channels) + * @tparam num_features_in_t number of input features + * @tparam kernel_size_time_t size of the convolution kernel (time axis) + * @tparam kernel_size_feature_t size of the convolution kernel (feature axis) + * @tparam dilation_rate_t dilation_rate (time axis) + * @tparam stride_t convolution stride (feature axis) + * @tparam valid_pad_t if true: pad is "valid". if false: pad is "same" + */ +template +class Conv2DT +{ +public: + static constexpr int num_features_out = Conv1DStateless::computeNumFeaturesOut(num_features_in_t, kernel_size_feature_t, stride_t, valid_pad_t); + static constexpr auto in_size = num_filters_in_t * num_features_in_t; + static constexpr auto out_size = num_filters_out_t * num_features_out; + + using bias_type = Eigen::Vector; + using input_type = Eigen::Matrix; + using input_type_flat = Eigen::Matrix; + using output_type = Eigen::Matrix; + using output_type_flat = Eigen::Matrix; + + static constexpr int receptive_field = 1 + (kernel_size_time_t - 1) * dilation_rate_t; + static constexpr int num_filters_in = num_filters_in_t; + static constexpr int num_features_in = num_features_in_t; + static constexpr int num_filters_out = num_filters_out_t; + static constexpr int kernel_size_time = kernel_size_time_t; + static constexpr int kernel_size_feature = kernel_size_feature_t; + static constexpr int dilation_rate = dilation_rate_t; + static constexpr int stride = stride_t; + static constexpr bool valid_pad = valid_pad_t; + + Conv2DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "conv2d"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Reset the layer's state */ + RTNEURAL_REALTIME void reset() + { + state_index = 0; + + for(int i = 0; i < receptive_field; i++) + { + state[i] = output_type::Zero(); + } + }; + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const input_type_flat& inMatrix) noexcept + { + const auto inMatrixReshaped = Eigen::Map(inMatrix.data()); + auto outMatrix = Eigen::Map(outs.data()); + + for(int i = 0; i < kernel_size_time_t; i++) + { + int state_idx_to_use = (state_index + (receptive_field - 1) - i * dilation_rate) % receptive_field; + + conv1dLayers[i].forward(inMatrixReshaped); + + state[state_idx_to_use] += Eigen::Map(conv1dLayers[i].outs); + } + + outMatrix = state[state_index].colwise() + bias; + + state[state_index].setZero(); + state_index = state_index == receptive_field - 1 ? 0 : state_index + 1; + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights [kernel_size_time][num_filters_out][num_filters_in][kernel_size_feature] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>>& inWeights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[num_filters_out] + */ + RTNEURAL_REALTIME void setBias(const std::vector& inBias); + + /** Returns the size of the convolution kernel (time axis). */ + RTNEURAL_REALTIME int getKernelSizeTime() const noexcept { return kernel_size_time_t; } + + /** Returns the size of the convolution kernel (feature axis). */ + RTNEURAL_REALTIME int getKernelSizeFeature() const noexcept { return kernel_size_feature_t; } + + /** Returns the convolution stride */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride_t; } + + /** Returns the convolution dilation rate */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate_t; } + + Eigen::Map outs; + +private: + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[num_filters_out_t * num_features_out]; + + std::array, + kernel_size_time_t> + conv1dLayers; + + std::array state; + + int state_index = 0; + + bias_type bias; +}; + +} // RTNEURAL + +#endif // CONV2D_EIGEN_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_eigen.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_eigen.tpp new file mode 100644 index 0000000..7ede459 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_eigen.tpp @@ -0,0 +1,91 @@ +#include "conv2d_eigen.h" + +namespace RTNEURAL_NAMESPACE +{ +template +Conv2D::Conv2D(int in_num_filters_in, int in_num_filters_out, int in_num_features_in, int in_kernel_size_time, int in_kernel_size_feature, + int in_dilation_rate, int in_stride, bool in_valid_pad) + : num_filters_in(in_num_filters_in) + , num_filters_out(in_num_filters_out) + , num_features_in(in_num_features_in) + , kernel_size_time(in_kernel_size_time) + , kernel_size_feature(in_kernel_size_feature) + , dilation_rate(in_dilation_rate) + , stride(in_stride) + , num_features_out(Conv1DStateless::computeNumFeaturesOut(in_num_features_in, in_kernel_size_feature, in_stride, in_valid_pad)) + , receptive_field(1 + (in_kernel_size_time - 1) * in_dilation_rate) // See "Dilated (atrous) convolution" note here: https://distill.pub/2019/computing-receptive-fields/ + , valid_pad(in_valid_pad) + , Layer(in_num_features_in * in_num_filters_in, Conv1DStateless::computeNumFeaturesOut(in_num_features_in, in_kernel_size_feature, in_stride, in_valid_pad) * in_num_filters_out) +{ + conv1dLayers.resize(kernel_size_time, Conv1DStateless(num_filters_in, num_features_in, num_filters_out, kernel_size_feature, stride, valid_pad)); + bias = Eigen::Vector::Zero(num_filters_out); + + state.resize(receptive_field, Eigen::Matrix::Zero(num_filters_out, num_features_out)); +} + +template +Conv2D::Conv2D(std::initializer_list sizes) + : Conv2D(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3), *(sizes.begin() + 4), + *(sizes.begin() + 5), *(sizes.begin() + 6), *(sizes.begin() + 7)) +{ +} + +template +Conv2D::Conv2D(const Conv2D& other) + : Conv2D(other.num_filters_in, other.num_filters_out, other.num_features_in, other.kernel_size_time, other.kernel_size_feature, + other.dilation_rate, other.stride, other.valid_pad) +{ +} + +template +Conv2D& Conv2D::operator=(const Conv2D& other) +{ + return *this = Conv2D(other); +} + +template +void Conv2D::setWeights(const std::vector>>>& inWeights) +{ + for(int i = 0; i < kernel_size_time; i++) + { + conv1dLayers[i].setWeights(inWeights[i]); + } +} + +template +void Conv2D::setBias(const std::vector& inBias) +{ + for(int i = 0; i < num_filters_out; i++) + { + bias(i) = inBias[i]; + } +} + +template +Conv2DT::Conv2DT() + : outs(outs_internal) +{ +} + +template +void Conv2DT::setWeights(const std::vector>>>& inWeights) +{ + for(int i = 0; i < kernel_size_time_t; i++) + { + conv1dLayers[i].setWeights(inWeights[i]); + } +} + +template +void Conv2DT::setBias(const std::vector& inBias) +{ + for(int i = 0; i < num_filters_out_t; i++) + { + bias(i) = inBias[i]; + } +} +} // RTNEURAL_NAMESPACE \ No newline at end of file diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_xsimd.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_xsimd.h new file mode 100644 index 0000000..5c783ff --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_xsimd.h @@ -0,0 +1,252 @@ +#ifndef RTNEURAL_CONV2D_XSIMD_H +#define RTNEURAL_CONV2D_XSIMD_H + +#include "../Layer.h" +#include "../config.h" +#include "../conv1d_stateless/conv1d_stateless.h" +#include + +namespace RTNEURAL_NAMESPACE +{ +/** + * Dynamic implementation of a 2-dimensional convolution layer with no activation. + * + * @tparam T Type of the layer (float, double, int ...) + */ +template +class Conv2D : public Layer +{ +public: + /** + * @param in_num_filters_in number of input filters (channels) + * @param in_num_filters_out number of output filters (channels) + * @param in_num_features_in number of input features + * @param in_kernel_size_time size of the convolution kernel (time axis) + * @param in_kernel_size_feature size of the convolution kernel (feature axis) + * @param in_dilation_rate dilation_rate (time axis) + * @param in_stride convolution stride (feature axis) + * @param in_valid_pad whether the padding is "valid" or not ("same" otherwise) + */ + Conv2D(int in_num_filters_in, int in_num_filters_out, int in_num_features_in, int in_kernel_size_time, int in_kernel_size_feature, int in_dilation_rate, int in_stride, bool in_valid_pad); + Conv2D(std::initializer_list sizes); + Conv2D(const Conv2D& other); + Conv2D& operator=(const Conv2D& other); + virtual ~Conv2D() = default; + + /** Reset the layer's state */ + RTNEURAL_REALTIME void reset() override + { + state_index = 0; + + for(int i = 0; i < receptive_field; i++) + { + std::fill(state[i].begin(), state[i].end(), (T)0); + } + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "conv2d"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* output) noexcept override + { + for(int i = 0; i < kernel_size_time; ++i) + { + int state_idx_to_use = (state_index + (receptive_field - 1) - i * dilation_rate) % receptive_field; + + conv1dLayers[i].forward(input, state[state_idx_to_use].data()); + } + + for(int i = 0; i < num_features_out; ++i) + { + const auto* stateCol = state[state_index].data() + i * num_filters_out; + auto* outCol = output + i * num_filters_out; + xsimd::transform(stateCol, stateCol + num_filters_out, bias.begin(), outCol, [](auto a, auto b) + { return a + b; }); + } + + std::fill(state[state_index].begin(), state[state_index].end(), (T)0); + state_index = state_index == receptive_field - 1 ? 0 : state_index + 1; + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights[num_filters_out][num_filters_in][kernel_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>>& inWeights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[num_filters_out] + */ + RTNEURAL_REALTIME void setBias(const std::vector& inBias); + + /** Returns the size of the convolution kernel (time axis). */ + RTNEURAL_REALTIME int getKernelSizeTime() const noexcept { return kernel_size_time; } + + /** Returns the size of the convolution kernel (feature axis). */ + RTNEURAL_REALTIME int getKernelSizeFeature() const noexcept { return kernel_size_feature; } + + /** Returns the convolution stride (feature axis) */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride; } + + /** Returns the convolution dilation rate (time axis) */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate; } + + const int num_filters_in; + const int num_features_in; + const int num_filters_out; + const int kernel_size_time; + const int kernel_size_feature; + const int dilation_rate; + const int stride; + const int num_features_out; + const int receptive_field; + const bool valid_pad; + +private: + std::vector> conv1dLayers; + + std::vector>> state; + + int state_index = 0; + + std::vector> bias; +}; + +//==================================================== + +/** + * Static implementation of a 2-dimensional convolution layer with no activation. + * + * @tparam T Type of the layer (float, double, int ...) + * @tparam num_filters_in_t number of input filters (channels) + * @tparam num_filters_out_t number of output filters (channels) + * @tparam num_features_in_t number of input features + * @tparam kernel_size_time_t size of the convolution kernel (time axis) + * @tparam kernel_size_feature_t size of the convolution kernel (feature axis) + * @tparam dilation_rate_t dilation_rate (time axis) + * @tparam stride_t convolution stride (feature axis) + * @tparam valid_pad_t if true: pad is "valid". if false: pad is "same" + */ +template +class Conv2DT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_num_filters_in = ceil_div(num_filters_in_t, v_size); + static constexpr auto v_num_filters_out = ceil_div(num_filters_out_t, v_size); + static constexpr auto v_in_size = v_num_filters_in * num_features_in_t; + +public: + static constexpr int num_features_out = Conv1DStateless::computeNumFeaturesOut(num_features_in_t, kernel_size_feature_t, stride_t, valid_pad_t); + static constexpr auto in_size = num_filters_in_t * num_features_in_t; + static constexpr auto out_size = num_filters_out_t * num_features_out; + static constexpr auto v_out_size = v_num_filters_out * num_features_out; + + using output_type = std::array; + + static constexpr int receptive_field = 1 + (kernel_size_time_t - 1) * dilation_rate_t; + static constexpr int num_filters_in = num_filters_in_t; + static constexpr int num_features_in = num_features_in_t; + static constexpr int num_filters_out = num_filters_out_t; + static constexpr int kernel_size_time = kernel_size_time_t; + static constexpr int kernel_size_feature = kernel_size_feature_t; + static constexpr int dilation_rate = dilation_rate_t; + static constexpr int stride = stride_t; + static constexpr bool valid_pad = valid_pad_t; + + Conv2DT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "conv2d"; } + + /** Returns false since convolution is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Reset the layer's state */ + RTNEURAL_REALTIME void reset() + { + state_index = 0; + + for(int i = 0; i < receptive_field; i++) + { + std::fill(state[i].begin(), state[i].end(), (T)0); + } + } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const v_type (&ins)[v_in_size]) noexcept + { + for(int i = 0; i < kernel_size_time; ++i) + { + int state_idx_to_use = (state_index + (receptive_field - 1) - i * dilation_rate) % receptive_field; + + std::fill(std::begin(conv1dLayers[i].outs), std::end(conv1dLayers[i].outs), (T)0); + conv1dLayers[i].forward(ins); + + for(int j = 0; j < state[state_idx_to_use].size(); ++j) + state[state_idx_to_use][j] += conv1dLayers[i].outs[j]; + } + + for(int i = 0; i < num_features_out; ++i) + { + for(int j = 0; j < v_num_filters_out; ++j) + { + outs[i * v_num_filters_out + j] = state[state_index][i * v_num_filters_out + j] + bias[j]; + } + } + + std::fill(state[state_index].begin(), state[state_index].end(), (T)0); + state_index = state_index == receptive_field - 1 ? 0 : state_index + 1; + } + + /** + * Sets the layer weights. + * + * The weights vector must have size weights [kernel_size_time][num_filters_out][num_filters_in][kernel_size_feature] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>>>& inWeights); + + /** + * Sets the layer biases. + * + * The bias vector must have size bias[num_filters_out] + */ + RTNEURAL_REALTIME void setBias(const std::vector& inBias); + + /** Returns the size of the convolution kernel (time axis). */ + RTNEURAL_REALTIME int getKernelSizeTime() const noexcept { return kernel_size_time_t; } + + /** Returns the size of the convolution kernel (feature axis). */ + RTNEURAL_REALTIME int getKernelSizeFeature() const noexcept { return kernel_size_feature_t; } + + /** Returns the convolution stride */ + RTNEURAL_REALTIME int getStride() const noexcept { return stride_t; } + + /** Returns the convolution dilation rate */ + RTNEURAL_REALTIME int getDilationRate() const noexcept { return dilation_rate_t; } + + v_type outs[v_out_size]; + +private: + std::array, + kernel_size_time_t> + conv1dLayers; + + std::array state; + + int state_index = 0; + + v_type bias[v_num_filters_out]; +}; + +} // RTNEURAL + +#endif // RTNEURAL_CONV2D_XSIMD_H diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_xsimd.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_xsimd.tpp new file mode 100644 index 0000000..c6fbbce --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/conv2d/conv2d_xsimd.tpp @@ -0,0 +1,88 @@ +#include "conv2d_xsimd.h" + +namespace RTNEURAL_NAMESPACE +{ +template +Conv2D::Conv2D(int in_num_filters_in, int in_num_filters_out, int in_num_features_in, int in_kernel_size_time, int in_kernel_size_feature, + int in_dilation_rate, int in_stride, bool in_valid_pad) + : num_filters_in(in_num_filters_in) + , num_filters_out(in_num_filters_out) + , num_features_in(in_num_features_in) + , kernel_size_time(in_kernel_size_time) + , kernel_size_feature(in_kernel_size_feature) + , dilation_rate(in_dilation_rate) + , stride(in_stride) + , num_features_out(Conv1DStateless::computeNumFeaturesOut(in_num_features_in, in_kernel_size_feature, in_stride, in_valid_pad)) + , receptive_field(1 + (in_kernel_size_time - 1) * in_dilation_rate) // See "Dilated (atrous) convolution" note here: https://distill.pub/2019/computing-receptive-fields/ + , valid_pad(in_valid_pad) + , Layer(in_num_features_in * in_num_filters_in, Conv1DStateless::computeNumFeaturesOut(in_num_features_in, in_kernel_size_feature, in_stride, in_valid_pad) * in_num_filters_out) +{ + conv1dLayers.resize(kernel_size_time, Conv1DStateless(num_filters_in, num_features_in, num_filters_out, kernel_size_feature, stride, valid_pad)); + bias.resize(num_filters_out, (T)0); + + state.resize(receptive_field); + for(auto& stateMat : state) + { + stateMat.resize(num_filters_out * num_features_out, (T)0); + } +} + +template +Conv2D::Conv2D(std::initializer_list sizes) + : Conv2D(*sizes.begin(), *(sizes.begin() + 1), *(sizes.begin() + 2), *(sizes.begin() + 3), *(sizes.begin() + 4), + *(sizes.begin() + 5), *(sizes.begin() + 6), *(sizes.begin() + 7)) +{ +} + +template +Conv2D::Conv2D(const Conv2D& other) + : Conv2D(other.num_filters_in, other.num_filters_out, other.num_features_in, other.kernel_size_time, other.kernel_size_feature, + other.dilation_rate, other.stride, other.valid_pad) +{ +} + +template +Conv2D& Conv2D::operator=(const Conv2D& other) +{ + return *this = Conv2D(other); +} + +template +void Conv2D::setWeights(const std::vector>>>& inWeights) +{ + for(int i = 0; i < kernel_size_time; i++) + { + conv1dLayers[i].setWeights(inWeights[i]); + } +} + +template +void Conv2D::setBias(const std::vector& inBias) +{ + std::copy(inBias.begin(), inBias.end(), bias.begin()); +} + +template +Conv2DT::Conv2DT() +{ +} + +template +void Conv2DT::setWeights(const std::vector>>>& inWeights) +{ + for(int i = 0; i < kernel_size_time_t; i++) + { + conv1dLayers[i].setWeights(inWeights[i]); + } +} + +template +void Conv2DT::setBias(const std::vector& inBias) +{ + std::copy(inBias.begin(), inBias.end(), reinterpret_cast(std::begin(bias))); +} +} // RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense.h new file mode 100644 index 0000000..9ab5973 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense.h @@ -0,0 +1,254 @@ +#ifndef DENSE_H_INCLUDED +#define DENSE_H_INCLUDED + +#include +#include +#include + +#if RTNEURAL_USE_EIGEN +#include "dense_eigen.h" +#elif RTNEURAL_USE_XSIMD +#include "dense_xsimd.h" +#else +#include "../Layer.h" +#include "../config.h" + +namespace RTNEURAL_NAMESPACE +{ + +#ifndef DOXYGEN +/** Single-output dense layer used internally */ +template +class Dense1 +{ +public: + explicit Dense1(int in_size) + : in_size(in_size) + { + weights = new T[in_size]; + } + + ~Dense1() { delete[] weights; } + + RTNEURAL_REALTIME inline T forward(const T* input) noexcept + { + return std::inner_product(weights, weights + in_size, input, (T)0) + bias; + } + + RTNEURAL_REALTIME void setWeights(const T* newWeights) + { + for(int i = 0; i < in_size; ++i) + weights[i] = newWeights[i]; + } + + RTNEURAL_REALTIME void setBias(T b) { bias = b; } + + RTNEURAL_REALTIME T getWeight(int i) const noexcept { return weights[i]; } + + RTNEURAL_REALTIME T getBias() const noexcept { return bias; } + +private: + const int in_size; + T bias; + + T* weights; +}; +#endif // DOXYGEN + +/** + * Dynamic implementation of a fully-connected (dense) layer, + * with no activation. + */ +template +class Dense final : public Layer +{ +public: + /** Constructs a dense layer for a given input and output size. */ + Dense(int in_size, int out_size) + : Layer(in_size, out_size) + { + subLayers = new Dense1*[out_size]; + for(int i = 0; i < out_size; ++i) + subLayers[i] = new Dense1(in_size); + } + + Dense(std::initializer_list sizes) + : Dense(*sizes.begin(), *(sizes.begin() + 1)) + { + } + + Dense(const Dense& other) + : Dense(other.in_size, other.out_size) + { + } + + Dense& operator=(const Dense& other) + { + return *this = Dense(other); + } + + virtual ~Dense() + { + for(int i = 0; i < Layer::out_size; ++i) + delete subLayers[i]; + + delete[] subLayers; + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "dense"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + for(int i = 0; i < Layer::out_size; ++i) + out[i] = subLayers[i]->forward(input); + } + + /** + * Sets the layer weights from a given vector. + * + * The dimension of the weights vector must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>& newWeights) + { + for(int i = 0; i < Layer::out_size; ++i) + subLayers[i]->setWeights(newWeights[i].data()); + } + + /** + * Sets the layer weights from a given array. + * + * The dimension of the weights array must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(T** newWeights) + { + for(int i = 0; i < Layer::out_size; ++i) + subLayers[i]->setWeights(newWeights[i]); + } + + /** + * Sets the layer bias from a given array of size + * bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const T* b) + { + for(int i = 0; i < Layer::out_size; ++i) + subLayers[i]->setBias(b[i]); + } + + /** Returns the weights value at the given indices. */ + RTNEURAL_REALTIME T getWeight(int i, int k) const noexcept + { + return subLayers[i]->getWeight(k); + } + + /** Returns the bias value at the given index. */ + RTNEURAL_REALTIME T getBias(int i) const noexcept { return subLayers[i]->getBias(); } + +private: + Dense1** subLayers; +}; + +//==================================================== +/** + * Static implementation of a fully-connected (dense) layer, + * with no activation. + */ +template +class DenseT +{ + static constexpr auto weights_size = in_sizet * out_sizet; + +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + + DenseT() + { + for(int i = 0; i < weights_size; ++i) + weights[i] = (T)0.0; + + for(int i = 0; i < out_size; ++i) + bias[i] = (T)0.0; + + for(int i = 0; i < out_size; ++i) + outs[i] = (T)0.0; + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "dense"; } + + /** Returns false since dense is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Reset is a no-op, since Dense does not have state. */ + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T (&ins)[in_size]) noexcept + { + for(int i = 0; i < out_size; ++i) + outs[i] = std::inner_product(ins, ins + in_size, &weights[i * in_size], (T)0) + bias[i]; + } + + /** + * Sets the layer weights from a given vector. + * + * The dimension of the weights vector must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>& newWeights) + { + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < in_size; ++k) + { + auto idx = i * in_size + k; + weights[idx] = newWeights[i][k]; + } + } + } + + /** + * Sets the layer weights from a given vector. + * + * The dimension of the weights array must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(T** newWeights) + { + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < in_size; ++k) + { + auto idx = i * in_size + k; + weights[idx] = newWeights[i][k]; + } + } + } + + /** + * Sets the layer bias from a given array of size + * bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const T* b) + { + for(int i = 0; i < out_size; ++i) + bias[i] = b[i]; + } + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + +private: + T bias[out_size]; + T weights[weights_size]; +}; + +} // namespace RTNEURAL_NAMESPACE + +#endif // RTNEURAL_USE_STL + +#endif // DENSE_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense_eigen.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense_eigen.h new file mode 100644 index 0000000..18b540e --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense_eigen.h @@ -0,0 +1,210 @@ +#ifndef DENSEEIGEN_H_INCLUDED +#define DENSEEIGEN_H_INCLUDED + +#include "../Layer.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ + +/** + * Dynamic implementation of a fully-connected (dense) layer, + * with no activation. + */ +template +class Dense : public Layer +{ +public: + /** Constructs a dense layer for a given input and output size. */ + Dense(int in_size, int out_size) + : Layer(in_size, out_size) + { + weights = Eigen::Matrix::Zero(out_size, in_size + 1); + + inVec = Eigen::Matrix::Zero(in_size + 1); + outVec = Eigen::Matrix::Zero(out_size); + + inVec(in_size, 0) = (T)1; + } + + Dense(std::initializer_list sizes) + : Dense(*sizes.begin(), *(sizes.begin() + 1)) + { + } + + Dense(const Dense& other) + : Dense(other.in_size, other.out_size) + { + } + + Dense& operator=(const Dense& other) + { + return *this = Dense(other); + } + + virtual ~Dense() = default; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "dense"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + for(int i = 0; i < Layer::in_size; ++i) + inVec(i, 0) = input[i]; + + /** + * out = | w b | * | input | + * | 1 | + */ + outVec.noalias() = weights * inVec; + + for(int i = 0; i < Layer::out_size; ++i) + out[i] = outVec(i, 0); + } + + /** + * Sets the layer weights from a given vector. + * + * The dimension of the weights vector must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>& newWeights) + { + for(int i = 0; i < Layer::out_size; ++i) + for(int k = 0; k < Layer::in_size; ++k) + weights(i, k) = newWeights[i][k]; + } + + /** + * Sets the layer weights from a given array. + * + * The dimension of the weights array must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(T** newWeights) + { + for(int i = 0; i < Layer::out_size; ++i) + for(int k = 0; k < Layer::in_size; ++k) + weights(i, k) = newWeights[i][k]; + } + + /** + * Sets the layer bias from a given array of size + * bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const T* b) + { + for(int i = 0; i < Layer::out_size; ++i) + weights(i, Layer::in_size) = b[i]; + } + + /** Returns the weights value at the given indices. */ + RTNEURAL_REALTIME T getWeight(int i, int k) const noexcept { return weights(i, k); } + + /** Returns the bias value at the given index. */ + RTNEURAL_REALTIME T getBias(int i) const noexcept { return weights(i, Layer::in_size); } + +private: + Eigen::Matrix weights; + + Eigen::Matrix inVec; + Eigen::Matrix outVec; +}; + +//==================================================== +/** + * Static implementation of a fully-connected (dense) layer, + * with no activation. + */ +template +class DenseT +{ + using out_vec_type = Eigen::Matrix; + using in_vec_type = Eigen::Matrix; + using mat_type = Eigen::Matrix; + +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + + DenseT() + : outs(outs_internal) + { + weights = mat_type::Zero(); + ins_internal = in_vec_type::Zero(); + ins_internal(in_size, 0) = (T)1; + outs = out_vec_type::Zero(); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "dense"; } + + /** Returns false since dense is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Reset is a no-op, since Dense does not have state. */ + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const Eigen::Matrix& ins) noexcept + { + for(int i = 0; i < in_size; ++i) + ins_internal(i, 0) = ins(i, 0); + + /** + * out = | w b | * | input | + * | 1 | + */ + outs.noalias() = weights * ins_internal; + } + + /** + * Sets the layer weights from a given vector. + * + * The dimension of the weights vector must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>& newWeights) + { + for(int i = 0; i < out_size; ++i) + for(int k = 0; k < in_size; ++k) + weights(i, k) = newWeights[i][k]; + } + + /** + * Sets the layer weights from a given vector. + * + * The dimension of the weights array must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(T** newWeights) + { + for(int i = 0; i < out_size; ++i) + for(int k = 0; k < in_size; ++k) + weights(i, k) = newWeights[i][k]; + } + + /** + * Sets the layer bias from a given array of size + * bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const T* b) + { + for(int i = 0; i < out_size; ++i) + weights(i, in_size) = b[i]; + } + + Eigen::Map outs; + +private: + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + in_vec_type ins_internal; + + mat_type weights; +}; + +} // namespace RTNEURAL_NAMESPACE + +#endif // DENSEEIGEN_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense_xsimd.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense_xsimd.h new file mode 100644 index 0000000..73c904b --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/dense/dense_xsimd.h @@ -0,0 +1,389 @@ +#ifndef DENSEXSIMD_H_INCLUDED +#define DENSEXSIMD_H_INCLUDED + +#include "../Layer.h" +#include "../config.h" +#include + +namespace RTNEURAL_NAMESPACE +{ + +/** + * Dynamic implementation of a fully-connected (dense) layer, + * with no activation. + */ +template +class Dense : public Layer +{ +public: + /** Constructs a dense layer for a given input and output size. */ + Dense(int in_size, int out_size) + : Layer(in_size, out_size) + { + prod.resize(in_size, (T)0); + weights = vec2_type(out_size, vec_type(in_size, (T)0)); + + bias.resize(out_size, (T)0); + sums.resize(out_size, (T)0); + } + + Dense(std::initializer_list sizes) + : Dense(*sizes.begin(), *(sizes.begin() + 1)) + { + } + + Dense(const Dense& other) + : Dense(other.in_size, other.out_size) + { + } + + Dense& operator=(const Dense& other) + { + return *this = Dense(other); + } + + virtual ~Dense() = default; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "dense"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* out) noexcept override + { + for(int l = 0; l < Layer::out_size; ++l) + { + xsimd::transform(input, &input[Layer::in_size], weights[l].data(), prod.data(), + [](auto const& a, auto const& b) + { return a * b; }); + + auto sum = xsimd::reduce(prod.begin(), prod.begin() + Layer::in_size, (T)0); + out[l] = sum + bias[l]; + } + } + + /** + * Sets the layer weights from a given vector. + * + * The dimension of the weights vector must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>& newWeights) + { + for(int i = 0; i < Layer::out_size; ++i) + for(int k = 0; k < Layer::in_size; ++k) + weights[i][k] = newWeights[i][k]; + } + + /** + * Sets the layer weights from a given array. + * + * The dimension of the weights array must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(T** newWeights) + { + for(int i = 0; i < Layer::out_size; ++i) + for(int k = 0; k < Layer::in_size; ++k) + weights[i][k] = newWeights[i][k]; + } + + /** + * Sets the layer bias from a given array of size + * bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const T* b) + { + for(int i = 0; i < Layer::out_size; ++i) + bias[i] = b[i]; + } + + /** Returns the weights value at the given indices. */ + RTNEURAL_REALTIME T getWeight(int i, int k) const noexcept { return weights[i][k]; } + + /** Returns the bias value at the given index. */ + RTNEURAL_REALTIME T getBias(int i) const noexcept { return bias[i]; } + +private: + using vec_type = std::vector>; + using vec2_type = std::vector; + + vec_type bias; + vec2_type weights; + vec_type prod; + vec_type sums; +}; + +//==================================================== +/** + * Static implementation of a fully-connected (dense) layer, + * with no activation. + */ +template +class DenseT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_in_size = ceil_div(in_sizet, v_size); + static constexpr auto v_out_size = ceil_div(out_sizet, v_size); + +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + + DenseT() + { + for(int i = 0; i < v_out_size; ++i) + for(int k = 0; k < in_size; ++k) + weights[k][i] = v_type((T)0.0); + + for(int i = 0; i < v_out_size; ++i) + bias[i] = v_type((T)0.0); + + for(int i = 0; i < v_out_size; ++i) + outs[i] = v_type((T)0.0); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "dense"; } + + /** Returns false since dense is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Reset is a no-op, since Dense does not have state. */ + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const v_type (&ins)[v_in_size]) noexcept + { + static constexpr auto v_size_inner = std::min(v_size, in_size); + + for(int i = 0; i < v_out_size; ++i) + outs[i] = bias[i]; + + T scalar_in alignas(RTNEURAL_DEFAULT_ALIGNMENT)[v_size] { (T)0 }; + for(int k = 0; k < v_in_size; ++k) + { + ins[k].store_aligned(scalar_in); + for(int i = 0; i < v_out_size; ++i) + { + for(int j = 0; j < v_size_inner; ++j) + outs[i] += scalar_in[j] * weights[k * v_size + j][i]; + } + } + } + + /** + * Sets the layer weights from a given vector. + * + * The dimension of the weights vector must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>& newWeights) + { + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < in_size; ++k) + { + weights[k][i / v_size] = set_value(weights[k][i / v_size], i % v_size, newWeights[i][k]); + } + } + } + + /** + * Sets the layer weights from a given vector. + * + * The dimension of the weights array must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(T** newWeights) + { + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < in_size; ++k) + { + weights[k][i / v_size] = set_value(weights[k][i / v_size], i % v_size, newWeights[i][k]); + } + } + } + + /** + * Sets the layer bias from a given array of size + * bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const T* b) + { + for(int i = 0; i < out_size; ++i) + bias[i / v_size] = set_value(bias[i / v_size], i % v_size, b[i]); + } + + v_type outs[v_out_size]; + +private: + v_type bias[v_out_size]; + v_type weights[in_size][v_out_size]; +}; + +/** + * Static implementation of a fully-connected (dense) layer, + * optimized for out_size=1. + */ +template +class DenseT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_in_size = ceil_div(in_sizet, v_size); + +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = 1; + + DenseT() + { + for(int i = 0; i < v_in_size; ++i) + weights[i] = v_type((T)0.0); + + outs[0] = v_type((T)0.0); + } + + std::string getName() const noexcept { return "dense"; } + constexpr bool isActivation() const noexcept { return false; } + + RTNEURAL_REALTIME void reset() { } + + RTNEURAL_REALTIME inline void forward(const v_type (&ins)[v_in_size]) noexcept + { + v_type y {}; + for(int k = 0; k < v_in_size; ++k) + y += ins[k] * weights[k]; + + outs[0] = v_type(xsimd::reduce_add(y) + bias); + } + + RTNEURAL_REALTIME void setWeights(const std::vector>& newWeights) + { + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < in_size; ++k) + { + auto idx = k / v_size; + weights[idx] = set_value(weights[idx], k % v_size, newWeights[i][k]); + } + } + } + + RTNEURAL_REALTIME void setWeights(T** newWeights) + { + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < in_size; ++k) + { + auto idx = k / v_size; + weights[idx] = set_value(weights[idx], k % v_size, newWeights[i][k]); + } + } + } + + RTNEURAL_REALTIME void setBias(const T* b) + { + bias = b[0]; + } + + v_type outs[1]; + +private: + T bias; + v_type weights[v_in_size]; +}; + +/** + * Static implementation of a fully-connected (dense) layer, + * optimized for in_size=1. + */ +template +class DenseT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_out_size = ceil_div(out_sizet, v_size); + +public: + static constexpr auto in_size = 1; + static constexpr auto out_size = out_sizet; + + DenseT() + { + for(int i = 0; i < v_out_size; ++i) + weights[i] = v_type((T)0.0); + + for(int i = 0; i < v_out_size; ++i) + bias[i] = v_type((T)0.0); + + for(int i = 0; i < v_out_size; ++i) + outs[i] = v_type((T)0.0); + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "dense"; } + + /** Returns false since dense is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Reset is a no-op, since Dense does not have state. */ + RTNEURAL_REALTIME void reset() { } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const v_type (&ins)[1]) noexcept + { + for(int i = 0; i < v_out_size; ++i) + outs[i] = bias[i]; + + const auto in = ins[0].get(0); + for(int i = 0; i < v_out_size; ++i) + outs[i] += in * weights[i]; + } + + /** + * Sets the layer weights from a given vector. + * + * The dimension of the weights vector must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(const std::vector>& newWeights) + { + for(int i = 0; i < out_size; ++i) + weights[i / v_size] = set_value(weights[i / v_size], i % v_size, newWeights[i][0]); + } + + /** + * Sets the layer weights from a given vector. + * + * The dimension of the weights array must be + * weights[out_size][in_size] + */ + RTNEURAL_REALTIME void setWeights(T** newWeights) + { + for(int i = 0; i < out_size; ++i) + weights[i / v_size] = set_value(weights[i / v_size], i % v_size, newWeights[i][0]); + } + + /** + * Sets the layer bias from a given array of size + * bias[out_size] + */ + RTNEURAL_REALTIME void setBias(const T* b) + { + for(int i = 0; i < out_size; ++i) + bias[i / v_size] = set_value(bias[i / v_size], i % v_size, b[i]); + } + + v_type outs[v_out_size]; + +private: + v_type bias[v_out_size]; + v_type weights[v_out_size]; +}; + +} // namespace RTNEURAL_NAMESPACE + +#endif // DENSEXSIMD_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru.h new file mode 100644 index 0000000..0e30040 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru.h @@ -0,0 +1,357 @@ +#ifndef GRU_H_INCLUDED +#define GRU_H_INCLUDED + +#include + +#if RTNEURAL_USE_EIGEN +#include "gru_eigen.h" +#include "gru_eigen.tpp" +#elif RTNEURAL_USE_XSIMD +#include "gru_xsimd.h" +#include "gru_xsimd.tpp" +#else +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include "../maths/maths_stl.h" +#include + +namespace RTNEURAL_NAMESPACE +{ + +/** + * Dynamic implementation of a gated recurrent unit (GRU) layer + * with tanh activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's GRU implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class GRULayer final : public Layer +{ +public: + /** Constructs a GRU layer for a given input and output size. */ + GRULayer(int in_size, int out_size); + GRULayer(std::initializer_list sizes); + GRULayer(const GRULayer& other); + GRULayer& operator=(const GRULayer& other); + virtual ~GRULayer(); + + /** Resets the state of the GRU. */ + RTNEURAL_REALTIME void reset() override { std::fill(ht1, ht1 + Layer::out_size, (T)0); } + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "gru"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override + { + for(int i = 0; i < Layer::out_size; ++i) + { + zVec[i] = MathsProvider::sigmoid(vMult(zWeights.W[i], input, Layer::in_size) + vMult(zWeights.U[i], ht1, Layer::out_size) + zWeights.b[0][i] + zWeights.b[1][i]); + rVec[i] = MathsProvider::sigmoid(vMult(rWeights.W[i], input, Layer::in_size) + vMult(rWeights.U[i], ht1, Layer::out_size) + rWeights.b[0][i] + rWeights.b[1][i]); + cVec[i] = MathsProvider::tanh(vMult(cWeights.W[i], input, Layer::in_size) + rVec[i] * (vMult(cWeights.U[i], ht1, Layer::out_size) + cWeights.b[1][i]) + cWeights.b[0][i]); + h[i] = ((T)1 - zVec[i]) * cVec[i] + zVec[i] * ht1[i]; + } + + std::copy(h, h + Layer::out_size, ht1); + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][3 * out_size] + */ + RTNEURAL_REALTIME void setWVals(T** wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][3 * out_size] + */ + RTNEURAL_REALTIME void setUVals(T** uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[2][3 * out_size] + */ + RTNEURAL_REALTIME void setBVals(T** bVals); + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][3 * out_size] + */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][3 * out_size] + */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[2][3 * out_size] + */ + RTNEURAL_REALTIME void setBVals(const std::vector>& bVals); + + /** Returns the kernel weight for the given indices. */ + RTNEURAL_REALTIME T getWVal(int i, int k) const noexcept; + + /** Returns the recurrent weight for the given indices. */ + RTNEURAL_REALTIME T getUVal(int i, int k) const noexcept; + + /** Returns the bias value for the given indices. */ + RTNEURAL_REALTIME T getBVal(int i, int k) const noexcept; + +protected: + T* ht1; + + /** Struct to hold layer weights (used internally) */ + struct WeightSet + { + WeightSet(int in_size, int out_size); + ~WeightSet(); + + T** W; // kernel weights + T** U; // recurrent weights + T** b; // bias + const int out_size; + }; + + WeightSet zWeights; + WeightSet rWeights; + WeightSet cWeights; + + T* zVec; + T* rVec; + T* cVec; + + static constexpr int kNumBiasLayers { 2 }; +}; + +//==================================================== +/** + * Static implementation of a gated recurrent unit (GRU) layer + * with tanh activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's GRU implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class GRULayerT +{ +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + + GRULayerT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "gru"; } + + /** Returns false since GRU is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Prepares the GRU to process with a given delay length. */ + template + std::enable_if_t + prepare(int delaySamples); + + /** Prepares the GRU to process with a given delay length. */ + template + std::enable_if_t + prepare(T delaySamples); + + /** Resets the state of the GRU. */ + RTNEURAL_REALTIME void reset(); + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if<(N > 1), void>::type + forward(const T (&ins)[in_size]) noexcept + { + // compute zt + recurrent_mat_mul(outs, Uz, zt); + kernel_mat_mul(ins, Wz, kernel_outs); + for(int i = 0; i < out_size; ++i) + zt[i] = MathsProvider::sigmoid(zt[i] + bz[i] + kernel_outs[i]); + + // compute rt + recurrent_mat_mul(outs, Ur, rt); + kernel_mat_mul(ins, Wr, kernel_outs); + for(int i = 0; i < out_size; ++i) + rt[i] = MathsProvider::sigmoid(rt[i] + br[i] + kernel_outs[i]); + + // compute h_hat + recurrent_mat_mul(outs, Uh, ct); + kernel_mat_mul(ins, Wh, kernel_outs); + for(int i = 0; i < out_size; ++i) + ht[i] = MathsProvider::tanh(rt[i] * (ct[i] + bh1[i]) + bh0[i] + kernel_outs[i]); + + computeOutput(); + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const T (&ins)[in_size]) noexcept + { + // compute zt + recurrent_mat_mul(outs, Uz, zt); + for(int i = 0; i < out_size; ++i) + zt[i] = MathsProvider::sigmoid(zt[i] + bz[i] + (Wz_1[i] * ins[0])); + + // compute rt + recurrent_mat_mul(outs, Ur, rt); + for(int i = 0; i < out_size; ++i) + rt[i] = MathsProvider::sigmoid(rt[i] + br[i] + (Wr_1[i] * ins[0])); + + // compute h_hat + recurrent_mat_mul(outs, Uh, ct); + for(int i = 0; i < out_size; ++i) + ht[i] = MathsProvider::tanh(rt[i] * (ct[i] + bh1[i]) + bh0[i] + (Wh_1[i] * ins[0])); + + computeOutput(); + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][3 * out_size] + */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][3 * out_size] + */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[2][3 * out_size] + */ + RTNEURAL_REALTIME void setBVals(const std::vector>& bVals); + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + +private: + template + inline std::enable_if_t + computeOutput() noexcept + { + for(int i = 0; i < out_size; ++i) + outs[i] = ((T)1.0 - zt[i]) * ht[i] + zt[i] * outs[i]; + } + + template + inline std::enable_if_t + computeOutput() noexcept + { + for(int i = 0; i < out_size; ++i) + outs_delayed[delayWriteIdx][i] = ((T)1.0 - zt[i]) * ht[i] + zt[i] * outs[i]; + + processDelay(outs_delayed, outs, delayWriteIdx); + } + + template + inline std::enable_if_t + processDelay(std::vector>& delayVec, T (&out)[out_size], int delayWriteIndex) noexcept + { + for(int i = 0; i < out_size; ++i) + out[i] = delayVec[0][i]; + + for(int j = 0; j < delayWriteIndex; ++j) + { + for(int i = 0; i < out_size; ++i) + delayVec[j][i] = delayVec[j + 1][i]; + } + } + + template + inline std::enable_if_t + processDelay(std::vector>& delayVec, T (&out)[out_size], int delayWriteIndex) noexcept + { + for(int i = 0; i < out_size; ++i) + out[i] = delayPlus1Mult * delayVec[0][i] + delayMult * delayVec[1][i]; + + for(int j = 0; j < delayWriteIndex; ++j) + { + for(int i = 0; i < out_size; ++i) + delayVec[j][i] = delayVec[j + 1][i]; + } + } + + static inline void recurrent_mat_mul(const T (&vec)[out_size], const T (&mat)[out_size][out_size], T (&out)[out_size]) noexcept + { + for(int j = 0; j < out_size; ++j) + out[j] = std::inner_product(mat[j], mat[j] + out_size, vec, (T)0); + } + + static inline void kernel_mat_mul(const T (&vec)[in_size], const T (&mat)[out_size][in_size], T (&out)[out_size]) noexcept + { + for(int j = 0; j < out_size; ++j) + out[j] = std::inner_product(mat[j], mat[j] + in_size, vec, (T)0); + } + + // kernel weights + T Wr alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][in_size]; + T Wz alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][in_size]; + T Wh alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][in_size]; + T kernel_outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + // single-input kernel weights + T Wz_1 alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T Wr_1 alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T Wh_1 alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + // recurrent weights + T Uz alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][out_size]; + T Ur alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][out_size]; + T Uh alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][out_size]; + + // biases + T bz alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T br alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T bh0 alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T bh1 alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + // intermediate vars + T zt alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T rt alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T ct alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T ht alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + // needed for delays when doing sample rate correction + std::vector> outs_delayed; + int delayWriteIdx = 0; + T delayMult = (T)1; + T delayPlus1Mult = (T)0; +}; + +} // namespace RTNEURAL_NAMESPACE + +#endif // RTNEURAL_USE_EIGEN + +#endif // GRU_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru.tpp new file mode 100644 index 0000000..1c3c76f --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru.tpp @@ -0,0 +1,364 @@ +#include "gru.h" + +namespace RTNEURAL_NAMESPACE +{ + +#if !RTNEURAL_USE_EIGEN && !RTNEURAL_USE_XSIMD +template +GRULayer::GRULayer(int in_size, int out_size) + : Layer(in_size, out_size) + , zWeights(in_size, out_size) + , rWeights(in_size, out_size) + , cWeights(in_size, out_size) +{ + ht1 = new T[out_size]; + zVec = new T[out_size]; + rVec = new T[out_size]; + cVec = new T[out_size]; +} + +template +GRULayer::GRULayer(std::initializer_list sizes) + : GRULayer(*sizes.begin(), *(sizes.begin() + 1)) +{ +} + +template +GRULayer::GRULayer(const GRULayer& other) + : GRULayer(other.in_size, other.out_size) +{ +} + +template +GRULayer& GRULayer::operator=(const GRULayer& other) +{ + if(&other != this) + *this = GRULayer(other); + return *this; +} + +template +GRULayer::~GRULayer() +{ + delete[] ht1; + delete[] zVec; + delete[] rVec; + delete[] cVec; +} + +template +GRULayer::WeightSet::WeightSet(int in_size, int out_size) + : out_size(out_size) +{ + W = new T*[out_size]; + U = new T*[out_size]; + b = new T*[kNumBiasLayers]; + + for(int i = 0; i < kNumBiasLayers; ++i) + { + b[i] = new T[out_size]; + } + + for(int i = 0; i < out_size; ++i) + { + W[i] = new T[in_size]; + U[i] = new T[out_size]; + } +} + +template +GRULayer::WeightSet::~WeightSet() +{ + for(int i = 0; i < kNumBiasLayers; ++i) + { + delete[] b[i]; + } + + for(int i = 0; i < out_size; ++i) + { + delete[] W[i]; + delete[] U[i]; + } + + delete[] b; + delete[] W; + delete[] U; +} + +template +void GRULayer::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < Layer::in_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.W[k][i] = wVals[i][k]; + rWeights.W[k][i] = wVals[i][k + Layer::out_size]; + cWeights.W[k][i] = wVals[i][k + Layer::out_size * 2]; + } + } +} + +template +void GRULayer::setWVals(T** wVals) +{ + for(int i = 0; i < Layer::in_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.W[k][i] = wVals[i][k]; + rWeights.W[k][i] = wVals[i][k + Layer::out_size]; + cWeights.W[k][i] = wVals[i][k + Layer::out_size * 2]; + } + } +} + +template +void GRULayer::setUVals(const std::vector>& uVals) +{ + for(int i = 0; i < Layer::out_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.U[k][i] = uVals[i][k]; + rWeights.U[k][i] = uVals[i][k + Layer::out_size]; + cWeights.U[k][i] = uVals[i][k + Layer::out_size * 2]; + } + } +} + +template +void GRULayer::setUVals(T** uVals) +{ + for(int i = 0; i < Layer::out_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.U[k][i] = uVals[i][k]; + rWeights.U[k][i] = uVals[i][k + Layer::out_size]; + cWeights.U[k][i] = uVals[i][k + Layer::out_size * 2]; + } + } +} + +template +void GRULayer::setBVals(const std::vector>& bVals) +{ + for(int i = 0; i < 2; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.b[i][k] = bVals[i][k]; + rWeights.b[i][k] = bVals[i][k + Layer::out_size]; + cWeights.b[i][k] = bVals[i][k + Layer::out_size * 2]; + } + } +} + +template +void GRULayer::setBVals(T** bVals) +{ + for(int i = 0; i < 2; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.b[i][k] = bVals[i][k]; + rWeights.b[i][k] = bVals[i][k + Layer::out_size]; + cWeights.b[i][k] = bVals[i][k + Layer::out_size * 2]; + } + } +} + +template +T GRULayer::getWVal(int i, int k) const noexcept +{ + T** set = zWeights.W; + if(k > 2 * Layer::out_size) + { + k -= 2 * Layer::out_size; + set = cWeights.W; + } + else if(k > Layer::out_size) + { + k -= Layer::out_size; + set = rWeights.W; + } + + return set[i][k]; +} + +template +T GRULayer::getUVal(int i, int k) const noexcept +{ + T** set = zWeights.U; + if(k > 2 * Layer::out_size) + { + k -= 2 * Layer::out_size; + set = cWeights.U; + } + else if(k > Layer::out_size) + { + k -= Layer::out_size; + set = rWeights.U; + } + + return set[i][k]; +} + +template +T GRULayer::getBVal(int i, int k) const noexcept +{ + T** set = zWeights.b; + if(k > 2 * Layer::out_size) + { + k -= 2 * Layer::out_size; + set = cWeights.b; + } + else if(k > Layer::out_size) + { + k -= Layer::out_size; + set = rWeights.b; + } + + return set[i][k]; +} + +//==================================================== +template +GRULayerT::GRULayerT() +{ + for(int i = 0; i < out_size; ++i) + { + // single-input kernel weights + Wz_1[i] = (T)0; + Wr_1[i] = (T)0; + Wh_1[i] = (T)0; + + // biases + bz[i] = (T)0; + br[i] = (T)0; + bh0[i] = (T)0; + bh1[i] = (T)0; + + // intermediate vars + zt[i] = (T)0; + rt[i] = (T)0; + ct[i] = (T)0; + ht[i] = (T)0; + } + + for(int i = 0; i < out_size; ++i) + { + // recurrent weights + for(int k = 0; k < out_size; ++k) + { + Uz[i][k] = (T)0; + Ur[i][k] = (T)0; + Uh[i][k] = (T)0; + } + + // kernel weights + for(int k = 0; k < in_size; ++k) + { + Wz[i][k] = (T)0; + Wr[i][k] = (T)0; + Wh[i][k] = (T)0; + } + } + + reset(); +} + +template +template +std::enable_if_t +GRULayerT::prepare(int delaySamples) +{ + delayWriteIdx = delaySamples - 1; + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +template +std::enable_if_t +GRULayerT::prepare(T delaySamples) +{ + const auto delayOffFactor = delaySamples - std::floor(delaySamples); + delayMult = (T)1 - delayOffFactor; + delayPlus1Mult = delayOffFactor; + + delayWriteIdx = (int)std::ceil(delaySamples) - (int)std::ceil(delayOffFactor); + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +void GRULayerT::reset() +{ + if(sampleRateCorr != SampleRateCorrectionMode::None) + { + for(auto& vec : outs_delayed) + std::fill(vec.begin(), vec.end(), T {}); + } + + // reset output state + for(int i = 0; i < out_size; ++i) + outs[i] = (T)0; +} + +// kernel weights +template +void GRULayerT::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < in_size; ++i) + { + for(int j = 0; j < out_size; ++j) + { + Wz[j][i] = wVals[i][j]; + Wr[j][i] = wVals[i][j + out_size]; + Wh[j][i] = wVals[i][j + 2 * out_size]; + } + } + + for(int j = 0; j < out_size; ++j) + { + Wz_1[j] = wVals[0][j]; + Wr_1[j] = wVals[0][j + out_size]; + Wh_1[j] = wVals[0][j + 2 * out_size]; + } +} + +// recurrent weights +template +void GRULayerT::setUVals(const std::vector>& uVals) +{ + for(int i = 0; i < out_size; ++i) + { + for(int j = 0; j < out_size; ++j) + { + Uz[j][i] = uVals[i][j]; + Ur[j][i] = uVals[i][j + out_size]; + Uh[j][i] = uVals[i][j + 2 * out_size]; + } + } +} + +// biases +template +void GRULayerT::setBVals(const std::vector>& bVals) +{ + for(int k = 0; k < out_size; ++k) + { + bz[k] = bVals[0][k] + bVals[1][k]; + br[k] = bVals[0][k + out_size] + bVals[1][k + out_size]; + bh0[k] = bVals[0][k + 2 * out_size]; + bh1[k] = bVals[1][k + 2 * out_size]; + } +} + +#endif // !RTNEURAL_USE_EIGEN && !RTNEURAL_USE_XSIMD + +} // namespace RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_eigen.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_eigen.h new file mode 100644 index 0000000..c83acd1 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_eigen.h @@ -0,0 +1,346 @@ +#ifndef GRUEIGEN_H_INCLUDED +#define GRUEIGEN_H_INCLUDED + +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include "../maths/maths_eigen.h" + +namespace RTNEURAL_NAMESPACE +{ + +/** + * Dynamic implementation of a gated recurrent unit (GRU) layer + * with tanh activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's GRU implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class GRULayer : public Layer +{ +public: + /** Constructs a GRU layer for a given input and output size. */ + GRULayer(int in_size, int out_size); + GRULayer(std::initializer_list sizes); + GRULayer(const GRULayer& other); + GRULayer& operator=(const GRULayer& other); + virtual ~GRULayer() = default; + + /** Resets the state of the GRU. */ + RTNEURAL_REALTIME void reset() override + { + extendedHt1.setZero(); + extendedHt1(Layer::out_size) = (T)1; + } + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "gru"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override + { + for(int i = 0; i < Layer::in_size; ++i) + { + extendedInVec(i) = input[i]; + } + + /** + * | Wz bz[0] | | input | | Wz * input + bz[0] | + * alpha = | Wr br[0] | * | 1 | = | Wr * input + br[0] | + * | Wc bc[0] | | Wc * input + bc[0] | + * + * | Uz bz[1] | | h(t-1) | | Uz * h(t-1) + bz[1] | + * beta = | Ur br[1] | * | 1 | = | Ur * h(t-1) + br[1] | + * | Uc bc[1] | | Uc * h(t-1) + bc[1] | + */ + alphaVec.noalias() = wCombinedWeights * extendedInVec; + betaVec.noalias() = uCombinedWeights * extendedHt1; + + /** + * gamma = sigmoid( | z | = sigmoid(alpha[0 : 2*out_sizet] + beta[0 : 2*out_sizet]) + * | r | ) + */ + gammaVec.noalias() = alphaVec.segment(0, 2 * Layer::out_size) + betaVec.segment(0, 2 * Layer::out_size); + gammaVec = MathsProvider::sigmoid(gammaVec); + + /** + * c = tanh( alpha[2*out_sizet : 3*out_sizet] + r.cwiseProduct(beta[2*out_sizet : 3*out_sizet] ) + * i.e. c = tanh( Wc * input + bc[0] + r.cwiseProduct(Uc * h(t-1) + bc[1]) ) + */ + cVec.noalias() = alphaVec.segment(2 * Layer::out_size, Layer::out_size) + gammaVec.segment(Layer::out_size, Layer::out_size).cwiseProduct(betaVec.segment(2 * Layer::out_size, Layer::out_size)); + cVec = MathsProvider::tanh(cVec); + + /** + * h(t-1) = (1 - z).cwiseProduct(c) + z.cwiseProduct(h(t-1)) + * = c - z.cwiseProduct(c) + z.cwiseProduct(ht(t-1)) + * = c + z.cwiseProduct(h(t-1) - c) + */ + extendedHt1.segment(0, Layer::out_size) = cVec + gammaVec.segment(0, Layer::out_size).cwiseProduct(extendedHt1.segment(0, Layer::out_size) - cVec); + + for(int i = 0; i < Layer::out_size; ++i) + { + h[i] = extendedHt1(i); + } + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][3 * out_size] + */ + RTNEURAL_REALTIME void setWVals(T** wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][3 * out_size] + */ + RTNEURAL_REALTIME void setUVals(T** uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[2][3 * out_size] + */ + RTNEURAL_REALTIME void setBVals(T** bVals); + + /** Returns the kernel weight for the given indices. */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** Returns the recurrent weight for the given indices. */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** Returns the bias value for the given indices. */ + RTNEURAL_REALTIME void setBVals(const std::vector>& bVals); + + RTNEURAL_REALTIME T getWVal(int i, int k) const noexcept; + RTNEURAL_REALTIME T getUVal(int i, int k) const noexcept; + RTNEURAL_REALTIME T getBVal(int i, int k) const noexcept; + +private: + // Kernels + // | Wz bz0 | + // | Wr br0 | + // | Wc bc0 | + Eigen::Matrix wCombinedWeights; + + // | Uz bz1 | + // | Ur br1 | + // | Uc bc1 | + Eigen::Matrix uCombinedWeights; + + // Input vec + Eigen::Matrix extendedInVec; + + // h(t-1) vec + Eigen::Matrix extendedHt1; + + // Scratch memory + Eigen::Matrix alphaVec; + Eigen::Matrix betaVec; + Eigen::Matrix gammaVec; + Eigen::Matrix cVec; +}; + +//==================================================== +/** + * Static implementation of a gated recurrent unit (GRU) layer + * with tanh activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's GRU implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class GRULayerT +{ + using in_type = Eigen::Matrix; + using extended_in_type = Eigen::Matrix; + using out_type = Eigen::Matrix; + using extended_out_type = Eigen::Matrix; + + using w_k_type = Eigen::Matrix; + using u_k_type = Eigen::Matrix; + + using three_out_type = Eigen::Matrix; + using two_out_type = Eigen::Matrix; + +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + + GRULayerT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "gru"; } + + /** Returns false since GRU is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Prepares the GRU to process with a given delay length. */ + template + std::enable_if_t + prepare(int delaySamples); + + /** Prepares the GRU to process with a given delay length. */ + template + std::enable_if_t + prepare(T delaySamples); + + /** Resets the state of the GRU. */ + RTNEURAL_REALTIME void reset(); + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const in_type& ins) noexcept + { + for(int i = 0; i < in_sizet; ++i) + { + extendedInVec(i) = ins(i); + } + + /** + * | Wz bz[0] | | input | | Wz * input + bz[0] | + * alpha = | Wr br[0] | * | 1 | = | Wr * input + br[0] | + * | Wc bc[0] | | Wc * input + bc[0] | + * + * | Uz bz[1] | | h(t-1) | | Uz * h(t-1) + bz[1] | + * beta = | Ur br[1] | * | 1 | = | Ur * h(t-1) + br[1] | + * | Uc bc[1] | | Uc * h(t-1) + bc[1] | + */ + alphaVec.noalias() = wCombinedWeights * extendedInVec; + betaVec.noalias() = uCombinedWeights * extendedHt1; + + /** + * gamma = sigmoid( | z | = sigmoid(alpha[0 : 2*out_sizet] + beta[0 : 2*out_sizet]) + * | r | ) + */ + gammaVec = MathsProvider::sigmoid(alphaVec.segment(0, 2 * out_sizet) + betaVec.segment(0, 2 * out_sizet)); + + /** + * c = tanh( alpha[2*out_sizet : 3*out_sizet] + r.cwiseProduct(beta[2*out_sizet : 3*out_sizet] ) + * i.e. c = tanh( Wc * input + bc[0] + r.cwiseProduct(Uc * h(t-1) + bc[1]) ) + */ + cVec.noalias() = alphaVec.segment(2 * out_sizet, out_sizet) + gammaVec.segment(out_sizet, out_sizet).cwiseProduct(betaVec.segment(2 * out_sizet, out_sizet)); + cVec = MathsProvider::tanh(cVec); + + /** + * h(t-1) = (1 - z).cwiseProduct(c) + z.cwiseProduct(h(t-1)) + * = c - z.cwiseProduct(c) + z.cwiseProduct(ht(t-1)) + * = c + z.cwiseProduct(h(t-1) - c) + */ + extendedHt1.segment(0, out_sizet) = cVec + gammaVec.segment(0, out_sizet).cwiseProduct(extendedHt1.segment(0, out_sizet) - cVec); + + computeOutput(); + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][3 * out_size] + */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][3 * out_size] + */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[2][3 * out_size] + */ + RTNEURAL_REALTIME void setBVals(const std::vector>& bVals); + + Eigen::Map outs; + +private: + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + template + inline std::enable_if_t + computeOutput() noexcept + { + for(int i = 0; i < out_sizet; ++i) + { + outs(i) = extendedHt1(i); + } + } + + template + inline std::enable_if_t + computeOutput() noexcept + { + for(int i = 0; i < out_sizet; ++i) + { + outs_delayed[delayWriteIdx][i] = extendedHt1(i); + } + + processDelay(outs_delayed, outs, delayWriteIdx); + + for(int i = 0; i < out_sizet; ++i) + { + extendedHt1(i) = outs(i); + } + } + + template + inline std::enable_if_t + processDelay(std::vector& delayVec, OutVec& out, int delayWriteIndex) noexcept + { + out = delayVec[0]; + + for(int j = 0; j < delayWriteIndex; ++j) + delayVec[j] = delayVec[j + 1]; + } + + template + inline std::enable_if_t + processDelay(std::vector& delayVec, OutVec& out, int delayWriteIndex) noexcept + { + out = delayPlus1Mult * delayVec[0] + delayMult * delayVec[1]; + + for(int j = 0; j < delayWriteIndex; ++j) + delayVec[j] = delayVec[j + 1]; + } + + // kernel weights + w_k_type wCombinedWeights; + u_k_type uCombinedWeights; + + // scratch memory + three_out_type alphaVec; + three_out_type betaVec; + two_out_type gammaVec; + + // input, output, memory + out_type cVec; + extended_in_type extendedInVec; + extended_out_type extendedHt1; + + // needed for delays when doing sample rate correction + std::vector outs_delayed; + int delayWriteIdx = 0; + T delayMult = (T)1; + T delayPlus1Mult = (T)0; +}; + +} // namespace RTNEURAL_NAMESPACE + +#endif // GRUEIGEN_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_eigen.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_eigen.tpp new file mode 100644 index 0000000..ab07b8e --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_eigen.tpp @@ -0,0 +1,240 @@ +#if RTNEURAL_USE_EIGEN + +#include "gru_eigen.h" + +namespace RTNEURAL_NAMESPACE +{ + +template +GRULayer::GRULayer(int in_size, int out_size) + : Layer(in_size, out_size) +{ + wCombinedWeights = Eigen::Matrix::Zero(3 * out_size, in_size + 1); + uCombinedWeights = Eigen::Matrix::Zero(3 * out_size, out_size + 1); + extendedInVec = Eigen::Matrix::Zero(in_size + 1); + extendedHt1 = Eigen::Matrix::Zero(out_size + 1); + extendedInVec(Layer::in_size) = (T)1; + extendedHt1(Layer::out_size) = (T)1; + + alphaVec = Eigen::Matrix::Zero(3 * out_size); + betaVec = Eigen::Matrix::Zero(3 * out_size); + gammaVec = Eigen::Matrix::Zero(2 * out_size); + cVec = Eigen::Matrix::Zero(out_size); +} + +template +GRULayer::GRULayer(std::initializer_list sizes) + : GRULayer(*sizes.begin(), *(sizes.begin() + 1)) +{ +} + +template +GRULayer::GRULayer(const GRULayer& other) + : GRULayer(other.in_size, other.out_size) +{ +} + +template +GRULayer& GRULayer::operator=(const GRULayer& other) +{ + return *this = GRULayer(other); +} + +template +void GRULayer::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < Layer::in_size; ++i) + { + for(int k = 0; k < Layer::out_size * 3; ++k) + { + wCombinedWeights(k, i) = wVals[i][k]; + } + } +} + +template +void GRULayer::setWVals(T** wVals) +{ + for(int i = 0; i < Layer::in_size; ++i) + { + for(int k = 0; k < Layer::out_size * 3; ++k) + { + wCombinedWeights(k, i) = wVals[i][k]; + } + } +} + +template +void GRULayer::setUVals(const std::vector>& uVals) +{ + for(int i = 0; i < Layer::out_size; ++i) + { + for(int k = 0; k < Layer::out_size * 3; ++k) + { + uCombinedWeights(k, i) = uVals[i][k]; + } + } +} + +template +void GRULayer::setUVals(T** uVals) +{ + for(int i = 0; i < Layer::out_size; ++i) + { + for(int k = 0; k < Layer::out_size * 3; ++k) + { + uCombinedWeights(k, i) = uVals[i][k]; + } + } +} + +template +void GRULayer::setBVals(const std::vector>& bVals) +{ + for(int k = 0; k < Layer::out_size * 3; ++k) + { + wCombinedWeights(k, Layer::in_size) = bVals[0][k]; + uCombinedWeights(k, Layer::out_size) = bVals[1][k]; + } +} + +template +void GRULayer::setBVals(T** bVals) +{ + for(int k = 0; k < Layer::out_size * 3; ++k) + { + wCombinedWeights(k, Layer::in_size) = bVals[0][k]; + uCombinedWeights(k, Layer::out_size) = bVals[1][k]; + } +} + +template +T GRULayer::getWVal(int i, int k) const noexcept +{ + return wCombinedWeights[k][i]; +} + +template +T GRULayer::getUVal(int i, int k) const noexcept +{ + return uCombinedWeights[k][i]; +} + +template +T GRULayer::getBVal(int i, int k) const noexcept +{ + T val; + if(i == 0) + { + val = wCombinedWeights[k][Layer::in_size]; + } + else + { + val = uCombinedWeights[k][Layer::out_size]; + } + return val; +} + +//==================================================== +template +GRULayerT::GRULayerT() + : outs(outs_internal) +{ + wCombinedWeights = w_k_type::Zero(); + uCombinedWeights = u_k_type::Zero(); + alphaVec = three_out_type::Zero(); + betaVec = three_out_type::Zero(); + gammaVec = two_out_type::Zero(); + cVec = out_type::Zero(); + extendedInVec = extended_in_type::Zero(); + extendedHt1 = extended_out_type::Zero(); + + extendedInVec(in_sizet) = (T)1; + extendedHt1(out_sizet) = (T)1; + + reset(); +} + +template +template +std::enable_if_t +GRULayerT::prepare(int delaySamples) +{ + delayWriteIdx = delaySamples - 1; + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +template +std::enable_if_t +GRULayerT::prepare(T delaySamples) +{ + const auto delayOffFactor = delaySamples - std::floor(delaySamples); + delayMult = (T)1 - delayOffFactor; + delayPlus1Mult = delayOffFactor; + + delayWriteIdx = (int)std::ceil(delaySamples) - (int)std::ceil(delayOffFactor); + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +void GRULayerT::reset() +{ + if(sampleRateCorr != SampleRateCorrectionMode::None) + { + for(auto& vec : outs_delayed) + vec = out_type::Zero(); + } + + // reset output state + outs = out_type::Zero(); + + // reset extended internal state + extendedHt1.setZero(); + extendedHt1(out_sizet) = (T)1; +} + +// kernel weights +template +void GRULayerT::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < in_size; ++i) + { + for(int k = 0; k < out_size * 3; ++k) + { + wCombinedWeights(k, i) = wVals[i][k]; + } + } +} + +// recurrent weights +template +void GRULayerT::setUVals(const std::vector>& uVals) +{ + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < out_size * 3; ++k) + { + uCombinedWeights(k, i) = uVals[i][k]; + } + } +} + +// biases +template +void GRULayerT::setBVals(const std::vector>& bVals) +{ + for(int k = 0; k < out_size * 3; ++k) + { + wCombinedWeights(k, in_sizet) = bVals[0][k]; + uCombinedWeights(k, out_sizet) = bVals[1][k]; + } +} + +} // namespace RTNEURAL_NAMESPACE + +#endif // RTNEURAL_USE_EIGEN diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_xsimd.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_xsimd.h new file mode 100644 index 0000000..212baed --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_xsimd.h @@ -0,0 +1,396 @@ +#ifndef GRUXSIMD_H_INCLUDED +#define GRUXSIMD_H_INCLUDED + +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include "../maths/maths_xsimd.h" +#include +namespace RTNEURAL_NAMESPACE +{ + +/** + * Dynamic implementation of a gated recurrent unit (GRU) layer + * with tanh activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's GRU implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class GRULayer : public Layer +{ +public: + /** Constructs a GRU layer for a given input and output size. */ + GRULayer(int in_size, int out_size); + GRULayer(std::initializer_list sizes); + GRULayer(const GRULayer& other); + GRULayer& operator=(const GRULayer& other); + virtual ~GRULayer(); + + /** Resets the state of the GRU. */ + RTNEURAL_REALTIME void reset() override { std::fill(ht1.begin(), ht1.end(), (T)0); } + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "gru"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override + { + for(int i = 0; i < Layer::out_size; ++i) + { + zVec[i] = vMult(zWeights.W[i].data(), input, prod_in.data(), Layer::in_size) + vMult(zWeights.U[i].data(), ht1.data(), prod_out.data(), Layer::out_size); + rVec[i] = vMult(rWeights.W[i].data(), input, prod_in.data(), Layer::in_size) + vMult(rWeights.U[i].data(), ht1.data(), prod_out.data(), Layer::out_size); + cVec[i] = vMult(cWeights.W[i].data(), input, prod_in.data(), Layer::in_size); + cTmp[i] = vMult(cWeights.U[i].data(), ht1.data(), prod_out.data(), Layer::out_size); + } + + vAdd(zVec.data(), zWeights.b[0].data(), zVec.data(), Layer::out_size); + vAdd(zVec.data(), zWeights.b[1].data(), zVec.data(), Layer::out_size); + sigmoid(zVec.data(), zVec.data(), Layer::out_size); + + vAdd(rVec.data(), rWeights.b[0].data(), rVec.data(), Layer::out_size); + vAdd(rVec.data(), rWeights.b[1].data(), rVec.data(), Layer::out_size); + sigmoid(rVec.data(), rVec.data(), Layer::out_size); + + vAdd(cTmp.data(), cWeights.b[1].data(), cTmp.data(), Layer::out_size); + vProd(cTmp.data(), rVec.data(), cTmp.data(), Layer::out_size); + vAdd(cTmp.data(), cVec.data(), cVec.data(), Layer::out_size); + vAdd(cVec.data(), cWeights.b[0].data(), cVec.data(), Layer::out_size); + tanh(cVec.data(), cVec.data(), Layer::out_size); + + vSub(ones.data(), zVec.data(), h, Layer::out_size); + vProd(h, cVec.data(), h, Layer::out_size); + vProd(zVec.data(), ht1.data(), prod_out.data(), Layer::out_size); + vAdd(h, prod_out.data(), h, Layer::out_size); + + vCopy(h, ht1.data(), Layer::out_size); + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][3 * out_size] + */ + RTNEURAL_REALTIME void setWVals(T** wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][3 * out_size] + */ + RTNEURAL_REALTIME void setUVals(T** uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[2][3 * out_size] + */ + RTNEURAL_REALTIME void setBVals(T** bVals); + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][3 * out_size] + */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][3 * out_size] + */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[2][3 * out_size] + */ + RTNEURAL_REALTIME void setBVals(const std::vector>& bVals); + + /** Returns the kernel weight for the given indices. */ + RTNEURAL_REALTIME T getWVal(int i, int k) const noexcept; + + /** Returns the recurrent weight for the given indices. */ + RTNEURAL_REALTIME T getUVal(int i, int k) const noexcept; + + /** Returns the bias value for the given indices. */ + RTNEURAL_REALTIME T getBVal(int i, int k) const noexcept; + +protected: + using vec_type = std::vector>; + using vec2_type = std::vector; + + vec_type ht1; + + struct WeightSet + { + WeightSet(int in_size, int out_size); + ~WeightSet(); + + vec2_type W; // kernel weights + vec2_type U; // recurrent weights + vec_type b[2]; // bias + const int out_size; + }; + + WeightSet zWeights; + WeightSet rWeights; + WeightSet cWeights; + + vec_type zVec; + vec_type rVec; + vec_type cVec; + vec_type cTmp; + + vec_type prod_in; + vec_type prod_out; + vec_type ones; +}; + +//==================================================== +/** + * Static implementation of a gated recurrent unit (GRU) layer + * with tanh activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's GRU implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class GRULayerT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_in_size = ceil_div(in_sizet, v_size); + static constexpr auto v_out_size = ceil_div(out_sizet, v_size); + +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + + GRULayerT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "gru"; } + + /** Returns false since GRU is not an activation layer. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Prepares the GRU to process with a given delay length. */ + template + std::enable_if_t + prepare(int delaySamples); + + /** Prepares the GRU to process with a given delay length. */ + template + std::enable_if_t + prepare(T delaySamples); + + /** Resets the state of the GRU. */ + RTNEURAL_REALTIME void reset(); + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if<(N > 1), void>::type + forward(const v_type (&ins)[v_in_size]) noexcept + { + // compute zt + recurrent_mat_mul(outs, Uz, zt); + kernel_mat_mul(ins, Wz, kernel_outs); + for(int i = 0; i < v_out_size; ++i) + zt[i] = MathsProvider::sigmoid(zt[i] + bz[i] + kernel_outs[i]); + + // compute rt + recurrent_mat_mul(outs, Ur, rt); + kernel_mat_mul(ins, Wr, kernel_outs); + for(int i = 0; i < v_out_size; ++i) + rt[i] = MathsProvider::sigmoid(rt[i] + br[i] + kernel_outs[i]); + + // compute h_hat + recurrent_mat_mul(outs, Uh, ct); + kernel_mat_mul(ins, Wh, kernel_outs); + for(int i = 0; i < v_out_size; ++i) + ht[i] = MathsProvider::tanh(xsimd::fma(rt[i], ct[i] + bh1[i], bh0[i] + kernel_outs[i])); + + computeOutput(); + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type (&ins)[v_in_size]) noexcept + { + // compute zt + recurrent_mat_mul(outs, Uz, zt); + for(int i = 0; i < v_out_size; ++i) + zt[i] = MathsProvider::sigmoid(xsimd::fma(Wz_1[i], ins[0], zt[i] + bz[i])); + + // compute rt + recurrent_mat_mul(outs, Ur, rt); + for(int i = 0; i < v_out_size; ++i) + rt[i] = MathsProvider::sigmoid(xsimd::fma(Wr_1[i], ins[0], rt[i] + br[i])); + + // compute h_hat + recurrent_mat_mul(outs, Uh, ct); + for(int i = 0; i < v_out_size; ++i) + ht[i] = MathsProvider::tanh(xsimd::fma(rt[i], ct[i] + bh1[i], xsimd::fma(Wh_1[i], ins[0], bh0[i]))); + + computeOutput(); + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][3 * out_size] + */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][3 * out_size] + */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[2][3 * out_size] + */ + RTNEURAL_REALTIME void setBVals(const std::vector>& bVals); + + v_type outs[v_out_size]; + +private: + template + inline std::enable_if_t + computeOutput() noexcept + { + for(int i = 0; i < v_out_size; ++i) + outs[i] = xsimd::fma((v_type((T)1.0) - zt[i]), ht[i], zt[i] * outs[i]); + } + + template + inline std::enable_if_t + computeOutput() noexcept + { + for(int i = 0; i < v_out_size; ++i) + outs_delayed[delayWriteIdx][i] = xsimd::fma((v_type((T)1.0) - zt[i]), ht[i], zt[i] * outs[i]); + + processDelay(outs_delayed, outs, delayWriteIdx); + } + + template + inline std::enable_if_t + processDelay(std::vector>& delayVec, v_type (&out)[v_out_size], int delayWriteIndex) noexcept + { + for(int i = 0; i < v_out_size; ++i) + out[i] = delayVec[0][i]; + + for(int j = 0; j < delayWriteIndex; ++j) + { + for(int i = 0; i < v_out_size; ++i) + delayVec[j][i] = delayVec[j + 1][i]; + } + } + + template + inline std::enable_if_t + processDelay(std::vector>& delayVec, v_type (&out)[v_out_size], int delayWriteIndex) noexcept + { + for(int i = 0; i < v_out_size; ++i) + out[i] = delayPlus1Mult * delayVec[0][i] + delayMult * delayVec[1][i]; + + for(int j = 0; j < delayWriteIndex; ++j) + { + for(int i = 0; i < v_out_size; ++i) + delayVec[j][i] = delayVec[j + 1][i]; + } + } + + static inline void recurrent_mat_mul(const v_type (&vec)[v_out_size], const v_type (&mat)[out_size][v_out_size], v_type (&out)[v_out_size]) noexcept + { + for(int i = 0; i < v_out_size; ++i) + out[i] = v_type(0); + + T scalar_in alignas(RTNEURAL_DEFAULT_ALIGNMENT)[v_size] { (T)0 }; + for(int k = 0; k < v_out_size; ++k) + { + vec[k].store_aligned(scalar_in); + for(int i = 0; i < v_out_size; ++i) + { + for(int j = 0; j < v_size; ++j) + out[i] += scalar_in[j] * mat[k * v_size + j][i]; + } + } + } + + static inline void kernel_mat_mul(const v_type (&vec)[v_in_size], const v_type (&mat)[in_size][v_out_size], v_type (&out)[v_out_size]) noexcept + { + for(int i = 0; i < v_out_size; ++i) + out[i] = v_type(0); + + T scalar_in alignas(RTNEURAL_DEFAULT_ALIGNMENT)[v_size] { (T)0 }; + for(int k = 0; k < v_in_size; ++k) + { + vec[k].store_aligned(scalar_in); + for(int i = 0; i < v_out_size; ++i) + { + for(int j = 0; j < v_size; ++j) + out[i] += scalar_in[j] * mat[k * v_size + j][i]; + } + } + } + + // kernel weights + v_type Wz[in_size][v_out_size]; + v_type Wr[in_size][v_out_size]; + v_type Wh[in_size][v_out_size]; + v_type kernel_outs[v_out_size]; + + // single-input kernel weights + v_type Wz_1[v_out_size]; + v_type Wr_1[v_out_size]; + v_type Wh_1[v_out_size]; + + // recurrent weights + v_type Uz[out_size][v_out_size]; + v_type Ur[out_size][v_out_size]; + v_type Uh[out_size][v_out_size]; + + // biases + v_type bz[v_out_size]; + v_type br[v_out_size]; + v_type bh0[v_out_size]; + v_type bh1[v_out_size]; + + // intermediate vars + v_type zt[v_out_size]; + v_type rt[v_out_size]; + v_type ct[v_out_size]; + v_type ht[v_out_size]; + + // needed for delays when doing sample rate correction + std::vector> outs_delayed; + int delayWriteIdx = 0; + v_type delayMult = (T)1; + v_type delayPlus1Mult = (T)0; +}; + +} // namespace RTNEURAL_NAMESPACE + +#endif // GRUXSIMD_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_xsimd.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_xsimd.tpp new file mode 100644 index 0000000..2c797f3 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/gru/gru_xsimd.tpp @@ -0,0 +1,337 @@ +#include "gru_xsimd.h" + +namespace RTNEURAL_NAMESPACE +{ + +template +GRULayer::GRULayer(int in_size, int out_size) + : Layer(in_size, out_size) + , zWeights(in_size, out_size) + , rWeights(in_size, out_size) + , cWeights(in_size, out_size) +{ + ht1.resize(out_size, (T)0); + zVec.resize(out_size, (T)0); + rVec.resize(out_size, (T)0); + cVec.resize(out_size, (T)0); + cTmp.resize(out_size, (T)0); + + prod_in.resize(in_size, (T)0); + prod_out.resize(out_size, (T)0); + + ones.resize(out_size, (T)1); +} + +template +GRULayer::GRULayer(std::initializer_list sizes) + : GRULayer(*sizes.begin(), *(sizes.begin() + 1)) +{ +} + +template +GRULayer::GRULayer(const GRULayer& other) + : GRULayer(other.in_size, other.out_size) +{ +} + +template +GRULayer& GRULayer::operator=(const GRULayer& other) +{ + return *this = GRULayer(other); +} + +template +GRULayer::~GRULayer() = default; + +template +GRULayer::WeightSet::WeightSet(int in_size, int out_size) + : out_size(out_size) +{ + W = vec2_type(out_size, vec_type(in_size, (T)0)); + U = vec2_type(out_size, vec_type(out_size, (T)0)); + + b[0].resize(out_size, (T)0); + b[1].resize(out_size, (T)0); +} + +template +GRULayer::WeightSet::~WeightSet() = default; + +template +void GRULayer::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < Layer::in_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.W[k][i] = wVals[i][k]; + rWeights.W[k][i] = wVals[i][k + Layer::out_size]; + cWeights.W[k][i] = wVals[i][k + Layer::out_size * 2]; + } + } +} + +template +void GRULayer::setWVals(T** wVals) +{ + for(int i = 0; i < Layer::in_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.W[k][i] = wVals[i][k]; + rWeights.W[k][i] = wVals[i][k + Layer::out_size]; + cWeights.W[k][i] = wVals[i][k + Layer::out_size * 2]; + } + } +} + +template +void GRULayer::setUVals(const std::vector>& uVals) +{ + for(int i = 0; i < Layer::out_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.U[k][i] = uVals[i][k]; + rWeights.U[k][i] = uVals[i][k + Layer::out_size]; + cWeights.U[k][i] = uVals[i][k + Layer::out_size * 2]; + } + } +} + +template +void GRULayer::setUVals(T** uVals) +{ + for(int i = 0; i < Layer::out_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.U[k][i] = uVals[i][k]; + rWeights.U[k][i] = uVals[i][k + Layer::out_size]; + cWeights.U[k][i] = uVals[i][k + Layer::out_size * 2]; + } + } +} + +template +void GRULayer::setBVals(const std::vector>& bVals) +{ + for(int i = 0; i < 2; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.b[i][k] = bVals[i][k]; + rWeights.b[i][k] = bVals[i][k + Layer::out_size]; + cWeights.b[i][k] = bVals[i][k + Layer::out_size * 2]; + } + } +} + +template +void GRULayer::setBVals(T** bVals) +{ + for(int i = 0; i < 2; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + zWeights.b[i][k] = bVals[i][k]; + rWeights.b[i][k] = bVals[i][k + Layer::out_size]; + cWeights.b[i][k] = bVals[i][k + Layer::out_size * 2]; + } + } +} + +template +T GRULayer::getWVal(int i, int k) const noexcept +{ + T** set = zWeights.W; + if(k > 2 * Layer::out_size) + { + k -= 2 * Layer::out_size; + set = cWeights.W; + } + else if(k > Layer::out_size) + { + k -= Layer::out_size; + set = rWeights.W; + } + + return set[i][k]; +} + +template +T GRULayer::getUVal(int i, int k) const noexcept +{ + T** set = zWeights.U; + if(k > 2 * Layer::out_size) + { + k -= 2 * Layer::out_size; + set = cWeights.U; + } + else if(k > Layer::out_size) + { + k -= Layer::out_size; + set = rWeights.U; + } + + return set[i][k]; +} + +template +T GRULayer::getBVal(int i, int k) const noexcept +{ + T** set = zWeights.b; + if(k > 2 * Layer::out_size) + { + k -= 2 * Layer::out_size; + set = cWeights.b; + } + else if(k > Layer::out_size) + { + k -= Layer::out_size; + set = rWeights.b; + } + + return set[i][k]; +} + +//==================================================== +template +GRULayerT::GRULayerT() +{ + for(int i = 0; i < v_out_size; ++i) + { + // single-input kernel weights + Wz_1[i] = v_type((T)0); + Wr_1[i] = v_type((T)0); + Wh_1[i] = v_type((T)0); + + // biases + bz[i] = v_type((T)0); + br[i] = v_type((T)0); + bh0[i] = v_type((T)0); + bh1[i] = v_type((T)0); + + // intermediate vars + zt[i] = v_type((T)0); + rt[i] = v_type((T)0); + ct[i] = v_type((T)0); + ht[i] = v_type((T)0); + } + + // kernel weights + for(int k = 0; k < in_size; ++k) + { + for(int i = 0; i < v_out_size; ++i) + { + Wz[k][i] = v_type((T)0); + Wr[k][i] = v_type((T)0); + Wh[k][i] = v_type((T)0); + } + } + + // recurrent weights + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < v_out_size; ++k) + { + Uz[i][k] = v_type((T)0); + Ur[i][k] = v_type((T)0); + Uh[i][k] = v_type((T)0); + } + } + + reset(); +} + +template +template +std::enable_if_t +GRULayerT::prepare(int delaySamples) +{ + delayWriteIdx = delaySamples - 1; + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +template +std::enable_if_t +GRULayerT::prepare(T delaySamples) +{ + const auto delayOffFactor = delaySamples - std::floor(delaySamples); + delayMult = (T)1 - delayOffFactor; + delayPlus1Mult = delayOffFactor; + + delayWriteIdx = (int)std::ceil(delaySamples) - (int)std::ceil(delayOffFactor); + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +void GRULayerT::reset() +{ + if(sampleRateCorr != SampleRateCorrectionMode::None) + { + for(auto& vec : outs_delayed) + std::fill(vec.begin(), vec.end(), v_type {}); + } + + // reset output state + for(int i = 0; i < v_out_size; ++i) + outs[i] = v_type((T)0); +} + +// kernel weights +template +void GRULayerT::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < in_size; ++k) + { + Wz[k][i / v_size] = set_value(Wz[k][i / v_size], i % v_size, wVals[k][i]); + Wr[k][i / v_size] = set_value(Wr[k][i / v_size], i % v_size, wVals[k][i + out_size]); + Wh[k][i / v_size] = set_value(Wh[k][i / v_size], i % v_size, wVals[k][i + 2 * out_size]); + } + } + + for(int j = 0; j < out_size; ++j) + { + Wz_1[j / v_size] = set_value(Wz_1[j / v_size], j % v_size, wVals[0][j]); + Wr_1[j / v_size] = set_value(Wr_1[j / v_size], j % v_size, wVals[0][j + out_size]); + Wh_1[j / v_size] = set_value(Wh_1[j / v_size], j % v_size, wVals[0][j + 2 * out_size]); + } +} + +// recurrent weights +template +void GRULayerT::setUVals(const std::vector>& uVals) +{ + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < out_size; ++k) + { + Uz[k][i / v_size] = set_value(Uz[k][i / v_size], i % v_size, uVals[k][i]); + Ur[k][i / v_size] = set_value(Ur[k][i / v_size], i % v_size, uVals[k][i + out_size]); + Uh[k][i / v_size] = set_value(Uh[k][i / v_size], i % v_size, uVals[k][i + 2 * out_size]); + } + } +} + +// biases +template +void GRULayerT::setBVals(const std::vector>& bVals) +{ + for(int k = 0; k < out_size; ++k) + { + bz[k / v_size] = set_value(bz[k / v_size], k % v_size, bVals[0][k] + bVals[1][k]); + br[k / v_size] = set_value(br[k / v_size], k % v_size, bVals[0][k + out_size] + bVals[1][k + out_size]); + bh0[k / v_size] = set_value(bh0[k / v_size], k % v_size, bVals[0][k + 2 * out_size]); + bh1[k / v_size] = set_value(bh1[k / v_size], k % v_size, bVals[1][k + 2 * out_size]); + } +} + +} // namespace RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm.h new file mode 100644 index 0000000..3d6cab4 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm.h @@ -0,0 +1,363 @@ +#ifndef LSTM_H_INCLUDED +#define LSTM_H_INCLUDED + +#if RTNEURAL_USE_EIGEN +#include "lstm_eigen.h" +#include "lstm_eigen.tpp" +#elif RTNEURAL_USE_XSIMD +#include "lstm_xsimd.h" +#include "lstm_xsimd.tpp" +#else +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include "../maths/maths_stl.h" +#include + +namespace RTNEURAL_NAMESPACE +{ + +/** + * Dynamic implementation of a LSTM layer with tanh + * activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's LSTM implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class LSTMLayer final : public Layer +{ +public: + /** Constructs a LSTM layer for a given input and output size. */ + LSTMLayer(int in_size, int out_size); + LSTMLayer(std::initializer_list sizes); + LSTMLayer(const LSTMLayer& other); + LSTMLayer& operator=(const LSTMLayer& other); + virtual ~LSTMLayer(); + + /** Resets the state of the LSTM. */ + RTNEURAL_REALTIME void reset() override; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "lstm"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override + { + for(int i = 0; i < Layer::out_size; ++i) + { + fVec[i] = MathsProvider::sigmoid(vMult(fWeights.W[i], input, Layer::in_size) + vMult(fWeights.U[i], ht1, Layer::out_size) + fWeights.b[i]); + iVec[i] = MathsProvider::sigmoid(vMult(iWeights.W[i], input, Layer::in_size) + vMult(iWeights.U[i], ht1, Layer::out_size) + iWeights.b[i]); + oVec[i] = MathsProvider::sigmoid(vMult(oWeights.W[i], input, Layer::in_size) + vMult(oWeights.U[i], ht1, Layer::out_size) + oWeights.b[i]); + ctVec[i] = MathsProvider::tanh(vMult(cWeights.W[i], input, Layer::in_size) + vMult(cWeights.U[i], ht1, Layer::out_size) + cWeights.b[i]); + cVec[i] = fVec[i] * ct1[i] + iVec[i] * ctVec[i]; + h[i] = oVec[i] * MathsProvider::tanh(cVec[i]); + } + + std::copy(cVec, cVec + Layer::out_size, ct1); + std::copy(h, h + Layer::out_size, ht1); + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][4 * out_size] + */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][4 * out_size] + */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[4 * out_size] + */ + RTNEURAL_REALTIME void setBVals(const std::vector& bVals); + +protected: + T* ht1; + T* ct1; + + /** Struct to hold layer weights (used internally) */ + struct WeightSet + { + WeightSet(int in_size, int out_size); + ~WeightSet(); + + T** W; // kernel weights + T** U; // recurrent weights + T* b; // bias + const int out_size; + }; + + WeightSet fWeights; + WeightSet iWeights; + WeightSet oWeights; + WeightSet cWeights; + + T* fVec; + T* iVec; + T* oVec; + T* ctVec; + T* cVec; +}; + +//==================================================== +/** + * Static implementation of a LSTM layer with tanh + * activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's LSTM implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class LSTMLayerT +{ +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + + LSTMLayerT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "lstm"; } + + /** Returns false since LSTM is not an activation. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Prepares the LSTM to process with a given delay length. */ + template + std::enable_if_t + prepare(int delaySamples); + + /** Prepares the LSTM to process with a given delay length. */ + template + std::enable_if_t + prepare(T delaySamples); + + /** Resets the state of the LSTM. */ + RTNEURAL_REALTIME void reset(); + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if<(N > 1), void>::type + forward(const T (&ins)[in_size]) noexcept + { + // compute ft + recurrent_mat_mul(outs, Uf, ft); + kernel_mat_mul(ins, Wf, kernel_outs); + for(int i = 0; i < out_size; ++i) + ft[i] = MathsProvider::sigmoid(ft[i] + bf[i] + kernel_outs[i]); + + // compute it + recurrent_mat_mul(outs, Ui, it); + kernel_mat_mul(ins, Wi, kernel_outs); + for(int i = 0; i < out_size; ++i) + it[i] = MathsProvider::sigmoid(it[i] + bi[i] + kernel_outs[i]); + + // compute ot + recurrent_mat_mul(outs, Uo, ot); + kernel_mat_mul(ins, Wo, kernel_outs); + for(int i = 0; i < out_size; ++i) + ot[i] = MathsProvider::sigmoid(ot[i] + bo[i] + kernel_outs[i]); + + computeOutputs(ins); + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const T (&ins)[in_size]) noexcept + { + // compute ft + recurrent_mat_mul(outs, Uf, ft); + for(int i = 0; i < out_size; ++i) + ft[i] = MathsProvider::sigmoid(ft[i] + bf[i] + (Wf_1[i] * ins[0])); + + // compute it + recurrent_mat_mul(outs, Ui, it); + for(int i = 0; i < out_size; ++i) + it[i] = MathsProvider::sigmoid(it[i] + bi[i] + (Wi_1[i] * ins[0])); + + // compute ot + recurrent_mat_mul(outs, Uo, ot); + for(int i = 0; i < out_size; ++i) + ot[i] = MathsProvider::sigmoid(ot[i] + bo[i] + (Wo_1[i] * ins[0])); + + computeOutputs(ins); + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][4 * out_size] + */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][4 * out_size] + */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[4 * out_size] + */ + RTNEURAL_REALTIME void setBVals(const std::vector& bVals); + + T outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + +private: + template + inline std::enable_if_t + computeOutputs(const T (&ins)[in_size]) noexcept + { + computeOutputsInternal(ins, ct, outs); + } + + template + inline std::enable_if_t + computeOutputs(const T (&ins)[in_size]) noexcept + { + computeOutputsInternal(ins, ct_delayed[delayWriteIdx], outs_delayed[delayWriteIdx]); + + processDelay(ct_delayed, ct, delayWriteIdx); + processDelay(outs_delayed, outs, delayWriteIdx); + } + + template + inline std::enable_if_t<(N > 1), void> + computeOutputsInternal(const T (&ins)[in_size], VecType& ctVec, VecType& outsVec) noexcept + { + // compute ct + recurrent_mat_mul(outs, Uc, ht); + kernel_mat_mul(ins, Wc, kernel_outs); + for(int i = 0; i < out_size; ++i) + ctVec[i] = it[i] * MathsProvider::tanh(ht[i] + bc[i] + kernel_outs[i]) + ft[i] * ct[i]; + + // compute output + for(int i = 0; i < out_size; ++i) + outsVec[i] = ot[i] * MathsProvider::tanh(ctVec[i]); + } + + template + inline std::enable_if_t + computeOutputsInternal(const T (&ins)[in_size], VecType& ctVec, VecType& outsVec) noexcept + { + // compute ct + recurrent_mat_mul(outs, Uc, ht); + for(int i = 0; i < out_size; ++i) + ctVec[i] = it[i] * MathsProvider::tanh(ht[i] + bc[i] + (Wc_1[i] * ins[0])) + ft[i] * ct[i]; + + // compute output + for(int i = 0; i < out_size; ++i) + outsVec[i] = ot[i] * MathsProvider::tanh(ctVec[i]); + } + + template + inline std::enable_if_t + processDelay(std::vector>& delayVec, T (&out)[out_size], int delayWriteIndex) noexcept + { + for(int i = 0; i < out_size; ++i) + out[i] = delayVec[0][i]; + + for(int j = 0; j < delayWriteIndex; ++j) + { + for(int i = 0; i < out_size; ++i) + delayVec[j][i] = delayVec[j + 1][i]; + } + } + + template + inline std::enable_if_t + processDelay(std::vector>& delayVec, T (&out)[out_size], int delayWriteIndex) noexcept + { + for(int i = 0; i < out_size; ++i) + out[i] = delayPlus1Mult * delayVec[0][i] + delayMult * delayVec[1][i]; + + for(int j = 0; j < delayWriteIndex; ++j) + { + for(int i = 0; i < out_size; ++i) + delayVec[j][i] = delayVec[j + 1][i]; + } + } + + static inline void recurrent_mat_mul(const T (&vec)[out_size], const T (&mat)[out_size][out_size], T (&out)[out_size]) noexcept + { + for(int j = 0; j < out_size; ++j) + out[j] = std::inner_product(mat[j], mat[j] + out_size, vec, (T)0); + } + + static inline void kernel_mat_mul(const T (&vec)[in_size], const T (&mat)[out_size][in_size], T (&out)[out_size]) noexcept + { + for(int j = 0; j < out_size; ++j) + out[j] = std::inner_product(mat[j], mat[j] + in_size, vec, (T)0); + } + + // kernel weights + T Wf alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][in_size]; + T Wi alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][in_size]; + T Wo alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][in_size]; + T Wc alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][in_size]; + T kernel_outs alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + // single-input kernel weights + T Wf_1 alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T Wi_1 alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T Wo_1 alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T Wc_1 alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + // recurrent weights + T Uf alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][out_size]; + T Ui alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][out_size]; + T Uo alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][out_size]; + T Uc alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size][out_size]; + + // biases + T bf alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T bi alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T bo alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T bc alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + // intermediate vars + T ft alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T it alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T ot alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T ht alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + T ct alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + // needed for delays when doing sample rate correction + std::vector> ct_delayed; + std::vector> outs_delayed; + int delayWriteIdx = 0; + T delayMult = (T)1; + T delayPlus1Mult = (T)0; +}; + +} // namespace RTNEURAL_NAMESPACE + +#endif + +#endif // LSTM_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm.tpp new file mode 100644 index 0000000..4e6b29e --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm.tpp @@ -0,0 +1,287 @@ +#include "lstm.h" + +namespace RTNEURAL_NAMESPACE +{ + +#if !RTNEURAL_USE_EIGEN && !RTNEURAL_USE_XSIMD + +template +LSTMLayer::LSTMLayer(int in_size, int out_size) + : Layer(in_size, out_size) + , fWeights(in_size, out_size) + , iWeights(in_size, out_size) + , oWeights(in_size, out_size) + , cWeights(in_size, out_size) +{ + ht1 = new T[out_size]; + ct1 = new T[out_size]; + + fVec = new T[out_size]; + iVec = new T[out_size]; + oVec = new T[out_size]; + ctVec = new T[out_size]; + cVec = new T[out_size]; +} + +template +LSTMLayer::LSTMLayer(std::initializer_list sizes) + : LSTMLayer(*sizes.begin(), *(sizes.begin() + 1)) +{ +} + +template +LSTMLayer::LSTMLayer(const LSTMLayer& other) + : LSTMLayer(other.in_size, other.out_size) +{ +} + +template +LSTMLayer& LSTMLayer::operator=(const LSTMLayer& other) +{ + if(&other != this) + *this = LSTMLayer(other); + return *this; +} + +template +LSTMLayer::~LSTMLayer() +{ + delete[] ht1; + delete[] ct1; + + delete[] fVec; + delete[] iVec; + delete[] oVec; + delete[] ctVec; + delete[] cVec; +} + +template +void LSTMLayer::reset() +{ + std::fill(ht1, ht1 + Layer::out_size, (T)0); + std::fill(ct1, ct1 + Layer::out_size, (T)0); +} + +template +LSTMLayer::WeightSet::WeightSet(int in_size, int out_size) + : out_size(out_size) +{ + W = new T*[out_size]; + U = new T*[out_size]; + b = new T[out_size]; + + for(int i = 0; i < out_size; ++i) + { + W[i] = new T[in_size]; + U[i] = new T[out_size]; + } +} + +template +LSTMLayer::WeightSet::~WeightSet() +{ + delete[] b; + + for(int i = 0; i < out_size; ++i) + { + delete[] W[i]; + delete[] U[i]; + } + + delete[] W; + delete[] U; +} + +template +void LSTMLayer::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < Layer::in_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + iWeights.W[k][i] = wVals[i][k]; + fWeights.W[k][i] = wVals[i][k + Layer::out_size]; + cWeights.W[k][i] = wVals[i][k + Layer::out_size * 2]; + oWeights.W[k][i] = wVals[i][k + Layer::out_size * 3]; + } + } +} + +template +void LSTMLayer::setUVals(const std::vector>& uVals) +{ + for(int i = 0; i < Layer::out_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + iWeights.U[k][i] = uVals[i][k]; + fWeights.U[k][i] = uVals[i][k + Layer::out_size]; + cWeights.U[k][i] = uVals[i][k + Layer::out_size * 2]; + oWeights.U[k][i] = uVals[i][k + Layer::out_size * 3]; + } + } +} + +template +void LSTMLayer::setBVals(const std::vector& bVals) +{ + for(int k = 0; k < Layer::out_size; ++k) + { + iWeights.b[k] = bVals[k]; + fWeights.b[k] = bVals[k + Layer::out_size]; + cWeights.b[k] = bVals[k + Layer::out_size * 2]; + oWeights.b[k] = bVals[k + Layer::out_size * 3]; + } +} + +//==================================================== +template +LSTMLayerT::LSTMLayerT() +{ + for(int i = 0; i < out_size; ++i) + { + // single-input kernel weights + Wf_1[i] = (T)0; + Wi_1[i] = (T)0; + Wo_1[i] = (T)0; + Wc_1[i] = (T)0; + + // biases + bf[i] = (T)0; + bi[i] = (T)0; + bo[i] = (T)0; + bc[i] = (T)0; + + // intermediate vars + ft[i] = (T)0; + it[i] = (T)0; + ot[i] = (T)0; + ht[i] = (T)0; + } + + for(int i = 0; i < out_size; ++i) + { + // recurrent weights + for(int k = 0; k < out_size; ++k) + { + Uf[i][k] = (T)0; + Ui[i][k] = (T)0; + Uo[i][k] = (T)0; + Uc[i][k] = (T)0; + } + + // kernel weights + for(int k = 0; k < in_size; ++k) + { + Wf[i][k] = (T)0; + Wi[i][k] = (T)0; + Wo[i][k] = (T)0; + Wc[i][k] = (T)0; + } + } + + reset(); +} + +template +template +std::enable_if_t +LSTMLayerT::prepare(int delaySamples) +{ + delayWriteIdx = delaySamples - 1; + ct_delayed.resize(delayWriteIdx + 1, {}); + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +template +std::enable_if_t +LSTMLayerT::prepare(T delaySamples) +{ + const auto delayOffFactor = delaySamples - std::floor(delaySamples); + delayMult = (T)1 - delayOffFactor; + delayPlus1Mult = delayOffFactor; + + delayWriteIdx = (int)std::ceil(delaySamples) - (int)std::ceil(delayOffFactor); + ct_delayed.resize(delayWriteIdx + 1, {}); + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +void LSTMLayerT::reset() +{ + if(sampleRateCorr != SampleRateCorrectionMode::None) + { + for(auto& x : ct_delayed) + std::fill(x.begin(), x.end(), T {}); + + for(auto& x : outs_delayed) + std::fill(x.begin(), x.end(), T {}); + } + + // reset output state + for(int i = 0; i < out_size; ++i) + { + ct[i] = (T)0; + outs[i] = (T)0; + } +} + +template +void LSTMLayerT::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < in_size; ++i) + { + for(int j = 0; j < out_size; ++j) + { + Wi[j][i] = wVals[i][j]; + Wf[j][i] = wVals[i][j + out_size]; + Wc[j][i] = wVals[i][j + 2 * out_size]; + Wo[j][i] = wVals[i][j + 3 * out_size]; + } + } + + for(int j = 0; j < out_size; ++j) + { + Wi_1[j] = wVals[0][j]; + Wf_1[j] = wVals[0][j + out_size]; + Wc_1[j] = wVals[0][j + 2 * out_size]; + Wo_1[j] = wVals[0][j + 3 * out_size]; + } +} + +template +void LSTMLayerT::setUVals(const std::vector>& uVals) +{ + for(int i = 0; i < out_size; ++i) + { + for(int j = 0; j < out_size; ++j) + { + Ui[j][i] = uVals[i][j]; + Uf[j][i] = uVals[i][j + out_size]; + Uc[j][i] = uVals[i][j + 2 * out_size]; + Uo[j][i] = uVals[i][j + 3 * out_size]; + } + } +} + +template +void LSTMLayerT::setBVals(const std::vector& bVals) +{ + for(int k = 0; k < out_size; ++k) + { + bi[k] = bVals[k]; + bf[k] = bVals[k + out_size]; + bc[k] = bVals[k + 2 * out_size]; + bo[k] = bVals[k + 3 * out_size]; + } +} + +#endif // !RTNEURAL_USE_EIGEN && !RTNEURAL_USE_XSIMD + +} // namespace RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_eigen.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_eigen.h new file mode 100644 index 0000000..c8f3d94 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_eigen.h @@ -0,0 +1,291 @@ +#ifndef LSTM_EIGEN_INCLUDED +#define LSTM_EIGEN_INCLUDED + +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include "../maths/maths_eigen.h" + +namespace RTNEURAL_NAMESPACE +{ + +/** + * Dynamic implementation of a LSTM layer with tanh + * activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's LSTM implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class LSTMLayer : public Layer +{ +public: + /** Constructs a LSTM layer for a given input and output size. */ + LSTMLayer(int in_size, int out_size); + LSTMLayer(std::initializer_list sizes); + LSTMLayer(const LSTMLayer& other); + LSTMLayer& operator=(const LSTMLayer& other); + virtual ~LSTMLayer() = default; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "lstm"; } + + /** Resets the state of the LSTM. */ + RTNEURAL_REALTIME void reset() override; + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override + { + for(int i = 0; i < Layer::in_size; ++i) + { + extendedInVecHt1(i) = input[i]; + } + + /** + * | f | | Wf Uf Bf | | input | + * | i | = | Wi Ui Bi | * | ht1 | + * | o | | Wo Uo Bo | | 1 | + * | ct | | Wct Uct Bct | + */ + fioctVecs.noalias() = combinedWeights * extendedInVecHt1; + + fioVecs = fioctVecs.segment(0, Layer::out_size * 3); + ctVec = MathsProvider::tanh(fioctVecs.segment(Layer::out_size * 3, Layer::out_size)); + + fioVecs = MathsProvider::sigmoid(fioVecs); + + ct1 = fioVecs.segment(0, Layer::out_size).cwiseProduct(ct1) + fioVecs.segment(Layer::out_size, Layer::out_size).cwiseProduct(ctVec); + cTanhVec = MathsProvider::tanh(ct1); + + ht1 = fioVecs.segment(Layer::out_size * 2, Layer::out_size).cwiseProduct(cTanhVec); + + for(int i = 0; i < Layer::out_size; ++i) + { + h[i] = extendedInVecHt1(Layer::in_size + i) = ht1(i); + } + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][4 * out_size] + */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][4 * out_size] + */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[4 * out_size] + */ + RTNEURAL_REALTIME void setBVals(const std::vector& bVals); + +private: + Eigen::Matrix combinedWeights; + + Eigen::Matrix extendedInVecHt1; + + Eigen::Matrix fioctVecs; + Eigen::Matrix fioVecs; + Eigen::Matrix ctVec; + + Eigen::Matrix cTanhVec; + + Eigen::Matrix ht1; + Eigen::Matrix ct1; +}; + +//==================================================== +/** + * Static implementation of a LSTM layer with tanh + * activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's LSTM implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class LSTMLayerT +{ + using weights_combined_type = Eigen::Matrix; + using extended_in_out_type = Eigen::Matrix; + using four_out_type = Eigen::Matrix; + using three_out_type = Eigen::Matrix; + + using in_type = Eigen::Matrix; + using out_type = Eigen::Matrix; + +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + + LSTMLayerT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "lstm"; } + + /** Returns false since LSTM is not an activation. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Prepares the LSTM to process with a given delay length. */ + template + std::enable_if_t + prepare(int delaySamples); + + /** Prepares the LSTM to process with a given delay length. */ + template + std::enable_if_t + prepare(T delaySamples); + + /** Resets the state of the LSTM. */ + RTNEURAL_REALTIME void reset(); + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const in_type& ins) noexcept + { + for(int i = 0; i < in_sizet; ++i) + { + extendedInHt1Vec(i) = ins(i); + } + + /** + * | f | | Wf Uf Bf | | input | + * | i | = | Wi Ui Bi | * | ht1 | + * | o | | Wo Uo Bo | | 1 | + * | ct | | Wct Uct Bct | + */ + fioctsVecs.noalias() = combinedWeights * extendedInHt1Vec; + + fioVecs = MathsProvider::sigmoid(fioctsVecs.segment(0, 3 * out_sizet)); + ctVec = MathsProvider::tanh(fioctsVecs.segment(3 * out_sizet, out_sizet)); + + computeOutputs(); + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][4 * out_size] + */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][4 * out_size] + */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[4 * out_size] + */ + RTNEURAL_REALTIME void setBVals(const std::vector& bVals); + + Eigen::Map outs; + +private: + T outs_internal alignas(RTNEURAL_DEFAULT_ALIGNMENT)[out_size]; + + template + inline std::enable_if_t + computeOutputs() noexcept + { + computeOutputsInternal(cVec, outs); + + for(int i = 0; i < out_sizet; ++i) + { + extendedInHt1Vec(in_sizet + i) = outs(i); + } + } + + template + inline std::enable_if_t + computeOutputs() noexcept + { + computeOutputsInternal(ct_delayed[delayWriteIdx], outs_delayed[delayWriteIdx]); + + processDelay(ct_delayed, cVec, delayWriteIdx); + processDelay(outs_delayed, outs, delayWriteIdx); + + for(int i = 0; i < out_sizet; ++i) + { + extendedInHt1Vec(in_sizet + i) = outs(i); + } + } + + template + inline void computeOutputsInternal(VecType1& cVecLocal, VecType2& outsVec) noexcept + { + cVecLocal.noalias() + = fioVecs.segment(0, out_sizet) + .cwiseProduct(cVec) + + fioVecs.segment(out_sizet, out_sizet) + .cwiseProduct(ctVec); + + cTanhVec = MathsProvider::tanh(cVecLocal); + outsVec.noalias() = fioVecs.segment(out_sizet * 2, out_sizet).cwiseProduct(cTanhVec); + } + + template + inline std::enable_if_t + processDelay(std::vector& delayVec, OutVec& out, int delayWriteIndex) noexcept + { + out = delayVec[0]; + + for(int j = 0; j < delayWriteIndex; ++j) + delayVec[j] = delayVec[j + 1]; + } + + template + inline std::enable_if_t + processDelay(std::vector& delayVec, OutVec& out, int delayWriteIndex) noexcept + { + out = delayPlus1Mult * delayVec[0] + delayMult * delayVec[1]; + + for(int j = 0; j < delayWriteIndex; ++j) + delayVec[j] = delayVec[j + 1]; + } + + // kernel weights + weights_combined_type combinedWeights; + extended_in_out_type extendedInHt1Vec; + four_out_type fioctsVecs; + three_out_type fioVecs; + out_type cTanhVec; + + // intermediate values + out_type ctVec; + out_type cVec; + + // needed for delays when doing sample rate correction + std::vector ct_delayed; + std::vector outs_delayed; + int delayWriteIdx = 0; + T delayMult = (T)1; + T delayPlus1Mult = (T)0; +}; + +} // namespace RTNEURAL_NAMESPACE + +#endif // LSTM_EIGEN_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_eigen.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_eigen.tpp new file mode 100644 index 0000000..c48301d --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_eigen.tpp @@ -0,0 +1,208 @@ +#include "lstm_eigen.h" + +namespace RTNEURAL_NAMESPACE +{ + +template +LSTMLayer::LSTMLayer(int in_size, int out_size) + : Layer(in_size, out_size) +{ + combinedWeights = Eigen::Matrix::Zero(4 * out_size, in_size + out_size + 1); + extendedInVecHt1 = Eigen::Matrix::Zero(in_size + out_size + 1); + extendedInVecHt1(in_size + out_size) = (T)1; + + fioctVecs = Eigen::Matrix::Zero(4 * out_size); + fioVecs = Eigen::Matrix::Zero(3 * out_size); + ctVec = Eigen::Matrix::Zero(out_size); + + cTanhVec = Eigen::Matrix::Zero(out_size, 1); + + ht1 = Eigen::Matrix::Zero(out_size); + ct1 = Eigen::Matrix::Zero(out_size); +} + +template +LSTMLayer::LSTMLayer(std::initializer_list sizes) + : LSTMLayer(*sizes.begin(), *(sizes.begin() + 1)) +{ +} + +template +LSTMLayer::LSTMLayer(const LSTMLayer& other) + : LSTMLayer(other.in_size, other.out_size) +{ +} + +template +LSTMLayer& LSTMLayer::operator=(const LSTMLayer& other) +{ + return *this = LSTMLayer(other); +} + +template +void LSTMLayer::reset() +{ + ht1.setZero(); + ct1.setZero(); + extendedInVecHt1.setZero(); + extendedInVecHt1(Layer::in_size + Layer::out_size) = (T)1; +} + +template +void LSTMLayer::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < Layer::in_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + combinedWeights(k, i) = wVals[i][k + Layer::out_size]; // Wf + combinedWeights(k + Layer::out_size, i) = wVals[i][k]; // Wi + combinedWeights(k + Layer::out_size * 2, i) = wVals[i][k + Layer::out_size * 3]; // Wo + combinedWeights(k + Layer::out_size * 3, i) = wVals[i][k + Layer::out_size * 2]; // Wc + } + } +} + +template +void LSTMLayer::setUVals(const std::vector>& uVals) +{ + int col; + for(int i = 0; i < Layer::out_size; ++i) + { + col = i + Layer::in_size; + for(int k = 0; k < Layer::out_size; ++k) + { + combinedWeights(k, col) = uVals[i][k + Layer::out_size]; // Uf + combinedWeights(k + Layer::out_size, col) = uVals[i][k]; // Ui + combinedWeights(k + Layer::out_size * 2, col) = uVals[i][k + Layer::out_size * 3]; // Uo + combinedWeights(k + Layer::out_size * 3, col) = uVals[i][k + Layer::out_size * 2]; // Uc + } + } +} + +template +void LSTMLayer::setBVals(const std::vector& bVals) +{ + int col = Layer::in_size + Layer::out_size; + for(int k = 0; k < Layer::out_size; ++k) + { + combinedWeights(k, col) = bVals[k + Layer::out_size]; // Bf + combinedWeights(k + Layer::out_size, col) = bVals[k]; // Bi + combinedWeights(k + Layer::out_size * 2, col) = bVals[k + Layer::out_size * 3]; // Bo + combinedWeights(k + Layer::out_size * 3, col) = bVals[k + Layer::out_size * 2]; // Bc + } +} + +//==================================================== +template +LSTMLayerT::LSTMLayerT() + : outs(outs_internal) +{ + combinedWeights = weights_combined_type::Zero(); + extendedInHt1Vec = extended_in_out_type::Zero(); + fioctsVecs = four_out_type::Zero(); + fioVecs = three_out_type::Zero(); + + ctVec = out_type::Zero(); + cTanhVec = out_type::Zero(); + + reset(); +} + +template +template +std::enable_if_t +LSTMLayerT::prepare(int delaySamples) +{ + delayWriteIdx = delaySamples - 1; + ct_delayed.resize(delayWriteIdx + 1, {}); + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +template +std::enable_if_t +LSTMLayerT::prepare(T delaySamples) +{ + const auto delayOffFactor = delaySamples - std::floor(delaySamples); + delayMult = (T)1 - delayOffFactor; + delayPlus1Mult = delayOffFactor; + + delayWriteIdx = (int)std::ceil(delaySamples) - (int)std::ceil(delayOffFactor); + ct_delayed.resize(delayWriteIdx + 1, {}); + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +void LSTMLayerT::reset() +{ + if(sampleRateCorr != SampleRateCorrectionMode::None) + { + for(auto& x : ct_delayed) + x = out_type::Zero(); + + for(auto& x : outs_delayed) + x = out_type::Zero(); + } + + // reset output state + extendedInHt1Vec.setZero(); + extendedInHt1Vec(in_sizet + out_sizet) = (T)1; + outs = out_type::Zero(); + cVec = out_type::Zero(); + ctVec.setZero(); +} + +// kernel weights +template +void LSTMLayerT::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < in_size; ++i) + { + for(int k = 0; k < out_size; ++k) + { + combinedWeights(k, i) = wVals[i][k + out_sizet]; // Wf + combinedWeights(k + out_sizet, i) = wVals[i][k]; // Wi + combinedWeights(k + out_sizet * 2, i) = wVals[i][k + out_sizet * 3]; // Wo + combinedWeights(k + out_sizet * 3, i) = wVals[i][k + out_sizet * 2]; // Wc + } + } +} + +// recurrent weights +template +void LSTMLayerT::setUVals(const std::vector>& uVals) +{ + int col; + for(int i = 0; i < out_size; ++i) + { + col = i + in_sizet; + for(int k = 0; k < out_size; ++k) + { + combinedWeights(k, col) = uVals[i][k + out_sizet]; // Uf + combinedWeights(k + out_sizet, col) = uVals[i][k]; // Ui + combinedWeights(k + out_sizet * 2, col) = uVals[i][k + out_sizet * 3]; // Uo + combinedWeights(k + out_sizet * 3, col) = uVals[i][k + out_sizet * 2]; // Uc + } + } +} + +// biases +template +void LSTMLayerT::setBVals(const std::vector& bVals) +{ + int col = in_size + out_size; + for(int k = 0; k < out_size; ++k) + { + combinedWeights(k, col) = bVals[k + out_sizet]; // Bf + combinedWeights(k + out_sizet, col) = bVals[k]; // Bi + combinedWeights(k + out_sizet * 2, col) = bVals[k + out_sizet * 3]; // Bo + combinedWeights(k + out_sizet * 3, col) = bVals[k + out_sizet * 2]; // Bc + } +} + +} // namespace RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_xsimd.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_xsimd.h new file mode 100644 index 0000000..973235f --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_xsimd.h @@ -0,0 +1,408 @@ +#ifndef LSTM_XSIMD_H_INCLUDED +#define LSTM_XSIMD_H_INCLUDED + +#include "../Layer.h" +#include "../common.h" +#include "../config.h" +#include "../maths/maths_xsimd.h" +#include + +namespace RTNEURAL_NAMESPACE +{ + +/** + * Dynamic implementation of a LSTM layer with tanh + * activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's LSTM implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class LSTMLayer : public Layer +{ +public: + /** Constructs a LSTM layer for a given input and output size. */ + LSTMLayer(int in_size, int out_size); + LSTMLayer(std::initializer_list sizes); + LSTMLayer(const LSTMLayer& other); + LSTMLayer& operator=(const LSTMLayer& other); + virtual ~LSTMLayer(); + + /** Resets the state of the LSTM. */ + RTNEURAL_REALTIME void reset() override; + + /** Returns the name of this layer. */ + std::string getName() const noexcept override { return "lstm"; } + + /** Performs forward propagation for this layer. */ + RTNEURAL_REALTIME inline void forward(const T* input, T* h) noexcept override + { + for(int i = 0; i < Layer::out_size; ++i) + { + fVec[i] = vMult(fWeights.W[i].data(), input, prod_in.data(), Layer::in_size) + vMult(fWeights.U[i].data(), ht1.data(), prod_out.data(), Layer::out_size); + iVec[i] = vMult(iWeights.W[i].data(), input, prod_in.data(), Layer::in_size) + vMult(iWeights.U[i].data(), ht1.data(), prod_out.data(), Layer::out_size); + oVec[i] = vMult(oWeights.W[i].data(), input, prod_in.data(), Layer::in_size) + vMult(oWeights.U[i].data(), ht1.data(), prod_out.data(), Layer::out_size); + ctVec[i] = vMult(cWeights.W[i].data(), input, prod_in.data(), Layer::in_size) + vMult(cWeights.U[i].data(), ht1.data(), prod_out.data(), Layer::out_size); + } + + vAdd(fVec.data(), fWeights.b.data(), fVec.data(), Layer::out_size); + sigmoid(fVec.data(), fVec.data(), Layer::out_size); + + vAdd(iVec.data(), iWeights.b.data(), iVec.data(), Layer::out_size); + sigmoid(iVec.data(), iVec.data(), Layer::out_size); + + vAdd(oVec.data(), oWeights.b.data(), oVec.data(), Layer::out_size); + sigmoid(oVec.data(), oVec.data(), Layer::out_size); + + vAdd(ctVec.data(), cWeights.b.data(), ctVec.data(), Layer::out_size); + tanh(ctVec.data(), ctVec.data(), Layer::out_size); + + vProd(fVec.data(), ct1.data(), cVec.data(), Layer::out_size); + vProd(iVec.data(), ctVec.data(), prod_out.data(), Layer::out_size); + vAdd(cVec.data(), prod_out.data(), cVec.data(), Layer::out_size); + + tanh(cVec.data(), h, Layer::out_size); + vProd(h, oVec.data(), h, Layer::out_size); + + vCopy(cVec.data(), ct1.data(), Layer::out_size); + vCopy(h, ht1.data(), Layer::out_size); + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][4 * out_size] + */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][4 * out_size] + */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[4 * out_size] + */ + RTNEURAL_REALTIME void setBVals(const std::vector& bVals); + +protected: + using vec_type = std::vector>; + using vec2_type = std::vector; + + vec_type ht1; + vec_type ct1; + + struct WeightSet + { + WeightSet(int in_size, int out_size); + ~WeightSet(); + + vec2_type W; // kernel weights + vec2_type U; // recurrent weights + vec_type b; // bias + const int out_size; + }; + + WeightSet fWeights; + WeightSet iWeights; + WeightSet oWeights; + WeightSet cWeights; + + vec_type fVec; + vec_type iVec; + vec_type oVec; + vec_type ctVec; + vec_type cVec; + + vec_type prod_in; + vec_type prod_out; +}; + +//==================================================== +/** + * Static implementation of a LSTM layer with tanh + * activation and sigmoid recurrent activation. + * + * To ensure that the recurrent state is initialized to zero, + * please make sure to call `reset()` before your first call to + * the `forward()` method. + * + * Compared to TensorFlow's LSTM implementation, this layer will + * behave by default as if the parameter `stateful=True`. A "stateless" + * GRU can be achieved by calling the `reset()` function in between + * calls to `forward()`. + */ +template +class LSTMLayerT +{ + using v_type = xsimd::simd_type; + static constexpr auto v_size = (int)v_type::size; + static constexpr auto v_in_size = ceil_div(in_sizet, v_size); + static constexpr auto v_out_size = ceil_div(out_sizet, v_size); + +public: + static constexpr auto in_size = in_sizet; + static constexpr auto out_size = out_sizet; + + LSTMLayerT(); + + /** Returns the name of this layer. */ + std::string getName() const noexcept { return "lstm"; } + + /** Returns false since LSTM is not an activation. */ + constexpr bool isActivation() const noexcept { return false; } + + /** Prepares the LSTM to process with a given delay length. */ + template + std::enable_if_t + prepare(int delaySamples); + + /** Prepares the LSTM to process with a given delay length. */ + template + std::enable_if_t + prepare(T delaySamples); + + /** Resets the state of the LSTM. */ + RTNEURAL_REALTIME void reset(); + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if<(N > 1), void>::type + forward(const v_type (&ins)[v_in_size]) noexcept + { + // compute ft + recurrent_mat_mul(outs, Uf, ft); + kernel_mat_mul(ins, Wf, kernel_outs); + for(int i = 0; i < v_out_size; ++i) + ft[i] = MathsProvider::sigmoid(ft[i] + bf[i] + kernel_outs[i]); + + // compute it + recurrent_mat_mul(outs, Ui, it); + kernel_mat_mul(ins, Wi, kernel_outs); + for(int i = 0; i < v_out_size; ++i) + it[i] = MathsProvider::sigmoid(it[i] + bi[i] + kernel_outs[i]); + + // compute ot + recurrent_mat_mul(outs, Uo, ot); + kernel_mat_mul(ins, Wo, kernel_outs); + for(int i = 0; i < v_out_size; ++i) + ot[i] = MathsProvider::sigmoid(ot[i] + bo[i] + kernel_outs[i]); + + computeOutputs(ins); + } + + /** Performs forward propagation for this layer. */ + template + RTNEURAL_REALTIME inline typename std::enable_if::type + forward(const v_type (&ins)[v_in_size]) noexcept + { + // compute ft + recurrent_mat_mul(outs, Uf, ft); + for(int i = 0; i < v_out_size; ++i) + ft[i] = MathsProvider::sigmoid(xsimd::fma(Wf_1[i], ins[0], ft[i] + bf[i])); + + // compute it + recurrent_mat_mul(outs, Ui, it); + for(int i = 0; i < v_out_size; ++i) + it[i] = MathsProvider::sigmoid(xsimd::fma(Wi_1[i], ins[0], it[i] + bi[i])); + + // compute ot + recurrent_mat_mul(outs, Uo, ot); + for(int i = 0; i < v_out_size; ++i) + ot[i] = MathsProvider::sigmoid(xsimd::fma(Wo_1[i], ins[0], ot[i] + bo[i])); + + computeOutputs(ins); + } + + /** + * Sets the layer kernel weights. + * + * The weights vector must have size weights[in_size][4 * out_size] + */ + RTNEURAL_REALTIME void setWVals(const std::vector>& wVals); + + /** + * Sets the layer recurrent weights. + * + * The weights vector must have size weights[out_size][4 * out_size] + */ + RTNEURAL_REALTIME void setUVals(const std::vector>& uVals); + + /** + * Sets the layer bias. + * + * The bias vector must have size weights[4 * out_size] + */ + RTNEURAL_REALTIME void setBVals(const std::vector& bVals); + + v_type outs[v_out_size]; + +private: + template + inline std::enable_if_t + computeOutputs(const v_type (&ins)[v_in_size]) noexcept + { + computeOutputsInternal(ins, ct, outs); + } + + template + inline std::enable_if_t + computeOutputs(const v_type (&ins)[v_in_size]) noexcept + { + computeOutputsInternal(ins, ct_delayed[delayWriteIdx], outs_delayed[delayWriteIdx]); + + processDelay(ct_delayed, ct, delayWriteIdx); + processDelay(outs_delayed, outs, delayWriteIdx); + } + + template + inline std::enable_if_t<(N > 1), void> + computeOutputsInternal(const v_type (&ins)[v_in_size], VecType& ctVec, VecType& outsVec) noexcept + { + // compute ct + recurrent_mat_mul(outs, Uc, ht); + kernel_mat_mul(ins, Wc, kernel_outs); + for(int i = 0; i < v_out_size; ++i) + ctVec[i] = xsimd::fma(it[i], MathsProvider::tanh(ht[i] + bc[i] + kernel_outs[i]), ft[i] * ct[i]); + + // compute output + for(int i = 0; i < v_out_size; ++i) + outsVec[i] = ot[i] * MathsProvider::tanh(ctVec[i]); + } + + template + inline std::enable_if_t + computeOutputsInternal(const v_type (&ins)[v_in_size], VecType& ctVec, VecType& outsVec) noexcept + { + // compute ct + recurrent_mat_mul(outs, Uc, ht); + for(int i = 0; i < v_out_size; ++i) + ctVec[i] = xsimd::fma(it[i], MathsProvider::tanh(xsimd::fma(Wc_1[i], ins[0], ht[i] + bc[i])), ft[i] * ct[i]); + + // compute output + for(int i = 0; i < v_out_size; ++i) + outsVec[i] = ot[i] * MathsProvider::tanh(ctVec[i]); + } + + template + inline std::enable_if_t + processDelay(std::vector>& delayVec, v_type (&out)[v_out_size], int delayWriteIndex) noexcept + { + for(int i = 0; i < v_out_size; ++i) + out[i] = delayVec[0][i]; + + for(int j = 0; j < delayWriteIndex; ++j) + { + for(int i = 0; i < v_out_size; ++i) + delayVec[j][i] = delayVec[j + 1][i]; + } + } + + template + inline std::enable_if_t + processDelay(std::vector>& delayVec, v_type (&out)[v_out_size], int delayWriteIndex) noexcept + { + for(int i = 0; i < v_out_size; ++i) + out[i] = delayPlus1Mult * delayVec[0][i] + delayMult * delayVec[1][i]; + + for(int j = 0; j < delayWriteIndex; ++j) + { + for(int i = 0; i < v_out_size; ++i) + delayVec[j][i] = delayVec[j + 1][i]; + } + } + + static inline void recurrent_mat_mul(const v_type (&vec)[v_out_size], const v_type (&mat)[out_size][v_out_size], v_type (&out)[v_out_size]) noexcept + { + for(int i = 0; i < v_out_size; ++i) + out[i] = v_type(0); + + T scalar_in alignas(RTNEURAL_DEFAULT_ALIGNMENT)[v_size] { (T)0 }; + for(int k = 0; k < v_out_size; ++k) + { + vec[k].store_aligned(scalar_in); + for(int i = 0; i < v_out_size; ++i) + { + for(int j = 0; j < v_size; ++j) + out[i] += scalar_in[j] * mat[k * v_size + j][i]; + } + } + } + + static inline void kernel_mat_mul(const v_type (&vec)[v_in_size], const v_type (&mat)[in_size][v_out_size], v_type (&out)[v_out_size]) noexcept + { + for(int i = 0; i < v_out_size; ++i) + out[i] = v_type(0); + + T scalar_in alignas(RTNEURAL_DEFAULT_ALIGNMENT)[v_size] { (T)0 }; + for(int k = 0; k < v_in_size; ++k) + { + vec[k].store_aligned(scalar_in); + for(int i = 0; i < v_out_size; ++i) + { + for(int j = 0; j < v_size; ++j) + out[i] += scalar_in[j] * mat[k * v_size + j][i]; + } + } + } + + static inline v_type sigmoid(v_type x) noexcept + { + return (T)1.0 / ((T)1.0 + xsimd::exp(-x)); + } + + // kernel weights + v_type Wf[in_size][v_out_size]; + v_type Wi[in_size][v_out_size]; + v_type Wo[in_size][v_out_size]; + v_type Wc[in_size][v_out_size]; + v_type kernel_outs[v_out_size]; + + // single-input kernel weights + v_type Wf_1[v_out_size]; + v_type Wi_1[v_out_size]; + v_type Wo_1[v_out_size]; + v_type Wc_1[v_out_size]; + + // recurrent weights + v_type Uf[out_size][v_out_size]; + v_type Ui[out_size][v_out_size]; + v_type Uo[out_size][v_out_size]; + v_type Uc[out_size][v_out_size]; + + // biases + v_type bf[v_out_size]; + v_type bi[v_out_size]; + v_type bo[v_out_size]; + v_type bc[v_out_size]; + + // intermediate vars + v_type ft[v_out_size]; + v_type it[v_out_size]; + v_type ot[v_out_size]; + v_type ht[v_out_size]; + v_type ct[v_out_size]; + + // needed for delays when doing sample rate correction + std::vector> ct_delayed; + std::vector> outs_delayed; + int delayWriteIdx = 0; + v_type delayMult = (T)1; + v_type delayPlus1Mult = (T)0; +}; + +} // namespace RTNEURAL_NAMESPACE + +#endif // LSTM_XSIMD_H_INCLUDED diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_xsimd.tpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_xsimd.tpp new file mode 100644 index 0000000..ad1f1b9 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/lstm/lstm_xsimd.tpp @@ -0,0 +1,259 @@ +#include "lstm_xsimd.h" + +namespace RTNEURAL_NAMESPACE +{ + +template +LSTMLayer::LSTMLayer(int in_size, int out_size) + : Layer(in_size, out_size) + , fWeights(in_size, out_size) + , iWeights(in_size, out_size) + , oWeights(in_size, out_size) + , cWeights(in_size, out_size) +{ + ht1.resize(out_size, (T)0); + ct1.resize(out_size, (T)0); + + fVec.resize(out_size, (T)0); + iVec.resize(out_size, (T)0); + oVec.resize(out_size, (T)0); + ctVec.resize(out_size, (T)0); + cVec.resize(out_size, (T)0); + + prod_in.resize(in_size, (T)0); + prod_out.resize(out_size, (T)0); +} + +template +LSTMLayer::LSTMLayer(std::initializer_list sizes) + : LSTMLayer(*sizes.begin(), *(sizes.begin() + 1)) +{ +} + +template +LSTMLayer::LSTMLayer(const LSTMLayer& other) + : LSTMLayer(other.in_size, other.out_size) +{ +} + +template +LSTMLayer& LSTMLayer::operator=(const LSTMLayer& other) +{ + return *this = LSTMLayer(other); +} + +template +LSTMLayer::~LSTMLayer() = default; + +template +void LSTMLayer::reset() +{ + std::fill(ht1.begin(), ht1.end(), (T)0); + std::fill(ct1.begin(), ct1.end(), (T)0); +} + +template +LSTMLayer::WeightSet::WeightSet(int in_size, int out_size) + : out_size(out_size) +{ + W = vec2_type(out_size, vec_type(in_size, (T)0)); + U = vec2_type(out_size, vec_type(out_size, (T)0)); + b.resize(out_size, (T)0); +} + +template +LSTMLayer::WeightSet::~WeightSet() = default; + +template +void LSTMLayer::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < Layer::in_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + iWeights.W[k][i] = wVals[i][k]; + fWeights.W[k][i] = wVals[i][k + Layer::out_size]; + cWeights.W[k][i] = wVals[i][k + Layer::out_size * 2]; + oWeights.W[k][i] = wVals[i][k + Layer::out_size * 3]; + } + } +} + +template +void LSTMLayer::setUVals(const std::vector>& uVals) +{ + for(int i = 0; i < Layer::out_size; ++i) + { + for(int k = 0; k < Layer::out_size; ++k) + { + iWeights.U[k][i] = uVals[i][k]; + fWeights.U[k][i] = uVals[i][k + Layer::out_size]; + cWeights.U[k][i] = uVals[i][k + Layer::out_size * 2]; + oWeights.U[k][i] = uVals[i][k + Layer::out_size * 3]; + } + } +} + +template +void LSTMLayer::setBVals(const std::vector& bVals) +{ + for(int k = 0; k < Layer::out_size; ++k) + { + iWeights.b[k] = bVals[k]; + fWeights.b[k] = bVals[k + Layer::out_size]; + cWeights.b[k] = bVals[k + Layer::out_size * 2]; + oWeights.b[k] = bVals[k + Layer::out_size * 3]; + } +} + +//==================================================== +template +LSTMLayerT::LSTMLayerT() +{ + for(int i = 0; i < v_out_size; ++i) + { + // single-input kernel weights + Wf_1[i] = v_type((T)0); + Wi_1[i] = v_type((T)0); + Wo_1[i] = v_type((T)0); + Wc_1[i] = v_type((T)0); + + // biases + bf[i] = v_type((T)0); + bi[i] = v_type((T)0); + bo[i] = v_type((T)0); + bc[i] = v_type((T)0); + + // intermediate vars + ft[i] = v_type((T)0); + it[i] = v_type((T)0); + ot[i] = v_type((T)0); + ht[i] = v_type((T)0); + } + + // kernel weights + for(int k = 0; k < in_size; ++k) + { + for(int i = 0; i < v_out_size; ++i) + { + Wf[k][i] = v_type((T)0); + Wi[k][i] = v_type((T)0); + Wo[k][i] = v_type((T)0); + Wc[k][i] = v_type((T)0); + } + } + + // recurrent weights + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < v_out_size; ++k) + { + Uf[i][k] = v_type((T)0); + Ui[i][k] = v_type((T)0); + Uo[i][k] = v_type((T)0); + Uc[i][k] = v_type((T)0); + } + } + + reset(); +} + +template +template +std::enable_if_t +LSTMLayerT::prepare(int delaySamples) +{ + delayWriteIdx = delaySamples - 1; + ct_delayed.resize(delayWriteIdx + 1, {}); + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +template +std::enable_if_t +LSTMLayerT::prepare(T delaySamples) +{ + const auto delayOffFactor = delaySamples - std::floor(delaySamples); + delayMult = (T)1 - delayOffFactor; + delayPlus1Mult = delayOffFactor; + + delayWriteIdx = (int)std::ceil(delaySamples) - (int)std::ceil(delayOffFactor); + ct_delayed.resize(delayWriteIdx + 1, {}); + outs_delayed.resize(delayWriteIdx + 1, {}); + + reset(); +} + +template +void LSTMLayerT::reset() +{ + if constexpr(sampleRateCorr != SampleRateCorrectionMode::None) + { + for(auto& x : ct_delayed) + std::fill(x.begin(), x.end(), v_type {}); + + for(auto& x : outs_delayed) + std::fill(x.begin(), x.end(), v_type {}); + } + + // reset output state + for(int i = 0; i < v_out_size; ++i) + { + ct[i] = v_type((T)0); + outs[i] = v_type((T)0); + } +} + +template +void LSTMLayerT::setWVals(const std::vector>& wVals) +{ + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < in_size; ++k) + { + Wi[k][i / v_size] = set_value(Wi[k][i / v_size], i % v_size, wVals[k][i]); + Wf[k][i / v_size] = set_value(Wf[k][i / v_size], i % v_size, wVals[k][i + out_size]); + Wc[k][i / v_size] = set_value(Wc[k][i / v_size], i % v_size, wVals[k][i + 2 * out_size]); + Wo[k][i / v_size] = set_value(Wo[k][i / v_size], i % v_size, wVals[k][i + 3 * out_size]); + } + } + + for(int j = 0; j < out_size; ++j) + { + Wi_1[j / v_size] = set_value(Wi_1[j / v_size], j % v_size, wVals[0][j]); + Wf_1[j / v_size] = set_value(Wf_1[j / v_size], j % v_size, wVals[0][j + out_size]); + Wc_1[j / v_size] = set_value(Wc_1[j / v_size], j % v_size, wVals[0][j + 2 * out_size]); + Wo_1[j / v_size] = set_value(Wo_1[j / v_size], j % v_size, wVals[0][j + 3 * out_size]); + } +} + +template +void LSTMLayerT::setUVals(const std::vector>& uVals) +{ + for(int i = 0; i < out_size; ++i) + { + for(int k = 0; k < out_size; ++k) + { + Ui[k][i / v_size] = set_value(Ui[k][i / v_size], i % v_size, uVals[k][i]); + Uf[k][i / v_size] = set_value(Uf[k][i / v_size], i % v_size, uVals[k][i + out_size]); + Uc[k][i / v_size] = set_value(Uc[k][i / v_size], i % v_size, uVals[k][i + 2 * out_size]); + Uo[k][i / v_size] = set_value(Uo[k][i / v_size], i % v_size, uVals[k][i + 3 * out_size]); + } + } +} + +template +void LSTMLayerT::setBVals(const std::vector& bVals) +{ + for(int k = 0; k < out_size; ++k) + { + bi[k / v_size] = set_value(bi[k / v_size], k % v_size, bVals[k]); + bf[k / v_size] = set_value(bf[k / v_size], k % v_size, bVals[k + out_size]); + bc[k / v_size] = set_value(bc[k / v_size], k % v_size, bVals[k + 2 * out_size]); + bo[k / v_size] = set_value(bo[k / v_size], k % v_size, bVals[k + 3 * out_size]); + } +} + +} // namespace RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_eigen.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_eigen.h new file mode 100644 index 0000000..3f7c8be --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_eigen.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +namespace RTNEURAL_NAMESPACE +{ +struct DefaultMathsProvider +{ + template + static auto tanh(const Matrix& x) + { + return x.array().tanh(); + } + + template + static auto sigmoid(const Matrix& x) + { + using T = typename Matrix::Scalar; + return (T)1 / (((T)-1 * x.array()).array().exp() + (T)1); + } + + template + static auto exp(const Matrix& x) + { + return x.array().exp(); + } +}; +} diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_stl.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_stl.h new file mode 100644 index 0000000..ae9d76c --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_stl.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +namespace RTNEURAL_NAMESPACE +{ +struct DefaultMathsProvider +{ + template + static T tanh(T x) + { + return std::tanh(x); + } + + template + static T sigmoid(T x) + { + return (T)1 / ((T)1 + std::exp(-x)); + } + + template + static T exp(T x) + { + return std::exp(x); + } +}; +} diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_xsimd.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_xsimd.h new file mode 100644 index 0000000..45a6177 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/maths/maths_xsimd.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +namespace RTNEURAL_NAMESPACE +{ +struct DefaultMathsProvider +{ + template + static T tanh(T x) + { + using std::tanh; + using xsimd::tanh; + return tanh(x); + } + + template + static T sigmoid(T x) + { + using std::exp; + using xsimd::exp; + return (T)1 / ((T)1 + exp(-x)); + } + + template + static T exp(T x) + { + using std::exp; + using xsimd::exp; + return exp(x); + } +}; +} diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/model_loader.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/model_loader.h new file mode 100644 index 0000000..a727e73 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/model_loader.h @@ -0,0 +1,728 @@ +#pragma once + +#include +#include "Model.h" +#include +#include +#include + +#if !RTNEURAL_NO_DEBUG +#include +#endif + +#if defined(__cplusplus) && __cplusplus >= 201703L +#define RTNEURAL_MAYBE_UNUSED [[maybe_unused]] +#else +#define RTNEURAL_MAYBE_UNUSED +#endif + +namespace RTNEURAL_NAMESPACE +{ +/** Utility functions for loading model weights from their json representation. */ +namespace json_parser +{ +#if RTNEURAL_NO_DEBUG + RTNEURAL_MAYBE_UNUSED static void debug_print(const std::string&, bool) + { + } +#else + RTNEURAL_MAYBE_UNUSED static void debug_print(const std::string& str, bool debug) + { + if(debug) + std::cout << str << std::endl; + } +#endif + + /** Loads weights for a Dense (or DenseT) layer from a json representation of the layer weights. */ + template + void loadDense(DenseType& dense, const nlohmann::json& weights) + { + // load weights + std::vector> denseWeights(dense.out_size); + for(auto& w : denseWeights) + w.resize(dense.in_size, (T)0); + + auto layerWeights = weights.at(0); + for(size_t i = 0; i < layerWeights.size(); ++i) + { + auto lw = layerWeights.at(i); + for(size_t j = 0; j < lw.size(); ++j) + denseWeights.at(j).at(i) = lw.at(j).get(); + } + + dense.setWeights(denseWeights); + + // load biases + std::vector denseBias = weights.at(1).get>(); + dense.setBias(denseBias.data()); + } + + /** Creates a Dense layer from a json representation of the layer weights. */ + template + std::unique_ptr> createDense(int in_size, int out_size, const nlohmann::json& weights) + { + auto dense = std::make_unique>(in_size, out_size); + loadDense(*dense.get(), weights); + return std::move(dense); + } + + /** Checks that a Dense (or DenseT) layer has the given dimensions. */ + template + bool checkDense(const DenseType& dense, const std::string& type, int layerDims, const bool debug) + { + if(type != "dense" && type != "time-distributed-dense") + { + debug_print("Wrong layer type! Expected: Dense", debug); + return false; + } + + if(layerDims != dense.out_size) + { + debug_print("Wrong layer size! Expected: " + std::to_string(dense.out_size), debug); + return false; + } + + return true; + } + + /** Loads weights for a Conv1D (or Conv1DT) layer from a json representation of the layer weights. */ + template + void loadConv1D(Conv1DType& conv, int kernel_size, int /*dilation*/, const nlohmann::json& weights) + { + // load weights + std::vector>> convWeights(conv.out_size); + for(auto& wIn : convWeights) + { + wIn.resize(conv.in_size / conv.getGroups()); + + for(auto& w : wIn) + w.resize(kernel_size, (T)0); + } + + auto layerWeights = weights.at(0); + for(size_t i = 0; i < layerWeights.size(); ++i) + { + auto lw = layerWeights.at(i); + for(size_t j = 0; j < lw.size(); ++j) + { + auto l = lw.at(j); + for(size_t k = 0; k < l.size(); ++k) + convWeights.at(k).at(j).at(kernel_size - 1 - i) = l.at(k).get(); + } + } + + conv.setWeights(convWeights); + + // load biases + std::vector convBias = weights.at(1).get>(); + conv.setBias(convBias); + } + + /** Loads weights for a Conv2D (or Conv2DT) layer from a json representation of the layer weights. */ + template + void loadConv2D(Conv2DType& conv2d, const nlohmann::json& weights) + { + // load weights + std::vector>>> convWeights(conv2d.kernel_size_time); + for(auto& wOut : convWeights) + { + wOut.resize(conv2d.num_filters_out); + + for(auto& wIn : wOut) + { + wIn.resize(conv2d.num_filters_in); + + for(auto& w : wIn) + w.resize(conv2d.kernel_size_feature, (T)0); + } + } + + // In Tensorflow (JSON file): [kernel_size_time, kernel_size_feature, num_filters_in, num_filters_out] + // In RTNeural conv2d::setWeights: [kernel_size_time, num_filters_out, num_filters_in, kernel_size_feature] + auto layerWeights = weights.at(0); + // Kernel Size Time + for(size_t i = 0; i < layerWeights.size(); ++i) + { + auto l1 = layerWeights.at(i); + // Kernel Size feature + for(size_t j = 0; j < l1.size(); ++j) + { + auto l2 = l1.at(j); + // Num filters in + for(size_t k = 0; k < l2.size(); ++k) + { + auto l3 = l2.at(k); + // Num filters out + for(size_t p = 0; p < l3.size(); ++p) + convWeights.at(i).at(p).at(k).at(j) = l3.at(p).get(); + } + } + } + + conv2d.setWeights(convWeights); + + // load biases + std::vector convBias = weights.at(1).get>(); + conv2d.setBias(convBias); + } + + /** Creates a Conv1D layer from a json representation of the layer weights. */ + template + std::unique_ptr> createConv1D(int in_size, int out_size, + int kernel_size, int dilation, int groups, const nlohmann::json& weights) + { + auto conv = std::make_unique>(in_size, out_size, kernel_size, dilation, groups); + loadConv1D(*conv.get(), kernel_size, dilation, weights); + return std::move(conv); + } + + /** Checks that a Conv1D (or Conv1DT) layer has the given dimensions. */ + template + bool checkConv1D(const Conv1DType& conv, const std::string& type, int layerDims, + int kernel_size, int dilation_rate, int groups, const bool debug) + { + if(type != "conv1d") + { + debug_print("Wrong layer type! Expected: Conv1D", debug); + return false; + } + + if(layerDims != conv.out_size) + { + debug_print("Wrong layer size! Expected: " + std::to_string(conv.out_size), debug); + return false; + } + + if(kernel_size != conv.getKernelSize()) + { + debug_print("Wrong kernel size! Expected: " + std::to_string(conv.getKernelSize()), debug); + return false; + } + + if(dilation_rate != conv.getDilationRate()) + { + debug_print("Wrong dilation_rate! Expected: " + std::to_string(conv.getDilationRate()), debug); + return false; + } + + if(groups != conv.getGroups()) + { + debug_print("Wrong number of groups! Expected: " + std::to_string(conv.getGroups()), debug); + return false; + } + + return true; + } + + template + std::unique_ptr> createConv2D(int num_filters_in, int num_features_in, int num_filters_out, + int kernel_size_time, int kernel_size_feature, int dilation, int stride, bool valid_pad, const nlohmann::json& weights) + { + auto conv = std::make_unique>(num_filters_in, num_filters_out, num_features_in, kernel_size_time, kernel_size_feature, dilation, stride, valid_pad); + loadConv2D(*conv.get(), weights); + return std::move(conv); + } + + /** Checks that a Conv2D (or Conv2DT) layer has the given dimensions. */ + template + bool checkConv2D(const Conv2DType& conv, const std::string& type, int layerDims, + int kernel_size_time, int kernel_size_feature, int dilation_rate, int stride, bool /*valid_pad*/, const bool debug) + { + if(type != "conv2d") + { + debug_print("Wrong layer type! Expected: Conv2D", debug); + return false; + } + + if(layerDims != conv.out_size) + { + debug_print("Wrong layer size! Expected: " + std::to_string(conv.out_size), debug); + return false; + } + + if(kernel_size_time != conv.getKernelSizeTime()) + { + debug_print("Wrong kernel size time! Expected: " + std::to_string(conv.getKernelSizeTime()), debug); + return false; + } + + if(kernel_size_feature != conv.getKernelSizeFeature()) + { + debug_print("Wrong kernel size feature! Expected: " + std::to_string(conv.getKernelSizeFeature()), debug); + return false; + } + + if(stride != conv.getStride()) + { + debug_print("Wrong stride! Expected: " + std::to_string(conv.getStride()), debug); + return false; + } + + if(dilation_rate != conv.getDilationRate()) + { + debug_print("Wrong dilation_rate! Expected: " + std::to_string(conv.getDilationRate()), debug); + return false; + } + + return true; + } + + /** Loads weights for a GRULayer (or GRULayerT) from a json representation of the layer weights. */ + template + void loadGRU(GRUType& gru, const nlohmann::json& weights) + { + // load kernel weights + std::vector> kernelWeights(gru.in_size); + for(auto& w : kernelWeights) + w.resize(3 * gru.out_size, (T)0); + + auto layerWeights = weights.at(0); + for(size_t i = 0; i < layerWeights.size(); ++i) + { + auto lw = layerWeights.at(i); + for(size_t j = 0; j < lw.size(); ++j) + kernelWeights.at(i).at(j) = lw.at(j).get(); + } + + gru.setWVals(kernelWeights); + + // load recurrent weights + std::vector> recurrentWeights(gru.out_size); + for(auto& w : recurrentWeights) + w.resize(3 * gru.out_size, (T)0); + + auto layerWeights2 = weights.at(1); + for(size_t i = 0; i < layerWeights2.size(); ++i) + { + auto lw = layerWeights2.at(i); + for(size_t j = 0; j < lw.size(); ++j) + recurrentWeights.at(i).at(j) = lw.at(j).get(); + } + + gru.setUVals(recurrentWeights); + + // load biases + std::vector> gruBias(2); + for(auto& b : gruBias) + b.resize(3 * gru.out_size, (T)0); + + auto layerBias = weights.at(2); + for(size_t i = 0; i < layerBias.size(); ++i) + { + auto lw = layerBias.at(i); + for(size_t j = 0; j < lw.size(); ++j) + gruBias.at(i).at(j) = lw.at(j).get(); + } + + gru.setBVals(gruBias); + } + + /** Creates a GRULayer from a json representation of the layer weights. */ + template + std::unique_ptr> createGRU(int in_size, int out_size, const nlohmann::json& weights) + { + auto gru = std::make_unique>(in_size, out_size); + loadGRU(*gru.get(), weights); + return std::move(gru); + } + + /** Checks that a GRULayer (or GRULayerT) has the given dimensions. */ + template + bool checkGRU(const GRUType& gru, const std::string& type, int layerDims, const bool debug) + { + if(type != "gru") + { + debug_print("Wrong layer type! Expected: GRU", debug); + return false; + } + + if(layerDims != gru.out_size) + { + debug_print("Wrong layer size! Expected: " + std::to_string(gru.out_size), debug); + return false; + } + + return true; + } + + /** Loads weights for a LSTMLayer (or LSTMLayerT) from a json representation of the layer weights. */ + template + void loadLSTM(LSTMType& lstm, const nlohmann::json& weights) + { + // load kernel weights + std::vector> kernelWeights(lstm.in_size); + for(auto& w : kernelWeights) + w.resize(4 * lstm.out_size, (T)0); + + auto layerWeights = weights.at(0); + for(size_t i = 0; i < layerWeights.size(); ++i) + { + auto lw = layerWeights.at(i); + for(size_t j = 0; j < lw.size(); ++j) + kernelWeights.at(i).at(j) = lw.at(j).get(); + } + + lstm.setWVals(kernelWeights); + + // load recurrent weights + std::vector> recurrentWeights(lstm.out_size); + for(auto& w : recurrentWeights) + w.resize(4 * lstm.out_size, (T)0); + + auto layerWeights2 = weights.at(1); + for(size_t i = 0; i < layerWeights2.size(); ++i) + { + auto lw = layerWeights2.at(i); + for(size_t j = 0; j < lw.size(); ++j) + recurrentWeights.at(i).at(j) = lw.at(j).get(); + } + + lstm.setUVals(recurrentWeights); + + // load biases + std::vector lstmBias = weights.at(2).get>(); + lstm.setBVals(lstmBias); + } + + /** Creates a LSTMLayer from a json representation of the layer weights. */ + template + std::unique_ptr> createLSTM(int in_size, int out_size, const nlohmann::json& weights) + { + auto lstm = std::make_unique>(in_size, out_size); + loadLSTM(*lstm.get(), weights); + return std::move(lstm); + } + + /** Checks that a LSTMLayer (or LSTMLayerT) has the given dimensions. */ + template + bool checkLSTM(const LSTMType& lstm, const std::string& type, int layerDims, const bool debug) + { + if(type != "lstm") + { + debug_print("Wrong layer type! Expected: LSTM", debug); + return false; + } + + if(layerDims != lstm.out_size) + { + debug_print("Wrong layer size! Expected: " + std::to_string(lstm.out_size), debug); + return false; + } + + return true; + } + + /** Loads weights for a PReLUActivation (or PReLUActivationT) from a json representation of the layer weights. */ + template + void loadPReLU(PReLUType& prelu, const nlohmann::json& weights) + { + std::vector preluWeights = weights.at(0).at(0).get>(); + prelu.setAlphaVals(preluWeights); + } + + /** Creates a PReLUActivation from a json representation of the layer weights. */ + template + std::unique_ptr> createPReLU(int in_size, const nlohmann::json& weights) + { + auto prelu = std::make_unique>(in_size); + loadPReLU(*prelu.get(), weights); + return std::move(prelu); + } + + /** Checks that a PReLUActivation (or PReLUActivationT) has the given dimensions. */ + template + bool checkPReLU(const PReLUType& prelu, const std::string& type, int layerDims, const bool debug) + { + if(type != "prelu") + { + debug_print("Wrong layer type! Expected: PReLU", debug); + return false; + } + + if(layerDims != prelu.out_size) + { + debug_print("Wrong layer size! Expected: " + std::to_string(prelu.out_size), debug); + return false; + } + + return true; + } + + /** Loads weights for a BatchNorm1DLayer (or BatchNorm1DT) or BatchNorm2DLayer (or BatchNorm2DT) from a json representation of the layer weights. */ + template + void loadBatchNorm(BatchNormType& batch_norm, const nlohmann::json& weights, bool affine) + { + if(affine) + { + batch_norm.setGamma(weights.at(0).get>()); + batch_norm.setBeta(weights.at(1).get>()); + batch_norm.setRunningMean(weights.at(2).get>()); + batch_norm.setRunningVariance(weights.at(3).get>()); + } + else + { + batch_norm.setRunningMean(weights.at(0).get>()); + batch_norm.setRunningVariance(weights.at(1).get>()); + } + } + + /** Loads weights for a BatchNorm1DLayer (or BatchNorm1DT) from a json representation of the layer weights. */ + template + void loadBatchNorm(BatchNormType& batch_norm, const nlohmann::json& weights) + { + loadBatchNorm(batch_norm, weights, BatchNormType::is_affine); + } + + /** Creates a BatchNorm1DLayer from a json representation of the layer weights. */ + template + std::unique_ptr> createBatchNorm(int size, const nlohmann::json& weights, T epsilon) + { + auto batch_norm = std::make_unique>(size); + loadBatchNorm(*batch_norm.get(), weights, weights.size() == 4); + batch_norm->setEpsilon(epsilon); + return std::move(batch_norm); + } + + template + std::unique_ptr> createBatchNorm2D(int num_filters_in, int num_features_in, const nlohmann::json& weights, T epsilon) + { + auto batch_norm = std::make_unique>(num_filters_in, num_features_in); + loadBatchNorm(*batch_norm.get(), weights, weights.size() == 4); + batch_norm->setEpsilon(epsilon); + return std::move(batch_norm); + } + + /** Checks that a BatchNorm1DLayer (or BatchNorm1DT) has the given dimensions. */ + template + bool checkBatchNorm(const BatchNormType& batch_norm, const std::string& type, int layerDims, const nlohmann::json& weights, const bool debug) + { + if(type != "batchnorm") + { + debug_print("Wrong layer type! Expected: BatchNorm", debug); + return false; + } + + if(BatchNormType::is_affine && weights.size() != 4) + { + debug_print("Wrong layer type! Expected: \"affine\" BatchNorm", debug); + return false; + } + + if(!BatchNormType::is_affine && weights.size() != 2) + { + debug_print("Wrong layer type! Expected: non-\"affine\" BatchNorm", debug); + return false; + } + + if(layerDims != batch_norm.out_size) + { + debug_print("Wrong layer size! Expected: " + std::to_string(batch_norm.out_size), debug); + return false; + } + return true; + } + + /** Checks that a BatchNorm2DLayer (or BatchNorm2DT) has the given dimensions. */ + template + bool checkBatchNorm2D(const BatchNormType& batch_norm, const std::string& type, int layerDims, const nlohmann::json& weights, const bool debug) + { + if(type != "batchnorm2d") + { + debug_print("Wrong layer type! Expected: BatchNorm2D", debug); + return false; + } + + if(BatchNormType::is_affine && weights.size() != 4) + { + debug_print("Wrong layer type! Expected: \"affine\" BatchNorm2D", debug); + return false; + } + + if(!BatchNormType::is_affine && weights.size() != 2) + { + debug_print("Wrong layer type! Expected: non-\"affine\" BatchNorm2D", debug); + return false; + } + + if(layerDims != batch_norm.out_size) + { + debug_print("Wrong layer size! Expected: " + std::to_string(batch_norm.out_size), debug); + return false; + } + + if(weights[0].size() != batch_norm.num_filters) + { + debug_print("Wrong weight dimension! Expected: " + std::to_string(batch_norm.num_features), debug); + return false; + } + + return true; + } + + /** Creates an activation layer of a given type. */ + template + std::unique_ptr> + createActivation(const std::string& activationType, int dims) + { + if(activationType == "tanh") + return std::make_unique>(dims); + + if(activationType == "relu") + return std::make_unique>(dims); + + if(activationType == "sigmoid") + return std::make_unique>(dims); + + if(activationType == "softmax") + return std::make_unique>(dims); + + if(activationType == "elu") + return std::make_unique>(dims); + + return {}; + } + + /** Checks that an Activation layer has the given dimensions */ + template + bool checkActivation(const LayerType& actLayer, const std::string& activationType, int dims, const bool debug) + { + if(dims != actLayer.out_size) + { + debug_print("Wrong layer size! Expected: " + std::to_string(actLayer.out_size), debug); + return false; + } + + if(activationType != actLayer.getName()) + { + debug_print("Wrong layer type! Expected: " + actLayer.getName(), debug); + return false; + } + + return true; + } + + /** Creates a neural network model from a json stream. */ + template + std::unique_ptr> parseJson(const nlohmann::json& parent, const bool debug = false) + { + auto shape = parent.at("in_shape"); + auto layers = parent.at("layers"); + + if(!shape.is_array() || !layers.is_array()) + return {}; + + const int nDims = shape.size() == 4 ? shape[2].get() * shape[3].get() : shape.back().get(); + + debug_print("# dimensions: " + std::to_string(nDims), debug); + + auto model = std::make_unique>(nDims); + + for(const auto& l : layers) + { + const auto type = l.at("type").get(); + debug_print("Layer: " + type, debug); + + const auto layerShape = l.at("shape"); + + // In case of 4 dimensional input (conv2d): multiply channel axis and feature axis to get layer dim + const int layerDims = layerShape.size() == 4 ? layerShape[2].get() * layerShape[3].get() : layerShape.back().get(); + + debug_print(" Dims: " + std::to_string(layerDims), debug); + + const auto weights = l.at("weights"); + + auto add_activation = [=](std::unique_ptr>& _model, const nlohmann::json& _l) + { + if(_l.contains("activation")) + { + const auto activationType = _l["activation"].get(); + if(!activationType.empty()) + { + debug_print(" activation: " + activationType, debug); + auto activation = createActivation(activationType, layerDims); + _model->addLayer(activation.release()); + } + } + }; + + if(type == "dense" || type == "time-distributed-dense") + { + auto dense = createDense(model->getNextInSize(), layerDims, weights); + model->addLayer(dense.release()); + add_activation(model, l); + } + else if(type == "conv1d") + { + const auto kernel_size = l.at("kernel_size").back().get(); + const auto dilation = l.at("dilation").back().get(); + const auto groups = l.value("groups", 1); + + auto conv = createConv1D(model->getNextInSize(), layerDims, kernel_size, dilation, groups, weights); + model->addLayer(conv.release()); + add_activation(model, l); + } + else if(type == "conv2d") + { + const auto kernel_size_time = l.at("kernel_size_time").back().get(); + const auto kernel_size_feature = l.at("kernel_size_feature").back().get(); + const auto dilation = l.at("dilation").back().get(); + const auto stride = l.at("strides").back().get(); + const auto num_filters_in = l.at("num_filters_in").back().get(); + const auto num_features_in = l.at("num_features_in").back().get(); + const auto num_filters_out = l.at("num_filters_out").back().get(); + const bool valid_pad = l.at("padding").get() == "valid"; + + auto conv = createConv2D(num_filters_in, num_features_in, num_filters_out, kernel_size_time, kernel_size_feature, dilation, stride, valid_pad, weights); + + // Check the layer + if(!checkConv2D(*conv, "conv2d", layerDims, kernel_size_time, kernel_size_feature, dilation, stride, valid_pad, debug)) + return {}; + + model->addLayer(conv.release()); + add_activation(model, l); + } + else if(type == "gru") + { + auto gru = createGRU(model->getNextInSize(), layerDims, weights); + model->addLayer(gru.release()); + } + else if(type == "lstm") + { + auto lstm = createLSTM(model->getNextInSize(), layerDims, weights); + model->addLayer(lstm.release()); + } + else if(type == "prelu") + { + auto prelu = createPReLU(model->getNextInSize(), weights); + model->addLayer(prelu.release()); + } + else if(type == "batchnorm") + { + auto batch_norm = createBatchNorm(model->getNextInSize(), weights, l.at("epsilon").get()); + model->addLayer(batch_norm.release()); + } + else if(type == "batchnorm2d") + { + auto batch_norm = createBatchNorm2D(l.at("num_filters_in"), l.at("num_features_in"), weights, l.at("epsilon").get()); + model->addLayer(batch_norm.release()); + } + else if(type == "activation") + { + add_activation(model, l); + } + } + + return std::move(model); + } + + /** Creates a neural network model from a json stream. */ + template + std::unique_ptr> parseJson(std::ifstream& jsonStream, const bool debug = false) + { + nlohmann::json parent; + jsonStream >> parent; + return parseJson(parent, debug); + } + +} // namespace json_parser +} // namespace RTNEURAL_NAMESPACE diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/torch_helpers.h b/JammLab/Transcription/Native/ThirdParty/RTNeural/torch_helpers.h new file mode 100644 index 0000000..25401fc --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/torch_helpers.h @@ -0,0 +1,151 @@ +#pragma once + +#include "model_loader.h" + +namespace RTNEURAL_NAMESPACE +{ +namespace torch_helpers +{ + namespace detail + { + /** Torch Conv1D layers store their kernel weights in reverse order. */ + template + void reverseKernels(std::vector>>& conv_weights) + { + for(auto& channel_weights : conv_weights) + { + for(auto& kernel : channel_weights) + { + std::reverse(kernel.begin(), kernel.end()); + } + } + } + + /** Transposes the rows and columns of a matrix stored as a 2D vector. */ + template + std::vector> transpose(const std::vector>& x) + { + auto outer_size = x.size(); + auto inner_size = x[0].size(); + std::vector> y(inner_size, std::vector(outer_size, (T)0)); + + for(size_t i = 0; i < outer_size; ++i) + { + for(size_t j = 0; j < inner_size; ++j) + y[j][i] = x[i][j]; + } + + return y; + } + + /** Swaps the "r" and "z" indexes of a GRU layer weights. */ + template + void swap_rz(std::vector>& vec2d, int size) + { + for(auto& vec : vec2d) + std::swap_ranges(vec.begin(), vec.begin() + size, vec.begin() + size); + } + } + + /** Loads a Dense layer from a JSON object containing a PyTorch state_dict. */ + template + void loadDense(const nlohmann::json& modelJson, const std::string& layerPrefix, DenseType& dense, bool hasBias = true) + { + const std::vector> dense_weights = modelJson.at(layerPrefix + "weight"); + dense.setWeights(dense_weights); + + if(hasBias) + { + const std::vector dense_bias = modelJson.at(layerPrefix + "bias"); + dense.setBias(dense_bias.data()); + } + else + { + const std::vector dense_bias((size_t)dense.out_size, (T)0); + dense.setBias(dense_bias.data()); + } + } + + /** Loads a Conv1D layer from a JSON object containing a PyTorch state_dict. */ + template + void loadConv1D(const nlohmann::json& modelJson, const std::string& layerPrefix, Conv1DType& conv, bool hasBias = true) + { + std::vector>> conv_weights = modelJson.at(layerPrefix + "weight"); + detail::reverseKernels(conv_weights); + conv.setWeights(conv_weights); + + if(hasBias) + { + std::vector conv_bias = modelJson.at(layerPrefix + "bias"); + conv.setBias(conv_bias); + } + else + { + std::vector conv_bias((size_t)conv.out_size, (T)0); + conv.setBias(conv_bias); + } + } + + /** Loads a GRU layer from a JSON object containing a PyTorch state_dict. */ + template + void loadGRU(const nlohmann::json& modelJson, const std::string& layerPrefix, GRUType& gru, bool hasBias = true) + { + // For the kernel and recurrent weights, PyTorch stores the weights similar to the + // Tensorflow format, but transposed, and with the "r" and "z" indexes swapped. + + const std::vector> gru_ih_weights = modelJson.at(layerPrefix + "weight_ih_l0"); + auto wVals = detail::transpose(gru_ih_weights); + detail::swap_rz(wVals, gru.out_size); + gru.setWVals(wVals); + + const std::vector> gru_hh_weights = modelJson.at(layerPrefix + "weight_hh_l0"); + auto uVals = detail::transpose(gru_hh_weights); + detail::swap_rz(uVals, gru.out_size); + gru.setUVals(uVals); + + // PyTorch stores the GRU bias pretty much the same as TensorFlow as well, + // just in two separate vectors. And again, we need to swap the "r" and "z" parts. + + if(hasBias) + { + const std::vector gru_ih_bias = modelJson.at(layerPrefix + "bias_ih_l0"); + const std::vector gru_hh_bias = modelJson.at(layerPrefix + "bias_hh_l0"); + std::vector> gru_bias { gru_ih_bias, gru_hh_bias }; + detail::swap_rz(gru_bias, gru.out_size); + gru.setBVals(gru_bias); + } + else + { + const std::vector gru_ih_bias((size_t)gru.out_size * 3, (T)0); + const std::vector gru_hh_bias((size_t)gru.out_size * 3, (T)0); + std::vector> gru_bias { gru_ih_bias, gru_hh_bias }; + gru.setBVals(gru_bias); + } + } + + /** Loads a LSTM layer from a JSON object containing a PyTorch state_dict. */ + template + void loadLSTM(const nlohmann::json& modelJson, const std::string& layerPrefix, LSTMType& lstm, bool hasBias = true) + { + const std::vector> lstm_weights_ih = modelJson.at(layerPrefix + "weight_ih_l0"); + lstm.setWVals(detail::transpose(lstm_weights_ih)); + + const std::vector> lstm_weights_hh = modelJson.at(layerPrefix + "weight_hh_l0"); + lstm.setUVals(detail::transpose(lstm_weights_hh)); + + if(hasBias) + { + std::vector lstm_bias_ih = modelJson.at(layerPrefix + "bias_ih_l0"); + std::vector lstm_bias_hh = modelJson.at(layerPrefix + "bias_hh_l0"); + for(size_t i = 0; i < lstm_bias_ih.size(); ++i) + lstm_bias_hh[i] += lstm_bias_ih[i]; + lstm.setBVals(lstm_bias_hh); + } + else + { + std::vector lstm_bias_hh((size_t)lstm.out_size * 4, (T)0); + lstm.setBVals(lstm_bias_hh); + } + } +} +} diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/xsimd-legacy/README.md b/JammLab/Transcription/Native/ThirdParty/RTNeural/xsimd-legacy/README.md new file mode 100644 index 0000000..c46749a --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/xsimd-legacy/README.md @@ -0,0 +1,7 @@ +algorithms.hpp used to exist, and some functions are used some functions +in RTNeural. Later on, xsimd removed this file from the public API as +they were alternative implementation for C++17/20 standard APIs. +We are not replacing those functions to maintain compatibility with +older C++ versions. Hence importing this file here. + +For more details, see: https://github.com/jatinchowdhury18/RTNeural/pull/81 diff --git a/JammLab/Transcription/Native/ThirdParty/RTNeural/xsimd-legacy/algorithms/algorithms.hpp b/JammLab/Transcription/Native/ThirdParty/RTNeural/xsimd-legacy/algorithms/algorithms.hpp new file mode 100644 index 0000000..8831c30 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/RTNeural/xsimd-legacy/algorithms/algorithms.hpp @@ -0,0 +1,201 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_ALGORITHMS_HPP +#define XSIMD_ALGORITHMS_HPP + +#include +#include +#include +#include + +#include + +namespace xsimd +{ + template + void transform(I1 first, I2 last, O1 out_first, UF&& f) noexcept + { + using value_type = typename std::decay::type; + using batch_type = batch; + + std::size_t size = static_cast(std::distance(first, last)); + std::size_t simd_size = batch_type::size; + + const auto* ptr_begin = &(*first); + auto* ptr_out = &(*out_first); + + std::size_t align_begin = xsimd::get_alignment_offset(ptr_begin, size, simd_size); + std::size_t out_align = xsimd::get_alignment_offset(ptr_out, size, simd_size); + std::size_t align_end = align_begin + ((size - align_begin) & ~(simd_size - 1)); + + if (align_begin == out_align) + { + for (std::size_t i = 0; i < align_begin; ++i) + { + out_first[i] = f(first[i]); + } + + for (std::size_t i = align_begin; i < align_end; i += simd_size) + { + batch_type batch = batch_type::load_aligned(&first[i]); + xsimd::store_aligned(&out_first[i], f(batch)); + } + + for (std::size_t i = align_end; i < size; ++i) + { + out_first[i] = f(first[i]); + } + } + else + { + for (std::size_t i = 0; i < align_begin; ++i) + { + out_first[i] = f(first[i]); + } + + for (std::size_t i = align_begin; i < align_end; i += simd_size) + { + batch_type batch = batch_type::load_aligned(&first[i]); + xsimd::store_unaligned(&out_first[i], f(batch)); + } + + for (std::size_t i = align_end; i < size; ++i) + { + out_first[i] = f(first[i]); + } + } + } + + template + void transform(I1 first_1, I2 last_1, I3 first_2, O1 out_first, UF&& f) noexcept + { + using value_type = typename std::decay::type; + using batch_type = batch; + + std::size_t size = static_cast(std::distance(first_1, last_1)); + std::size_t simd_size = batch_type::size; + + const auto* ptr_begin_1 = &(*first_1); + const auto* ptr_begin_2 = &(*first_2); + auto* ptr_out = &(*out_first); + + std::size_t align_begin_1 = xsimd::get_alignment_offset(ptr_begin_1, size, simd_size); + std::size_t align_begin_2 = xsimd::get_alignment_offset(ptr_begin_2, size, simd_size); + std::size_t out_align = xsimd::get_alignment_offset(ptr_out, size, simd_size); + std::size_t align_end = align_begin_1 + ((size - align_begin_1) & ~(simd_size - 1)); + +#define XSIMD_LOOP_MACRO(A1, A2, A3) \ + for (std::size_t i = 0; i < align_begin_1; ++i) \ + { \ + out_first[i] = f(first_1[i], first_2[i]); \ + } \ + \ + batch_type batch_1, batch_2; \ + for (std::size_t i = align_begin_1; i < align_end; i += simd_size) \ + { \ + batch_1 = batch_type::A1(&first_1[i]); \ + batch_2 = batch_type::A2(&first_2[i]); \ + xsimd::A3(&out_first[i], f(batch_1, batch_2)); \ + } \ + \ + for (std::size_t i = align_end; i < size; ++i) \ + { \ + out_first[i] = f(first_1[i], first_2[i]); \ + } + + if (align_begin_1 == out_align && align_begin_1 == align_begin_2) + { + XSIMD_LOOP_MACRO(load_aligned, load_aligned, store_aligned); + } + else if (align_begin_1 == out_align && align_begin_1 != align_begin_2) + { + XSIMD_LOOP_MACRO(load_aligned, load_unaligned, store_aligned); + } + else if (align_begin_1 != out_align && align_begin_1 == align_begin_2) + { + XSIMD_LOOP_MACRO(load_aligned, load_aligned, store_unaligned); + } + else if (align_begin_1 != out_align && align_begin_1 != align_begin_2) + { + XSIMD_LOOP_MACRO(load_aligned, load_unaligned, store_unaligned); + } + +#undef XSIMD_LOOP_MACRO + } + + // TODO: Remove this once we drop C++11 support + namespace detail + { + struct plus + { + template + auto operator()(X&& x, Y&& y) noexcept -> decltype(x + y) { return x + y; } + }; + } + + template + Init reduce(Iterator1 first, Iterator2 last, Init init, BinaryFunction&& binfun = detail::plus {}) noexcept + { + using value_type = typename std::decay::type; + using batch_type = batch; + + std::size_t size = static_cast(std::distance(first, last)); + constexpr std::size_t simd_size = batch_type::size; + + if (size < simd_size) + { + while (first != last) + { + init = binfun(init, *first++); + } + return init; + } + + const auto* const ptr_begin = &(*first); + + std::size_t align_begin = xsimd::get_alignment_offset(ptr_begin, size, simd_size); + std::size_t align_end = align_begin + ((size - align_begin) & ~(simd_size - 1)); + + // reduce initial unaligned part + for (std::size_t i = 0; i < align_begin; ++i) + { + init = binfun(init, first[i]); + } + + // reduce aligned part + auto ptr = ptr_begin + align_begin; + batch_type batch_init = batch_type::load_aligned(ptr); + ptr += simd_size; + for (auto const end = ptr_begin + align_end; ptr < end; ptr += simd_size) + { + batch_type batch = batch_type::load_aligned(ptr); + batch_init = binfun(batch_init, batch); + } + + // reduce across batch + alignas(batch_type) std::array arr; + xsimd::store_aligned(arr.data(), batch_init); + for (auto x : arr) + init = binfun(init, x); + + // reduce final unaligned part + for (std::size_t i = align_end; i < size; ++i) + { + init = binfun(init, first[i]); + } + + return init; + } + +} + +#endif diff --git a/JammLab/Transcription/Native/ThirdParty/nlohmann/json.hpp b/JammLab/Transcription/Native/ThirdParty/nlohmann/json.hpp new file mode 100644 index 0000000..beee013 --- /dev/null +++ b/JammLab/Transcription/Native/ThirdParty/nlohmann/json.hpp @@ -0,0 +1,24441 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + +/****************************************************************************\ + * Note on documentation: The source files contain links to the online * + * documentation of the public API at https://json.nlohmann.me. This URL * + * contains the most recent documentation and should also be applicable to * + * previous versions; documentation for deprecated functions is not * + * removed, but marked deprecated. See "Generate documentation" section in * + * file docs/README.md. * +\****************************************************************************/ + +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ + +#include // all_of, find, for_each +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#ifndef JSON_NO_IO + #include // istream, ostream +#endif // JSON_NO_IO +#include // random_access_iterator_tag +#include // unique_ptr +#include // accumulate +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// This file contains all macro definitions affecting or depending on the ABI + +#ifndef JSON_SKIP_LIBRARY_VERSION_CHECK + #if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH) + #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 11 || NLOHMANN_JSON_VERSION_PATCH != 1 + #warning "Already included a different version of the library!" + #endif + #endif +#endif + +#define NLOHMANN_JSON_VERSION_MAJOR 3 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_MINOR 11 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_PATCH 1 // NOLINT(modernize-macro-to-enum) + +#ifndef JSON_DIAGNOSTICS + #define JSON_DIAGNOSTICS 0 +#endif + +#ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON + #define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0 +#endif + +#if JSON_DIAGNOSTICS + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag +#else + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS +#endif + +#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON + #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp +#else + #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON +#endif + +#define NLOHMANN_JSON_ABI_PREFIX_EX(major, minor, patch) \ + json_v ## major ## _ ## minor ## _ ## patch +#define NLOHMANN_JSON_ABI_PREFIX(major, minor, patch) \ + NLOHMANN_JSON_ABI_PREFIX_EX(major, minor, patch) + +#define NLOHMANN_JSON_ABI_CONCAT_EX(a, b, c) a ## b ## c +#define NLOHMANN_JSON_ABI_CONCAT(a, b, c) \ + NLOHMANN_JSON_ABI_CONCAT_EX(a, b, c) + +#define NLOHMANN_JSON_ABI_STRING \ + NLOHMANN_JSON_ABI_CONCAT( \ + NLOHMANN_JSON_ABI_PREFIX( \ + NLOHMANN_JSON_VERSION_MAJOR, \ + NLOHMANN_JSON_VERSION_MINOR, \ + NLOHMANN_JSON_VERSION_PATCH), \ + NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS, \ + NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON) + +#ifndef NLOHMANN_JSON_NAMESPACE + #define NLOHMANN_JSON_NAMESPACE nlohmann::NLOHMANN_JSON_ABI_STRING +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_BEGIN +#define NLOHMANN_JSON_NAMESPACE_BEGIN \ + namespace nlohmann \ + { \ + inline namespace NLOHMANN_JSON_ABI_STRING \ + { +#endif + +#ifndef NLOHMANN_JSON_NAMESPACE_END +#define NLOHMANN_JSON_NAMESPACE_END \ + } /* namespace (abi_string) */ \ + } /* namespace nlohmann */ +#endif + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // transform +#include // array +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // nullptr_t +#include // exception +#include // runtime_error +#include // to_string +#include // vector + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // array +#include // size_t +#include // uint8_t +#include // string + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // declval, pair +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +// https://en.cppreference.com/w/cpp/experimental/is_detected +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; + void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template class Op, class... Args> +using is_detected = typename detector::value_t; + +template class Op, class... Args> +struct is_detected_lazy : is_detected { }; + +template class Op, class... Args> +using detected_t = typename detector::type; + +template class Op, class... Args> +using detected_or = detector; + +template class Op, class... Args> +using detected_or_t = typename detected_or::type; + +template class Op, class... Args> +using is_detected_exact = std::is_same>; + +template class Op, class... Args> +using is_detected_convertible = + std::is_convertible, To>; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + + +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2016-2021 Evan Nemerson +// SPDX-License-Identifier: MIT + +/* Hedley - https://nemequ.github.io/hedley + * Created by Evan Nemerson + */ + +#if !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < 15) +#if defined(JSON_HEDLEY_VERSION) + #undef JSON_HEDLEY_VERSION +#endif +#define JSON_HEDLEY_VERSION 15 + +#if defined(JSON_HEDLEY_STRINGIFY_EX) + #undef JSON_HEDLEY_STRINGIFY_EX +#endif +#define JSON_HEDLEY_STRINGIFY_EX(x) #x + +#if defined(JSON_HEDLEY_STRINGIFY) + #undef JSON_HEDLEY_STRINGIFY +#endif +#define JSON_HEDLEY_STRINGIFY(x) JSON_HEDLEY_STRINGIFY_EX(x) + +#if defined(JSON_HEDLEY_CONCAT_EX) + #undef JSON_HEDLEY_CONCAT_EX +#endif +#define JSON_HEDLEY_CONCAT_EX(a,b) a##b + +#if defined(JSON_HEDLEY_CONCAT) + #undef JSON_HEDLEY_CONCAT +#endif +#define JSON_HEDLEY_CONCAT(a,b) JSON_HEDLEY_CONCAT_EX(a,b) + +#if defined(JSON_HEDLEY_CONCAT3_EX) + #undef JSON_HEDLEY_CONCAT3_EX +#endif +#define JSON_HEDLEY_CONCAT3_EX(a,b,c) a##b##c + +#if defined(JSON_HEDLEY_CONCAT3) + #undef JSON_HEDLEY_CONCAT3 +#endif +#define JSON_HEDLEY_CONCAT3(a,b,c) JSON_HEDLEY_CONCAT3_EX(a,b,c) + +#if defined(JSON_HEDLEY_VERSION_ENCODE) + #undef JSON_HEDLEY_VERSION_ENCODE +#endif +#define JSON_HEDLEY_VERSION_ENCODE(major,minor,revision) (((major) * 1000000) + ((minor) * 1000) + (revision)) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MAJOR) + #undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_MINOR) + #undef JSON_HEDLEY_VERSION_DECODE_MINOR +#endif +#define JSON_HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000) + +#if defined(JSON_HEDLEY_VERSION_DECODE_REVISION) + #undef JSON_HEDLEY_VERSION_DECODE_REVISION +#endif +#define JSON_HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000) + +#if defined(JSON_HEDLEY_GNUC_VERSION) + #undef JSON_HEDLEY_GNUC_VERSION +#endif +#if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#elif defined(__GNUC__) + #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0) +#endif + +#if defined(JSON_HEDLEY_GNUC_VERSION_CHECK) + #undef JSON_HEDLEY_GNUC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GNUC_VERSION) + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION) + #undef JSON_HEDLEY_MSVC_VERSION +#endif +#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, (_MSC_FULL_VER % 100000) / 100) +#elif defined(_MSC_FULL_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10) +#elif defined(_MSC_VER) && !defined(__ICL) + #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_MSVC_VERSION_CHECK) + #undef JSON_HEDLEY_MSVC_VERSION_CHECK +#endif +#if !defined(JSON_HEDLEY_MSVC_VERSION) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (0) +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch))) +#elif defined(_MSC_VER) && (_MSC_VER >= 1200) + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch))) +#else + #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_VER >= ((major * 100) + (minor))) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION) + #undef JSON_HEDLEY_INTEL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE) +#elif defined(__INTEL_COMPILER) && !defined(__ICL) + #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_VERSION) + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #undef JSON_HEDLEY_INTEL_CL_VERSION +#endif +#if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) && defined(__ICL) + #define JSON_HEDLEY_INTEL_CL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER, __INTEL_COMPILER_UPDATE, 0) +#endif + +#if defined(JSON_HEDLEY_INTEL_CL_VERSION_CHECK) + #undef JSON_HEDLEY_INTEL_CL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_INTEL_CL_VERSION) + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_CL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_INTEL_CL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION) + #undef JSON_HEDLEY_PGI_VERSION +#endif +#if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__) + #define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__) +#endif + +#if defined(JSON_HEDLEY_PGI_VERSION_CHECK) + #undef JSON_HEDLEY_PGI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PGI_VERSION) + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #undef JSON_HEDLEY_SUNPRO_VERSION +#endif +#if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10) +#elif defined(__SUNPRO_C) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C) & 0xf) +#elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), (__SUNPRO_CC & 0xf) * 10) +#elif defined(__SUNPRO_CC) + #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC) & 0xf) +#endif + +#if defined(JSON_HEDLEY_SUNPRO_VERSION_CHECK) + #undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_SUNPRO_VERSION) + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#endif +#if defined(__EMSCRIPTEN__) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__) +#endif + +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK) + #undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION) + #undef JSON_HEDLEY_ARM_VERSION +#endif +#if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, (__ARMCOMPILER_VERSION % 10000) / 100) +#elif defined(__CC_ARM) && defined(__ARMCC_VERSION) + #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, (__ARMCC_VERSION % 10000) / 100) +#endif + +#if defined(JSON_HEDLEY_ARM_VERSION_CHECK) + #undef JSON_HEDLEY_ARM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_ARM_VERSION) + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION) + #undef JSON_HEDLEY_IBM_VERSION +#endif +#if defined(__ibmxl__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__) +#elif defined(__xlC__) && defined(__xlC_ver__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff) +#elif defined(__xlC__) + #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0) +#endif + +#if defined(JSON_HEDLEY_IBM_VERSION_CHECK) + #undef JSON_HEDLEY_IBM_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IBM_VERSION) + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_VERSION) + #undef JSON_HEDLEY_TI_VERSION +#endif +#if \ + defined(__TI_COMPILER_VERSION__) && \ + ( \ + defined(__TMS470__) || defined(__TI_ARM__) || \ + defined(__MSP430__) || \ + defined(__TMS320C2000__) \ + ) +#if (__TI_COMPILER_VERSION__ >= 16000000) + #define JSON_HEDLEY_TI_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif +#endif + +#if defined(JSON_HEDLEY_TI_VERSION_CHECK) + #undef JSON_HEDLEY_TI_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_VERSION) + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #undef JSON_HEDLEY_TI_CL2000_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C2000__) + #define JSON_HEDLEY_TI_CL2000_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL2000_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL2000_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL2000_VERSION) + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL2000_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL2000_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #undef JSON_HEDLEY_TI_CL430_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__MSP430__) + #define JSON_HEDLEY_TI_CL430_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL430_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL430_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL430_VERSION) + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL430_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL430_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #undef JSON_HEDLEY_TI_ARMCL_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && (defined(__TMS470__) || defined(__TI_ARM__)) + #define JSON_HEDLEY_TI_ARMCL_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION_CHECK) + #undef JSON_HEDLEY_TI_ARMCL_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_ARMCL_VERSION) + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_ARMCL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #undef JSON_HEDLEY_TI_CL6X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__TMS320C6X__) + #define JSON_HEDLEY_TI_CL6X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL6X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL6X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL6X_VERSION) + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL6X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL6X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #undef JSON_HEDLEY_TI_CL7X_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__C7000__) + #define JSON_HEDLEY_TI_CL7X_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CL7X_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CL7X_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CL7X_VERSION) + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CL7X_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CL7X_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #undef JSON_HEDLEY_TI_CLPRU_VERSION +#endif +#if defined(__TI_COMPILER_VERSION__) && defined(__PRU__) + #define JSON_HEDLEY_TI_CLPRU_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#endif + +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION_CHECK) + #undef JSON_HEDLEY_TI_CLPRU_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TI_CLPRU_VERSION) + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_CLPRU_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION) + #undef JSON_HEDLEY_CRAY_VERSION +#endif +#if defined(_CRAYC) + #if defined(_RELEASE_PATCHLEVEL) + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL) + #else + #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_CRAY_VERSION_CHECK) + #undef JSON_HEDLEY_CRAY_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_CRAY_VERSION) + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION) + #undef JSON_HEDLEY_IAR_VERSION +#endif +#if defined(__IAR_SYSTEMS_ICC__) + #if __VER__ > 1000 + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000)) + #else + #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(__VER__ / 100, __VER__ % 100, 0) + #endif +#endif + +#if defined(JSON_HEDLEY_IAR_VERSION_CHECK) + #undef JSON_HEDLEY_IAR_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_IAR_VERSION) + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION) + #undef JSON_HEDLEY_TINYC_VERSION +#endif +#if defined(__TINYC__) + #define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100) +#endif + +#if defined(JSON_HEDLEY_TINYC_VERSION_CHECK) + #undef JSON_HEDLEY_TINYC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION) + #undef JSON_HEDLEY_DMC_VERSION +#endif +#if defined(__DMC__) + #define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf) +#endif + +#if defined(JSON_HEDLEY_DMC_VERSION_CHECK) + #undef JSON_HEDLEY_DMC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_DMC_VERSION) + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #undef JSON_HEDLEY_COMPCERT_VERSION +#endif +#if defined(__COMPCERT_VERSION__) + #define JSON_HEDLEY_COMPCERT_VERSION JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, __COMPCERT_VERSION__ % 100) +#endif + +#if defined(JSON_HEDLEY_COMPCERT_VERSION_CHECK) + #undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_COMPCERT_VERSION) + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION) + #undef JSON_HEDLEY_PELLES_VERSION +#endif +#if defined(__POCC__) + #define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0) +#endif + +#if defined(JSON_HEDLEY_PELLES_VERSION_CHECK) + #undef JSON_HEDLEY_PELLES_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_PELLES_VERSION) + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #undef JSON_HEDLEY_MCST_LCC_VERSION +#endif +#if defined(__LCC__) && defined(__LCC_MINOR__) + #define JSON_HEDLEY_MCST_LCC_VERSION JSON_HEDLEY_VERSION_ENCODE(__LCC__ / 100, __LCC__ % 100, __LCC_MINOR__) +#endif + +#if defined(JSON_HEDLEY_MCST_LCC_VERSION_CHECK) + #undef JSON_HEDLEY_MCST_LCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_MCST_LCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_MCST_LCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION) + #undef JSON_HEDLEY_GCC_VERSION +#endif +#if \ + defined(JSON_HEDLEY_GNUC_VERSION) && \ + !defined(__clang__) && \ + !defined(JSON_HEDLEY_INTEL_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_ARM_VERSION) && \ + !defined(JSON_HEDLEY_CRAY_VERSION) && \ + !defined(JSON_HEDLEY_TI_VERSION) && \ + !defined(JSON_HEDLEY_TI_ARMCL_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL430_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL2000_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL6X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CL7X_VERSION) && \ + !defined(JSON_HEDLEY_TI_CLPRU_VERSION) && \ + !defined(__COMPCERT__) && \ + !defined(JSON_HEDLEY_MCST_LCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION +#endif + +#if defined(JSON_HEDLEY_GCC_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_VERSION_CHECK +#endif +#if defined(JSON_HEDLEY_GCC_VERSION) + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#else + #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_ATTRIBUTE +#endif +#if \ + defined(__has_attribute) && \ + ( \ + (!defined(JSON_HEDLEY_IAR_VERSION) || JSON_HEDLEY_IAR_VERSION_CHECK(8,5,9)) \ + ) +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute) +#else +# define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#endif +#if defined(__has_attribute) + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#endif +#if \ + defined(__has_cpp_attribute) && \ + defined(__cplusplus) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS) + #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS +#endif +#if !defined(__cplusplus) || !defined(__has_cpp_attribute) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#elif \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION) && \ + (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) && \ + (!defined(JSON_HEDLEY_MSVC_VERSION) || JSON_HEDLEY_MSVC_VERSION_CHECK(19,20,0)) + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(ns::attribute) +#else + #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(ns,attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#endif +#if defined(__has_cpp_attribute) && defined(__cplusplus) + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_BUILTIN) + #undef JSON_HEDLEY_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin) +#else + #define JSON_HEDLEY_HAS_BUILTIN(builtin) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_BUILTIN) + #undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_BUILTIN) + #undef JSON_HEDLEY_GCC_HAS_BUILTIN +#endif +#if defined(__has_builtin) + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#else + #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_FEATURE) + #undef JSON_HEDLEY_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature) +#else + #define JSON_HEDLEY_HAS_FEATURE(feature) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_FEATURE) + #undef JSON_HEDLEY_GNUC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_FEATURE) + #undef JSON_HEDLEY_GCC_HAS_FEATURE +#endif +#if defined(__has_feature) + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#else + #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_EXTENSION) + #undef JSON_HEDLEY_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension) +#else + #define JSON_HEDLEY_HAS_EXTENSION(extension) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_EXTENSION) + #undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_EXTENSION) + #undef JSON_HEDLEY_GCC_HAS_EXTENSION +#endif +#if defined(__has_extension) + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#else + #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#endif +#if defined(__has_declspec_attribute) + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#else + #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_HAS_WARNING) + #undef JSON_HEDLEY_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning) +#else + #define JSON_HEDLEY_HAS_WARNING(warning) (0) +#endif + +#if defined(JSON_HEDLEY_GNUC_HAS_WARNING) + #undef JSON_HEDLEY_GNUC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_GCC_HAS_WARNING) + #undef JSON_HEDLEY_GCC_HAS_WARNING +#endif +#if defined(__has_warning) + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#else + #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + defined(__clang__) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,17) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(8,0,0) || \ + (JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) && defined(__C99_PRAGMA_OPERATOR)) + #define JSON_HEDLEY_PRAGMA(value) _Pragma(#value) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_PRAGMA(value) __pragma(value) +#else + #define JSON_HEDLEY_PRAGMA(value) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_PUSH) + #undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#endif +#if defined(JSON_HEDLEY_DIAGNOSTIC_POP) + #undef JSON_HEDLEY_DIAGNOSTIC_POP +#endif +#if defined(__clang__) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push)) + #define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop)) +#elif JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("pop") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") + #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_PUSH + #define JSON_HEDLEY_DIAGNOSTIC_POP +#endif + +/* JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_ +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat") +# if JSON_HEDLEY_HAS_WARNING("-Wc++17-extensions") +# if JSON_HEDLEY_HAS_WARNING("-Wc++1z-extensions") +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + _Pragma("clang diagnostic ignored \"-Wc++1z-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + _Pragma("clang diagnostic ignored \"-Wc++17-extensions\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# else +# define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(xpr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat\"") \ + xpr \ + JSON_HEDLEY_DIAGNOSTIC_POP +# endif +# endif +#endif +#if !defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(x) x +#endif + +#if defined(JSON_HEDLEY_CONST_CAST) + #undef JSON_HEDLEY_CONST_CAST +#endif +#if defined(__cplusplus) +# define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast(expr)) +#elif \ + JSON_HEDLEY_HAS_WARNING("-Wcast-qual") || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_CONST_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_CONST_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_REINTERPRET_CAST) + #undef JSON_HEDLEY_REINTERPRET_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast(expr)) +#else + #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_STATIC_CAST) + #undef JSON_HEDLEY_STATIC_CAST +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast(expr)) +#else + #define JSON_HEDLEY_STATIC_CAST(T, expr) ((T) (expr)) +#endif + +#if defined(JSON_HEDLEY_CPP_CAST) + #undef JSON_HEDLEY_CPP_CAST +#endif +#if defined(__cplusplus) +# if JSON_HEDLEY_HAS_WARNING("-Wold-style-cast") +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wold-style-cast\"") \ + ((T) (expr)) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# elif JSON_HEDLEY_IAR_VERSION_CHECK(8,3,0) +# define JSON_HEDLEY_CPP_CAST(T, expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("diag_suppress=Pe137") \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_CPP_CAST(T, expr) ((T) (expr)) +# endif +#else +# define JSON_HEDLEY_CPP_CAST(T, expr) (expr) +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wdeprecated-declarations") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:1478 1786)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1216,1444,1445") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:4996)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && !defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:161)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:4068)) +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(16,9,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 161") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-attributes") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("clang diagnostic ignored \"-Wunknown-attributes\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("warning(disable:1292)") +#elif JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:1292)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES __pragma(warning(disable:5030)) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(20,7,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097,1098") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("error_messages(off,attrskipunsup)") +#elif \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1173") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress=Pe1097") +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES _Pragma("diag_suppress 1097") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_CPP_ATTRIBUTES +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wcast-qual") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#endif + +#if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION) + #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunused-function") + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("clang diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("GCC diagnostic ignored \"-Wunused-function\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(1,0,0) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION __pragma(warning(disable:4505)) +#elif JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION _Pragma("diag_suppress 3142") +#else + #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNUSED_FUNCTION +#endif + +#if defined(JSON_HEDLEY_DEPRECATED) + #undef JSON_HEDLEY_DEPRECATED +#endif +#if defined(JSON_HEDLEY_DEPRECATED_FOR) + #undef JSON_HEDLEY_DEPRECATED_FOR +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " # since)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement)) +#elif \ + (JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(18,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,3,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,3,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since))) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__("Since " #since "; use " #replacement))) +#elif defined(__cplusplus) && (__cplusplus >= 201402L) + #define JSON_HEDLEY_DEPRECATED(since) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since)]]) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[deprecated("Since " #since "; use " #replacement)]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__)) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_PELLES_VERSION_CHECK(6,50,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_DEPRECATED(since) _Pragma("deprecated") + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated") +#else + #define JSON_HEDLEY_DEPRECATED(since) + #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) +#endif + +#if defined(JSON_HEDLEY_UNAVAILABLE) + #undef JSON_HEDLEY_UNAVAILABLE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warning) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since))) +#else + #define JSON_HEDLEY_UNAVAILABLE(available_since) +#endif + +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT +#endif +#if defined(JSON_HEDLEY_WARN_UNUSED_RESULT_MSG) + #undef JSON_HEDLEY_WARN_UNUSED_RESULT_MSG +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) __attribute__((__warn_unused_result__)) +#elif (JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) >= 201907L) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard(msg)]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(nodiscard) + #define JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[nodiscard]]) +#elif defined(_Check_return_) /* SAL */ + #define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_ + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) _Check_return_ +#else + #define JSON_HEDLEY_WARN_UNUSED_RESULT + #define JSON_HEDLEY_WARN_UNUSED_RESULT_MSG(msg) +#endif + +#if defined(JSON_HEDLEY_SENTINEL) + #undef JSON_HEDLEY_SENTINEL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position))) +#else + #define JSON_HEDLEY_SENTINEL(position) +#endif + +#if defined(JSON_HEDLEY_NO_RETURN) + #undef JSON_HEDLEY_NO_RETURN +#endif +#if JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NO_RETURN __noreturn +#elif \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L + #define JSON_HEDLEY_NO_RETURN _Noreturn +#elif defined(__cplusplus) && (__cplusplus >= 201103L) + #define JSON_HEDLEY_NO_RETURN JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[noreturn]]) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,2,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_NO_RETURN _Pragma("does_not_return") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NO_RETURN __attribute((noreturn)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#else + #define JSON_HEDLEY_NO_RETURN +#endif + +#if defined(JSON_HEDLEY_NO_ESCAPE) + #undef JSON_HEDLEY_NO_ESCAPE +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(noescape) + #define JSON_HEDLEY_NO_ESCAPE __attribute__((__noescape__)) +#else + #define JSON_HEDLEY_NO_ESCAPE +#endif + +#if defined(JSON_HEDLEY_UNREACHABLE) + #undef JSON_HEDLEY_UNREACHABLE +#endif +#if defined(JSON_HEDLEY_UNREACHABLE_RETURN) + #undef JSON_HEDLEY_UNREACHABLE_RETURN +#endif +#if defined(JSON_HEDLEY_ASSUME) + #undef JSON_HEDLEY_ASSUME +#endif +#if \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_ASSUME(expr) __assume(expr) +#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_assume) + #define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr) +#elif \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #if defined(__cplusplus) + #define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr) + #else + #define JSON_HEDLEY_ASSUME(expr) _nassert(expr) + #endif +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,10,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,5) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(10,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable() +#elif defined(JSON_HEDLEY_ASSUME) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif +#if !defined(JSON_HEDLEY_ASSUME) + #if defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, ((expr) ? 1 : (JSON_HEDLEY_UNREACHABLE(), 1))) + #else + #define JSON_HEDLEY_ASSUME(expr) JSON_HEDLEY_STATIC_CAST(void, expr) + #endif +#endif +#if defined(JSON_HEDLEY_UNREACHABLE) + #if \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (JSON_HEDLEY_STATIC_CAST(void, JSON_HEDLEY_ASSUME(0)), (value)) + #else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE() + #endif +#else + #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return (value) +#endif +#if !defined(JSON_HEDLEY_UNREACHABLE) + #define JSON_HEDLEY_UNREACHABLE() JSON_HEDLEY_ASSUME(0) +#endif + +JSON_HEDLEY_DIAGNOSTIC_PUSH +#if JSON_HEDLEY_HAS_WARNING("-Wpedantic") + #pragma clang diagnostic ignored "-Wpedantic" +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wc++98-compat-pedantic") && defined(__cplusplus) + #pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +#endif +#if JSON_HEDLEY_GCC_HAS_WARNING("-Wvariadic-macros",4,0,0) + #if defined(__clang__) + #pragma clang diagnostic ignored "-Wvariadic-macros" + #elif defined(JSON_HEDLEY_GCC_VERSION) + #pragma GCC diagnostic ignored "-Wvariadic-macros" + #endif +#endif +#if defined(JSON_HEDLEY_NON_NULL) + #undef JSON_HEDLEY_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__))) +#else + #define JSON_HEDLEY_NON_NULL(...) +#endif +JSON_HEDLEY_DIAGNOSTIC_POP + +#if defined(JSON_HEDLEY_PRINTF_FORMAT) + #undef JSON_HEDLEY_PRINTF_FORMAT +#endif +#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && !defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(ms_printf, string_idx, first_to_check))) +#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && defined(__USE_MINGW_ANSI_STDIO) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(gnu_printf, string_idx, first_to_check))) +#elif \ + JSON_HEDLEY_HAS_ATTRIBUTE(format) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(__printf__, string_idx, first_to_check))) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6,0,0) + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __declspec(vaformat(printf,string_idx,first_to_check)) +#else + #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) +#endif + +#if defined(JSON_HEDLEY_CONSTEXPR) + #undef JSON_HEDLEY_CONSTEXPR +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_CONSTEXPR JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(constexpr) + #endif +#endif +#if !defined(JSON_HEDLEY_CONSTEXPR) + #define JSON_HEDLEY_CONSTEXPR +#endif + +#if defined(JSON_HEDLEY_PREDICT) + #undef JSON_HEDLEY_PREDICT +#endif +#if defined(JSON_HEDLEY_LIKELY) + #undef JSON_HEDLEY_LIKELY +#endif +#if defined(JSON_HEDLEY_UNLIKELY) + #undef JSON_HEDLEY_UNLIKELY +#endif +#if defined(JSON_HEDLEY_UNPREDICTABLE) + #undef JSON_HEDLEY_UNPREDICTABLE +#endif +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_unpredictable) + #define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable((expr)) +#endif +#if \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) && !defined(JSON_HEDLEY_PGI_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(9,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability( (expr), (value), (probability)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1 , (probability)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0 , (probability)) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect (!!(expr), 1 ) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect (!!(expr), 0 ) +#elif \ + (JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,7,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,27) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PREDICT(expr, expected, probability) \ + (((probability) >= 0.9) ? __builtin_expect((expr), (expected)) : (JSON_HEDLEY_STATIC_CAST(void, expected), (expr))) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \ + })) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \ + (__extension__ ({ \ + double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \ + })) +# define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) +# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#else +# define JSON_HEDLEY_PREDICT(expr, expected, probability) (JSON_HEDLEY_STATIC_CAST(void, expected), (expr)) +# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr)) +# define JSON_HEDLEY_LIKELY(expr) (!!(expr)) +# define JSON_HEDLEY_UNLIKELY(expr) (!!(expr)) +#endif +#if !defined(JSON_HEDLEY_UNPREDICTABLE) + #define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5) +#endif + +#if defined(JSON_HEDLEY_MALLOC) + #undef JSON_HEDLEY_MALLOC +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_MALLOC __attribute__((__malloc__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_MALLOC _Pragma("returns_new_memory") +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_MALLOC __declspec(restrict) +#else + #define JSON_HEDLEY_MALLOC +#endif + +#if defined(JSON_HEDLEY_PURE) + #undef JSON_HEDLEY_PURE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(pure) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,96,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PURE __attribute__((__pure__)) +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) +# define JSON_HEDLEY_PURE _Pragma("does_not_write_global_data") +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(2,0,1) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) \ + ) +# define JSON_HEDLEY_PURE _Pragma("FUNC_IS_PURE;") +#else +# define JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_CONST) + #undef JSON_HEDLEY_CONST +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(const) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(2,5,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_CONST __attribute__((__const__)) +#elif \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) + #define JSON_HEDLEY_CONST _Pragma("no_side_effect") +#else + #define JSON_HEDLEY_CONST JSON_HEDLEY_PURE +#endif + +#if defined(JSON_HEDLEY_RESTRICT) + #undef JSON_HEDLEY_RESTRICT +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT restrict +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,4) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ + defined(__clang__) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RESTRICT __restrict +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,3,0) && !defined(__cplusplus) + #define JSON_HEDLEY_RESTRICT _Restrict +#else + #define JSON_HEDLEY_RESTRICT +#endif + +#if defined(JSON_HEDLEY_INLINE) + #undef JSON_HEDLEY_INLINE +#endif +#if \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ + (defined(__cplusplus) && (__cplusplus >= 199711L)) + #define JSON_HEDLEY_INLINE inline +#elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(6,2,0) + #define JSON_HEDLEY_INLINE __inline__ +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,1,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(3,1,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,2,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(8,0,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_INLINE __inline +#else + #define JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_ALWAYS_INLINE) + #undef JSON_HEDLEY_ALWAYS_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) +# define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_ALWAYS_INLINE __forceinline +#elif defined(__cplusplus) && \ + ( \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) \ + ) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_ALWAYS_INLINE _Pragma("inline=forced") +#else +# define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE +#endif + +#if defined(JSON_HEDLEY_NEVER_INLINE) + #undef JSON_HEDLEY_NEVER_INLINE +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(15,12,0) || \ + (JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(4,8,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_ARMCL_VERSION_CHECK(5,2,0) || \ + (JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL2000_VERSION_CHECK(6,4,0) || \ + (JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,0,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(4,3,0) || \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) || \ + JSON_HEDLEY_TI_CL7X_VERSION_CHECK(1,2,0) || \ + JSON_HEDLEY_TI_CLPRU_VERSION_CHECK(2,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8,10,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(10,2,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("noinline") +#elif JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,0,0) && defined(__cplusplus) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) + #define JSON_HEDLEY_NEVER_INLINE _Pragma("inline=never") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) + #define JSON_HEDLEY_NEVER_INLINE __attribute((noinline)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) + #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#else + #define JSON_HEDLEY_NEVER_INLINE +#endif + +#if defined(JSON_HEDLEY_PRIVATE) + #undef JSON_HEDLEY_PRIVATE +#endif +#if defined(JSON_HEDLEY_PUBLIC) + #undef JSON_HEDLEY_PUBLIC +#endif +#if defined(JSON_HEDLEY_IMPORT) + #undef JSON_HEDLEY_IMPORT +#endif +#if defined(_WIN32) || defined(__CYGWIN__) +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC __declspec(dllexport) +# define JSON_HEDLEY_IMPORT __declspec(dllimport) +#else +# if \ + JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + ( \ + defined(__TI_EABI__) && \ + ( \ + (JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,2,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(7,5,0) \ + ) \ + ) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) +# define JSON_HEDLEY_PRIVATE __attribute__((__visibility__("hidden"))) +# define JSON_HEDLEY_PUBLIC __attribute__((__visibility__("default"))) +# else +# define JSON_HEDLEY_PRIVATE +# define JSON_HEDLEY_PUBLIC +# endif +# define JSON_HEDLEY_IMPORT extern +#endif + +#if defined(JSON_HEDLEY_NO_THROW) + #undef JSON_HEDLEY_NO_THROW +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__)) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) + #define JSON_HEDLEY_NO_THROW __declspec(nothrow) +#else + #define JSON_HEDLEY_NO_THROW +#endif + +#if defined(JSON_HEDLEY_FALL_THROUGH) + #undef JSON_HEDLEY_FALL_THROUGH +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(fallthrough) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(7,0,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__)) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE_NS(clang,fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[clang::fallthrough]]) +#elif JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough) + #define JSON_HEDLEY_FALL_THROUGH JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_([[fallthrough]]) +#elif defined(__fallthrough) /* SAL */ + #define JSON_HEDLEY_FALL_THROUGH __fallthrough +#else + #define JSON_HEDLEY_FALL_THROUGH +#endif + +#if defined(JSON_HEDLEY_RETURNS_NON_NULL) + #undef JSON_HEDLEY_RETURNS_NON_NULL +#endif +#if \ + JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__)) +#elif defined(_Ret_notnull_) /* SAL */ + #define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_ +#else + #define JSON_HEDLEY_RETURNS_NON_NULL +#endif + +#if defined(JSON_HEDLEY_ARRAY_PARAM) + #undef JSON_HEDLEY_ARRAY_PARAM +#endif +#if \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ + !defined(__STDC_NO_VLA__) && \ + !defined(__cplusplus) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_TINYC_VERSION) + #define JSON_HEDLEY_ARRAY_PARAM(name) (name) +#else + #define JSON_HEDLEY_ARRAY_PARAM(name) +#endif + +#if defined(JSON_HEDLEY_IS_CONSTANT) + #undef JSON_HEDLEY_IS_CONSTANT +#endif +#if defined(JSON_HEDLEY_REQUIRE_CONSTEXPR) + #undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#endif +/* JSON_HEDLEY_IS_CONSTEXPR_ is for + HEDLEY INTERNAL USE ONLY. API subject to change without notice. */ +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #undef JSON_HEDLEY_IS_CONSTEXPR_ +#endif +#if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,19) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_TI_CL6X_VERSION_CHECK(6,1,0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) && !defined(__cplusplus)) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_MCST_LCC_VERSION_CHECK(1,25,10) + #define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr) +#endif +#if !defined(__cplusplus) +# if \ + JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,24) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0)), int*) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((intptr_t) ((expr) * 0)) : (int*) 0)), int*) +#endif +# elif \ + ( \ + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ + !defined(JSON_HEDLEY_SUNPRO_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && \ + !defined(JSON_HEDLEY_IAR_VERSION)) || \ + (JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) && !defined(JSON_HEDLEY_IAR_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5,3,0) +#if defined(__INTPTR_TYPE__) + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0), int*: 1, void*: 0) +#else + #include + #define JSON_HEDLEY_IS_CONSTEXPR_(expr) _Generic((1 ? (void*) ((intptr_t) * 0) : (int*) 0), int*: 1, void*: 0) +#endif +# elif \ + defined(JSON_HEDLEY_GCC_VERSION) || \ + defined(JSON_HEDLEY_INTEL_VERSION) || \ + defined(JSON_HEDLEY_TINYC_VERSION) || \ + defined(JSON_HEDLEY_TI_ARMCL_VERSION) || \ + JSON_HEDLEY_TI_CL430_VERSION_CHECK(18,12,0) || \ + defined(JSON_HEDLEY_TI_CL2000_VERSION) || \ + defined(JSON_HEDLEY_TI_CL6X_VERSION) || \ + defined(JSON_HEDLEY_TI_CL7X_VERSION) || \ + defined(JSON_HEDLEY_TI_CLPRU_VERSION) || \ + defined(__clang__) +# define JSON_HEDLEY_IS_CONSTEXPR_(expr) ( \ + sizeof(void) != \ + sizeof(*( \ + 1 ? \ + ((void*) ((expr) * 0L) ) : \ +((struct { char v[sizeof(void) * 2]; } *) 1) \ + ) \ + ) \ + ) +# endif +#endif +#if defined(JSON_HEDLEY_IS_CONSTEXPR_) + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY_IS_CONSTEXPR_(expr) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY_IS_CONSTEXPR_(expr) ? (expr) : (-1)) +#else + #if !defined(JSON_HEDLEY_IS_CONSTANT) + #define JSON_HEDLEY_IS_CONSTANT(expr) (0) + #endif + #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr) +#endif + +#if defined(JSON_HEDLEY_BEGIN_C_DECLS) + #undef JSON_HEDLEY_BEGIN_C_DECLS +#endif +#if defined(JSON_HEDLEY_END_C_DECLS) + #undef JSON_HEDLEY_END_C_DECLS +#endif +#if defined(JSON_HEDLEY_C_DECL) + #undef JSON_HEDLEY_C_DECL +#endif +#if defined(__cplusplus) + #define JSON_HEDLEY_BEGIN_C_DECLS extern "C" { + #define JSON_HEDLEY_END_C_DECLS } + #define JSON_HEDLEY_C_DECL extern "C" +#else + #define JSON_HEDLEY_BEGIN_C_DECLS + #define JSON_HEDLEY_END_C_DECLS + #define JSON_HEDLEY_C_DECL +#endif + +#if defined(JSON_HEDLEY_STATIC_ASSERT) + #undef JSON_HEDLEY_STATIC_ASSERT +#endif +#if \ + !defined(__cplusplus) && ( \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || \ + (JSON_HEDLEY_HAS_FEATURE(c_static_assert) && !defined(JSON_HEDLEY_INTEL_CL_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(6,0,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ + defined(_Static_assert) \ + ) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message) +#elif \ + (defined(__cplusplus) && (__cplusplus >= 201103L)) || \ + JSON_HEDLEY_MSVC_VERSION_CHECK(16,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(static_assert(expr, message)) +#else +# define JSON_HEDLEY_STATIC_ASSERT(expr, message) +#endif + +#if defined(JSON_HEDLEY_NULL) + #undef JSON_HEDLEY_NULL +#endif +#if defined(__cplusplus) + #if __cplusplus >= 201103L + #define JSON_HEDLEY_NULL JSON_HEDLEY_DIAGNOSTIC_DISABLE_CPP98_COMPAT_WRAP_(nullptr) + #elif defined(NULL) + #define JSON_HEDLEY_NULL NULL + #else + #define JSON_HEDLEY_NULL JSON_HEDLEY_STATIC_CAST(void*, 0) + #endif +#elif defined(NULL) + #define JSON_HEDLEY_NULL NULL +#else + #define JSON_HEDLEY_NULL ((void*) 0) +#endif + +#if defined(JSON_HEDLEY_MESSAGE) + #undef JSON_HEDLEY_MESSAGE +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_MESSAGE(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(message msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg) +#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,0,0) +# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_WARNING) + #undef JSON_HEDLEY_WARNING +#endif +#if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") +# define JSON_HEDLEY_WARNING(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ + JSON_HEDLEY_PRAGMA(clang warning msg) \ + JSON_HEDLEY_DIAGNOSTIC_POP +#elif \ + JSON_HEDLEY_GCC_VERSION_CHECK(4,8,0) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg) +#elif \ + JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#else +# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg) +#endif + +#if defined(JSON_HEDLEY_REQUIRE) + #undef JSON_HEDLEY_REQUIRE +#endif +#if defined(JSON_HEDLEY_REQUIRE_MSG) + #undef JSON_HEDLEY_REQUIRE_MSG +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(diagnose_if) +# if JSON_HEDLEY_HAS_WARNING("-Wgcc-compat") +# define JSON_HEDLEY_REQUIRE(expr) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), #expr, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ + __attribute__((diagnose_if(!(expr), msg, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP +# else +# define JSON_HEDLEY_REQUIRE(expr) __attribute__((diagnose_if(!(expr), #expr, "error"))) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) __attribute__((diagnose_if(!(expr), msg, "error"))) +# endif +#else +# define JSON_HEDLEY_REQUIRE(expr) +# define JSON_HEDLEY_REQUIRE_MSG(expr,msg) +#endif + +#if defined(JSON_HEDLEY_FLAGS) + #undef JSON_HEDLEY_FLAGS +#endif +#if JSON_HEDLEY_HAS_ATTRIBUTE(flag_enum) && (!defined(__cplusplus) || JSON_HEDLEY_HAS_WARNING("-Wbitfield-enum-conversion")) + #define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__)) +#else + #define JSON_HEDLEY_FLAGS +#endif + +#if defined(JSON_HEDLEY_FLAGS_CAST) + #undef JSON_HEDLEY_FLAGS_CAST +#endif +#if JSON_HEDLEY_INTEL_VERSION_CHECK(19,0,0) +# define JSON_HEDLEY_FLAGS_CAST(T, expr) (__extension__ ({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("warning(disable:188)") \ + ((T) (expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ + })) +#else +# define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr) +#endif + +#if defined(JSON_HEDLEY_EMPTY_BASES) + #undef JSON_HEDLEY_EMPTY_BASES +#endif +#if \ + (JSON_HEDLEY_MSVC_VERSION_CHECK(19,0,23918) && !JSON_HEDLEY_MSVC_VERSION_CHECK(20,0,0)) || \ + JSON_HEDLEY_INTEL_CL_VERSION_CHECK(2021,1,0) + #define JSON_HEDLEY_EMPTY_BASES __declspec(empty_bases) +#else + #define JSON_HEDLEY_EMPTY_BASES +#endif + +/* Remaining macros are deprecated. */ + +#if defined(JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK) + #undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#endif +#if defined(__clang__) + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) (0) +#else + #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#endif + +#if defined(JSON_HEDLEY_CLANG_HAS_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_BUILTIN) + #undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#endif +#define JSON_HEDLEY_CLANG_HAS_BUILTIN(builtin) JSON_HEDLEY_HAS_BUILTIN(builtin) + +#if defined(JSON_HEDLEY_CLANG_HAS_FEATURE) + #undef JSON_HEDLEY_CLANG_HAS_FEATURE +#endif +#define JSON_HEDLEY_CLANG_HAS_FEATURE(feature) JSON_HEDLEY_HAS_FEATURE(feature) + +#if defined(JSON_HEDLEY_CLANG_HAS_EXTENSION) + #undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#endif +#define JSON_HEDLEY_CLANG_HAS_EXTENSION(extension) JSON_HEDLEY_HAS_EXTENSION(extension) + +#if defined(JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE) + #undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#endif +#define JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) + +#if defined(JSON_HEDLEY_CLANG_HAS_WARNING) + #undef JSON_HEDLEY_CLANG_HAS_WARNING +#endif +#define JSON_HEDLEY_CLANG_HAS_WARNING(warning) JSON_HEDLEY_HAS_WARNING(warning) + +#endif /* !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < X) */ + + +// This file contains all internal macro definitions (except those affecting ABI) +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// #include + + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// C++ language standard detection +// if the user manually specified the used c++ version this is skipped +#if !defined(JSON_HAS_CPP_20) && !defined(JSON_HAS_CPP_17) && !defined(JSON_HAS_CPP_14) && !defined(JSON_HAS_CPP_11) + #if (defined(__cplusplus) && __cplusplus >= 202002L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) + #define JSON_HAS_CPP_20 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 + #elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 + #endif + // the cpp 11 flag is always specified because it is the minimal required version + #define JSON_HAS_CPP_11 +#endif + +#ifdef __has_include + #if __has_include() + #include + #endif +#endif + +#if !defined(JSON_HAS_FILESYSTEM) && !defined(JSON_HAS_EXPERIMENTAL_FILESYSTEM) + #ifdef JSON_HAS_CPP_17 + #if defined(__cpp_lib_filesystem) + #define JSON_HAS_FILESYSTEM 1 + #elif defined(__cpp_lib_experimental_filesystem) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif !defined(__has_include) + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_FILESYSTEM 1 + #elif __has_include() + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 1 + #endif + + // std::filesystem does not work on MinGW GCC 8: https://sourceforge.net/p/mingw-w64/bugs/737/ + #if defined(__MINGW32__) && defined(__GNUC__) && __GNUC__ == 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before GCC 8: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 8 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before Clang 7: https://en.cppreference.com/w/cpp/compiler_support + #if defined(__clang_major__) && __clang_major__ < 7 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before MSVC 19.14: https://en.cppreference.com/w/cpp/compiler_support + #if defined(_MSC_VER) && _MSC_VER < 1914 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before iOS 13 + #if defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED < 130000 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + + // no filesystem support before macOS Catalina + #if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500 + #undef JSON_HAS_FILESYSTEM + #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #endif + #endif +#endif + +#ifndef JSON_HAS_EXPERIMENTAL_FILESYSTEM + #define JSON_HAS_EXPERIMENTAL_FILESYSTEM 0 +#endif + +#ifndef JSON_HAS_FILESYSTEM + #define JSON_HAS_FILESYSTEM 0 +#endif + +#ifndef JSON_HAS_THREE_WAY_COMPARISON + #if defined(__cpp_impl_three_way_comparison) && __cpp_impl_three_way_comparison >= 201907L \ + && defined(__cpp_lib_three_way_comparison) && __cpp_lib_three_way_comparison >= 201907L + #define JSON_HAS_THREE_WAY_COMPARISON 1 + #else + #define JSON_HAS_THREE_WAY_COMPARISON 0 + #endif +#endif + +#ifndef JSON_HAS_RANGES + // ranges header shipping in GCC 11.1.0 (released 2021-04-27) has syntax error + #if defined(__GLIBCXX__) && __GLIBCXX__ == 20210427 + #define JSON_HAS_RANGES 0 + #elif defined(__cpp_lib_ranges) + #define JSON_HAS_RANGES 1 + #else + #define JSON_HAS_RANGES 0 + #endif +#endif + +#ifdef JSON_HAS_CPP_17 + #define JSON_INLINE_VARIABLE inline +#else + #define JSON_INLINE_VARIABLE +#endif + +#if JSON_HEDLEY_HAS_ATTRIBUTE(no_unique_address) + #define JSON_NO_UNIQUE_ADDRESS [[no_unique_address]] +#else + #define JSON_NO_UNIQUE_ADDRESS +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdocumentation" + #pragma clang diagnostic ignored "-Wdocumentation-unknown-command" +#endif + +// allow disabling exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #include + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// allow overriding assert +#if !defined(JSON_ASSERT) + #include // assert + #define JSON_ASSERT(x) assert(x) +#endif + +// allow to access some private functions (needed by the test suite) +#if defined(JSON_TESTS_PRIVATE) + #define JSON_PRIVATE_UNLESS_TESTED public +#else + #define JSON_PRIVATE_UNLESS_TESTED private +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [&j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer, \ + class BinaryType> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// Macros to simplify conversion from/to types + +#define NLOHMANN_JSON_EXPAND( x ) x +#define NLOHMANN_JSON_GET_MACRO(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, _62, _63, _64, NAME,...) NAME +#define NLOHMANN_JSON_PASTE(...) NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_GET_MACRO(__VA_ARGS__, \ + NLOHMANN_JSON_PASTE64, \ + NLOHMANN_JSON_PASTE63, \ + NLOHMANN_JSON_PASTE62, \ + NLOHMANN_JSON_PASTE61, \ + NLOHMANN_JSON_PASTE60, \ + NLOHMANN_JSON_PASTE59, \ + NLOHMANN_JSON_PASTE58, \ + NLOHMANN_JSON_PASTE57, \ + NLOHMANN_JSON_PASTE56, \ + NLOHMANN_JSON_PASTE55, \ + NLOHMANN_JSON_PASTE54, \ + NLOHMANN_JSON_PASTE53, \ + NLOHMANN_JSON_PASTE52, \ + NLOHMANN_JSON_PASTE51, \ + NLOHMANN_JSON_PASTE50, \ + NLOHMANN_JSON_PASTE49, \ + NLOHMANN_JSON_PASTE48, \ + NLOHMANN_JSON_PASTE47, \ + NLOHMANN_JSON_PASTE46, \ + NLOHMANN_JSON_PASTE45, \ + NLOHMANN_JSON_PASTE44, \ + NLOHMANN_JSON_PASTE43, \ + NLOHMANN_JSON_PASTE42, \ + NLOHMANN_JSON_PASTE41, \ + NLOHMANN_JSON_PASTE40, \ + NLOHMANN_JSON_PASTE39, \ + NLOHMANN_JSON_PASTE38, \ + NLOHMANN_JSON_PASTE37, \ + NLOHMANN_JSON_PASTE36, \ + NLOHMANN_JSON_PASTE35, \ + NLOHMANN_JSON_PASTE34, \ + NLOHMANN_JSON_PASTE33, \ + NLOHMANN_JSON_PASTE32, \ + NLOHMANN_JSON_PASTE31, \ + NLOHMANN_JSON_PASTE30, \ + NLOHMANN_JSON_PASTE29, \ + NLOHMANN_JSON_PASTE28, \ + NLOHMANN_JSON_PASTE27, \ + NLOHMANN_JSON_PASTE26, \ + NLOHMANN_JSON_PASTE25, \ + NLOHMANN_JSON_PASTE24, \ + NLOHMANN_JSON_PASTE23, \ + NLOHMANN_JSON_PASTE22, \ + NLOHMANN_JSON_PASTE21, \ + NLOHMANN_JSON_PASTE20, \ + NLOHMANN_JSON_PASTE19, \ + NLOHMANN_JSON_PASTE18, \ + NLOHMANN_JSON_PASTE17, \ + NLOHMANN_JSON_PASTE16, \ + NLOHMANN_JSON_PASTE15, \ + NLOHMANN_JSON_PASTE14, \ + NLOHMANN_JSON_PASTE13, \ + NLOHMANN_JSON_PASTE12, \ + NLOHMANN_JSON_PASTE11, \ + NLOHMANN_JSON_PASTE10, \ + NLOHMANN_JSON_PASTE9, \ + NLOHMANN_JSON_PASTE8, \ + NLOHMANN_JSON_PASTE7, \ + NLOHMANN_JSON_PASTE6, \ + NLOHMANN_JSON_PASTE5, \ + NLOHMANN_JSON_PASTE4, \ + NLOHMANN_JSON_PASTE3, \ + NLOHMANN_JSON_PASTE2, \ + NLOHMANN_JSON_PASTE1)(__VA_ARGS__)) +#define NLOHMANN_JSON_PASTE2(func, v1) func(v1) +#define NLOHMANN_JSON_PASTE3(func, v1, v2) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE2(func, v2) +#define NLOHMANN_JSON_PASTE4(func, v1, v2, v3) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE3(func, v2, v3) +#define NLOHMANN_JSON_PASTE5(func, v1, v2, v3, v4) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE4(func, v2, v3, v4) +#define NLOHMANN_JSON_PASTE6(func, v1, v2, v3, v4, v5) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE5(func, v2, v3, v4, v5) +#define NLOHMANN_JSON_PASTE7(func, v1, v2, v3, v4, v5, v6) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE6(func, v2, v3, v4, v5, v6) +#define NLOHMANN_JSON_PASTE8(func, v1, v2, v3, v4, v5, v6, v7) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE7(func, v2, v3, v4, v5, v6, v7) +#define NLOHMANN_JSON_PASTE9(func, v1, v2, v3, v4, v5, v6, v7, v8) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE8(func, v2, v3, v4, v5, v6, v7, v8) +#define NLOHMANN_JSON_PASTE10(func, v1, v2, v3, v4, v5, v6, v7, v8, v9) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE9(func, v2, v3, v4, v5, v6, v7, v8, v9) +#define NLOHMANN_JSON_PASTE11(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE10(func, v2, v3, v4, v5, v6, v7, v8, v9, v10) +#define NLOHMANN_JSON_PASTE12(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE11(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11) +#define NLOHMANN_JSON_PASTE13(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE12(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12) +#define NLOHMANN_JSON_PASTE14(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE13(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13) +#define NLOHMANN_JSON_PASTE15(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE14(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14) +#define NLOHMANN_JSON_PASTE16(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE15(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15) +#define NLOHMANN_JSON_PASTE17(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE16(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16) +#define NLOHMANN_JSON_PASTE18(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE17(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17) +#define NLOHMANN_JSON_PASTE19(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE18(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18) +#define NLOHMANN_JSON_PASTE20(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE19(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19) +#define NLOHMANN_JSON_PASTE21(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE20(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20) +#define NLOHMANN_JSON_PASTE22(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE21(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21) +#define NLOHMANN_JSON_PASTE23(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE22(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22) +#define NLOHMANN_JSON_PASTE24(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE23(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23) +#define NLOHMANN_JSON_PASTE25(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE24(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24) +#define NLOHMANN_JSON_PASTE26(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE25(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25) +#define NLOHMANN_JSON_PASTE27(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE26(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26) +#define NLOHMANN_JSON_PASTE28(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE27(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27) +#define NLOHMANN_JSON_PASTE29(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE28(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28) +#define NLOHMANN_JSON_PASTE30(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE29(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29) +#define NLOHMANN_JSON_PASTE31(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE30(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30) +#define NLOHMANN_JSON_PASTE32(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE31(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31) +#define NLOHMANN_JSON_PASTE33(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE32(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32) +#define NLOHMANN_JSON_PASTE34(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE33(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33) +#define NLOHMANN_JSON_PASTE35(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE34(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34) +#define NLOHMANN_JSON_PASTE36(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE35(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) +#define NLOHMANN_JSON_PASTE37(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE36(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) +#define NLOHMANN_JSON_PASTE38(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE37(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37) +#define NLOHMANN_JSON_PASTE39(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE38(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38) +#define NLOHMANN_JSON_PASTE40(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE39(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39) +#define NLOHMANN_JSON_PASTE41(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE40(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40) +#define NLOHMANN_JSON_PASTE42(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE41(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41) +#define NLOHMANN_JSON_PASTE43(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE42(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42) +#define NLOHMANN_JSON_PASTE44(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE43(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43) +#define NLOHMANN_JSON_PASTE45(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE44(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44) +#define NLOHMANN_JSON_PASTE46(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE45(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45) +#define NLOHMANN_JSON_PASTE47(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE46(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46) +#define NLOHMANN_JSON_PASTE48(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE47(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47) +#define NLOHMANN_JSON_PASTE49(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE48(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48) +#define NLOHMANN_JSON_PASTE50(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE49(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49) +#define NLOHMANN_JSON_PASTE51(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE50(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50) +#define NLOHMANN_JSON_PASTE52(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE51(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51) +#define NLOHMANN_JSON_PASTE53(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE52(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52) +#define NLOHMANN_JSON_PASTE54(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE53(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53) +#define NLOHMANN_JSON_PASTE55(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE54(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54) +#define NLOHMANN_JSON_PASTE56(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE55(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55) +#define NLOHMANN_JSON_PASTE57(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE56(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56) +#define NLOHMANN_JSON_PASTE58(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE57(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57) +#define NLOHMANN_JSON_PASTE59(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE58(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58) +#define NLOHMANN_JSON_PASTE60(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE59(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59) +#define NLOHMANN_JSON_PASTE61(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE60(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60) +#define NLOHMANN_JSON_PASTE62(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE61(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61) +#define NLOHMANN_JSON_PASTE63(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE62(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62) +#define NLOHMANN_JSON_PASTE64(func, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) NLOHMANN_JSON_PASTE2(func, v1) NLOHMANN_JSON_PASTE63(func, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63) + +#define NLOHMANN_JSON_TO(v1) nlohmann_json_j[#v1] = nlohmann_json_t.v1; +#define NLOHMANN_JSON_FROM(v1) nlohmann_json_j.at(#v1).get_to(nlohmann_json_t.v1); +#define NLOHMANN_JSON_FROM_WITH_DEFAULT(v1) nlohmann_json_t.v1 = nlohmann_json_j.value(#v1, nlohmann_json_default_obj.v1); + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_INTRUSIVE +@since version 3.9.0 +*/ +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE(Type, ...) \ + friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +#define NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + friend void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + friend void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + +/*! +@brief macro +@def NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE +@since version 3.9.0 +*/ +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM, __VA_ARGS__)) } + +#define NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(Type, ...) \ + inline void to_json(nlohmann::json& nlohmann_json_j, const Type& nlohmann_json_t) { NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_TO, __VA_ARGS__)) } \ + inline void from_json(const nlohmann::json& nlohmann_json_j, Type& nlohmann_json_t) { Type nlohmann_json_default_obj; NLOHMANN_JSON_EXPAND(NLOHMANN_JSON_PASTE(NLOHMANN_JSON_FROM_WITH_DEFAULT, __VA_ARGS__)) } + + +// inspired from https://stackoverflow.com/a/26745591 +// allows to call any std function as if (e.g. with begin): +// using std::begin; begin(x); +// +// it allows using the detected idiom to retrieve the return type +// of such an expression +#define NLOHMANN_CAN_CALL_STD_FUNC_IMPL(std_name) \ + namespace detail { \ + using std::std_name; \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + } \ + \ + namespace detail2 { \ + struct std_name##_tag \ + { \ + }; \ + \ + template \ + std_name##_tag std_name(T&&...); \ + \ + template \ + using result_of_##std_name = decltype(std_name(std::declval()...)); \ + \ + template \ + struct would_call_std_##std_name \ + { \ + static constexpr auto const value = ::nlohmann::detail:: \ + is_detected_exact::value; \ + }; \ + } /* namespace detail2 */ \ + \ + template \ + struct would_call_std_##std_name : detail2::would_call_std_##std_name \ + { \ + } + +#ifndef JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_USE_IMPLICIT_CONVERSIONS 1 +#endif + +#if JSON_USE_IMPLICIT_CONVERSIONS + #define JSON_EXPLICIT +#else + #define JSON_EXPLICIT explicit +#endif + +#ifndef JSON_DISABLE_ENUM_SERIALIZATION + #define JSON_DISABLE_ENUM_SERIALIZATION 0 +#endif + +#ifndef JSON_USE_GLOBAL_UDLS + #define JSON_USE_GLOBAL_UDLS 1 +#endif + +#if JSON_HAS_THREE_WAY_COMPARISON + #include // partial_ordering +#endif + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/////////////////////////// +// JSON type enumeration // +/////////////////////////// + +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa see @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type + +@since version 1.0.0 +*/ +enum class value_t : std::uint8_t +{ + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + binary, ///< binary array (ordered collection of bytes) + discarded ///< discarded by the parser callback function +}; + +/*! +@brief comparison operator for JSON types + +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string < binary +- furthermore, each type is not smaller than itself +- discarded values are not comparable +- binary is represented as a b"" string in python and directly comparable to a + string; however, making a binary array directly comparable with a string would + be surprising behavior in a JSON file. + +@since version 1.0.0 +*/ +#if JSON_HAS_THREE_WAY_COMPARISON + inline std::partial_ordering operator<=>(const value_t lhs, const value_t rhs) noexcept // *NOPAD* +#else + inline bool operator<(const value_t lhs, const value_t rhs) noexcept +#endif +{ + static constexpr std::array order = {{ + 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, + 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */, + 6 /* binary */ + } + }; + + const auto l_index = static_cast(lhs); + const auto r_index = static_cast(rhs); +#if JSON_HAS_THREE_WAY_COMPARISON + if (l_index < order.size() && r_index < order.size()) + { + return order[l_index] <=> order[r_index]; // *NOPAD* + } + return std::partial_ordering::unordered; +#else + return l_index < order.size() && r_index < order.size() && order[l_index] < order[r_index]; +#endif +} + +// GCC selects the built-in operator< over an operator rewritten from +// a user-defined spaceship operator +// Clang, MSVC, and ICC select the rewritten candidate +// (see GCC bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105200) +#if JSON_HAS_THREE_WAY_COMPARISON && defined(__GNUC__) +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + return std::is_lt(lhs <=> rhs); // *NOPAD* +} +#endif + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief replace all occurrences of a substring by another string + +@param[in,out] s the string to manipulate; changed so that all + occurrences of @a f are replaced with @a t +@param[in] f the substring to replace with @a t +@param[in] t the string to replace @a f + +@pre The search string @a f must not be empty. **This precondition is +enforced with an assertion.** + +@since version 2.0.0 +*/ +template +inline void replace_substring(StringType& s, const StringType& f, + const StringType& t) +{ + JSON_ASSERT(!f.empty()); + for (auto pos = s.find(f); // find first occurrence of f + pos != StringType::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t, and + pos = s.find(f, pos + t.size())) // find next occurrence of f + {} +} + +/*! + * @brief string escaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to escape + * @return escaped string + * + * Note the order of escaping "~" to "~0" and "/" to "~1" is important. + */ +template +inline StringType escape(StringType s) +{ + replace_substring(s, StringType{"~"}, StringType{"~0"}); + replace_substring(s, StringType{"/"}, StringType{"~1"}); + return s; +} + +/*! + * @brief string unescaping as described in RFC 6901 (Sect. 4) + * @param[in] s string to unescape + * @return unescaped string + * + * Note the order of escaping "~1" to "/" and "~0" to "~" is important. + */ +template +static void unescape(StringType& s) +{ + replace_substring(s, StringType{"~1"}, StringType{"/"}); + replace_substring(s, StringType{"~0"}, StringType{"~"}); +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-FileCopyrightText: 2018 The Abseil Authors +// SPDX-License-Identifier: MIT + + + +#include // array +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type +#include // index_sequence, make_index_sequence, index_sequence_for + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template +using uncvref_t = typename std::remove_cv::type>::type; + +#ifdef JSON_HAS_CPP_14 + +// the following utilities are natively available in C++14 +using std::enable_if_t; +using std::index_sequence; +using std::make_index_sequence; +using std::index_sequence_for; + +#else + +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +// The following code is taken from https://github.com/abseil/abseil-cpp/blob/10cb35e459f5ecca5b2ff107635da0bfa41011b4/absl/utility/utility.h +// which is part of Google Abseil (https://github.com/abseil/abseil-cpp), licensed under the Apache License 2.0. + +//// START OF CODE FROM GOOGLE ABSEIL + +// integer_sequence +// +// Class template representing a compile-time integer sequence. An instantiation +// of `integer_sequence` has a sequence of integers encoded in its +// type through its template arguments (which is a common need when +// working with C++11 variadic templates). `absl::integer_sequence` is designed +// to be a drop-in replacement for C++14's `std::integer_sequence`. +// +// Example: +// +// template< class T, T... Ints > +// void user_function(integer_sequence); +// +// int main() +// { +// // user_function's `T` will be deduced to `int` and `Ints...` +// // will be deduced to `0, 1, 2, 3, 4`. +// user_function(make_integer_sequence()); +// } +template +struct integer_sequence +{ + using value_type = T; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +// index_sequence +// +// A helper template for an `integer_sequence` of `size_t`, +// `absl::index_sequence` is designed to be a drop-in replacement for C++14's +// `std::index_sequence`. +template +using index_sequence = integer_sequence; + +namespace utility_internal +{ + +template +struct Extend; + +// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. +template +struct Extend, SeqSize, 0> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)... >; +}; + +template +struct Extend, SeqSize, 1> +{ + using type = integer_sequence < T, Ints..., (Ints + SeqSize)..., 2 * SeqSize >; +}; + +// Recursion helper for 'make_integer_sequence'. +// 'Gen::type' is an alias for 'integer_sequence'. +template +struct Gen +{ + using type = + typename Extend < typename Gen < T, N / 2 >::type, N / 2, N % 2 >::type; +}; + +template +struct Gen +{ + using type = integer_sequence; +}; + +} // namespace utility_internal + +// Compile-time sequences of integers + +// make_integer_sequence +// +// This template alias is equivalent to +// `integer_sequence`, and is designed to be a drop-in +// replacement for C++14's `std::make_integer_sequence`. +template +using make_integer_sequence = typename utility_internal::Gen::type; + +// make_index_sequence +// +// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, +// and is designed to be a drop-in replacement for C++14's +// `std::make_index_sequence`. +template +using make_index_sequence = make_integer_sequence; + +// index_sequence_for +// +// Converts a typename pack into an index sequence of the same length, and +// is designed to be a drop-in replacement for C++14's +// `std::index_sequence_for()` +template +using index_sequence_for = make_index_sequence; + +//// END OF CODE FROM GOOGLE ABSEIL + +#endif + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static JSON_INLINE_VARIABLE constexpr T value{}; +}; + +#ifndef JSON_HAS_CPP_17 + template + constexpr T static_const::value; +#endif + +template +inline constexpr std::array make_array(Args&& ... args) +{ + return std::array {{static_cast(std::forward(args))...}}; +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // numeric_limits +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval +#include // tuple + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // random_access_iterator_tag + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN + +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin); + +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN + +NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end); + +NLOHMANN_JSON_NAMESPACE_END + +// #include + +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.1 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT + +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ + #define INCLUDE_NLOHMANN_JSON_FWD_HPP_ + + #include // int64_t, uint64_t + #include // map + #include // allocator + #include // string + #include // vector + + // #include + + + /*! + @brief namespace for Niels Lohmann + @see https://github.com/nlohmann + @since version 1.0.0 + */ + NLOHMANN_JSON_NAMESPACE_BEGIN + + /*! + @brief default JSONSerializer template argument + + This serializer ignores the template arguments and uses ADL + ([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) + for serialization. + */ + template + struct adl_serializer; + + /// a class to store JSON values + /// @sa https://json.nlohmann.me/api/basic_json/ + template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer, + class BinaryType = std::vector> + class basic_json; + + /// @brief JSON Pointer defines a string syntax for identifying a specific value within a JSON document + /// @sa https://json.nlohmann.me/api/json_pointer/ + template + class json_pointer; + + /*! + @brief default specialization + @sa https://json.nlohmann.me/api/json/ + */ + using json = basic_json<>; + + /// @brief a minimal map-like container that preserves insertion order + /// @sa https://json.nlohmann.me/api/ordered_map/ + template + struct ordered_map; + + /// @brief specialization that maintains the insertion order of object keys + /// @sa https://json.nlohmann.me/api/ordered_json/ + using ordered_json = basic_json; + + NLOHMANN_JSON_NAMESPACE_END + +#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ + + +NLOHMANN_JSON_NAMESPACE_BEGIN +/*! +@brief detail namespace with internal helper functions + +This namespace collects functions that should not be exposed, +implementations of some @ref basic_json methods, and meta-programming helpers. + +@since version 2.1.0 +*/ +namespace detail +{ + +///////////// +// helpers // +///////////// + +// Note to maintainers: +// +// Every trait in this file expects a non CV-qualified type. +// The only exceptions are in the 'aliases for detected' section +// (i.e. those of the form: decltype(T::member_function(std::declval()))) +// +// In this case, T has to be properly CV-qualified to constraint the function arguments +// (e.g. to_json(BasicJsonType&, const T&)) + +template struct is_basic_json : std::false_type {}; + +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct is_basic_json : std::true_type {}; + +// used by exceptions create() member functions +// true_type for pointer to possibly cv-qualified basic_json or std::nullptr_t +// false_type otherwise +template +struct is_basic_json_context : + std::integral_constant < bool, + is_basic_json::type>::type>::value + || std::is_same::value > +{}; + +////////////////////// +// json_ref helpers // +////////////////////// + +template +class json_ref; + +template +struct is_json_ref : std::false_type {}; + +template +struct is_json_ref> : std::true_type {}; + +////////////////////////// +// aliases for detected // +////////////////////////// + +template +using mapped_type_t = typename T::mapped_type; + +template +using key_type_t = typename T::key_type; + +template +using value_type_t = typename T::value_type; + +template +using difference_type_t = typename T::difference_type; + +template +using pointer_t = typename T::pointer; + +template +using reference_t = typename T::reference; + +template +using iterator_category_t = typename T::iterator_category; + +template +using to_json_function = decltype(T::to_json(std::declval()...)); + +template +using from_json_function = decltype(T::from_json(std::declval()...)); + +template +using get_template_function = decltype(std::declval().template get()); + +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json : std::false_type {}; + +// trait checking if j.get is valid +// use this trait instead of std::is_constructible or std::is_convertible, +// both rely on, or make use of implicit conversions, and thus fail when T +// has several constructors/operator= (see https://github.com/nlohmann/json/issues/958) +template +struct is_getable +{ + static constexpr bool value = is_detected::value; +}; + +template +struct has_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json : std::false_type {}; + +template +struct has_non_default_from_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +// This trait checks if BasicJsonType::json_serializer::to_json exists +// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion. +template +struct has_to_json : std::false_type {}; + +template +struct has_to_json < BasicJsonType, T, enable_if_t < !is_basic_json::value >> +{ + using serializer = typename BasicJsonType::template json_serializer; + + static constexpr bool value = + is_detected_exact::value; +}; + +template +using detect_key_compare = typename T::key_compare; + +template +struct has_key_compare : std::integral_constant::value> {}; + +// obtains the actual object key comparator +template +struct actual_object_comparator +{ + using object_t = typename BasicJsonType::object_t; + using object_comparator_t = typename BasicJsonType::default_object_comparator_t; + using type = typename std::conditional < has_key_compare::value, + typename object_t::key_compare, object_comparator_t>::type; +}; + +template +using actual_object_comparator_t = typename actual_object_comparator::type; + +/////////////////// +// is_ functions // +/////////////////// + +// https://en.cppreference.com/w/cpp/types/conjunction +template struct conjunction : std::true_type { }; +template struct conjunction : B { }; +template +struct conjunction +: std::conditional(B::value), conjunction, B>::type {}; + +// https://en.cppreference.com/w/cpp/types/negation +template struct negation : std::integral_constant < bool, !B::value > { }; + +// Reimplementation of is_constructible and is_default_constructible, due to them being broken for +// std::pair and std::tuple until LWG 2367 fix (see https://cplusplus.github.io/LWG/lwg-defects.html#2367). +// This causes compile errors in e.g. clang 3.5 or gcc 4.9. +template +struct is_default_constructible : std::is_default_constructible {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction, is_default_constructible> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + +template +struct is_default_constructible> + : conjunction...> {}; + + +template +struct is_constructible : std::is_constructible {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + +template +struct is_constructible> : is_default_constructible> {}; + + +template +struct is_iterator_traits : std::false_type {}; + +template +struct is_iterator_traits> +{ + private: + using traits = iterator_traits; + + public: + static constexpr auto value = + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value && + is_detected::value; +}; + +template +struct is_range +{ + private: + using t_ref = typename std::add_lvalue_reference::type; + + using iterator = detected_t; + using sentinel = detected_t; + + // to be 100% correct, it should use https://en.cppreference.com/w/cpp/iterator/input_or_output_iterator + // and https://en.cppreference.com/w/cpp/iterator/sentinel_for + // but reimplementing these would be too much work, as a lot of other concepts are used underneath + static constexpr auto is_iterator_begin = + is_iterator_traits>::value; + + public: + static constexpr bool value = !std::is_same::value && !std::is_same::value && is_iterator_begin; +}; + +template +using iterator_t = enable_if_t::value, result_of_begin())>>; + +template +using range_value_t = value_type_t>>; + +// The following implementation of is_complete_type is taken from +// https://blogs.msdn.microsoft.com/vcblog/2015/12/02/partial-support-for-expression-sfinae-in-vs-2015-update-1/ +// and is written by Xiang Fan who agreed to using it in this library. + +template +struct is_complete_type : std::false_type {}; + +template +struct is_complete_type : std::true_type {}; + +template +struct is_compatible_object_type_impl : std::false_type {}; + +template +struct is_compatible_object_type_impl < + BasicJsonType, CompatibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + // macOS's is_constructible does not play well with nonesuch... + static constexpr bool value = + is_constructible::value && + is_constructible::value; +}; + +template +struct is_compatible_object_type + : is_compatible_object_type_impl {}; + +template +struct is_constructible_object_type_impl : std::false_type {}; + +template +struct is_constructible_object_type_impl < + BasicJsonType, ConstructibleObjectType, + enable_if_t < is_detected::value&& + is_detected::value >> +{ + using object_t = typename BasicJsonType::object_t; + + static constexpr bool value = + (is_default_constructible::value && + (std::is_move_assignable::value || + std::is_copy_assignable::value) && + (is_constructible::value && + std::is_same < + typename object_t::mapped_type, + typename ConstructibleObjectType::mapped_type >::value)) || + (has_from_json::value || + has_non_default_from_json < + BasicJsonType, + typename ConstructibleObjectType::mapped_type >::value); +}; + +template +struct is_constructible_object_type + : is_constructible_object_type_impl {}; + +template +struct is_compatible_string_type +{ + static constexpr auto value = + is_constructible::value; +}; + +template +struct is_constructible_string_type +{ + // launder type through decltype() to fix compilation failure on ICPC +#ifdef __INTEL_COMPILER + using laundered_type = decltype(std::declval()); +#else + using laundered_type = ConstructibleStringType; +#endif + + static constexpr auto value = + conjunction < + is_constructible, + is_detected_exact>::value; +}; + +template +struct is_compatible_array_type_impl : std::false_type {}; + +template +struct is_compatible_array_type_impl < + BasicJsonType, CompatibleArrayType, + enable_if_t < + is_detected::value&& + is_iterator_traits>>::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 + !std::is_same>::value >> +{ + static constexpr bool value = + is_constructible>::value; +}; + +template +struct is_compatible_array_type + : is_compatible_array_type_impl {}; + +template +struct is_constructible_array_type_impl : std::false_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t::value >> + : std::true_type {}; + +template +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t < !std::is_same::value&& + !is_compatible_string_type::value&& + is_default_constructible::value&& +(std::is_move_assignable::value || + std::is_copy_assignable::value)&& +is_detected::value&& +is_iterator_traits>>::value&& +is_detected::value&& +// special case for types like std::filesystem::path whose iterator's value_type are themselves +// c.f. https://github.com/nlohmann/json/pull/3073 +!std::is_same>::value&& + is_complete_type < + detected_t>::value >> +{ + using value_type = range_value_t; + + static constexpr bool value = + std::is_same::value || + has_from_json::value || + has_non_default_from_json < + BasicJsonType, + value_type >::value; +}; + +template +struct is_constructible_array_type + : is_constructible_array_type_impl {}; + +template +struct is_compatible_integer_type_impl : std::false_type {}; + +template +struct is_compatible_integer_type_impl < + RealIntegerType, CompatibleNumberIntegerType, + enable_if_t < std::is_integral::value&& + std::is_integral::value&& + !std::is_same::value >> +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits; + using CompatibleLimits = std::numeric_limits; + + static constexpr auto value = + is_constructible::value && + CompatibleLimits::is_integer && + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template +struct is_compatible_integer_type + : is_compatible_integer_type_impl {}; + +template +struct is_compatible_type_impl: std::false_type {}; + +template +struct is_compatible_type_impl < + BasicJsonType, CompatibleType, + enable_if_t::value >> +{ + static constexpr bool value = + has_to_json::value; +}; + +template +struct is_compatible_type + : is_compatible_type_impl {}; + +template +struct is_constructible_tuple : std::false_type {}; + +template +struct is_constructible_tuple> : conjunction...> {}; + +template +struct is_json_iterator_of : std::false_type {}; + +template +struct is_json_iterator_of : std::true_type {}; + +template +struct is_json_iterator_of : std::true_type +{}; + +// checks if a given type T is a template specialization of Primary +template