diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..d2e4e420 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "CMSIS_5"] + path = CMSIS_5 + url = git@github.com:ARM-software/CMSIS_5.git +[submodule "CMSIS"] + path = CMSIS + url = https://github.com/ARM-software/CMSIS_5.git diff --git a/CMSIS b/CMSIS new file mode 160000 index 00000000..4ed57307 --- /dev/null +++ b/CMSIS @@ -0,0 +1 @@ +Subproject commit 4ed5730787a0f98e8858c9998e406c608ab10b28 diff --git a/CMakeLists.txt b/CMakeLists.txt index 51aa345c..651bbf0f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,15 +18,35 @@ set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} -T\"${CMAKE_CURRENT_LIST_DIR}/ld/n set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -T\"${CMAKE_CURRENT_LIST_DIR}/ld/nrf52833.ld\"" PARENT_SCOPE) set(CMAKE_SYSTEM_PROCESSOR "armv7-m" PARENT_SCOPE) + +set(ROOT "${CMAKE_CURRENT_LIST_DIR}/CMSIS/") +list(APPEND INCLUDE_DIRS "${ROOT}/CMSIS/Core/Include/") + + +# Define the path to CMSIS-DSP (ROOT is defined on command line when using cmake) +set(DSP ${ROOT}/CMSIS/DSP) + +include(${DSP}/Toolchain/GCC.cmake) + # add them + include_directories(${INCLUDE_DIRS}) +# Load CMSIS-DSP definitions. Libraries will be built in bin_dsp +add_subdirectory(${DSP}/Source bin_dsp) + # create our target add_library(codal-microbit-v2 ${SOURCE_FILES}) target_link_libraries( codal-microbit-v2 codal-nrf52 + CMSISDSPSupport + CMSISDSPTransform + CMSISDSPCommon + CMSISDSPComplexMath + CMSISDSPFastMath + CMSISDSPStatistics codal-core codal-microbit-nrf5sdk ${LIB_OBJECT_FILES} diff --git a/inc/EmojiRecogniser.h b/inc/EmojiRecogniser.h new file mode 100644 index 00000000..3c0a744b --- /dev/null +++ b/inc/EmojiRecogniser.h @@ -0,0 +1,78 @@ + +#ifndef EMOJI_RECOGNISER_H +#define EMOJI_RECOGNISER_H + +/* + * + * The emoji recogniser is a subclass of sound recogniser that defines + * the actual samples for the emoji sounds. They are just parts of the + * emoji sounds that can be recognised: remain quite consistent across + * multiple plays of the sound. + * + * + * Example + * + * Taking the happy sound as an example, there are a few constants defined: + * + * happy_sequences the number of sequences in the happy sound + * + * happy_max_deviations the maximum number of deviations in the + * sound - i.e. a deviation is considered + * a data point that is more than the allowed + * threshold off the sampled frequency + * + * happy_samples a 3-dimensional array with the sampled sound: + * - the first dimension is the different + * sequences + * - the second is the samples in each sequence + * - the third is the data points in each sample + * of each sequence + * + * happy_thresholds an array with the thresholds for each of the + * sequences + * + * happy_deviations an array with the maximum deviations for each + * sequence + * + * happy_nr_samples an array with the number of samples in each + * sequence + * + * All these are packaged in a Sound struct. + */ + +#include "MicroBitSoundRecogniser.h" + +#define DEVICE_EMOJI_RECOGNISER_EVT_HAPPY 1 +#define DEVICE_EMOJI_RECOGNISER_EVT_HELLO 2 +#define DEVICE_EMOJI_RECOGNISER_EVT_SAD 3 +#define DEVICE_EMOJI_RECOGNISER_EVT_SOARING 4 +#define DEVICE_EMOJI_RECOGNISER_EVT_TWINKLE 5 + +// 37 is the first unused id in CodalComponent. +// It might be better for this to be in CodalComponent. +#define DEVICE_ID_EMOJI_RECOGNISER 37 + +class EmojiRecogniser : public MicroBitSoundRecogniser +{ + void addHappySound(); + void addHelloSound(); + void addSadSound(); + void addSoaringSound(); + void addTwinkleSound(); + + protected: + /* + * The function to call when a sound is recognised. + */ + void recognisedSound(uint16_t id); + + public: + EmojiRecogniser(MicroBitAudioProcessor& processor); + + /* + * Converts from id to sound name. + */ + static ManagedString getSoundName(Event& evnt); +}; + +#endif \ No newline at end of file diff --git a/inc/MicroBitAudioProcessor.h b/inc/MicroBitAudioProcessor.h new file mode 100644 index 00000000..708c6ff8 --- /dev/null +++ b/inc/MicroBitAudioProcessor.h @@ -0,0 +1,188 @@ +/* +The MIT License (MIT) +Copyright (c) 2020 Arm Limited. +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + +#include "MicroBit.h" +#include "DataStream.h" +#define ARM_MATH_CM4 +#include "arm_math.h" + +#ifndef MICROBIT_AUDIO_PROCESSOR_H +#define MICROBIT_AUDIO_PROCESSOR_H + +/* + * Provides the fundamental frequencies in the microphone data. + * + * It takes in the microphone data (sampled at MIC_SAMPLE_RATE Hz + * which is ~11000 Hz now) and produces AudioFrameAnalysis data. + * +*/ + +// Default configuration values +#define MIC_SAMPLE_RATE (1000000 / MIC_SAMPLE_DELTA) + +#define RECOGNITION_START_FREQ 1700 +#define RECOGNITION_END_FREQ 5400 +#define ANALYSIS_STD_MULT_THRESHOLD 2 + +#define MAXIMUM_NUMBER_OF_FREQUENCIES 4 +#define SIMILAR_FREQ_THRESHOLD 100 + +// Sampling more often - ~22000 Hz, allows better detection of emoji +// sounds and gives relibale results for morse code +#if MIC_SAMPLE_DELTA == 45 + +#define DEFAULT_AUDIO_SAMPLES_NUMBER 512 +#define EMOJI_AUDIO_SAMPLES_NUMBER 512 +#define MORSE_AUDIO_SAMPLES_NUMBER 256 + +#define DEFAULT_STD_THRESHOLD 140 +#define EMOJI_STD_THRESHOLD 80 +#define MORSE_STD_THRESHOLD 200 + +// If sampling at 11000 Hz the emoji detection would still work - not +// as good, but the morse code would require with longer durations. +#elif MIC_SAMPLE_DELTA == 91 + +#define DEFAULT_AUDIO_SAMPLES_NUMBER 512 +#define EMOJI_AUDIO_SAMPLES_NUMBER 512 +#define MORSE_AUDIO_SAMPLES_NUMBER 256 + +#define DEFAULT_STD_THRESHOLD 70 +#define EMOJI_STD_THRESHOLD 45 +#define MORSE_STD_THRESHOLD 75 + +#endif + +class MicroBitAudioProcessor : public DataSink, public DataSource +{ +public: + + /* + * An AudioFrameAnalysis has the fundamental frequencies of a + * frame - maximum MAXIMUM_NUMBER_OF_FREQUENCIES and ordered + * from the most likely to the least. + */ + struct AudioFrameAnalysis { + uint8_t size; + uint16_t buf[MAXIMUM_NUMBER_OF_FREQUENCIES]; + }; + +private: + + DataSource &audiostream; // the stream of data to analyse + DataSink *recogniser; // the recogniser the frequencies should be send to + uint16_t audio_samples_number; // the number of samples to collect before analysing a frame + uint16_t std_threshold; // the threshold for the standard deviation + arm_rfft_fast_instance_f32 fft_instance; // the instance of CMSIS fft that is used to run fft + float *buf; // the buffer to store the incoming data + float *fft_output; // an array to store the result of the fft + float *mag; // an array to store the magnitudes of the frequencies + + uint16_t buf_len; // the length of the incoming buffer + bool recording; // whether it should analyse the data or be idle + + AudioFrameAnalysis output; // the result of the analysis + + /* + * Converts from frequency to the index in the array. + * + * @param freq a frequency in the range 0 - 5000 Hz. + * + * @return the index to the frequency bucket freq is in + * as it comes out of the fft + */ + uint16_t frequencyToIndex(int freq); + + /* + * Converts from the index in the array to frequency. + * + * @param index a index in the range 0 - audio_samples_number / 2. + * + * @return the avg frequency in the bucket + */ + float32_t indexToFrequency(int index); + + public: + + /* + * Constructor. + * + * Initialize the MicroBitAduioProcessor. + */ + MicroBitAudioProcessor(DataSource& source, uint16_t audio_samples_number = DEFAULT_AUDIO_SAMPLES_NUMBER, uint16_t std_threshold = DEFAULT_STD_THRESHOLD); + + /* + * Destructor. + * + * Deallocates all the memory allocated dynamically. + */ + ~MicroBitAudioProcessor(); + + /* + * A callback for when the data is ready. + * + * Analyses the data when enough of it comes in, using + * the following algorithm: + * + * The audio processor accumulates microphone data as it comes + * in and after getting audio_samples_number of them it process + * the frame. + * + * It transforms the date from time domain to frequency domain + * using the CMSIS fft. + * + * If the mean of the magnitudes of frequnecies is lower than + * ANALYSIS_MEAN_THRESHOLD or the standard deviation (std) is + * lower than ANALYSIS_STD_THRESHOLD then the frame is considered + * silence - no fundamental frequency. + * + * It then filters out the frequencies that have a magnitude lower + * than the mean + ANALYSIS_STD_MULT_THRESHOLD * std. This ensures + * that only outlier frequencies are being considered. + * + * It then filters out the neighbour frequencies around the peaks. + * + * Some of these operations are implemented together to optimize the + * algorithm. + */ + virtual int pullRequest(); + + /* + * Allow out downstream component to register itself with us + */ + void connect(DataSink *downstream); + + /* + * Provides the next available data to the downstream caller. + */ + virtual ManagedBuffer pull(); + + /* + * Starts recording and analysing. + */ + void startRecording(); + + /* + * Stops from recording and analysing. + */ + void stopRecording(); +}; + +#endif \ No newline at end of file diff --git a/inc/MicroBitMorseInterpreter.h b/inc/MicroBitMorseInterpreter.h new file mode 100644 index 00000000..88ffe172 --- /dev/null +++ b/inc/MicroBitMorseInterpreter.h @@ -0,0 +1,62 @@ +#ifndef MICROBIT_MORSE_INTERPRETER_H +#define MICROBIT_MORSE_INTERPRETER_H + +#include "MicroBit.h" +#include "DataStream.h" +#include "MicroBitMorseRecogniser.h" +#include "MorseEncoder.h" + +#define DEVICE_MORSE_INTERPRETER_EVT_NEW_MESSAGE 1 + +// It might be better for this to be in CodalComponent. +#define DEVICE_ID_MORSE_INTERPRETER 38 + +/* + * This class takes morse data from a MicroBitMorseCodeRecogniser and uses a MorseEncoder to decode it. + * It then calls an event signalling the fact that it is done processing some data. + * The last processed data can then be taken from lastMessage. + */ +class MicroBitMorseInterpreter: public DataSink { + + private: + + MicroBitMorseRecogniser& recogniser; // recogniser that this takes data from + MicroBit& uBit; // the microbit - used in order to send an event in the message bus + MorseEncoder encoder; // encoder used for decoding received data + bool interpreting; // wether the Interpreter is currently interpreting or not + + public: + + /* + * Last processed message + */ + ManagedString lastMessage; + + /* + * Constructor. + * + * Initializes the interpreter. + * + * @param rec is the recogniser this will receive data from + * + * @param bit is the micro:bit + */ + MicroBitMorseInterpreter(MicroBitMorseRecogniser& rec, MicroBit& bit); + + /* + * Callback for when the data is ready. + */ + virtual int pullRequest(); + + /* + * Starts interpreting and also starts the associated recogniser. + */ + void startInterpreting(); + + /* + * Stops interpreting and also stops the associated recogniser. + */ + void stopInterpreting(); +}; + +#endif \ No newline at end of file diff --git a/inc/MicroBitMorsePlayer.h b/inc/MicroBitMorsePlayer.h new file mode 100644 index 00000000..755b0312 --- /dev/null +++ b/inc/MicroBitMorsePlayer.h @@ -0,0 +1,66 @@ +#ifndef MICROBIT_MORSE_PLAYER_H +#define MICROBIT_MORSE_PLAYER_H + +#include "MicroBit.h" +#include "ManagedString.h" + +#define DEFAULT_DURATION 186 +#define DEFAULT_FREQUENCY 2000 +#define DEFAULT_RANDOMNESS 0 + +/* + * The morse player takes strings of characters and converts them into morse code using a morse encoder. + * Then, it takes every element one by one and plays it over the speaker. + * It also adds 'element space' farmes between consecutive dots and dashes, in order to help differentiate between audible elements. + * The frequency and time unit of these sounds are set in the constructor. + * + * + * Durations for morse elements: + * dot - unit + * dash - unit*3 + * space between dots and dashes - unit + * space between characters - unit*3 + * space between words - unit*7 + * end of transmission - unit*10 + */ +class MicroBitMorsePlayer { + + private: + MicroBit& uBit; + int frequency; // frequency the sounds are played at + int duration; // time unit for elements + int randomness; // randomness of sounds - this is currently not settable and defaults to 0 + + ManagedString dotFrame; // dot frame + ManagedString dashFrame; // dash frame + ManagedString pause1Frame; // element space frame + ManagedString pause3Frame; // character space frame + ManagedString pause7Frame; // word space frame + ManagedString pause10Frame; // end of transmission frame + + void createFrames(); // generates frames for all the morse elements from frequency, duration and randomness + + public: + + /* + * Constructor + * + * Initializes the player. + * + * @param bit is the micro:bit + * + * @param freq is the frequency the sounds will be played at + * + * @param dur is the time unit for playing the sounds + */ + MicroBitMorsePlayer(MicroBit& bit, int freq, int dur); + + /* + * Converts a string into morse and then plays it over the speaker. + * + * @param in is the input string + */ + void play(const char* in); +}; + +#endif \ No newline at end of file diff --git a/inc/MicroBitMorseRecogniser.h b/inc/MicroBitMorseRecogniser.h new file mode 100644 index 00000000..17f9c174 --- /dev/null +++ b/inc/MicroBitMorseRecogniser.h @@ -0,0 +1,148 @@ + +#ifndef MICROBIT_MORSE_RECOGNISER_H +#define MICROBIT_MORSE_RECOGNISER_H + +#include "DataStream.h" +#include "MicroBitAudioProcessor.h" +#include "arm_math.h" + +/* + * The sound recogniser takes in data from the audio processor - in + * form of AudioFrameAnalysis. It then tries to identify whether + * the listening frequency was played during that frame. Based on + * the last frames it sends dots ('.'), dashes ('-'), letter spaces + * (' '), word spaces (';'), and end-of-transmission ('#') to the + * data sink connected. + * + * Algorithm + * + * The recogniser first converts the AudioFrameAnalysis data in + * booleans which represent whether or not the listening frequency was + * being played in that frame. A timeUnit is calculated based on the + * sampling frequency and the number of samples taken before analysis + * by the audio processor - it represents the number of frames a dot + * should take. + * + * It has 2 values: zeros and ones which counts for how many timeUnits + * the frequency wasn't being played/ was being played. The value + * synchronised keeps track of the last transition - true if the last + * transition was from the frequency not being played to it being played, + * false otherwise. At each transition, the values of zeros or ones are + * being interpreted into the right character and then reset. + * + * All characters are being accumulated into a buffer and send together + * when the end-of-transmission character ('#') is found. + */ + +// Configuration vairables +#define MORSE_FRAME_TRUE_RATE_THRESHOLD 0.8 +#define MORSE_FRAME_FALSE_RATE_THRESHOLD 0.6 +#define DETECTION_THRESHOLD 130 +#define MAX_TIME_UNIT 500 +#define MAX_MESSAGE 500 + +class MicroBitMorseRecogniser : public DataSink, public DataSource +{ + MicroBitAudioProcessor& audio_proceesor; // the audio processor it takes data from + DataSink* interpreter; // the interpreter where it sends the data when ready + + uint16_t timeUnit; // the number of frames for one dot + uint16_t frequency; // the frequency it uses for communication + + bool analysing; // whether it should analyse the data coming in + bool synchronised; // whether it synchrnosied with a stream on ones - should be the status of last transition + + uint16_t zeros; // the number of zero frames it received since last transition + uint16_t ones; // the number on one frames it received since last transition + + uint8_t output[MAX_MESSAGE]; // the ouput buffer + bool buffer[2 * MAX_TIME_UNIT]; // the input buffer + uint16_t output_len; // the length of the output buffer + uint16_t buffer_len; // the length of the input buffer + + + /* + * Processes a incoming frame. + * + * @param frame the frame it should process + */ + void processFrame(MicroBitAudioProcessor::AudioFrameAnalysis* frame); + + /* + * Checks if the number of ones in the last timeUnit frames up to + * a given position in buffer fit in a given threshold. + * + * @param to specifies the interval [to - timeUnit, to) which is + * analysed + * + * @param threshold the minimum number of ones in the interval + * + * @return number of trues in buffer[to - timeUnit, to) >= threshold + * + * @note if threshold is 255 thet it will use + * timeUnit * MORSE_FRAME_TRUE_RATE_THRESHOLD as the threshold + */ + bool recogniseLastMorseFrame(uint16_t to, uint16_t threshold ); + + + /* + * Adds a character to the output buffer. If it adds the + * end-of-transmission ('#') character, then it sends the buffer + * to the next component in the pipeline - 'interpreter'. + * + * @param c the character to be added to the out buffer + */ + void pushOut(char c); + + public: + + /* + * Constructor. + * + * Initializes the MicroBitMorseRecogniser. + * + * @param the audio processor it should take data from + * + * @param freq the frequency it should listen for + * + * @param timeUnit the amount of time in ms it takes a dot + * to be transmitted + */ + MicroBitMorseRecogniser(MicroBitAudioProcessor& processor, uint16_t freq, uint16_t timeUnit) ; + + /* + * Destructor. + * + * Deallocates all the memory allocated dynamically. + */ + ~MicroBitMorseRecogniser(); + + /* + * A callback for when the data is ready. + */ + virtual int pullRequest(); + + /* + * Starts analysing the data that comes in. + */ + void startAnalysing(); + + /* + * Stops analysing the data and also stops the audio processor + * from receiving. + */ + void stopAnalysing(); + + /* + * Provides the next available data to the downstream caller. + */ + virtual ManagedBuffer pull(); + + /* + * Allow out downstream component to register itself with us + */ + virtual void connect(DataSink *sink); +}; + + +#endif \ No newline at end of file diff --git a/inc/MicroBitSoundRecogniser.h b/inc/MicroBitSoundRecogniser.h new file mode 100644 index 00000000..97857e12 --- /dev/null +++ b/inc/MicroBitSoundRecogniser.h @@ -0,0 +1,221 @@ + +#ifndef MICROBIT_SOUND_RECOGNISER_H +#define MICROBIT_SOUND_RECOGNISER_H + +#include "MicroBit.h" +#include "DataStream.h" +#include "MicroBitAudioProcessor.h" +#include "arm_math.h" + +/* + * + * The sound recogniser takes in data from the audio processor - in + * form of AudioFrameAnalysis. It then tries to match the history + * against samples of sounds. + * + * + * Sound fingerprint + * + * A sound has multiple sequences, each sequence having multiple samples + * to account for the randomness. + * + * Each sequence also has: + * a threshold - the maximum absolute difference from the sampled + * frequency and the heard frequency. + * maximum number of deviations - the maximum number of datapoints + * that can be more than the threshold away from the + * sampled frequency + * + * A sound also has a maximum number of deviations - the total maximum + * deviations across all sequences. + */ + + +/* + * The maximum length of the buffer. + * + * It should be >= than the maximum length of datapoints in + * any sample. + */ +#define HISTORY_LENGTH 30 + +class MicroBitSoundRecogniser : public DataSink +{ + private: + + MicroBitAudioProcessor& audio_proceesor; // the audio processor it takes data from + + bool analysing; // whether it should analyse the data or be idle + + MicroBitAudioProcessor::AudioFrameAnalysis buffer[2 * HISTORY_LENGTH]; // the buffer to collect the incoming data in + uint8_t buffer_len; // the length of the buffer + + protected: + + /* + * A struct to package a sample. + */ + struct SoundSample { + SoundSample(const uint16_t* _frames, uint8_t size); + ~SoundSample(); + + uint8_t size; // the number of data points in the sample + const uint16_t* frames; // the data points + }; + + + /* + * A struct to package a sequence. + */ + struct SoundSequence { + SoundSequence(uint8_t size, uint32_t threshold, uint8_t deviation); + ~SoundSequence(); + + uint8_t size; // the number of samples + uint32_t threshold; // the threshold for the sequence + uint8_t deviation; // the maximum number of deviations allowed for the sequence + SoundSample** samples; // ponter to the array of samples + }; + + + /* + * A struct to package a sound. + */ + struct Sound { + Sound(uint8_t size, uint8_t max_deviation, uint8_t max_history_len, bool consider_all_frequencies); + ~Sound(); + + bool consider_all_frequencies; // whether or not to consider all frequencies detected or just the dominant one + uint8_t max_deviation; // the maximum total number of deviations allowed + uint8_t size; // the number of sequences in the sound + SoundSequence** sequences; // pointer to the array of sequences + + /* + * Update called when new data comes in. + * + * @param buffer the buffer with the last data points that came in + * + * @note This keeps the history updated which is needed for the + * dynamic programming approach chosen for matching + */ + void update( MicroBitAudioProcessor::AudioFrameAnalysis* buffer, + uint8_t buffer_len ); + + /* + * Whether or not the sound matched in the last frame. + * + * @return whether or not the sound matched in the last frame. + * + * @note Should be called after update when new data comes in. + */ + bool matched(); + + /* + * Resets the history buffer. + * + * @note Used when the data stops coming in - e.g. when the analyser + * is paused + */ + void resetHistory(); + + private: + + /* + * Matches a sequence to the last couple of data points in the buffer. + * + * @param seq_id the id of the sequence to try to match + * + * @param buffer the buffer of data points that came in + * + * @param buffer_len the length of the buffer + * + * @return the number of deviations in the last data points to the sound up to + * the seq_id sequence or 255 if it's above the maximums allowed. + */ + uint8_t matchSequence( uint8_t seq_id, + MicroBitAudioProcessor::AudioFrameAnalysis* buffer, + uint8_t buffer_len) const; + + /* + * Getter for the internal history buffer of the sound. + * + * @param frames_ago the number of frames ago for which the + * query was made + * + * @param seq_id the id of the sequence to get the deviation for + * + * @return the deviation (or 255 if it didn't match) up to the + * sequence seq_id that was frames_ago frames ago. + * + * @note used to check if the sound matched up to the first seq_id + * sequences so that the matching doesn't need to recheck + * those sequences. + */ + uint8_t getDeviation(uint8_t frames_ago, uint8_t seq_id) const; + + /* + * Adds to the history buffer a deviation for a certain sequence. + * + * @param seq_id the id of the sequence to add the value to. + * + * @param value the value to be added to the history buffer + */ + void addToHistory(uint8_t seq_id, uint8_t value); + + /* + * Ends a history frame, increasing the length of the buffer. + */ + void endHistoryFrame(); + + uint8_t* history; // the array of to keep the history buffer in + uint8_t history_len; // the size of the history buffer + uint8_t max_history_len; // the maximum length of the history buffer. Used for double buffering + }; + + /* + * Constructor. + * + * Initialize the MicroBitSoundRecogniser. + * + * @note is protected to make the class abstract. + */ + MicroBitSoundRecogniser(MicroBitAudioProcessor& processor); + + /* + * Virtual function to call when a sound is recognised. + */ + virtual void recognisedSound(uint16_t id) = 0; + + Sound** sounds; // pointer to the array of sounds to recognise + uint16_t* sound_ids; // the array of sound ids + uint8_t sounds_size; // the number of sounds to try to recognise + + public: + + /* + * Destructor. + * + * Deallocates all the memory allocated dynamically. + */ + ~MicroBitSoundRecogniser(); + + + /* + * A callback for when the data is ready. + */ + virtual int pullRequest(); + + /* + * Starts analysing the data that comes in. + */ + void startAnalysing(); + + + /* + * Stops analysing the data and also stops the audio processor + * from receiving. + */ + void stopAnalysing(); +}; + +#endif diff --git a/inc/MorseEncoder.h b/inc/MorseEncoder.h new file mode 100644 index 00000000..3a5dc782 --- /dev/null +++ b/inc/MorseEncoder.h @@ -0,0 +1,80 @@ +#ifndef MORSE_ENCODER_H +#define MORSE_ENCODER_H + +#include "ManagedString.h" +#include + + +/* + * Comparator for ManagedString. + */ +struct cmpString { + bool operator()(ManagedString a, ManagedString b) const { + return (a < b); + } +}; + +/* + * The morse encoder has two uses. + * Firstly, it can encode a string of characters into its morse representation. + * This is done by transforming every character using a map and adding spaces between characters and words when appropriate. + * At the end, the `end of transmission` character is added. + * Whenever a lowercase letter is encountered, it is converted to uppercase. + * Whenever an unknown character is encountered, it is converted to '&'. + * + * Secondly, it can decode a series of morse elements into the corresponding string. + * Decoding takes every continuous sequence of dots and dashes and transforms it into the appropriate character using another map. + * This is done by building up every continuous sequence of dots and dashes into a small buffer and, when another element is encountered, + * transforming it into the appropriate character using a map. The buffer is reset every time this happens. + * All of the generated chracyters are added to the output string. + * + * + * This uses the following elements for morse code: + * dot - '.' + * dash - '-' + * element gap - not represented + * letter gap - ' ' + * word gap - ';' + * end of transmission - '#' + * + * The reason element gap is not represented is that it can be easily added appropriately at the time that the morse code is played over the speaker. + * When interpreting morse played by another source, it can just be ignored. +*/ +class MorseEncoder { + + private: + static std::map toStr; // map from characters to their morse encoding + std::map toChar; // map from morse encodings to their characters + + public: + /* + * Constructor + * + * Initializes the encoder. + */ + MorseEncoder(); + + /* + * Encodes a string into morse code. + * + * @param in is the input string to be encoded + * + * @param out is the output string that the encoding is written into + * + * @note out should be at least 6 times the size of in + */ + void encode(const char* in, char* out); + + /* + * Decodes morse code into a string. + * + * @param in is the input string containing morse code + * + * @param out is the output string that the morse code is decoded into + * + * @note out should be at least the same size as in + */ + void decode(const char* in, char* out); +}; + +#endif \ No newline at end of file diff --git a/model/MicroBit.cpp b/model/MicroBit.cpp index f8ceb8a7..f1794e20 100644 --- a/model/MicroBit.cpp +++ b/model/MicroBit.cpp @@ -60,7 +60,7 @@ MicroBit::MicroBit() : capTouchTimer(NRF_TIMER3, TIMER3_IRQn), timer(systemTimer), messageBus(), - adc(adcTimer, 91), + adc(adcTimer, MIC_SAMPLE_DELTA), touchSensor(capTouchTimer), io(adc, touchSensor), serial(io.usbTx, io.usbRx, NRF_UARTE0), diff --git a/model/MicroBit.h b/model/MicroBit.h index d47f5ed2..186e8a2e 100644 --- a/model/MicroBit.h +++ b/model/MicroBit.h @@ -87,7 +87,7 @@ DEALINGS IN THE SOFTWARE. //#include "MicroBitLightSensor.h" - +#define MIC_SAMPLE_DELTA 45 // Status flag values #define DEVICE_INITIALIZED 0x01 diff --git a/source/EmojiRecogniser.cpp b/source/EmojiRecogniser.cpp new file mode 100644 index 00000000..be355f82 --- /dev/null +++ b/source/EmojiRecogniser.cpp @@ -0,0 +1,608 @@ + +#include "EmojiRecogniser.h" + +EmojiRecogniser::EmojiRecogniser( MicroBitAudioProcessor& processor ) + : MicroBitSoundRecogniser(processor){ + sounds = new Sound* [5]; + sound_ids = new uint16_t[5]; + addHappySound(); + addHelloSound(); + addSadSound(); + addSoaringSound(); + addTwinkleSound(); +} + +/* + * The function to call when a sound is recognised. + */ +void EmojiRecogniser::recognisedSound(uint16_t id){ + Event evnt(DEVICE_ID_EMOJI_RECOGNISER, id); +} + +/* + * Converts from id to sound name. + */ +ManagedString EmojiRecogniser::getSoundName(Event& evnt){ + if(evnt.source != DEVICE_ID_EMOJI_RECOGNISER) + return ManagedString(""); + switch (evnt.value) { + case DEVICE_EMOJI_RECOGNISER_EVT_HAPPY: + return ManagedString("happy"); + case DEVICE_EMOJI_RECOGNISER_EVT_HELLO: + return ManagedString("hello"); + case DEVICE_EMOJI_RECOGNISER_EVT_SAD: + return ManagedString("sad"); + case DEVICE_EMOJI_RECOGNISER_EVT_SOARING: + return ManagedString("soaring"); + case DEVICE_EMOJI_RECOGNISER_EVT_TWINKLE: + return ManagedString("twinkle"); + } + return ManagedString(""); +} + +// HAPPY Sound ---- + +#if MIC_SAMPLE_DELTA == 91 + +const uint8_t happy_sequences = 2; +const uint8_t happy_max_deviations = 2; + +uint16_t happy_samples[happy_sequences][2][8] = { + { + { 4, 2121, 2394, 2646, 2646}, + { 5, 2121, 2373, 2373, 2646, 2646} + }, + { + { 7, 2646, 2835, 2646, 2646, 2394, 2394, 2394}, + { 7, 2646, 2835, 2835, 2646, 2394, 2373, 2394} + } +}; + +uint16_t happy_thresholds[happy_sequences] = { + 40, + 50 +}; + +uint8_t happy_deviations[happy_sequences] = { + 1, + 2 +}; + +uint8_t happy_nr_samples[happy_sequences] = { + 2, + 2 +}; + +#elif MIC_SAMPLE_DELTA == 45 + +const uint8_t happy_sequences = 3; +const uint8_t happy_max_deviations = 7; + +uint16_t happy_samples[happy_sequences][2][10] = { + { + { 6, 2107, 2107, 2365, 2365, 2365, 2623}, + { 7, 2107, 2107, 2365, 2365, 2365, 2365, 2623} + }, + { + { 9, 2623, 2623, 2623, 2623, 2623, 2795, 2795, 2795, 2623}, + { 9, 2623, 2623, 2623, 2623, 2666, 2795, 2795, 2666, 2623} + }, + { + { 8, 2623, 2623, 2623, 2365, 2365, 2365, 2365, 2365}, + { 8, 2623, 2623, 2365, 2365, 2365, 2365, 2365, 2365} + } +}; + +uint16_t happy_thresholds[happy_sequences] = { + 100, + 100, + 70 +}; + +uint8_t happy_deviations[happy_sequences] = { + 3, + 4, + 4 +}; + +uint8_t happy_nr_samples[happy_sequences] = { + 2, + 2, + 2 +}; + +#endif + + +void EmojiRecogniser::addHappySound() { + + uint8_t it = sounds_size; + sounds_size ++; + sound_ids[it] = DEVICE_EMOJI_RECOGNISER_EVT_HAPPY; + + uint8_t history = 0; + + for(uint8_t i = 0; i < happy_sequences; i++) + for(uint8_t j = 0; j < happy_nr_samples[i]; j ++) + history = max(history, happy_samples[i][j][0] + 4); + + sounds[it] = new Sound(happy_sequences, happy_max_deviations, history, true); + + for(uint8_t i = 0; i < happy_sequences; i++){ + sounds[it] -> sequences[i] = new SoundSequence(happy_nr_samples[i], happy_thresholds[i], happy_deviations[i]); + for(uint8_t j = 0; j < happy_nr_samples[i]; j ++) + sounds[it] -> sequences[i] -> samples[j] = new SoundSample(happy_samples[i][j] + 1, happy_samples[i][j][0]); + } + +} + +// HELLO Sound ---- + + +#if MIC_SAMPLE_DELTA == 91 + +const uint8_t hello_sequences = 2; +const uint8_t hello_max_deviations = 3; + +uint16_t hello_samples[hello_sequences][3][10] = { + // First sequence + { + { 3, 2688, 2919, 3528}, + { 3, 2751, 2919, 3528}, + { 3, 2688, 3402, 3528} + }, + { + { 9, 3507, 3150, 3087, 3003, 2961, 2940, 2940, 2961, 2940}, + { 9, 3339, 3108, 3024, 2982, 2940, 2940, 2961, 2961, 2940}, + { 9, 3381, 3150, 3087, 3003, 2961, 2940, 2940, 2961, 2961} + } +}; + +uint16_t hello_thresholds[hello_sequences] = { + 50, + 80 +}; + +uint8_t hello_deviations[hello_sequences] = { + 2, + 3 +}; + +uint8_t hello_nr_samples[hello_sequences] = { + 3, + 3 +}; + +#elif MIC_SAMPLE_DELTA == 45 + +const uint8_t hello_sequences = 3; +const uint8_t hello_max_deviations = 3; + +uint16_t hello_samples[hello_sequences][3][10] = { + { + { 6, 2881, 2623, 2838, 3139, 3397, 3526}, + { 6, 2838, 2623, 2881, 3268, 3440, 3526}, + { 7, 2795, 2494, 2666, 3053, 3268, 3483, 3526} + }, + { + { 4, 3526, 3440, 3311, 3139}, + { 4, 3483, 3311, 3182, 3139} + }, + { + { 9, 3139, 3096, 3053, 3010, 2967, 2967, 2924, 2924, 2924}, + { 9, 3096, 3053, 3010, 2967, 2967, 2924, 2924, 2924, 2924} + } +}; + +uint16_t hello_thresholds[hello_sequences] = { + 100, + 80, + 80 +}; + +uint8_t hello_deviations[hello_sequences] = { + 3, + 2, + 3 +}; + +uint8_t hello_nr_samples[hello_sequences] = { + 3, + 2, + 2 +}; + +#endif + +void EmojiRecogniser::addHelloSound() { + uint8_t it = sounds_size; + sounds_size ++; + sound_ids[it] = DEVICE_EMOJI_RECOGNISER_EVT_HELLO; + + uint8_t history = 0; + + for(uint8_t i = 0; i < hello_sequences; i++) + for(uint8_t j = 0; j < hello_nr_samples[i]; j ++) + history = max(history, hello_samples[i][j][0] + 4); + + sounds[it] = new Sound(hello_sequences, hello_max_deviations, history, true); + + for(uint8_t i = 0; i < hello_sequences; i++){ + sounds[it] -> sequences[i] = new SoundSequence(hello_nr_samples[i], hello_thresholds[i], hello_deviations[i]); + for(uint8_t j = 0; j < hello_nr_samples[i]; j ++) + sounds[it] -> sequences[i] -> samples[j] = new SoundSample(hello_samples[i][j] + 1, hello_samples[i][j][0]); + } +} + + +// SAD Sound ---- + +#if MIC_SAMPLE_DELTA == 91 + +const uint8_t sad_sequences = 2; +const uint8_t sad_max_deviations = 5; + +uint16_t sad_samples[sad_sequences][3][16] = { + // First sequence + { + { 8, 3423, 3339, 3255, 3087, 2961, 2856, 2709, 2604}, + { 8, 3381, 3318, 3192, 3045, 2898, 2793, 2625, 2520}, + { 8, 3318, 3255, 3087, 2919, 2793, 2688, 2562, 2436} + }, + { + { 15, 3591, 3423, 3318, 3171, 3024, 2940, 2877, 2835, 2814, 2814, 2835, 2898, 2940, 3045, 2898}, + { 15, 3507, 3423, 3213, 3087, 3003, 2919, 2856, 2814, 2814, 2814, 2856, 2919, 3003, 3150, 2898}, + { 14, 3402, 3234, 3108, 3045, 2940, 2856, 2814, 2814, 2814, 2856, 2898, 2982, 3066, 2898} + } +}; + +uint16_t sad_thresholds[sad_sequences] = { + 50, + 100 +}; + +uint8_t sad_deviations[sad_sequences] = { + 4, + 6 +}; + +uint8_t sad_nr_samples[sad_sequences] = { + 3, + 3 +}; + +#elif MIC_SAMPLE_DELTA == 45 + +const uint8_t sad_sequences = 4; +const uint8_t sad_max_deviations = 9; + +uint16_t sad_samples[sad_sequences][3][13] = { + { + { 11, 5289, 5246, 5203, 5117, 5074, 5031, 4945, 4902, 4816, 4730, 4601}, + { 10, 5332, 5289, 5246, 5160, 5117, 5074, 5031, 4945, 4902, 4816} + }, + { + { 11, 3096, 3010, 2924, 2881, 2795, 2709, 2666, 2580, 2537, 2451, 2408}, + { 9, 3010, 2924, 2838, 2752, 2709, 2623, 2580, 2494, 2451}, + { 11, 3139, 3053, 3010, 2924, 2838, 2795, 2709, 2623, 2580, 2494, 2451} + }, + { + { 11, 3440, 3397, 3311, 3225, 3139, 3096, 3053, 3010, 2967, 2924, 2881}, + { 11, 3483, 3397, 3354, 3268, 3182, 3139, 3096, 3010, 2967, 2924, 2881}, + { 12, 3569, 3483, 3397, 3311, 3268, 3182, 3139, 3053, 3010, 2967, 2924, 2881} + }, + { + { 12, 2838, 2838, 2838, 2795, 2795, 2795, 2795, 2838, 2838, 2881, 2924, 2967}, + { 11, 2881, 2838, 2838, 2795, 2795, 2795, 2795, 2838, 2838, 2881, 2881} + } +}; + +uint16_t sad_thresholds[sad_sequences] = { + 100, + 120, + 100, + 80, +}; + +uint8_t sad_deviations[sad_sequences] = { + 4, + 4, + 4, + 3 +}; + +uint8_t sad_nr_samples[sad_sequences] = { + 2, + 3, + 3, + 2 +}; +#endif + +void EmojiRecogniser::addSadSound() { + uint8_t it = sounds_size; + sounds_size ++; + sound_ids[it] = DEVICE_EMOJI_RECOGNISER_EVT_SAD; + + uint8_t history = 0; + + for(uint8_t i = 0; i < sad_sequences; i++) + for(uint8_t j = 0; j < sad_nr_samples[i]; j ++) + history = max(history, sad_samples[i][j][0] + 4); + + sounds[it] = new Sound(sad_sequences, sad_max_deviations, history, true); + + for(uint8_t i = 0; i < sad_sequences; i++){ + sounds[it] -> sequences[i] = new SoundSequence(sad_nr_samples[i], sad_thresholds[i], sad_deviations[i]); + for(uint8_t j = 0; j < sad_nr_samples[i]; j ++) + sounds[it] -> sequences[i] -> samples[j] = new SoundSample(sad_samples[i][j] + 1, sad_samples[i][j][0]); + } +} + + +// SOARING Sound ---- + +#if MIC_SAMPLE_DELTA == 91 + +const uint8_t soaring_sequences = 7; +const uint8_t soaring_max_deviations = 7; + +uint16_t soaring_samples[soaring_sequences][1][7] = { + { + { 5, 4179, 4179, 4179, 4179, 4179} + }, + { + { 5, 4284, 4284, 4284, 4284, 4284} + }, + { + { 5, 4389, 4389, 4389, 4389, 4389} + }, + { + { 5, 4494, 4494, 4494, 4494, 4494} + }, + { + { 5, 4599, 4599, 4599, 4599, 4599} + }, + { + { 5, 4704, 4704, 4704, 4704, 4704} + }, + { + { 5, 4809, 4809, 4809, 4809, 4809} + } +}; + +const uint16_t soaring_thresholds[soaring_sequences] = { + 100, + 100, + 100, + 100, + 100, + 100, + 100 +}; + +const uint8_t soaring_deviations[soaring_sequences] = { + 3, + 3, + 3, + 3, + 3, + 3, + 3 +}; + +const uint8_t soaring_nr_samples[soaring_sequences] = { + 1, + 1, + 1, + 1, + 1, + 1, + 1 +}; + +#elif MIC_SAMPLE_DELTA == 45 + +const uint8_t soaring_sequences = 7; +const uint8_t soaring_max_deviations = 10; + +uint16_t soaring_samples[soaring_sequences][1][11] = { + { + { 10, 4179, 4179, 4179, 4179, 4179, 4179, 4179, 4179, 4179, 4179} + }, + { + { 10, 4284, 4284, 4284, 4284, 4284, 4284, 4284, 4284, 4284, 4284} + }, + { + { 10, 4389, 4389, 4389, 4389, 4389, 4389, 4389, 4389, 4389, 4389} + }, + { + { 10, 4494, 4494, 4494, 4494, 4494, 4494, 4494, 4494, 4494, 4494} + }, + { + { 10, 4599, 4599, 4599, 4599, 4599, 4599, 4599, 4599, 4599, 4599} + }, + { + { 10, 4704, 4704, 4704, 4704, 4704, 4704, 4704, 4704, 4704, 4704} + }, + { + { 10, 4809, 4809, 4809, 4809, 4809, 4809, 4809, 4809, 4809, 4809} + } +}; + +const uint16_t soaring_thresholds[soaring_sequences] = { + 100, + 100, + 100, + 100, + 100, + 100, + 100 +}; + +const uint8_t soaring_deviations[soaring_sequences] = { + 3, + 3, + 3, + 3, + 3, + 3, + 3 +}; + +const uint8_t soaring_nr_samples[soaring_sequences] = { + 1, + 1, + 1, + 1, + 1, + 1, + 1 +}; + +#endif + + +void EmojiRecogniser::addSoaringSound() { + uint8_t it = sounds_size; + sounds_size ++; + sound_ids[it] = DEVICE_EMOJI_RECOGNISER_EVT_SOARING; + + uint8_t history = 0; + + for(uint8_t i = 0; i < soaring_sequences; i++) + for(uint8_t j = 0; j < soaring_nr_samples[i]; j ++) + history = max(history, soaring_samples[i][j][0] + 4); + + sounds[it] = new Sound(soaring_sequences, soaring_max_deviations, history, true); + + for(uint8_t i = 0; i < soaring_sequences; i++){ + sounds[it] -> sequences[i] = new SoundSequence(soaring_nr_samples[i], soaring_thresholds[i], soaring_deviations[i]); + for(uint8_t j = 0; j < soaring_nr_samples[i]; j ++) + sounds[it] -> sequences[i] -> samples[j] = new SoundSample(soaring_samples[i][j] + 1, soaring_samples[i][j][0]); + } +} + + +// TWINKLE Sound ---- + +#if MIC_SAMPLE_DELTA == 91 + +const uint8_t twinkle_sequences = 4; +const uint8_t twinkle_max_deviations = 5; + +uint16_t twinkle_samples[twinkle_sequences][5][8] = { + // First sequence + { + { 5, 1827, 2709, 3612, 4053, 4809}, + { 6, 1827, 2709, 2709, 3612, 4053, 4809}, + { 6, 1827, 2730, 3612, 4053, 4053, 4809}, + { 6, 1827, 2709, 3591, 3612, 4053, 4809} + }, + { + { 7, 4788, 4767, 4473, 4473, 4011, 3570, 3339}, + { 6, 4788, 4767, 4473, 4032, 3570, 3360}, + { 7, 4809, 4767, 4473, 4032, 3570, 3570, 3339} + }, + { + { 7, 1827, 2625, 2373, 2226, 2016, 2016, 1785}, + { 7, 1827, 1827, 2604, 2373, 2226, 2016, 1785}, + { 7, 1827, 1827, 2373, 2373, 2226, 2016, 1785}, + { 7, 4116, 1827, 2394, 2373, 2226, 2016, 1785}, + { 6, 4137, 2688, 2373, 2226, 2016, 1785} + }, + { + { 6, 2982, 2982, 2688, 2373, 2226, 2016}, + { 6, 3360, 2982, 2688, 2667, 2373, 2226}, + { 6, 3339, 2982, 2982, 2688, 2373, 2226} + } +}; + +const uint16_t twinkle_thresholds[twinkle_sequences] = { + 80, + 80, + 80, + 80 +}; + +const uint8_t twinkle_deviations[twinkle_sequences] = { + 3, + 3, + 3, + 3 +}; + +const uint8_t twinkle_nr_samples[twinkle_sequences] = { + 4, + 3, + 5, + 3 +}; + +#elif MIC_SAMPLE_DELTA == 45 + +const uint8_t twinkle_sequences = 4; +const uint8_t twinkle_max_deviations = 9; + +uint16_t twinkle_samples[twinkle_sequences][3][10] = { + { + { 9, 2236, 2236, 2021, 2021, 1763, 1763, 1763, 1677, 1677}, + { 9, 2236, 2236, 2021, 2021, 2021, 1763, 1763, 1677, 1677}, + { 9, 2236, 2236, 2236, 2021, 2021, 1763, 1763, 1677, 1677} + }, + { + { 8, 2967, 2967, 2666, 2666, 2666, 2365, 2365, 2236}, + { 8, 2967, 2967, 2967, 2666, 2666, 2365, 2365, 2236} + }, + { + { 7, 2236, 3010, 3010, 2666, 2666, 2666, 2494}, + { 7, 2236, 2236, 3010, 3010, 2666, 2666, 2494} + }, + { + { 5, 2494, 2967, 2967, 2666, 2666}, + { 5, 2494, 2494, 2967, 2967, 2666} + } +}; + +const uint16_t twinkle_thresholds[twinkle_sequences] = { + 100, + 100, + 100, + 100 +}; + +const uint8_t twinkle_deviations[twinkle_sequences] = { + 5, + 4, + 3, + 3 +}; + +const uint8_t twinkle_nr_samples[twinkle_sequences] = { + 3, + 2, + 2, + 2 +}; + +#endif + +void EmojiRecogniser::addTwinkleSound() { + uint8_t it = sounds_size; + sounds_size ++; + sound_ids[it] = DEVICE_EMOJI_RECOGNISER_EVT_TWINKLE; + + uint8_t history = 0; + + for(uint8_t i = 0; i < twinkle_sequences; i++) + for(uint8_t j = 0; j < twinkle_nr_samples[i]; j ++) + history = max(history, twinkle_samples[i][j][0] + 4); + + sounds[it] = new Sound(twinkle_sequences, twinkle_max_deviations, history, true); + + for(uint8_t i = 0; i < twinkle_sequences; i++){ + sounds[it] -> sequences[i] = new SoundSequence(twinkle_nr_samples[i], twinkle_thresholds[i], twinkle_deviations[i]); + for(uint8_t j = 0; j < twinkle_nr_samples[i]; j ++) + sounds[it] -> sequences[i] -> samples[j] = new SoundSample(twinkle_samples[i][j] + 1, twinkle_samples[i][j][0]); + } +} \ No newline at end of file diff --git a/source/MicroBitAudioProcessor.cpp b/source/MicroBitAudioProcessor.cpp new file mode 100644 index 00000000..aab4812d --- /dev/null +++ b/source/MicroBitAudioProcessor.cpp @@ -0,0 +1,241 @@ +/* +The MIT License (MIT) +Copyright (c) 2020 Arm Limited. +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + +#include "MicroBit.h" +#include "MicroBitAudioProcessor.h" + +#include +#include + +/* + * Constructor. + * + * Initialize the MicroBitAduioProcessor. + */ +MicroBitAudioProcessor::MicroBitAudioProcessor( DataSource& source, + uint16_t audio_samples_number, + uint16_t std_threshold) + : audiostream(source), + recogniser(NULL), + audio_samples_number(audio_samples_number), + std_threshold(std_threshold) { + + arm_rfft_fast_init_f32(&fft_instance, audio_samples_number); + + /* Double Buffering: We allocate twice the number of samples*/ + buf = new float[audio_samples_number * 2]; + fft_output = new float[audio_samples_number]; + mag = new float[audio_samples_number / 2]; + + memset(buf, 0, sizeof(buf)); + + buf_len = 0; + recording = false; + + if (buf == NULL || fft_output == NULL || mag == NULL) { + DMESG("DEVICE_NO_RESOURCES"); + target_panic(DEVICE_OOM); + } + DMESG("%s %p", "Audio Processor connecting to upstream, Pointer to me : ", this); + audiostream.connect(*this); +} + +/* + * Destructor. + * + * Deallocates all the memory allocated dynamically. + */ +MicroBitAudioProcessor::~MicroBitAudioProcessor() +{ + delete buf; + delete fft_output; + delete mag; +} + +/* + * Converts from frequency to the index in the array. + * + * @param freq a frequency in the range 0 - 5000 Hz. + * + * @return the index to the frequency bucket freq is in + * as it comes out of the fft + */ +uint16_t MicroBitAudioProcessor::frequencyToIndex(int freq) { + return (freq / ((uint32_t)MIC_SAMPLE_RATE / audio_samples_number)); +} + +/* + * Converts from the index in the array to frequency. + * + * @param index a index in the range 0 - audio_samples_number / 2. + * + * @return the avg frequency in the bucket + */ +float32_t MicroBitAudioProcessor::indexToFrequency(int index) { + return ((uint32_t)MIC_SAMPLE_RATE / audio_samples_number) * index; +} + + +/* + * Allow out downstream component to register itself with us + */ +void MicroBitAudioProcessor::connect(DataSink *downstream){ + recogniser = downstream; +} + + +/* + * Provides the next available data to the downstream caller. + */ +ManagedBuffer MicroBitAudioProcessor::pull() +{ + return ManagedBuffer(((uint8_t *) (&output)), (int) sizeof(AudioFrameAnalysis)); +} + + +/* + * A callback for when the data is ready. + * + * Analyses the data when enough of it comes in, using + * the following algorithm: + * + * The audio processor accumulates microphone data as it comes + * in and after getting audio_samples_number of them it process + * the frame. + * + * It transforms the date from time domain to frequency domain + * using the CMSIS fft. + * + * If the standard deviation (std) is lower than std_threshold then t + * he frame is considered silent - no fundamental frequency. + * + * It then filters out the frequencies that have the magnitude lower + * than the mean + ANALYSIS_STD_MULT_THRESHOLD * std. This ensures + * that only outlier frequencies are being considered. + * + * It then filters out the neighbour frequencies around the peaks. + * + * Some of these operations are implemented together to optimize the + * algorithm. + */ +int MicroBitAudioProcessor::pullRequest() +{ + + int s; + int result; + + auto mic_samples = audiostream.pull(); + + if (!recording) + return DEVICE_OK; + + int8_t *data = (int8_t *) &mic_samples[0]; + + int samples = mic_samples.length(); + + for (int i=0; i < samples; i++) + { + + + s = (int) *data; + result = s; + + data++; + buf[buf_len++] = (float)result; + + + if (!(buf_len % audio_samples_number)) + { + buf_len = 0; + + uint16_t from = frequencyToIndex(RECOGNITION_START_FREQ); + uint16_t to = min(frequencyToIndex(RECOGNITION_END_FREQ), audio_samples_number / 2); + + arm_rfft_fast_f32(&fft_instance, buf , fft_output, 0); + arm_cmplx_mag_f32(&fft_output[0], mag, to); + + float32_t mean = 0; + float32_t std = 0; + + arm_mean_f32 (&mag[from], to - from, &mean); + arm_std_f32 (&mag[from], to - from, &std); + + float32_t threshold = mean + std * ANALYSIS_STD_MULT_THRESHOLD; + + output.size = 0; + + if(std > std_threshold) { + std::vector> freq_played; + + for(uint16_t i=from; i < to; i++) + if(mag[i] > threshold) + freq_played.push_back(std::make_pair(indexToFrequency(i), mag[i])); + + sort(freq_played.begin(), freq_played.end(), + [&](std::pair a, + std::pair b) { + return a.second > b.second; + }); + + for(uint16_t i = 0; i < freq_played.size(); i++) { + if(output.size == 0) { + output.buf[output.size ++] = freq_played[i].first; + continue; + } + + bool similar_found = false; + for (uint16_t j = 0; j < output.size; j ++) + if(abs(output.buf[j] - freq_played[i].first) <= SIMILAR_FREQ_THRESHOLD) + similar_found = true; + + if(!similar_found) { + output.buf[output.size ++] = freq_played[i].first; + if(output.size >= MAXIMUM_NUMBER_OF_FREQUENCIES) + break; + } + } + } + + if(recogniser) + recogniser -> pullRequest(); + } + } + + return DEVICE_OK; +} + + +/* + * Starts recording and analysing. + */ +void MicroBitAudioProcessor::startRecording() +{ + this->recording = true; + DMESG("START RECORDING"); +} + +/* + * Stops from recording and analysing. + */ +void MicroBitAudioProcessor::stopRecording() +{ + this->recording = false; + DMESG("STOP RECORDING"); +} \ No newline at end of file diff --git a/source/MicroBitMorseInterpreter.cpp b/source/MicroBitMorseInterpreter.cpp new file mode 100644 index 00000000..9a703199 --- /dev/null +++ b/source/MicroBitMorseInterpreter.cpp @@ -0,0 +1,60 @@ +#include "MicroBitMorseInterpreter.h" + +/* + * This class takes morse data from a MicroBitMorseCodeRecogniser and uses a MorseEncoder to decode it. + * It then calls an event signalling the fact that it is done processing some data. + * The last processed data can then be taken from lastMessage. + */ + +/* + * Constructor. + * + * Initializes the interpreter. + * + * @param rec is the recogniser this will receive data from + * + * @param bit is the micro:bit + */ +MicroBitMorseInterpreter::MicroBitMorseInterpreter(MicroBitMorseRecogniser& rec, MicroBit& bit) + : recogniser(rec), uBit(bit){ + recogniser.connect(this); + interpreting = false; + encoder = MorseEncoder(); +} + +/* + * Callback for when the data is ready. + */ +int MicroBitMorseInterpreter::pullRequest() { + + ManagedBuffer data = recogniser.pull(); + + if(!interpreting) return DEVICE_OK; + + char* in = (char*)data.getBytes(); + char* out = new char[data.length()]; + encoder.decode(in, out); + + lastMessage = out; + + Event evnt(DEVICE_ID_MORSE_INTERPRETER, DEVICE_MORSE_INTERPRETER_EVT_NEW_MESSAGE); + + return DEVICE_OK; +} + + +/* + * Starts interpreting and also starts the associated recogniser. + */ +void MicroBitMorseInterpreter::startInterpreting() { + interpreting = true; + recogniser.startAnalysing(); +} + +/* + * Stops interpreting and also stops the associated recogniser. + */ +void MicroBitMorseInterpreter::stopInterpreting() { + interpreting = false; + recogniser.stopAnalysing(); +} diff --git a/source/MicroBitMorsePlayer.cpp b/source/MicroBitMorsePlayer.cpp new file mode 100644 index 00000000..d96ec8ef --- /dev/null +++ b/source/MicroBitMorsePlayer.cpp @@ -0,0 +1,140 @@ +#include "MicroBitMorsePlayer.h" +#include "MorseEncoder.h" + +/* + * The morse player takes strings of characters and converts them into morse code using a morse encoder. + * Then, it takes every element one by one and plays it over the speaker. + * It also adds 'element space' farmes between consecutive dots and dashes, in order to help differentiate between audible elements. + * The frequency and time unit of these sounds are set in the constructor. + * + * + * Durations for morse elements: + * dot - unit + * dash - unit*3 + * space between dots and dashes - unit + * space between characters - unit*3 + * space between words - unit*7 + * end of transmission - unit*10 + */ + +/* + * Constructor + * + * Initializes the player. + * + * @param bit is the micro:bit + * + * @param freq is the frequency the sounds will be played at + * + * @param dur is the time unit for playing the sounds + */ +MicroBitMorsePlayer::MicroBitMorsePlayer(MicroBit& bit, int freq, int dur) + :uBit(bit), frequency(freq), duration(dur), randomness(DEFAULT_RANDOMNESS){ + createFrames(); +} + +/* + * Converts a string into morse and then plays it over the speaker. + * + * @param in is the input string + */ +void MicroBitMorsePlayer::play(const char* in){ + int i = 0; + while (in[i]!=0) i++; + char* out = new char[i*7]; + + MorseEncoder me = MorseEncoder(); + me.encode(in, out); + + i = 0; + while (out[i]!=0){ + switch(out[i]){ + case '#': + uBit.audio.soundExpressions.play(pause10Frame); + break; + case ' ': + uBit.audio.soundExpressions.play(pause3Frame); + break; + case ';': + uBit.audio.soundExpressions.play(pause7Frame); + break; + case '.': + uBit.audio.soundExpressions.play(dotFrame); + // add space between dots and dashes + if (out[i+1] == '.' || out[i+1] == '-') + uBit.audio.soundExpressions.play(pause1Frame); + break; + case '-': + uBit.audio.soundExpressions.play(dashFrame); + // add space between dots and dashes + if (out[i+1] == '.' || out[i+1] == '-') + uBit.audio.soundExpressions.play(pause1Frame); + break; + } + i++; + } + + delete out; +} + +/* + * Turns an integer into a ManagedString of length 4 + */ +ManagedString fourString(int n){ + if (n < 10) + return ManagedString("000") + ManagedString(n); + if (n < 100) + return ManagedString("00") + ManagedString(n); + if (n < 1000) + return ManagedString("0") + ManagedString(n); + if (n > 9999) + return ManagedString("9999"); + return ManagedString(n); +} + +// generates frames for all the morse elements from frequency, duration and randomness +void MicroBitMorsePlayer::createFrames(){ + ManagedString freqString = fourString(frequency); + ManagedString oneFrameString = fourString(duration); + ManagedString threeFrameString = fourString(duration * 3); + ManagedString sevenFrameString = fourString(duration * 7); + ManagedString tenFrameString = fourString(duration * 10); + ManagedString randomString = fourString(randomness); + + + dotFrame = ManagedString("01023")+ + freqString+ + oneFrameString+ + ManagedString("02440")+ + freqString+ + ManagedString("0888102301280002000024")+ + randomString+ + ManagedString("000000000000000000000000"); + + dashFrame = ManagedString("01023")+ + freqString+ + threeFrameString+ + ManagedString("02440")+ + freqString+ + ManagedString("0888102301280002000024")+ + randomString+ + ManagedString("000000000000000000000000"); + + + pause1Frame = ManagedString("010230000")+ + oneFrameString+ + ManagedString("02440000008881023012800020000240000000000000000000000000000"); + + pause3Frame = ManagedString("010230000")+ + threeFrameString+ + ManagedString("02440000008881023012800020000240000000000000000000000000000"); + + pause7Frame = ManagedString("010230000")+ + sevenFrameString+ + ManagedString("02440000008881023012800020000240000000000000000000000000000"); + + pause10Frame = ManagedString("010230000")+ + tenFrameString+ + ManagedString("02440000008881023012800020000240000000000000000000000000000"); + +} \ No newline at end of file diff --git a/source/MicroBitMorseRecogniser.cpp b/source/MicroBitMorseRecogniser.cpp new file mode 100644 index 00000000..8a7688f8 --- /dev/null +++ b/source/MicroBitMorseRecogniser.cpp @@ -0,0 +1,231 @@ +#include "MicroBitMorseRecogniser.h" + +/* + * The sound recogniser takes in data from the audio processor - in + * form of AudioFrameAnalysis. It then tries to identify whether + * the listening frequency was played during that frame. Based on + * the last frames it sends dots ('.'), dashes ('-'), letter spaces + * (' '), word spaces (';'), and end-of-transmission ('#') to the + * data sink connected. + * + * Algorithm + * + * The recogniser first converts the AudioFrameAnalysis data in + * booleans which represent whether or not the listening frequency was + * being played in that frame. A timeUnit is calculated based on the + * sampling frequency and the number of samples taken before analysis + * by the audio processor - it represents the number of frames a dot + * should take. + * + * It has 2 values: zeros and ones which counts for how many timeUnits + * the frequency wasn't being played/ was being played. The value + * synchronised keeps track of the last transition - true if the last + * transition was from the frequency not being played to it being played, + * false otherwise. At each transition, the values of zeros or ones are + * being interpreted into the right character and then reset. + * + * All characters are being accumulated into a buffer and send together + * when the end-of-transmission character ('#') is found. + */ + +/* + * Constructor. + * + * Initializes the MicroBitMorseRecogniser. + * + * @param the audio processor it should take data from + * + * @param freq the frequency it should listen for + * + * @param timeUnit the amount of time in ms it takes a dot + * to be transmitted + */ +MicroBitMorseRecogniser::MicroBitMorseRecogniser(MicroBitAudioProcessor& audio_processor, uint16_t freq, uint16_t _timeUnit) + : audio_proceesor(audio_processor), frequency(freq) { + audio_proceesor.connect(this); + + analysing = false; + buffer_len = 0; + output_len = 0; + + timeUnit = 1.0 * MIC_SAMPLE_RATE * _timeUnit / 1000 / MORSE_AUDIO_SAMPLES_NUMBER + 0.5; + + memset(buffer, 0, sizeof(buffer)); + memset(output, 0, sizeof(output)); + + synchronised = false; + zeros = 0; + ones = 0; + + // Compensate for distorsion + frequency -= 50; + +} + +/* + * Destructor. + * + * Deallocates all the memory allocated dynamically. + */ +MicroBitMorseRecogniser::~MicroBitMorseRecogniser(){ +} + +/* + * Starts analysing the data that comes in. + */ +void MicroBitMorseRecogniser::startAnalysing(){ + analysing = true; + audio_proceesor.startRecording(); +} + +/* + * Stops analysing the data and also stops the audio processor + * from receiving. + */ +void MicroBitMorseRecogniser::stopAnalysing(){ + analysing = false; + buffer_len = 0; + audio_proceesor.stopRecording(); +} + +/* + * A callback for when the data is ready. + */ +int MicroBitMorseRecogniser::pullRequest() { + auto frames = audio_proceesor.pull(); + + if(!analysing) return DEVICE_OK; + + MicroBitAudioProcessor::AudioFrameAnalysis* buf = (MicroBitAudioProcessor::AudioFrameAnalysis* ) &frames[0]; + uint16_t size = frames.length() / sizeof(MicroBitAudioProcessor::AudioFrameAnalysis); + + for(uint16_t i = 0; i < size; i++) + processFrame(&buf[i]); + + return DEVICE_OK; +} + +/* + * Allow out downstream component to register itself with us + */ +void MicroBitMorseRecogniser::connect(DataSink *sink){ + interpreter = sink; +} + +/* + * Provides the next available data to the downstream caller. + */ +ManagedBuffer MicroBitMorseRecogniser::pull() +{ + uint16_t len = output_len; + output_len = 0; + return ManagedBuffer(((uint8_t *) (&output)), len*sizeof(uint8_t)); +} + +/* + * Checks if the number of ones in the last timeUnit frames up to + * a given position in buffer fit in a given threshold. + * + * @param to specifies the interval [to - timeUnit, to) which is + * analysed + * + * @param threshold the minimum number of ones in the interval + * + * @return number of trues in buffer[to - timeUnit, to) >= threshold + * + * @note if threshold is 255 thet it will use + * timeUnit * MORSE_FRAME_TRUE_RATE_THRESHOLD as the threshold + */ +bool MicroBitMorseRecogniser::recogniseLastMorseFrame(uint16_t to, uint16_t threshold = 255) { + if(to < timeUnit) return false; + if(threshold == 255) threshold = timeUnit * MORSE_FRAME_TRUE_RATE_THRESHOLD; + + uint16_t nr_true = 0; + for(uint16_t it = to - timeUnit; it < to; it ++) + if(buffer[it]) nr_true ++; + + return nr_true >= threshold; +} + +/* + * Adds a character to the output buffer. If it adds the + * end-of-transmission ('#') character, then it sends the buffer + * to the next component in the pipeline - 'interpreter'. + * + * @param c the character to be added to the out buffer + */ +void MicroBitMorseRecogniser::pushOut(char c){ + if (output_len == 0 && (c == ' ' || c == ';' || c == '#')) return; + output[output_len] = c; + output_len++; + if (c == '#') + interpreter->pullRequest(); +} + +/* + * Processes a incoming frame. + * + * @param frame the frame it should process + */ +void MicroBitMorseRecogniser::processFrame(MicroBitAudioProcessor::AudioFrameAnalysis* frame) { + + bool detected_freq = false; + for (uint16_t i = 0; i < frame -> size && !detected_freq; i++ ) + if(abs(frame -> buf[i] - frequency) < DETECTION_THRESHOLD) + detected_freq = true; + + buffer[buffer_len] = detected_freq; + buffer_len ++; + + if(buffer_len < timeUnit) return; + + uint16_t true_threshold = timeUnit * MORSE_FRAME_TRUE_RATE_THRESHOLD; + uint16_t false_threshold = timeUnit * MORSE_FRAME_FALSE_RATE_THRESHOLD; + + if(!synchronised && recogniseLastMorseFrame(buffer_len, true_threshold) && + buffer[buffer_len - timeUnit]) { + + zeros += (int)((0.5 + buffer_len) / timeUnit) - 1; + + if(zeros > 1 && zeros <= 4) pushOut(' '); + else if (zeros > 4 && zeros <= 8) pushOut(';'); + + buffer_len = 0; + zeros = 0; + ones = 1; + synchronised = true; + + return; + } + + if(synchronised && !recogniseLastMorseFrame(buffer_len, timeUnit - false_threshold) && + !buffer[buffer_len - timeUnit]) { + ones += (int)((0.5 + buffer_len) / timeUnit) - 1; + + if(ones > 2) + pushOut('-'); + else + pushOut('.'); + + buffer_len = 0; + synchronised = false; + zeros = 1; + ones = 0; + return; + } + + if(buffer_len == 2 * timeUnit) { + + if(zeros == 9) + pushOut('#'); + + if (synchronised) + ones ++; + else + zeros ++; + + memcpy(buffer, &buffer[timeUnit], sizeof(bool) * timeUnit); + buffer_len = timeUnit; + } +} + diff --git a/source/MicroBitSoundRecogniser.cpp b/source/MicroBitSoundRecogniser.cpp new file mode 100644 index 00000000..82472250 --- /dev/null +++ b/source/MicroBitSoundRecogniser.cpp @@ -0,0 +1,283 @@ + +#include "MicroBitSoundRecogniser.h" + +/* + * Constructor. + * + * Initialize the MicroBitSoundRecogniser. + */ +MicroBitSoundRecogniser::MicroBitSoundRecogniser(MicroBitAudioProcessor& audio_processor) + : audio_proceesor(audio_processor){ + analysing = false; + audio_proceesor.connect(this); + buffer_len = 0; + sounds_size = 0; +} + +/* + * Destructor. + * + * Deallocates all the memory allocated dynamically. + */ +MicroBitSoundRecogniser::~MicroBitSoundRecogniser(){ + if(sounds_size != 0){ + for(uint8_t i = 0; i < sounds_size; i++) { + delete sounds[i]; + } + delete [] sounds; + delete [] sound_ids; + } +} + +/* + * A callback for when the data is ready. + */ +int MicroBitSoundRecogniser::pullRequest(){ + + auto frames = audio_proceesor.pull(); + + if(!analysing) return DEVICE_OK; + + MicroBitAudioProcessor::AudioFrameAnalysis* buf = (MicroBitAudioProcessor::AudioFrameAnalysis* ) &frames[0]; + buffer[buffer_len].size = buf[0].size; + for(uint8_t i = 0; i update(buffer, buffer_len); + if(sounds[sound_it] -> matched()){ + recognisedSound(sound_ids[sound_it]); + return DEVICE_OK; + } + } + + return DEVICE_OK; +} + +/* + * Starts analysing the data that comes in. Also sets the callback. + * + * @TODO change it to send a message on the message bus + * rather than having a callback + */ +void MicroBitSoundRecogniser::startAnalysing(){ + analysing = true; + audio_proceesor.startRecording(); +} + +/* + * Stops analysing the data and also stops the audio processor + * from receiving. + */ +void MicroBitSoundRecogniser::stopAnalysing(){ + analysing = false; + buffer_len = 0; + audio_proceesor.stopRecording(); + for(uint8_t sound_it; sound_it < sounds_size; sound_it ++) + sounds[sound_it] -> resetHistory(); +} + + +MicroBitSoundRecogniser::SoundSample::SoundSample(const uint16_t* _frames, uint8_t size) + : size(size), frames(_frames) { } + +MicroBitSoundRecogniser::SoundSample::~SoundSample() { + delete[] frames; +} + +MicroBitSoundRecogniser::SoundSequence::SoundSequence( uint8_t size, + uint32_t threshold, + uint8_t deviation) + : size(size), threshold(threshold), + deviation(deviation) { + samples = new SoundSample* [size]; +} + +MicroBitSoundRecogniser::SoundSequence::~SoundSequence() { + for(uint8_t i = 0 ; i < size; i++) + delete samples[i]; + delete [] samples; +} + +MicroBitSoundRecogniser::Sound::Sound(uint8_t size, uint8_t max_deviation, uint8_t max_history_len, bool consider_all_frequencies) + : size(size), max_deviation(max_deviation), history_len(0), + max_history_len(max_history_len), consider_all_frequencies(consider_all_frequencies) { + sequences = new SoundSequence* [size]; + history = new uint8_t[2 * max_history_len * size]; +} + +MicroBitSoundRecogniser::Sound::~Sound() { + for(uint8_t i = 0 ; i < size; i++) + delete sequences[i]; + delete [] sequences; + delete [] history; +} + +/* + * Update called when new data comes in. + * + * @param buffer the buffer with the last data points that came in + * + * @note This keeps the history updated which is needed for the + * dynamic programming approach chosen for matching + */ +void MicroBitSoundRecogniser::Sound::update(MicroBitAudioProcessor::AudioFrameAnalysis* buffer, + uint8_t buffer_len){ + for(uint8_t seq_it = 0; seq_it < size; seq_it ++) { + uint8_t x = matchSequence(seq_it, buffer, buffer_len); + addToHistory(seq_it, x); + } + endHistoryFrame(); +} + +/* + * Whether or not the sound matched in the last frame. + * + * @return whether or not the sound matched in the last frame. + * + * @note Should be called after update when new data comes in. + */ +bool MicroBitSoundRecogniser::Sound::matched() { + if(getDeviation(1, size - 1) <= max_deviation){ + history_len = 0; + return true; + } + return false; +} + +/* + * Matches a sequence to the last couple of data points in the buffer. + * + * @param seq_id the id of the sequence to try to match + * + * @param buffer the buffer of data points that came in + * + * @param buffer_len the length of the buffer + * + * @return the number of deviations in the last data points to the sound up to + * the seq_id sequence or 255 if it's above the maximums allowed. + */ +uint8_t MicroBitSoundRecogniser::Sound::matchSequence(uint8_t seq_id, + MicroBitAudioProcessor::AudioFrameAnalysis* buffer, + uint8_t buffer_len) const { + SoundSequence* seq = sequences[seq_id]; + uint8_t min_dev = 255; + for(uint8_t sample_it = 0; sample_it < seq -> size; sample_it ++) { + uint8_t sample_len = seq -> samples[sample_it] -> size; + if(buffer_len < sample_len) continue; + + uint8_t deviation = 255; + if(seq_id == 0) deviation = 0; + else if (seq_id && deviation > getDeviation(sample_len, seq_id - 1)) + deviation = getDeviation(sample_len, seq_id - 1); + else if (seq_id && deviation > getDeviation(sample_len + 1, seq_id - 1)) + deviation = getDeviation(sample_len, seq_id - 1); + + if(deviation > max_deviation || deviation >= min_dev) continue; + + uint32_t diff = 0; + uint8_t nr_of_diffs = 0; + uint8_t deviations_left = seq->deviation; + + for(uint8_t i = 0; i < sample_len; i++) { + if(seq -> samples[sample_it] -> frames[i] == 0) continue; + if(buffer[buffer_len - sample_len + i].size == 0) { + deviation ++; + continue; + } + + uint16_t freq = seq -> samples[sample_it] -> frames[i]; + + diff = abs(freq - buffer[buffer_len - sample_len + i].buf[0]); + + if(consider_all_frequencies) + for(uint8_t j = 1; j < buffer[buffer_len - sample_len + i].size; j++) + diff = min(diff, abs(freq - buffer[buffer_len - sample_len + i].buf[j]) ); + + if(deviations_left && diff > seq -> threshold && deviation < max_deviation){ + deviations_left --; + deviation ++; + } + else if(diff > seq -> threshold ){ + deviation = 255; + break; + } + } + + if(deviation < min_dev && deviation <= max_deviation) + min_dev = deviation; + + } + + return min_dev; +} + +/* + * Getter for the internal history buffer of the sound. + * + * @param frames_ago the number of frames ago for which the + * query was made + * + * @param seq_id the id of the sequence to get the deviation for + * + * @return the deviation (or 255 if it didn't match) up to the + * sequence seq_id that was frames_ago frames ago. + * + * @note used to check if the sound matched up to the first seq_id + * sequences so that the matching doesn't need to recheck + * those sequences. + */ +uint8_t MicroBitSoundRecogniser::Sound::getDeviation(uint8_t frames_ago, uint8_t seq_id) const { + if(history_len < frames_ago) return 255; + return history[(history_len - frames_ago) * size + seq_id]; +} + +/* + * Adds to the history buffer a deviation for a certain sequence. + * + * @param seq_id the id of the sequence to add the value to. + * + * @param value the value to be added to the history buffer + */ +void MicroBitSoundRecogniser::Sound::addToHistory(uint8_t seq_id, uint8_t value){ + history[history_len * size + seq_id] = value; +} + +/* + * Ends a history frame, increasing the length of the buffer. + */ +void MicroBitSoundRecogniser::Sound::endHistoryFrame(){ + history_len ++; + // double buffering + if(history_len == 2 * max_history_len) { + memcpy(&history[0], &history[max_history_len * size], sizeof(uint8_t) * max_history_len * size); + history_len = max_history_len; + } +} + +/* + * Resets the history buffer. + * + * @note Used when the data stops coming in - e.g. when the analyser + * is paused + */ +void MicroBitSoundRecogniser::Sound::resetHistory(){ + history_len = 0; +} + + + diff --git a/source/MorseEncoder.cpp b/source/MorseEncoder.cpp new file mode 100644 index 00000000..3c446c30 --- /dev/null +++ b/source/MorseEncoder.cpp @@ -0,0 +1,128 @@ +#include "MicroBit.h" +#include "MorseEncoder.h" + +/* + * The morse encoder has two uses. + * Firstly, it can encode a string of characters into its morse representation. + * This is done by transforming every character using a map and adding spaces between characters and words when appropriate. + * At the end, the `end of transmission` character is added. + * Whenever a lowercase letter is encountered, it is converted to uppercase. + * Whenever an unknown character is encountered, it is converted to '&'. + * + * Secondly, it can decode a series of morse elements into the corresponding string. + * Decoding takes every continuous sequence of dots and dashes and transforms it into the appropriate character using another map. + * This is done by building up every continuous sequence of dots and dashes into a small buffer and, when another element is encountered, + * transforming it into the appropriate character using a map. The buffer is reset every time this happens. + * All of the generated chracyters are added to the output string. + * + * + * This uses the following elements for morse code: + * dot - '.' + * dash - '-' + * element gap - not represented + * letter gap - ' ' + * word gap - ';' + * end of transmission - '#' + * + * The reason element gap is not represented is that it can be easily added appropriately at the time that the morse code is played over the speaker. + * When interpreting morse played by another source, it can just be ignored. +*/ + +// Initialize map from characters to encodings +std::map MorseEncoder::toStr = { + {'A', ".-"}, {'B', "-..."}, {'C', "-.-."}, {'D', "-.."}, {'E', "."}, {'F', "..-."}, {'G', "--."}, {'H', "...."}, {'I', ".."}, {'J', ".---"}, + {'K', "-.-"}, {'L', ".-.."}, {'M', "--"}, {'N', "-."}, {'O', "---"}, {'P', ".--."}, {'Q', "--.-"}, {'R', ".-."}, {'S', "..."}, {'T', "-"}, + {'U', "..-"}, {'V', "...-"}, {'W', ".--"}, {'X', "-..-"}, {'Y', "-.--"}, {'Z', "--.."}, {'1', ".----"}, {'2', "..---"}, {'3', "...--"}, + {'4', "....-"}, {'5', "....."}, {'6', "-...."}, {'7', "--..."}, {'8', "---.."}, {'9', "----."}, {'0', "-----"}, {'&', ".-..."}}; + +/* +* Constructor +* +* Initializes the encoder. +*/ +MorseEncoder::MorseEncoder() { + // Generate toChar from toStr + for(std::map::iterator it = toStr.begin(); it != toStr.end(); ++it) { + toChar[it->second] = it->first; + } +} + +/* +* Encodes a string into morse code. +* +* @param in is the input string to be encoded +* +* @param out is the output string that the encoding is written into +* +* @note out should be at least 6 times the size of in +*/ +void MorseEncoder::encode(const char* in, char* out) { + int i = 0; // index over in + int j = 0; // index over out + char c; + while (in[i]!=0) { + c = in[i]; + // capitalize letters + if (('a' <= c) && (c <= 'z')) c -= ('a' - 'A'); + + // turn unknown characters into & + if (c != ' ' && (c < 'A' || c > 'Z') && (c < '0' || c > '9')) c = '&'; + + if (c == ' '){ + if (j>0 && out[j-1] == ' ') out[j-1] = ';'; + else {out[j] = ';'; j++;} + } + else{ + + ManagedString s = toStr[c]; + int k = 0; // index over s + while (s.charAt(k)!=0) { + out[j] = s.charAt(k); + j++;k++; + } + out[j] = ' ';j++; + } + i++; + } + if (j>0 && out[j-1] == ' ') out[j-1] = '#'; + else {out[j] = '#'; j++;} + out[j] = 0; +} + +/* +* Decodes morse code into a string. +* +* @param in is the input string containing morse code +* +* @param out is the output string that the morse code is decoded into +* +* @note out should be at least the same size as in +*/ +void MorseEncoder::decode(const char* in, char* out){ + int i = 0; // index over in + int j = 0; // index over out + int k = 0; // index over s + char s[10]; // current character buffer + char c; + + while (in[i]!=0){ + c = in[i]; + if (c == '.' || c == '-'){ + s[k] = c; k++; + } + else{ + if (k!=0){ + s[k] = 0; + k = 0; + out[j] = toChar[s]; + j++; + } + if (c == ';'){ + out[j] = ' '; + j++; + } + } + i++; + } + out[j] = 0; +} diff --git a/target.json b/target.json index c6a0a540..6a9e6bd2 100644 --- a/target.json +++ b/target.json @@ -49,11 +49,11 @@ "cmake_definitions":{ "MBED_LEGACY_TOOLCHAIN":"GCC_ARM;" }, - "cpu_opts":"-mcpu=cortex-m4 -mthumb -mfpu=fpv4-sp-d16 -mfloat-abi=softfp", + "cpu_opts":"-mcpu=cortex-m4 -mthumb -mfpu=fpv4-sp-d16 -mfloat-abi=hard", "asm_flags":"-fno-exceptions -fno-unwind-tables --specs=nosys.specs -mcpu=cortex-m4 -mthumb", "c_flags":"-std=c99 --specs=nosys.specs -Warray-bounds", "cpp_flags":"-std=c++11 -fwrapv -fno-rtti -fno-threadsafe-statics -fno-exceptions -fno-unwind-tables -Wl,--gc-sections -Wl,--sort-common -Wl,--sort-section=alignment -Wno-array-bounds", - "linker_flags":"-Wl,--no-wchar-size-warning -Wl,--gc-sections -Wl,--wrap,atexit -mcpu=cortex-m4 -mthumb -mfpu=fpv4-sp-d16 -mfloat-abi=softfp -Wl,--start-group -lstdc++ -lsupc++ -lm -lc -lgcc -lnosys -Wl,--end-group", + "linker_flags":"-Wl,--no-wchar-size-warning -Wl,--gc-sections -Wl,--wrap,atexit -mcpu=cortex-m4 -mthumb -mfpu=fpv4-sp-d16 -mfloat-abi=hard -Wl,--start-group -lstdc++ -lsupc++ -lm -lc -lgcc -lnosys -Wl,--end-group", "libraries":[ { "name":"codal-core",