diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..595c1b2 --- /dev/null +++ b/.clang-format @@ -0,0 +1,21 @@ +Standard: Cpp11 +BasedOnStyle: LLVM +IndentWidth: 4 +ColumnLimit: 120 +AccessModifierOffset: -4 +NamespaceIndentation: All +BreakBeforeBraces: Custom +BraceWrapping: + AfterEnum: true + AfterStruct: true + AfterClass: true + SplitEmptyFunction: true + AfterControlStatement: true + AfterNamespace: false + AfterFunction: true + AfterUnion: true + AfterExternBlock: false + BeforeCatch: false + BeforeElse: true + SplitEmptyRecord: true + SplitEmptyNamespace: true diff --git a/Makefile b/Makefile index a0efc89..0b1f4cf 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ CFLAGS += -O3 CXXFLAGS += -O3 -COMMON_OBJ = src/audio.o src/fifo.o src/pa_ringbuffer.o src/util.o +COMMON_OBJ = src/audio.o src/fifo.o src/pa_ringbuffer.o src/util.o src/auto_channel_detection.o EC_OBJ = $(COMMON_OBJ) src/ec.o EC_LOOPBACK_OBJ = $(COMMON_OBJ) src/ec_hw.o diff --git a/src/auto_channel_detection.c b/src/auto_channel_detection.c new file mode 100644 index 0000000..b00336c --- /dev/null +++ b/src/auto_channel_detection.c @@ -0,0 +1,264 @@ +#include +#include +#include +#include + +#include "audio.h" +#include "conf.h" + +#include "auto_channel_detection.h" + +extern int g_is_quit; + +#define BIPNORM_TO_INT16_SLOPE 32767.5f +#define INT16_TO_BIPNORM_SLOPE 1.0f / BIPNORM_TO_INT16_SLOPE + +typedef struct _env_chan_map +{ + unsigned channel; + float val; +} env_chan_map; + +float int16_to_bipNorm(int16_t input) +{ + float input_start = INT16_MIN; + float output_start = -1.0f; + return output_start + (INT16_TO_BIPNORM_SLOPE * (input - input_start)); +} + +int16_t bipNorm_to_int16(float input) +{ + float input_start = -1.0f; + float output_start = INT16_MIN; + return output_start + roundf(BIPNORM_TO_INT16_SLOPE * (input - input_start)); +} + +float fast_exp3(float x) { return (6 + x * (6 + x * (3 + x))) * 0.16666666f; } + +void peak_envelope_follower(env_chan_map *env, float s, float release) +{ + if (s > env->val) + env->val = s; + else + env->val = s + release * (env->val - s); +} + +void sort_smallest_k(env_chan_map env[], int n, int k) +{ + // For each arr[i] find whether + // it is a part of n-smallest + // with insertion sort concept + for (int i = k; i < n; ++i) + { + + // find largest from first k-elements + float max_var = env[k - 1].val; + int pos = k - 1; + + for (int j = k - 2; j >= 0; j--) + { + if (env[j].val > max_var) + { + max_var = env[j].val; + pos = j; + } + } + + // if largest is greater than arr[i] + // shift all element one place left + if (max_var > env[i].val) + { + + int j = pos; + while (j < k - 1) + { + env_chan_map tmp = env[j]; + env[j] = env[j + 1]; + env[j + 1] = tmp; + j++; + } + + // swap arr[k-1] and arr[i] + env_chan_map tmp = env[k - 1]; + env[k - 1] = env[i]; + env[i] = tmp; + } + } +} + +void init_envelopes(conf_t *config, env_chan_map *env) +{ + for (int channel = 0; channel < config->rec_channels; channel++) + { + env[channel].channel = channel; + env[channel].val = 0; + } +} + +int detect_loopback_channels(conf_t *config, int16_t *buf, int *loopback_list, env_chan_map *env, int frame_size, + float release) +{ + int loopback_list_tmp[32]; + int res = 0; + + // We loop over envelopes as they get reordered by volume on every pass! + for (int env_idx = 0; env_idx < config->rec_channels; env_idx++) + { + for (int i = 0; i < frame_size; ++i) + { + unsigned channel = env[env_idx].channel; + unsigned pos = config->rec_channels * i + channel; + float s = int16_to_bipNorm(buf[pos]); + peak_envelope_follower(&env[env_idx], fabs(s), release); + } + } + + sort_smallest_k(env, config->rec_channels, config->ref_channels); + + // printf("detected loopback channels:"); + for (int channel = 0; channel < config->ref_channels; channel++) + { + loopback_list_tmp[channel] = env[channel].channel; + // printf(" %d", loopback_list_tmp[channel]); + } + // printf("\n"); + + // selected channels might be in different order + for (int channel = 0; channel < config->ref_channels; channel++) + { + int in_prev_list = 0; + for (int i = 0; i < config->ref_channels; i++) + { + if (loopback_list[i] == loopback_list_tmp[channel]) + { + in_prev_list = 1; + break; + } + } + + if (!in_prev_list) + { + if (loopback_list[channel] != 32) + { + // printf("channel %d was not previously selected\n", loopback_list_tmp[channel]); + } + res++; + } + } + + if (res) + { + printf("Current loopback channel best candidates:"); + for (int channel = 0; channel < config->ref_channels; channel++) + { + loopback_list[channel] = loopback_list_tmp[channel]; + printf(" %d", loopback_list[channel]); + } + printf("\n"); + } + + return res; +} + +void auto_channel_detection(conf_t *config, int16_t *rec, int *mic_list, int *loopback_list, int frame_size, + int timeout, int coherence_window_size_ms, int total_window_size_ms, int envelope_ms, + int save_audio) +{ + int coherence_window_size = config->rate * coherence_window_size_ms / 1000; + int total_window_size = config->rate * total_window_size_ms / 1000; + int coherence_window_size_r = coherence_window_size; + int total_window_size_r = total_window_size; + env_chan_map *env = malloc(sizeof(env_chan_map) * config->rec_channels); + float env_frames = config->rate * (envelope_ms / 1000.0f); + float release = fast_exp3(-2.0f / env_frames); + FILE *fp_rec = NULL; + + if (save_audio) + { + fp_rec = fopen("/tmp/auto_channel_detection.raw", "wb"); + + if (fp_rec == NULL) + { + printf("Fail to open file(s)\n"); + exit(1); + } + } + + init_envelopes(config, env); + + for (int speaker = 0; speaker < config->ref_channels; speaker++) + { + loopback_list[speaker] = 32; + } + + printf("Detecting loopback channels...\n"); + + while (!g_is_quit && coherence_window_size_r > 0) + { + capture_read(rec, frame_size, timeout); + + int detect_res = detect_loopback_channels(config, rec, loopback_list, env, frame_size, release); + if (detect_res) + { + // we detected different channels. Let's start over + coherence_window_size_r = coherence_window_size; + } + else + { + // detected channels are the same as before + coherence_window_size_r -= frame_size; + } + total_window_size_r -= frame_size; + + if (fp_rec) + { + fwrite(rec, 2, frame_size * config->rec_channels, fp_rec); + } + + if (total_window_size_r <= 0) + { + fprintf(stderr, + "Failed to detect loopback channels within time limit\n" // + "Please make sure no audio is playing and try again\n" // + ); + exit(1); + } + }; + + printf("%10s|%10s\n", "channel", "volume"); + for (int i = 0; i < config->rec_channels; i++) + { + printf("%10u|%10f\n", env[i].channel, env[i].val); + } + + printf("Detected channels:\n"); + printf(" rec:"); + for (int i = 0; i < config->ref_channels; i++) + { + printf(" %u", loopback_list[i]); + } + printf("\n"); + + printf(" mic:"); + int mic_list_idx = 0; + for (int channel = 0; channel < config->rec_channels; channel++) + { + int found = 0; + for (int i = 0; i < config->ref_channels; i++) + { + if (channel == loopback_list[i]) + { + found = 1; + break; + } + } + if (found) + { + continue; + } + mic_list[mic_list_idx] = channel; + mic_list_idx++; + printf(" %d", channel); + } + printf("\n"); +} diff --git a/src/auto_channel_detection.h b/src/auto_channel_detection.h new file mode 100644 index 0000000..c28051c --- /dev/null +++ b/src/auto_channel_detection.h @@ -0,0 +1,17 @@ +#ifndef _AUTO_CHANNEL_DETECTION_H_ +#define _AUTO_CHANNEL_DETECTION_H_ + +void auto_channel_detection( + conf_t *config, // global config object + int16_t *rec, // pre-allocated rec buffer + int *mic_list, // buffer which will contain the list of the discovered mic channels + int *loopback_list, // buffer which will contain the list of the discovered loopbacks channels + int frame_size, // capturing buffer's frame size + int timeout, // capturing buffer's timeout + int coherence_window_size_ms, // how long should discovered the channels be consistent + int total_window_size_ms, // how long before we give up after discovering inconsistent results + int envelope_ms, // influences the decay rate of the envelope's window used to detect channel volume + int save_audio // save audio captured during channel detection operations +); + +#endif // _AUTO_CHANNEL_DETECTION_H_ diff --git a/src/ec.c b/src/ec.c index 93b2e00..ffc0530 100644 --- a/src/ec.c +++ b/src/ec.c @@ -1,39 +1,40 @@ // ec - echo canceller #include +#include +#include +#include +#include #include #include -#include #include -#include -#include -#include -#include #include +#include #include -#include "conf.h" #include "audio.h" +#include "conf.h" -const char *usage = - "Usage:\n %s [options]\n" - "Options:\n" - " -i PCM playback PCM (default)\n" - " -o PCM capture PCM (default)\n" - " -r rate sample rate (16000)\n" - " -c channels recording channels (2)\n" - " -b size buffer size (262144)\n" - " -d delay system delay between playback and capture (0)\n" - " -f filter_length AEC filter length (2048)\n" - " -s save audio to /tmp/playback.raw, /tmp/recording.raw and /tmp/out.raw\n" - " -D daemonize\n" - " -h display this help text\n" - "Note:\n" - " Access audio I/O through named pipes (/tmp/ec.input for playback and /tmp/ec.output for recording)\n" - " `cat audio.raw > /tmp/ec.input` to play audio\n" - " `cat /tmp/ec.output > out.raw` to get recording audio\n" - " Only support mono playback\n"; +const char *usage = // + "Usage:\n %s [options]\n" // + "Options:\n" // + " -i PCM playback PCM (default)\n" // + " -o PCM capture PCM (default)\n" // + " -r rate sample rate (16000)\n" // + " -c channels recording channels (2)\n" // + " -b size buffer size (262144)\n" // + " -d delay system delay between playback and capture (0)\n" // + " -f filter_length AEC filter length (2048)\n" // + " -s save audio to /tmp/playback.raw, /tmp/recording.raw and /tmp/out.raw\n" // + " -D daemonize\n" // + " -h display this help text\n" // + "Note:\n" // + " Access audio I/O through named pipes (/tmp/ec.input for playback and /tmp/ec.output for recording)\n" // + " `cat audio.raw > /tmp/ec.input` to play audio\n" // + " `cat /tmp/ec.output > out.raw` to get recording audio\n" // + " Only support mono playback\n" // + ; volatile int g_is_quit = 0; @@ -47,6 +48,45 @@ void int_handler(int signal) g_is_quit = 1; } +void daemonize(void) +{ + pid_t pid, sid; + + /* Fork off the parent process */ + pid = fork(); + if (pid < 0) + { + printf("fork() failed\n"); + exit(1); + } + /* If we got a good PID, then + we can exit the parent process. */ + if (pid > 0) + { + exit(0); + } + + /* Change the file mode mask */ + umask(0); + + /* Open any logs here */ + + /* Create a new SID for the child process */ + sid = setsid(); + if (sid < 0) + { + printf("setsid() failed\n"); + exit(1); + } + + /* Change the current working directory */ + if ((chdir("/")) < 0) + { + printf("chdir() failed\n"); + exit(1); + } +} + int main(int argc, char *argv[]) { SpeexEchoState *echo_state; @@ -60,7 +100,7 @@ int main(int argc, char *argv[]) int opt = 0; int delay = 0; int save_audio = 0; - int daemonize = 0; + int daemon = 0; conf_t config = { .rec_pcm = "default", @@ -75,7 +115,7 @@ int main(int argc, char *argv[]) .buffer_size = 1024 * 16, .playback_fifo_size = 1024 * 4, .filter_length = 4096, - .bypass = 1 + .bypass = 1, }; while ((opt = getopt(argc, argv, "b:c:d:Df:hi:o:r:s")) != -1) @@ -93,7 +133,7 @@ int main(int argc, char *argv[]) delay = atoi(optarg); break; case 'D': - daemonize = 1; + daemon = 1; break; case 'f': config.filter_length = atoi(optarg); @@ -122,43 +162,9 @@ int main(int argc, char *argv[]) } } - if (daemonize) + if (daemon) { - pid_t pid, sid; - - /* Fork off the parent process */ - pid = fork(); - if (pid < 0) - { - printf("fork() failed\n"); - exit(1); - } - /* If we got a good PID, then - we can exit the parent process. */ - if (pid > 0) - { - exit(0); - } - - /* Change the file mode mask */ - umask(0); - - /* Open any logs here */ - - /* Create a new SID for the child process */ - sid = setsid(); - if (sid < 0) - { - printf("setsid() failed\n"); - exit(1); - } - - /* Change the current working directory */ - if ((chdir("/")) < 0) - { - printf("chdir() failed\n"); - exit(1); - } + daemonize(); } int frame_size = config.rate * 10 / 1000; // 10 ms @@ -193,10 +199,7 @@ int main(int argc, char *argv[]) sig_int_handler.sa_flags = 0; sigaction(SIGINT, &sig_int_handler, NULL); - echo_state = speex_echo_state_init_mc(frame_size, - config.filter_length, - config.rec_channels, - config.ref_channels); + echo_state = speex_echo_state_init_mc(frame_size, config.filter_length, config.rec_channels, config.ref_channels); speex_echo_ctl(echo_state, SPEEX_ECHO_SET_SAMPLING_RATE, &(config.rate)); playback_start(&config); @@ -205,7 +208,7 @@ int main(int argc, char *argv[]) printf("Running... Press Ctrl+C to exit\n"); - int timeout = 200 * 1000 * frame_size / config.rate; // ms + int timeout = 200 * 1000 * frame_size / config.rate; // ms // system delay between recording and playback printf("skip frames %d\n", capture_skip(delay)); diff --git a/src/ec_hw.c b/src/ec_hw.c index 3aa32dc..83a92cc 100644 --- a/src/ec_hw.c +++ b/src/ec_hw.c @@ -1,40 +1,45 @@ -// ec - echo canceller +// ec_hw - hardware echo canceller #include +#include +#include +#include +#include +#include +#include #include #include -#include #include -#include -#include -#include -#include #include +#include -#include - -#include "conf.h" #include "audio.h" +#include "auto_channel_detection.h" +#include "conf.h" -const char *usage = - "Usage:\n %s -c {input channels} -l {loopback channel} -m {mic channel list} [options]\n" - "Options:\n" - " -i PCM playback PCM (default)\n" - // " -o PCM capture PCM (default)\n" - " -r rate sample rate (16000)\n" - " -c channels input channels\n" - " -b size buffer size (262144)\n" - // " -d delay system delay between playback and capture (0)\n" - " -f filter_length AEC filter length (2048)\n" - " -l loopback loopback channel\n" - " -m mic_channels microphone channel list\n" - " -s save audio to /tmp/recording.raw and /tmp/out.raw\n" - " -D daemonize\n" - " -h display this help text\n" - "Note:\n" - " Echo Cancellation with loopback channel\n" - " `cat /tmp/ec.output > out.raw` to get recording audio\n" - " Only support mono playback\n"; +const char *usage = // + "Usage:\n %s -c {input channels} -l {loopback channel} -m {mic channel list} [options]\n" // + "Options:\n" // + " -a enable automatic channel detection\n" // + " -i PCM playback PCM (default)\n" // + " -r rate sample rate (16000)\n" // + " -c channels input channels\n" // + " -b size buffer size (262144)\n" // + " -f filter_length AEC filter length (2048)\n" // + " -l loopback loopback channel list (or count if -a flag was passed)\n" // + " -m mic_channels microphone channel list (or count if -a flag was passed)\n" // + " -s save audio to /tmp/recording.raw and /tmp/out.raw\n" // + " -p enable speex's audio preprocessing (by default only residual echo cancelling)\n" // + " -D daemonize\n" // + " -h display this help text\n" // + "Note:\n" // + " Echo Cancellation with loopback channel\n" // + " `cat /tmp/ec.output > out.raw` to get recording audio\n" // + " Only support mono playback\n" // + "Usage examples:\n" // + " ec_hw -i plughw:seeed8micvoicec -c 8 -l 6,7 -m 0,1,2,3,4,5 -f 160 -p\n" // + " ec_hw -i plughw:seeed8micvoicec -c 8 -l 2 -m 6 -a -f 160 -p\n" // + ; volatile int g_is_quit = 0; @@ -87,24 +92,29 @@ void daemonize(void) } } - int main(int argc, char *argv[]) { SpeexEchoState *echo_state; + SpeexPreprocessState **preprocess_state = NULL; int16_t *rec = NULL; int16_t *near = NULL; int16_t *far = NULL; int16_t *out = NULL; + int16_t *mono = NULL; FILE *fp_rec = NULL; FILE *fp_out = NULL; int opt = 0; // int delay = 0; int save_audio = 0; + int save_audio_channel_detection = 0; int daemon = 0; char *mic_list_str = NULL; int mic_list[32]; - int loopback_channel = -1; + char *loopback_list_str = NULL; + int loopback_list[32]; + int enable_auto_channel_detection = 0; + int preprocess_audio = 0; conf_t config = { .rec_pcm = "default", @@ -119,13 +129,16 @@ int main(int argc, char *argv[]) .buffer_size = 1024 * 16, .playback_fifo_size = 1024 * 4, .filter_length = 4096, - .bypass = 0 + .bypass = 0, }; - while ((opt = getopt(argc, argv, "b:c:d:Df:hi:l:m:o:r:s")) != -1) + while ((opt = getopt(argc, argv, "ab:c:d:Df:hi:l:m:o:r:sxp")) != -1) { switch (opt) { + case 'a': + enable_auto_channel_detection = 1; + break; case 'b': config.buffer_size = atoi(optarg); break; @@ -148,7 +161,7 @@ int main(int argc, char *argv[]) config.rec_pcm = optarg; break; case 'l': - loopback_channel = atoi(optarg); + loopback_list_str = optarg; // loopback channel break; case 'm': @@ -164,6 +177,12 @@ int main(int argc, char *argv[]) case 's': save_audio = 1; break; + case 'x': + save_audio_channel_detection = 1; + break; + case 'p': + preprocess_audio = 1; + break; case '?': printf("\n"); printf(usage, argv[0]); @@ -173,40 +192,93 @@ int main(int argc, char *argv[]) } } - if (config.rec_channels <= 0) { + if (config.rec_channels <= 0) + { printf("Input channels is not set, use '-c' to set one\n"); exit(-1); } - if (loopback_channel < 0 || loopback_channel >= config.rec_channels) { - printf("The loopback channel %d is not valid\n", loopback_channel); - exit(-1); + if (enable_auto_channel_detection) + { + char *end; + config.ref_channels = strtol(loopback_list_str, &end, 10); + if (strlen(end)) + { + fprintf(stderr, + "Found additional content while trying to detect loopback_list channel count\n" // + "Please provide a single number representing the total channels amount\n" // + "The additional part found was: %s\n", // + end); + exit(1); + } + config.out_channels = strtol(mic_list_str, &end, 10); + if (strlen(end)) + { + fprintf(stderr, + "Found additional content while trying to detect mic_list channel count\n" // + "Please provide a single number representing the total channels amount\n" // + "The additional part found was: %s\n", // + end); + exit(1); + } } + else + { + char *mic_channel_str = strtok(mic_list_str, ","); + config.out_channels = 0; + while (mic_channel_str != NULL) + { + int channel = atoi(mic_channel_str); + if (channel >= config.rec_channels) + { + printf("The channel number %d must be less than input channels %d\n", // + channel, config.rec_channels); + exit(-1); + } - char *mic_channel_str = strtok(mic_list_str, ","); - config.out_channels = 0; - while (mic_channel_str != NULL) { - int channel = atoi(mic_channel_str); - if (channel >= config.rec_channels) { - printf("The channel number %d must be less than input channels %d\n", channel, config.rec_channels); - exit(-1); - } + mic_list[config.out_channels] = channel; + config.out_channels++; - mic_list[config.out_channels] = channel; - config.out_channels++; + if (config.out_channels >= config.rec_channels) + { + printf("The output channels %d must be less than input channels %d\n", // + config.out_channels, config.rec_channels); + exit(-1); + } - if (config.out_channels >= config.rec_channels) { - printf("The output channels %d must be less than input channels %d\n", config.out_channels, config.rec_channels); - exit(-1); + mic_channel_str = strtok(NULL, ","); } - mic_channel_str = strtok(NULL, ","); + char *loopback_channel_str = strtok(loopback_list_str, ","); + config.ref_channels = 0; + while (loopback_channel_str != NULL) + { + int channel = atoi(loopback_channel_str); + if (channel >= config.rec_channels) + { + printf("The channel number %d must be less than input channels %d\n", // + channel, config.rec_channels); + exit(-1); + } + + loopback_list[config.ref_channels] = channel; + config.ref_channels++; + + if (config.ref_channels >= config.rec_channels) + { + printf("The output channels %d must be less than input channels %d\n", // + config.out_channels, config.rec_channels); + exit(-1); + } + + loopback_channel_str = strtok(NULL, ","); + } } - if (daemon) { + if (daemon) + { daemonize(); } - int frame_size = config.rate * 10 / 1000; // 10 ms @@ -240,35 +312,102 @@ int main(int argc, char *argv[]) sig_int_handler.sa_flags = 0; sigaction(SIGINT, &sig_int_handler, NULL); - echo_state = speex_echo_state_init_mc(frame_size, - config.filter_length, - config.out_channels, - config.ref_channels); + echo_state = speex_echo_state_init_mc(frame_size, config.filter_length, config.out_channels, config.ref_channels); speex_echo_ctl(echo_state, SPEEX_ECHO_SET_SAMPLING_RATE, &(config.rate)); capture_start(&config); fifo_setup(&config); + printf("rec_channels: %u out_channels: %u ref_channels: %u\n", config.rec_channels, config.out_channels, + config.ref_channels); + printf("Running... Press Ctrl+C to exit\n"); - int timeout = 200 * 1000 * frame_size / config.rate; // ms + int timeout = 200 * 1000 * frame_size / config.rate; // ms + + if (enable_auto_channel_detection) + { + int coherence_window_size_ms = 300; + int total_window_size_ms = 2000; + int envelope_ms = 100; + auto_channel_detection(&config, rec, mic_list, loopback_list, frame_size, timeout, coherence_window_size_ms, + total_window_size_ms, envelope_ms, save_audio_channel_detection); + } + + if (preprocess_audio) + { + mono = (int16_t *)calloc(frame_size, sizeof(int16_t)); + preprocess_state = (SpeexPreprocessState **)malloc(sizeof(SpeexPreprocessState *) * config.rec_channels); + + if (mono == NULL || preprocess_state == NULL) + { + printf("Fail to allocate memory\n"); + exit(1); + } + for (int i = 0; i < config.out_channels; i++) + { + preprocess_state[i] = speex_preprocess_state_init(frame_size, config.rate); + + speex_preprocess_ctl(preprocess_state[i], SPEEX_PREPROCESS_SET_ECHO_STATE, echo_state); + + // Feel free to experiment and enable any of the following. + // Parametrizing each of them is a lot of work and + // might need a more complex input system other than optarg(e.g. config file) + // float f; + // int n; + // n = 1; + // speex_preprocess_ctl(preprocess_state[i], SPEEX_PREPROCESS_SET_DENOISE, &n); + // n = 0; + // speex_preprocess_ctl(preprocess_state[i], SPEEX_PREPROCESS_SET_AGC, &n); + // n = 8000; + // speex_preprocess_ctl(preprocess_state[i], SPEEX_PREPROCESS_SET_AGC_LEVEL, &n); + // n = 0; + // speex_preprocess_ctl(preprocess_state[i], SPEEX_PREPROCESS_SET_DEREVERB, &n); + // f = .0; + // speex_preprocess_ctl(preprocess_state[i], SPEEX_PREPROCESS_SET_DEREVERB_DECAY, &f); + // f = .0; + // speex_preprocess_ctl(preprocess_state[i], SPEEX_PREPROCESS_SET_DEREVERB_LEVEL, &f); + } + } while (!g_is_quit) { capture_read(rec, frame_size, timeout); - for (int i=0; i