Initial commit

.gitignore

@@ -0,0 +1 @@
+*~

Makefile

@@ -0,0 +1,10 @@
+
+CXXFLAGS = -Wall -O3 -Iinclude
+
+TARGETS = local_lat remote_lat local_thr remote_thr
+
+all: $(TARGETS)
+
+clean:
+	rm -f *~
+	rm -f $(TARGETS)

README

@@ -0,0 +1,45 @@
+NanoMQ
+======
+
+NanoMQ is a ultra low latency messaging kernel. It enables messaging
+between processes in much the same way as POSIX message queues but at
+sub-microsecond latencies. NanoMQ uses efficient wait-free ring
+buffers arranged in a complete graph. Each node can send messages to
+any other node, receiving nodes needs to exclusively own a CPU core or
+HyperThread. The ultra low latency can thus be achieved by avoiding
+context switches.
+
+Building
+--------
+
+Just run make. Requires recent GCC.
+
+Performance
+-----------
+
+On my Intel(R) Core(TM)2 Duo CPU T7500 @ 2.20GHz I get an average RTT
+of 250 ns and a throughput of 13M msg/s for a two node setup with 100
+byte messages. It will be interesting to see measurements on multi CPU
+systems and the latency depending on which cache the cores share.
+
+Use case
+--------
+
+In high frequency trading (HFT) systems you want to separate feed
+handlers and order management systems (OMS) from strategy code in
+order to increase fault tolerancy and support live deployment of bug
+fixes or new strategies. NanoMQ allows you to separate these parts of
+a trading system into separate processes while keeping communication
+latencies to a fraction of a microsecond.
+
+Resources
+---------
+
+Git repository: http://github.com/rigtorp/nanomq
+
+Copying
+-------
+
+Free use of this software is granted under the terms of the GNU
+General Public License (GPL). For details see the file `COPYING`
+included with the NanoMQ distribution.

TODO

@@ -0,0 +1,5 @@
+* Fair receiving when receiving from all nodes
+* Monitoring tool
+  * Monitor message throughput
+  * Ping nodes to check if they are running
+* Message framing, ie send message length

include/bones/barrier-x86.h

@@ -0,0 +1,41 @@
+/*
+   Copyright (C) 2007,2008 Qualcomm Incorporated. All rights reserved.
+   Written by Max Krasnyansky <[email protected]>
+
+   This file is part the Bones library. It is licensed under
+   Boost Software License - Version 1.0 - August 17th, 2003
+
+   Permission is hereby granted, free of charge, to any person or organization
+   obtaining a copy of the software and accompanying documentation covered by
+   this license (the "Software") to use, reproduce, display, distribute,
+   execute, and transmit the Software, and to prepare derivative works of the
+   Software, and to permit third-parties to whom the Software is furnished to
+   do so, all subject to the following:
+
+   The copyright notices in the Software and this entire statement, including
+   the above license grant, this restriction and the following disclaimer,
+   must be included in all copies of the Software, in whole or in part, and
+   all derivative works of the Software, unless such copies or derivative
+   works are solely in the form of machine-executable object code generated by
+   a source language processor.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+   SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+   FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+   DEALINGS IN THE SOFTWARE.
+*/
+
+/**
+ * @file bones/barrier-x86.h
+ * X86 (32 and 64 bit) specific barriers.
+ * @warning do not include directly. @see bones/barrier.h
+ */
+
+// Memory barriers
+// This version requires SSE capable CPU.
+static inline void force_inline memrw() { asm volatile("mfence":::"memory"); }
+static inline void force_inline memr()  { asm volatile("lfence":::"memory"); }
+static inline void force_inline memw()  { asm volatile("sfence":::"memory"); }

include/bones/barrier.h

@@ -0,0 +1,61 @@
+/*
+   Copyright (C) 2007,2008 Qualcomm Incorporated. All rights reserved.
+   Written by Max Krasnyansky <[email protected]>
+
+   This file is part the Bones library. It is licensed under
+   Boost Software License - Version 1.0 - August 17th, 2003
+
+   Permission is hereby granted, free of charge, to any person or organization
+   obtaining a copy of the software and accompanying documentation covered by
+   this license (the "Software") to use, reproduce, display, distribute,
+   execute, and transmit the Software, and to prepare derivative works of the
+   Software, and to permit third-parties to whom the Software is furnished to
+   do so, all subject to the following:
+
+   The copyright notices in the Software and this entire statement, including
+   the above license grant, this restriction and the following disclaimer,
+   must be included in all copies of the Software, in whole or in part, and
+   all derivative works of the Software, unless such copies or derivative
+   works are solely in the form of machine-executable object code generated by
+   a source language processor.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+   SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+   FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+   DEALINGS IN THE SOFTWARE.
+*/
+
+/**
+ * @file bones/barrier.h
+ * Compiler and memory barriers.
+ */
+
+#ifndef BONES2_BARRIER_H
+#define BONES2_BARRIER_H
+
+#include <bones/compiler.h>
+
+namespace bones {
+namespace barrier {
+
+// We're using inline function here instead of #defines to avoid 
+// name space clashes.
+
+/**
+ * Compiler barrier
+ */
+static inline void force_inline comp() { asm volatile("": : :"memory"); }
+
+#if (defined(__i386__) || defined(__x86_64__))
+#include <bones/barrier-x86.h>
+#else
+#error Unsupported CPU
+#endif
+
+} // namespace barrier
+} // namespace bones
+
+#endif // BONES2_BARRIER_H

include/bones/compiler.h

@@ -0,0 +1,57 @@
+/*
+   Copyright (C) 2007,2008 Qualcomm Incorporated. All rights reserved.
+   Written by Max Krasnyansky <[email protected]>
+
+   This file is part the Bones library. It is licensed under
+   Boost Software License - Version 1.0 - August 17th, 2003
+
+   Permission is hereby granted, free of charge, to any person or organization
+   obtaining a copy of the software and accompanying documentation covered by
+   this license (the "Software") to use, reproduce, display, distribute,
+   execute, and transmit the Software, and to prepare derivative works of the
+   Software, and to permit third-parties to whom the Software is furnished to
+   do so, all subject to the following:
+
+   The copyright notices in the Software and this entire statement, including
+   the above license grant, this restriction and the following disclaimer,
+   must be included in all copies of the Software, in whole or in part, and
+   all derivative works of the Software, unless such copies or derivative
+   works are solely in the form of machine-executable object code generated by
+   a source language processor.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+   SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+   FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+   DEALINGS IN THE SOFTWARE.
+*/
+
+/**
+ * @file bones/compiler.h
+ * Compiler related macros, shortcuts, etc.
+ */
+
+#ifndef BONES2_COMPILER_H
+#define BONES2_COMPILER_H
+
+// Note: Currently supported compilers are:
+// 	GCC 4.0 and above
+// 	Intel C++ compiler
+
+// Branch likelyhood links
+#ifndef likely
+#define likely(x)    __builtin_expect(!!(x), 1)
+#endif
+
+#ifndef unlikely
+#define unlikely(x)  __builtin_expect(!!(x), 0)
+#endif
+
+// Forced inlining 
+#ifndef force_inline
+#define force_inline __attribute__ ((__always_inline__))
+#endif
+
+#endif // BONES2_COMPILER_H

include/bones/cpu-x86.h

@@ -0,0 +1,62 @@
+/*
+   Copyright (C) 2007,2008 Qualcomm Incorporated. All rights reserved.
+   Written by Max Krasnyansky <[email protected]>
+
+   This file is part the Bones library. It is licensed under
+   Boost Software License - Version 1.0 - August 17th, 2003
+
+   Permission is hereby granted, free of charge, to any person or organization
+   obtaining a copy of the software and accompanying documentation covered by
+   this license (the "Software") to use, reproduce, display, distribute,
+   execute, and transmit the Software, and to prepare derivative works of the
+   Software, and to permit third-parties to whom the Software is furnished to
+   do so, all subject to the following:
+
+   The copyright notices in the Software and this entire statement, including
+   the above license grant, this restriction and the following disclaimer,
+   must be included in all copies of the Software, in whole or in part, and
+   all derivative works of the Software, unless such copies or derivative
+   works are solely in the form of machine-executable object code generated by
+   a source language processor.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+   SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+   FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+   DEALINGS IN THE SOFTWARE.
+*/
+
+/**
+ * @file bones/cpu-x86.h
+ * X86 (32 and 64 bit) specific parts of the CPU library.
+ * @see bones/cpu.h
+ */
+
+// mfence ensures that tsc reads are properly serialized
+// On Intel chips it's actually enough to just do lfence but
+// that would require some conditional logic. 
+
+#if defined __x86_64__
+static inline uint64_t __tsc(void)
+{
+   unsigned int a, d;
+   asm volatile ("rdtsc" : "=a" (a), "=d"(d));
+   return ((unsigned long) a) | (((unsigned long) d) << 32);
+}
+
+#else
+
+static inline uint64_t __tsc(void)
+{
+   uint64_t c;
+   asm volatile ("rdtsc" : "=A" (c));
+   return c;
+}
+
+#endif
+
+static inline void __relax()  { asm volatile ("pause":::"memory"); } 
+static inline void __lock()   { asm volatile ("cli" ::: "memory"); }
+static inline void __unlock() { asm volatile ("sti" ::: "memory"); }