Add worker and local dev nugget tools

.gitignore

@@ -29,6 +29,8 @@ compile
 depcomp
 missing
 src/dispatcher
+src/rzb-dev
+src/rzb-worker
 aclocal.m4
 libtool
 ltmain.sh
@@ -47,6 +49,17 @@ tests/test_string_list
 tests/test_uuids
 tests/test_config_file
 tests/test_timer
+tests/test_config_next
+tests/test_health
+tests/test_message_body
+tests/test_messages_next
+tests/test_runtime_next
+tests/test_schema_conformance
+tests/test_fileserver
+tests/smoke_inspector_next
+tests/smoke_operational_next
+tests/smoke_source_next
+tests/.smoke-operation.json
 tests/test_json_buffer_primitives
 tests/test_json_buffer_structs
 tests/test_json_buffer_invalid
@@ -55,4 +68,6 @@ tests/*.log
 tests/*.trs
 test-driver
 razorback.pc
+pacp
+txt
 /\.idea

.gitmodules

@@ -17,3 +17,6 @@
 [submodule "common_m4"]
 	path = common_m4
 	url = ../razorback-common-m4.git
+[submodule "schemas/razorback"]
+	path = schemas/razorback
+	url = ../razorback-schemas.git

Dockerfile

@@ -51,3 +51,4 @@ RUN cp /razorback/etc/razorback/magic.sample /razorback/etc/razorback/magic \
     && printf '/razorback/lib\n' > /etc/ld.so.conf.d/razorback.conf \
     && ldconfig
 WORKDIR /razorback
+USER razorback

Makefile.am

@@ -20,3 +20,8 @@ EXTRA_DIST = LICENSE README.md SECURITY.md
 
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = razorback.pc
+
+.PHONY: smoke-dispatcher-next
+
+smoke-dispatcher-next:
+	$(MAKE) -C tests smoke-dispatcher-next

README.md

@@ -70,6 +70,32 @@ int main(void)
 }
 ```
 
+## Tooling
+
+The API build now installs two helper binaries for nugget development and
+operation.
+
+`rzb-worker` loads a single nugget module through the existing `initNug()` /
+`shutdownNug()` interface and keeps it running with the normal Razorback
+runtime:
+
+```bash
+src/rzb-worker --debug --health-port=8080 /path/to/nugget.so
+```
+
+`rzb-dev` also loads a nugget through `initNug()` / `shutdownNug()`, but it
+enables an internal local-only dev mode first so the created context does not
+require broker connections or other external Razorback services. It runs one
+file through the nugget's inspection hook and prints the resulting judgments
+and submissions:
+
+```bash
+src/rzb-dev --type=image/svg+xml /path/to/nugget.so /path/to/input.file
+```
+
+If `--type` is omitted, `rzb-dev` will infer the datatype only when the nugget
+registers exactly one inspection datatype.
+
 ## Health Checks
 
 Razorback provides a small process-wide health subsystem for library consumers

api.vcxproj

@@ -219,9 +219,10 @@
     <ClCompile Include="src\block_pool.c" />
     <ClCompile Include="src\command_and_control.c" />
     <ClCompile Include="src\config_file.c" />
-    <ClCompile Include="src\connected_entity.c" />
-    <ClCompile Include="src\crypto.c" />
-    <ClCompile Include="src\daemon.c" />
+    <ClCompile Include="src\connected_entity.c" />
+    <ClCompile Include="src\crypto.c" />
+    <ClCompile Include="src\daemon.c" />
+    <ClCompile Include="src\dev_mode.c" />
     <ClCompile Include="src\event.c" />
     <ClCompile Include="src\fantasia.c" />
     <ClCompile Include="src\hash.c" />

api.vcxproj.filters

@@ -120,12 +120,15 @@
     <ClCompile Include="src\string_list.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="src\submission.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="src\thread.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
+    <ClCompile Include="src\submission.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\dev_mode.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="src\thread.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="src\thread_pool.c">
       <Filter>Source Files</Filter>
     </ClCompile>

configure.ac

@@ -53,6 +53,23 @@ m4_include([common_m4/platform_flags.m4])
 # Pull in the dependancies on on the API (use the nugget way).
 m4_include([common_m4/api_deps.m4])
 
+AC_CHECK_LIB([z], [compress2], [], [ZLIB="no"])
+AS_IF([test "x$ZLIB" = "xno"],
+          [AC_MSG_ERROR("zlib library not found")]
+     )
+AC_CHECK_HEADERS([zlib.h], [], [ZLIB_HEADER="no"])
+AS_IF([test "x$ZLIB_HEADER" = "xno"],
+          [AC_MSG_ERROR("zlib header file not found")]
+     )
+AC_CHECK_LIB([yaml], [yaml_parser_initialize], [], [YAML="no"])
+AS_IF([test "x$YAML" = "xno"],
+          [AC_MSG_ERROR("libyaml library not found")]
+     )
+AC_CHECK_HEADERS([yaml.h], [], [YAML_HEADER="no"])
+AS_IF([test "x$YAML_HEADER" = "xno"],
+          [AC_MSG_ERROR("libyaml header file not found")]
+     )
+
 # Pull in the standard header and function checks.
 m4_include([common_m4/standard_headers_functions.m4])
 

debian/control

@@ -1,7 +1,7 @@
 Source: librazorback
 Priority: extra
 Maintainer: Tom Judge <tjudge@sourcefire.com>
-Build-Depends: debhelper (>= 7.0.50~), autotools-dev, uuid-dev, libmagic-dev, libssl-dev, libconfig8-dev, libpcre3-dev, libmysqlclient-dev, libcurl4-openssl-dev, cdbs
+Build-Depends: debhelper (>= 7.0.50~), autotools-dev, uuid-dev, libmagic-dev, libssl-dev, zlib1g-dev, libyaml-dev, libconfig8-dev, libpcre3-dev, libmysqlclient-dev, libcurl4-openssl-dev, cdbs
 Standards-Version: 3.9.1
 Section: libs
 Homepage: http://razorbacktm.sourceforge.net/

docker-bake.hcl

@@ -25,7 +25,7 @@ variable "BASE_IMAGE" {
 }
 
 variable "REPO" {
-    default = "harbor.vrt.sourcefire.com/pinkhat/"
+    default = "ghcr.io/cisco-talos/razorback/"
 }
 
 variable "VERSION" {

etc/api.conf.sample

@@ -5,8 +5,6 @@ Global: {
     HelloTime=2;
     DeadTime=10;
     MessageFormat="json";
-    TransferPassword="razorback";
-    TransferMode="traditional";
 };
 
 Locality: {

include/Makefile.am

@@ -21,12 +21,14 @@ razorback/api.h \
 razorback/block.h \
 razorback/block_id.h \
 razorback/block_pool.h \
+razorback/config_next.h \
 razorback/config_file.h \
 razorback/connected_entity.h \
 razorback/daemon.h \
 razorback/debug.h \
 razorback/event.h \
 razorback/file.h \
+razorback/fileserver.h \
 razorback.h \
 razorback/hash.h \
 razorback/health.h \
@@ -37,21 +39,23 @@ razorback/list.h \
 razorback/lock.h \
 razorback/log.h \
 razorback/message_formats.h \
+razorback/message_body.h \
+razorback/messages_next.h \
 razorback/messages.h \
 razorback/metadata.h \
 razorback/ntlv.h \
 razorback/nugget.h \
 razorback/queue.h \
 razorback/queue_list.h \
 razorback/response_queue.h \
+razorback/runtime_next.h \
 razorback/socket.h \
 razorback/string_list.h \
 razorback/submission.h \
 razorback/telemetry.h \
 razorback/thread.h \
 razorback/thread_pool.h \
 razorback/timer.h \
-razorback/transfer.h \
 razorback/types.h \
 razorback/uuids.h \
 razorback/visibility.h

include/razorback.h

@@ -27,16 +27,20 @@
 #include <razorback/block.h>
 #include <razorback/block_id.h>
 #include <razorback/block_pool.h>
+#include <razorback/config_next.h>
 #include <razorback/config_file.h>
 #include <razorback/daemon.h>
 #include <razorback/debug.h>
 #include <razorback/hash.h>
 #include <razorback/health.h>
 #include <razorback/log.h>
+#include <razorback/message_body.h>
+#include <razorback/messages_next.h>
 #include <razorback/messages.h>
 #include <razorback/ntlv.h>
 #include <razorback/queue.h>
 #include <razorback/queue_list.h>
+#include <razorback/runtime_next.h>
 #include <razorback/socket.h>
 #include <razorback/telemetry.h>
 #include <razorback/thread.h>

include/razorback/api.h

@@ -105,6 +105,7 @@ struct RazorbackCommandAndControlHooks
 
 
 #define CONTEXT_FLAG_STAND_ALONE 0x00000001
+#define CONTEXT_FLAG_DEV_TOOL    0x00000002
 
 /** API Context
  */
@@ -251,12 +252,6 @@ SO_PUBLIC extern bool Razorback_Output_Launch(
     struct RazorbackOutputHooks *hooks
 );
 
-/**
- * Get the transfer server protocol.
- * @return The password.
- */
-SO_PUBLIC extern char * Razorback_Get_Transfer_Password(void);
-
 /* Make APIs standardized while keeping function naming convention */
 #define RZB_Register_Collector          Razorback_Init_Collection_Context
 #define RZB_DataBlock_Create            BlockPool_CreateItem

src/transfer/core.h → include/razorback/config_next.h

@@ -16,32 +16,41 @@
  *  MA 02110-1301, USA.
  */
 
-#ifndef TRANSFER_CORE_H
-#define TRANSFER_CORE_H
-#include <razorback/types.h>
-#include <razorback/connected_entity.h>
-#include <razorback/transfer.h>
+/** @file config_next.h
+ * Dispatcher-next YAML/env configuration helper.
+ */
+#ifndef RAZORBACK_CONFIG_NEXT_H
+#define RAZORBACK_CONFIG_NEXT_H
 
+#include <razorback/types.h>
+#include <razorback/visibility.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+enum RzbNextConfigType
+{
+    RZB_NEXT_CONFIG_STRING = 0,
+    RZB_NEXT_CONFIG_INT = 1,
+    RZB_NEXT_CONFIG_BOOL = 2
+};
+
+struct RzbNextConfigKey
+{
+    const char *key;
+    enum RzbNextConfigType type;
+    void *dest;
+};
+
+SO_PUBLIC extern bool RzbNextConfig_Load(
+    const char *baseFile,
+    const char *localFile,
+    const char *envPrefix,
+    const struct RzbNextConfigKey *keys
+);
 
-
-char * Transfer_generateFilename (struct Block *block);
-
-bool  Transport_IsSupported(uint8_t protocol);
-enum TransferStatus Transfer_Store(struct BlockPoolItem *item, struct ConnectedEntity *dispatcher);
-enum TransferStatus Transfer_Fetch(struct Block *block, struct ConnectedEntity *dispatcher);
-void Transfer_Free(struct Block *block, struct ConnectedEntity *dispatcher);
-
-
-// Init functions
-bool File_Init(void);
-bool SSH_Init(void);
-bool HTTP_Init(void);
 #ifdef __cplusplus
 }
 #endif
-#endif
+#endif /* RAZORBACK_CONFIG_NEXT_H */