Added TLS Block Analysis (#3016)

* Enabled TLS block analysis via --cfg=tls,blocks_analysis,1

* Added comment and optimization

* Updated output format

* Code cleanup

Author: lucaderi

example/ndpiReader.c

@@ -2513,9 +2513,8 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
 
     char unknown_cipher[8];
     if(flow->ssh_tls.server_cipher != '\0')
-      {
-	fprintf(out, "[Cipher: %s]", ndpi_cipher2str(flow->ssh_tls.server_cipher, unknown_cipher));
-      }
+      fprintf(out, "[Cipher: %s]", ndpi_cipher2str(flow->ssh_tls.server_cipher, unknown_cipher));
+    
     if(flow->bittorent_hash != NULL) fprintf(out, "[BT Hash: %s]", flow->bittorent_hash);
     if(flow->dhcp_fingerprint != NULL) fprintf(out, "[DHCP Fingerprint: %s]", flow->dhcp_fingerprint);
     if(flow->dhcp_class_ident) fprintf(out, "[DHCP Class Ident: %s]",
@@ -2531,6 +2530,17 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
     print_bin(out, "Plen Bins", &flow->payload_len_bin);
 #endif
 
+    if(flow->ssh_tls.num_blocks > 0) {
+      int i;
+
+      fprintf(out, "[TLS blocks: ");
+
+      for(i=0; i<flow->ssh_tls.num_blocks; i++)
+	fprintf(out, "%s%u/%d", (i > 0) ? "," : "", flow->ssh_tls.blocks[i].block_type, flow->ssh_tls.blocks[i].len);
+
+      fprintf(out, "]");
+    }
+
     if(flow->flow_payload && (flow->flow_payload_len > 0)) {
       u_int i;
 

example/reader_util.c

@@ -420,7 +420,7 @@ struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * pref
     LOG(NDPI_LOG_ERROR, "global structure initialization failed\n");
     return NULL;
   }
-  
+
   workflow = ndpi_calloc(1, sizeof(struct ndpi_workflow));
   if(workflow == NULL) {
     LOG(NDPI_LOG_ERROR, "global structure initialization failed\n");
@@ -1539,7 +1539,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
 
     if(flow->ndpi_flow->ndpi.fingerprint)
       flow->ndpi_fingerprint = ndpi_strdup(flow->ndpi_flow->ndpi.fingerprint);
-	
+
     if(flow->ndpi_flow->protos.tls_quic.ja4_client_raw)
       flow->ssh_tls.ja4_client_raw = ndpi_strdup(flow->ndpi_flow->protos.tls_quic.ja4_client_raw);
 
@@ -1579,16 +1579,21 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
 	correct_csv_data_field(flow->ssh_tls.negotiated_alpn);
     }
 
-    if(enable_doh_dot_detection) {
-      /* For TLS we use TLS block lenght instead of payload lenght */
-      ndpi_reset_bin(&flow->payload_len_bin);
-
-      for(i=0; i<flow->ndpi_flow->l4.tcp.tls.num_tls_blocks; i++) {
-	u_int16_t len = abs(flow->ndpi_flow->l4.tcp.tls.tls_application_blocks_len[i]);
-
-	/* printf("[TLS_LEN] %u\n", len); */
-	ndpi_inc_bin(&flow->payload_len_bin, plen2slot(len), 1);
+    if(flow->protocol == IPPROTO_TCP) {
+      if(enable_doh_dot_detection) {
+	/* For TLS we use TLS block lenght instead of payload lenght */
+	ndpi_reset_bin(&flow->payload_len_bin);
+	
+	for(i=0; i<flow->ndpi_flow->l4.tcp.tls.num_tls_blocks; i++) {
+	  u_int16_t len = abs(flow->ndpi_flow->l4.tcp.tls.tls_blocks[i].len);
+	  
+	  /* printf("[TLS_LEN] %u\n", len); */
+	  ndpi_inc_bin(&flow->payload_len_bin, plen2slot(len), 1);
+	}
       }
+      
+      flow->ssh_tls.num_blocks = flow->ndpi_flow->l4.tcp.tls.num_tls_blocks;
+      memcpy(flow->ssh_tls.blocks, flow->ndpi_flow->l4.tcp.tls.tls_blocks, sizeof(flow->ndpi_flow->l4.tcp.tls.tls_blocks));
     }
   }
   /* FASTCGI */
@@ -1616,14 +1621,14 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
 	     ndpi_print_os_hint(flow->ndpi_flow->tcp.os_hint));
     flow->tcp_fingerprint = ndpi_strdup(buf);
   }
-  
+
   /* HTTP metadata are "global" not in `flow->ndpi_flow->protos` union; for example, we can have
      HTTP/BitTorrent and in that case we want to export also HTTP attributes */
   if(ndpi_stack_is_http_like(&flow->detected_protocol.protocol_stack)) { /* HTTP, HTTP_PROXY, HTTP_CONNECT */
     if(flow->ndpi_flow->http.url != NULL) {
       ndpi_snprintf(flow->http.url, sizeof(flow->http.url), "%s", flow->ndpi_flow->http.url);
     }
-    
+
     flow->http.response_status_code = flow->ndpi_flow->http.response_status_code;
     ndpi_snprintf(flow->http.content_type, sizeof(flow->http.content_type), "%s", flow->ndpi_flow->http.content_type ? flow->ndpi_flow->http.content_type : "");
     ndpi_snprintf(flow->http.server, sizeof(flow->http.server), "%s", flow->ndpi_flow->http.server ? flow->ndpi_flow->http.server : "");
@@ -1636,7 +1641,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
 
   if(ndpi_stack_contains(&flow->detected_protocol.protocol_stack, NDPI_PROTOCOL_RTP))
     memcpy(&flow->rtp, &flow->ndpi_flow->rtp, sizeof(flow->rtp));
-     
+
   ndpi_snprintf(flow->http.user_agent,
                 sizeof(flow->http.user_agent),
                 "%s", (flow->ndpi_flow->http.user_agent ? flow->ndpi_flow->http.user_agent : ""));

example/reader_util.h

@@ -318,6 +318,9 @@ typedef struct ndpi_flow_info {
     ndpi_cipher_weakness client_unsafe_cipher, server_unsafe_cipher;
 
     u_int32_t quic_version;
+
+    u_int8_t num_blocks;
+    struct ndpi_tls_block blocks[NDPI_MAX_NUM_TLS_APPL_BLOCKS];
   } ssh_tls;
 
   struct {
@@ -357,7 +360,7 @@ typedef struct ndpi_flow_info {
 #else
   struct ndpi_bin payload_len_bin;
 #endif
-
+  
   /* Flow payload */
   u_int16_t flow_payload_len;
   char *flow_payload;  

src/include/ndpi_private.h

@@ -299,7 +299,7 @@ struct ndpi_detection_module_config_struct {
   int tls_ja4c_fingerprint_enabled;
   int tls_ja4r_fingerprint_enabled;
   int tls_subclassification_enabled;
-
+  int tls_blocks_analysis_enabled;
   int quic_subclassification_enabled;
 
   int smtp_opportunistic_tls_enabled;

src/include/ndpi_typedefs.h

@@ -830,9 +830,13 @@ struct ndpi_lru_cache {
 #define NDPI_HEURISTICS_TLS_OBFUSCATED_TLS		0x02 /* Enable heuristic to detect proxied/obfuscated TLS flows over TLS tunnels, i.e. TLS over TLS */
 #define NDPI_HEURISTICS_TLS_OBFUSCATED_HTTP		0x04 /* Enable heuristic to detect proxied/obfuscated TLS flows over HTTP/WebSocket */
 
-
 /* ************************************************** */
 
+struct ndpi_tls_block {
+  u_int8_t block_type; /* + = src->dst, - = dst->src */
+  int16_t len;
+};
+
 struct ndpi_flow_tcp_struct {
   /* TCP sequence number */
   u_int32_t next_tcp_seq_nr[2];
@@ -854,12 +858,10 @@ struct ndpi_flow_tcp_struct {
   struct {
     /* NDPI_PROTOCOL_TLS */
     u_int8_t app_data_seen[2];
-    u_int8_t num_tls_blocks;
-    int16_t tls_application_blocks_len[NDPI_MAX_NUM_TLS_APPL_BLOCKS]; /* + = src->dst, - = dst->src */
+    u_int8_t num_tls_blocks, num_processed_tls_blocks /* used internally for dissection */;
+    struct ndpi_tls_block tls_blocks[NDPI_MAX_NUM_TLS_APPL_BLOCKS];
   } tls;
 
-
-
   /* NDPI_PROTOCOL_MAIL_SMTP */
   u_int16_t smtp_command_bitmask;
 

src/lib/ndpi_main.c

@@ -13433,6 +13433,7 @@ static const struct cfg_param {
   { "tls",           "metadata.ja4c_fingerprint",               "enable", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(tls_ja4c_fingerprint_enabled), NULL },
   { "tls",           "metadata.ja4r_fingerprint",               "disable", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(tls_ja4r_fingerprint_enabled), NULL },
   { "tls",           "subclassification",                       "enable", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(tls_subclassification_enabled), NULL },
+  { "tls",           "blocks_analysis",                         "disable", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(tls_blocks_analysis_enabled), NULL },
 
   { "quic",          "subclassification",                       "enable", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(quic_subclassification_enabled), NULL },
 

src/lib/protocols/tls.c

@@ -123,8 +123,10 @@ static bool str_contains_digit(char *str) {
 
 /* TODO: rename */
 static int keep_extra_dissection_tcp(struct ndpi_detection_module_struct *ndpi_struct,
-                                     struct ndpi_flow_struct *flow)
-{
+                                     struct ndpi_flow_struct *flow) {
+  if(ndpi_struct->cfg.tls_blocks_analysis_enabled)
+    return(1); /* Process as much TLS blocks as the max packet number */
+  
   /* Common path: found handshake on both directions */
   if(
      (flow->tls_quic.certificate_processed == 1 && flow->protos.tls_quic.client_hello_processed)
@@ -1245,7 +1247,7 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct,
   }
 
   if((ndpi_struct->num_tls_blocks_to_follow != 0)
-     && (flow->l4.tcp.tls.num_tls_blocks >= ndpi_struct->num_tls_blocks_to_follow)) {
+     && (flow->l4.tcp.tls.num_processed_tls_blocks >= ndpi_struct->num_tls_blocks_to_follow)) {
 #ifdef DEBUG_TLS_BLOCKS
     printf("*** [TLS Block] Enough blocks dissected\n");
 #endif
@@ -1412,6 +1414,23 @@ int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 
     content_type = message->buffer[0];
 
+    if(ndpi_struct->cfg.tls_blocks_analysis_enabled) {
+      if(flow->l4.tcp.tls.num_tls_blocks < NDPI_MAX_NUM_TLS_APPL_BLOCKS) {
+	int16_t blen = len-5;
+	
+	/* Use positive values for c->s and negative for s->c */
+	if(packet->packet_direction != 0) blen = -blen;
+	
+	flow->l4.tcp.tls.tls_blocks[flow->l4.tcp.tls.num_tls_blocks].len = blen;
+	flow->l4.tcp.tls.tls_blocks[flow->l4.tcp.tls.num_tls_blocks++].block_type = content_type;
+	
+#ifdef DEBUG_TLS_BLOCKS
+	printf("*** [TLS Block] [len: %u][num_tls_blocks: %u/%u]\n",
+	       len-5, flow->l4.tcp.tls.num_tls_blocks, ndpi_struct->num_tls_blocks_to_follow);
+#endif
+      }
+    }
+
     /* Overwriting packet payload */
     p = packet->payload;
     p_len = packet->payload_packet_len; /* Backup */
@@ -1423,7 +1442,7 @@ int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	  so in this case we reset the number of observed
 	  TLS blocks
 	*/
-	flow->l4.tcp.tls.num_tls_blocks = 0;
+	flow->l4.tcp.tls.num_processed_tls_blocks = 0;
       }
       if(len == 6 &&
          message->buffer[1] == 0x03 && /* TLS >= 1.0 */
@@ -1439,8 +1458,15 @@ int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct,
         ndpi_int_tls_add_connection(ndpi_struct, flow);
         flow->l4.tcp.tls.app_data_seen[packet->packet_direction] = 1;
         /* Further data is encrypted so we are not able to parse it without
-           erros and without setting `something_went_wrong` variable */
-        break;
+           errors and without setting `something_went_wrong` variable */
+
+	if(!ndpi_struct->cfg.tls_blocks_analysis_enabled) {
+	  /*
+	    In case of TLS blocks analysis we want to analize all the blocks
+	    whereas in "standard" mode we can use this shortcut and break
+	  */
+	  break;
+	}
       }
     } else if(content_type == 0x15 /* Alert */) {
       /* https://techcommunity.microsoft.com/t5/iis-support-blog/ssl-tls-alert-protocol-and-the-alert-codes/ba-p/377132 */
@@ -1513,22 +1539,6 @@ int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	flow->l4.tcp.tls.app_data_seen[packet->packet_direction] = 1;
 	if(flow->l4.tcp.tls.app_data_seen[!packet->packet_direction] == 1)
 	  flow->tls_quic.certificate_processed = 1;
-
-	if(flow->tls_quic.certificate_processed) {
-	  if(flow->l4.tcp.tls.num_tls_blocks < ndpi_struct->num_tls_blocks_to_follow) {
-	    int16_t blen = len-5;
-
-	    /* Use positive values for c->s e negative for s->c */
-	    if(packet->packet_direction != 0) blen = -blen;
-
-	    flow->l4.tcp.tls.tls_application_blocks_len[flow->l4.tcp.tls.num_tls_blocks++] = blen;
-	  }
-
-#ifdef DEBUG_TLS_BLOCKS
-	  printf("*** [TLS Block] [len: %u][num_tls_blocks: %u/%u]\n",
-		 len-5, flow->l4.tcp.tls.num_tls_blocks, ndpi_struct->num_tls_blocks_to_follow);
-#endif
-	}
       }
     }
 
@@ -1552,7 +1562,7 @@ int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 
   if(something_went_wrong
      || ((ndpi_struct->num_tls_blocks_to_follow > 0)
-	 && (flow->l4.tcp.tls.num_tls_blocks == ndpi_struct->num_tls_blocks_to_follow))
+	 && (flow->l4.tcp.tls.num_processed_tls_blocks == ndpi_struct->num_tls_blocks_to_follow))
      || ((ndpi_struct->num_tls_blocks_to_follow == 0)
 	 && (!keep_extra_dissection_tcp(ndpi_struct, flow)))
      ) {