GenWQE: Collect temporary bytes from dictionary buffer

Signed-off-by: Frank Haverkamp <[email protected]>

include/libzHW.h

@@ -480,7 +480,10 @@ int zedc_inflateGetHeader(zedc_streamp strm, gzedc_headerp head);
 int zedc_inflateSaveBuffers(zedc_streamp strm, const char *prefix);
 void zedc_lib_debug(int onoff);	/* debug outputs on/off */
 void zedc_set_logfile(FILE *logfile);
+
 int zedc_inflate_pending_output(struct zedc_stream_s *strm);
+int zedc_read_pending_output(struct zedc_stream_s *strm,
+			uint8_t *buf, unsigned int len);
 
 /**
  * The application can compare zedc_Version and ZEDC_VERSION for

lib/hardware.c

@@ -79,7 +79,7 @@ struct hw_state {
  */
 static int output_buffer_empty(struct hw_state *s)
 {
-	return s->obuf_avail == s->obuf_total;
+	return (s->obuf_avail == s->obuf_total);
 }
 
 /**
@@ -1424,9 +1424,9 @@ int h_inflate(z_streamp strm, int flush)
 	/* No progress possible (no more input and no buffered output):
 	   Z_BUF_ERROR */
 	obuf_bytes = s->obuf - s->obuf_next; /* bytes in obuf */
-	if (obuf_bytes == 0) {
-		hw_trace("[%p] OBYTES_IN_DICT %d bytes\n", strm,
-			 h->obytes_in_dict);
+	if ((obuf_bytes == 0) && (zedc_inflate_pending_output(h) == 0)) {
+		hw_trace("[%p] OBYTES_IN_DICT %d bytes (1) This must be 0!\n",
+			strm, h->obytes_in_dict);
 
 		if (s->rc == Z_STREAM_END)   /* hardware saw FEOB */
 			return Z_STREAM_END; /* nothing to do anymore */
@@ -1453,46 +1453,33 @@ int h_inflate(z_streamp strm, int flush)
 		/* Give out what is already there */
 		obuf_bytes = h_flush_obuf(strm);
 
-		if ((s->rc == Z_STREAM_END) &&	/* hardware saw FEOB */
-		    (obuf_bytes == 0))		/* no more output in buf */
-			return Z_STREAM_END;	/* nothing to do anymore */
-
-		if (((obuf_bytes != 0) || zedc_inflate_pending_output(h)) &&
-		    (strm->avail_out == 0))
-			return Z_OK;		/* need new output buffer */
+		if ((s->rc == Z_STREAM_END) &&	/* hardware/sw saw FEOB */
+		    (obuf_bytes == 0)) {	/* no more output in buf */
+			unsigned int rem_bytes;
+
+			/* no more output in temp? */
+			rc = zedc_read_pending_output(h, strm->next_out,
+						strm->avail_out);
+			if (rc < 0) {
+				hw_trace("[%s] err: Read temp buffer rc=%d!\n",
+					__func__, rc);
+				return rc;
+			}
 
-#if 0
-/*
- *  FIXME We have put this now behind the hardware decompression, where
- * it should have been already from the beginning. Wonder if that
- * is passing all the testcases. If it does, we can hopefully remove
- * this.
- */
-		/*
-		 * Need more output space, just useful if Z_STREAM_END
-		 * not seen before.
-		 */
-		if ((s->rc != Z_STREAM_END) && (strm->avail_out == 0)) {
-			rc = Z_OK;
+			hw_trace("[%s] collected %d bytes from dict buffer\n",
+				__func__, rc);
+			strm->avail_out -= rc;
+			strm->total_out += rc;
 
-#ifdef CONFIG_CIRCUMVENTION_FOR_Z_STREAM_END	/* For MongoDB PoC */
+			rem_bytes = zedc_inflate_pending_output(h);
+			if (rem_bytes != 0)
+				return Z_OK;	/* call me again */
 
-			rc = __check_stream_end(strm);
-			if (rc == Z_STREAM_END) {
-				hw_trace("    Suppress Z_STREAM_END %zd %zd (1)\n",
-					 s->obuf_avail, s->obuf_total);
-				s->rc = Z_STREAM_END;
-				rc = Z_OK;
-			}
-
-			hw_trace("[%p] .......... flush=%s avail_in=%d "
-				 "avail_out=%d __check_stream=%s (1)\n", strm,
-				 flush_to_str(flush), strm->avail_in,
-				 strm->avail_out, ret_to_str(rc));
-#endif
-			return rc;
+			return Z_STREAM_END;	/* nothing to do anymore */
 		}
-#endif
+		if (((obuf_bytes != 0) || zedc_inflate_pending_output(h)) &&
+		    (strm->avail_out == 0))
+			return Z_OK;		/* need new output buffer */
 
 		/*
 		 * Original idea: Do not send 0 data to HW
@@ -1568,8 +1555,10 @@ int h_inflate(z_streamp strm, int flush)
 #ifdef CONFIG_CIRCUMVENTION_FOR_Z_STREAM_END	/* For MongoDB PoC */
 		/* FIXME Experimental check for Z_STREAM_END here */
 		if ((s->rc != Z_STREAM_END) && (strm->avail_out == 0)) {
-			rc = __check_stream_end(strm);
-			if (rc == Z_STREAM_END) {
+			int _rc;
+
+			_rc = __check_stream_end(strm);
+			if (_rc == Z_STREAM_END) {
 				hw_trace("    Suppress Z_STREAM_END %zd %zd (2)\n",
 					 s->obuf_avail, s->obuf_total);
 				s->rc = Z_STREAM_END;
@@ -1581,8 +1570,12 @@ int h_inflate(z_streamp strm, int flush)
 		}
 #endif
 		/* Hardware saw FEOB and output buffer is empty */
-		if ((s->rc == Z_STREAM_END) && output_buffer_empty(s))
+		if ((s->rc == Z_STREAM_END) &&  output_buffer_empty(s) &&
+		    (zedc_inflate_pending_output(h) == 0)) {
+			hw_trace("[%p] OBYTES_IN_DICT %d bytes (2) Must be 0!\n",
+				strm, h->obytes_in_dict);
 			return Z_STREAM_END;	/* nothing to do anymore */
+		}
 
 		if (strm->avail_out == 0)	/* need more output space */
 			return Z_OK;

lib/inflate.c

@@ -768,6 +768,37 @@ int zedc_inflate_pending_output(struct zedc_stream_s *strm)
 	return strm->obytes_in_dict;
 }
 
+/**
+ * @brief	Enable wrapper code to access internal buffer.
+ * 		If data is left from previous task due to insufficent
+ *		output buffer space, this data must first be stored
+ *		to the new output buffer.
+ * @param strm	decompression job context
+ */
+int zedc_read_pending_output(struct zedc_stream_s *strm,
+			uint8_t *buf, unsigned int len)
+{
+	uint8_t *pdict;
+	unsigned int _len = 0;
+
+	if (strm->obytes_in_dict == 0)
+		return ZEDC_OK;
+	if (strm->dict_len < strm->obytes_in_dict)
+		return ZEDC_ERR_DICT_OVERRUN;
+
+	/* obytes at end of dict */
+	pdict = strm->wsp->dict[strm->wsp_page] +
+		strm->out_dict_offs + strm->dict_len - strm->obytes_in_dict;
+
+	while (len && strm->obytes_in_dict) {
+		*buf++ = *pdict++;
+		strm->obytes_in_dict--;
+		len--;
+		_len++;
+	}
+	return _len;
+}
+
 /**
  * @brief	If data is left from previous task due to insufficent
  *		output buffer space, this data must first be stored

misc/zpipe_append.c

@@ -3,7 +3,7 @@
    Version 1.4	11 December 2005  Mark Adler */
 
 /*
- * Copyright 2016, International Business Machines
+ * Copyright 2016, 2017 International Business Machines
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -375,13 +375,14 @@ static void usage(char *prog)
 
 	fprintf(stderr, "%s usage: %s [-h] [-v]\n"
 		"    [-F, --format <ZLIB|DEFLATE|GZIP>]\n"
-		"    [-e, --excact-input]  input matches size of data\n"
+		"    [-e, --excact-input] input matches size of data\n"
 		"    [-E, --excact-output] output matches size of data\n"
 		"    [-f, --fush <Z_NO_FLUSH|Z_PARTIAL_FLUSH|Z_FULL_FLUSH>]\n"
 		"    [-i, --i_bufsize <i_bufsize>]\n"
 		"    [-o, --o_bufsize <o_bufsize>]\n"
 		"    [-p, --pattern <pattern>] pattern to generate test-data\n"
-		"    [-s, --size <data-size>]\n",
+		"    [-s, --size <data-size>]\n"
+		"    [-k, --keep] do not delete resulting files\n",
 		b, b);
 }
 
@@ -400,6 +401,7 @@ int main(int argc, char **argv)
 	int exact_input = 0, exact_output = 0;
 	size_t size = 256 * 1024;
 	int expect_z_stream_end = 0;
+	int keep = 0;
 
 	_pattern = getpid();
 
@@ -419,12 +421,13 @@ int main(int argc, char **argv)
 			{ "o_bufsize",	 required_argument, NULL, 'o' },
 			{ "size",	 required_argument, NULL, 's' },
 			{ "pattern",	 required_argument, NULL, 'p' },
+			{ "keep",	 no_argument,       NULL, 'k' },
 			{ "verbose",	 no_argument,	    NULL, 'v' },
 			{ "help",	 no_argument,	    NULL, 'h' },
 			{ 0,		 no_argument,	    NULL, 0   },
 		};
 
-		ch = getopt_long(argc, argv, "F:f:Eei:o:s:p:vh?",
+		ch = getopt_long(argc, argv, "kF:f:Eei:o:s:p:vh?",
 				 long_options, &option_index);
 		if (ch == -1)    /* all params processed ? */
 			break;
@@ -450,6 +453,9 @@ int main(int argc, char **argv)
 		case 'E':
 			exact_output = 1;
 			break;
+		case 'k':
+			keep = 1;
+			break;
 		case 'v':
 			verbose++;
 			break;
@@ -589,9 +595,11 @@ int main(int argc, char **argv)
 		exit(EXIT_FAILURE);
 	}
 
-	unlink(i_fname);
-	unlink(n_fname);
-	unlink(o_fname);
+	if (!keep) {
+		unlink(i_fname);
+		unlink(n_fname);
+		unlink(o_fname);
+	}
 
 	exit(EXIT_SUCCESS);
 }