From f364f1e2642335536a072ff731b9e4281744d646 Mon Sep 17 00:00:00 2001 From: Frank Haverkamp Date: Tue, 28 Nov 2017 17:15:22 +0100 Subject: [PATCH] GenWQE: Collect temporary bytes from dictionary buffer Signed-off-by: Frank Haverkamp --- include/libzHW.h | 3 ++ lib/hardware.c | 79 +++++++++++++++++++++------------------------ lib/inflate.c | 31 ++++++++++++++++++ misc/zpipe_append.c | 22 +++++++++---- 4 files changed, 85 insertions(+), 50 deletions(-) diff --git a/include/libzHW.h b/include/libzHW.h index e3a022d..aa9c63d 100644 --- a/include/libzHW.h +++ b/include/libzHW.h @@ -480,7 +480,10 @@ int zedc_inflateGetHeader(zedc_streamp strm, gzedc_headerp head); int zedc_inflateSaveBuffers(zedc_streamp strm, const char *prefix); void zedc_lib_debug(int onoff); /* debug outputs on/off */ void zedc_set_logfile(FILE *logfile); + int zedc_inflate_pending_output(struct zedc_stream_s *strm); +int zedc_read_pending_output(struct zedc_stream_s *strm, + uint8_t *buf, unsigned int len); /** * The application can compare zedc_Version and ZEDC_VERSION for diff --git a/lib/hardware.c b/lib/hardware.c index 12f2268..7fca44b 100644 --- a/lib/hardware.c +++ b/lib/hardware.c @@ -79,7 +79,7 @@ struct hw_state { */ static int output_buffer_empty(struct hw_state *s) { - return s->obuf_avail == s->obuf_total; + return (s->obuf_avail == s->obuf_total); } /** @@ -1424,9 +1424,9 @@ int h_inflate(z_streamp strm, int flush) /* No progress possible (no more input and no buffered output): Z_BUF_ERROR */ obuf_bytes = s->obuf - s->obuf_next; /* bytes in obuf */ - if (obuf_bytes == 0) { - hw_trace("[%p] OBYTES_IN_DICT %d bytes\n", strm, - h->obytes_in_dict); + if ((obuf_bytes == 0) && (zedc_inflate_pending_output(h) == 0)) { + hw_trace("[%p] OBYTES_IN_DICT %d bytes (1) This must be 0!\n", + strm, h->obytes_in_dict); if (s->rc == Z_STREAM_END) /* hardware saw FEOB */ return Z_STREAM_END; /* nothing to do anymore */ @@ -1453,46 +1453,33 @@ int h_inflate(z_streamp strm, int flush) /* Give out what is already there */ obuf_bytes = h_flush_obuf(strm); - if ((s->rc == Z_STREAM_END) && /* hardware saw FEOB */ - (obuf_bytes == 0)) /* no more output in buf */ - return Z_STREAM_END; /* nothing to do anymore */ - - if (((obuf_bytes != 0) || zedc_inflate_pending_output(h)) && - (strm->avail_out == 0)) - return Z_OK; /* need new output buffer */ + if ((s->rc == Z_STREAM_END) && /* hardware/sw saw FEOB */ + (obuf_bytes == 0)) { /* no more output in buf */ + unsigned int rem_bytes; + + /* no more output in temp? */ + rc = zedc_read_pending_output(h, strm->next_out, + strm->avail_out); + if (rc < 0) { + hw_trace("[%s] err: Read temp buffer rc=%d!\n", + __func__, rc); + return rc; + } -#if 0 -/* - * FIXME We have put this now behind the hardware decompression, where - * it should have been already from the beginning. Wonder if that - * is passing all the testcases. If it does, we can hopefully remove - * this. - */ - /* - * Need more output space, just useful if Z_STREAM_END - * not seen before. - */ - if ((s->rc != Z_STREAM_END) && (strm->avail_out == 0)) { - rc = Z_OK; + hw_trace("[%s] collected %d bytes from dict buffer\n", + __func__, rc); + strm->avail_out -= rc; + strm->total_out += rc; -#ifdef CONFIG_CIRCUMVENTION_FOR_Z_STREAM_END /* For MongoDB PoC */ + rem_bytes = zedc_inflate_pending_output(h); + if (rem_bytes != 0) + return Z_OK; /* call me again */ - rc = __check_stream_end(strm); - if (rc == Z_STREAM_END) { - hw_trace(" Suppress Z_STREAM_END %zd %zd (1)\n", - s->obuf_avail, s->obuf_total); - s->rc = Z_STREAM_END; - rc = Z_OK; - } - - hw_trace("[%p] .......... flush=%s avail_in=%d " - "avail_out=%d __check_stream=%s (1)\n", strm, - flush_to_str(flush), strm->avail_in, - strm->avail_out, ret_to_str(rc)); -#endif - return rc; + return Z_STREAM_END; /* nothing to do anymore */ } -#endif + if (((obuf_bytes != 0) || zedc_inflate_pending_output(h)) && + (strm->avail_out == 0)) + return Z_OK; /* need new output buffer */ /* * Original idea: Do not send 0 data to HW @@ -1568,8 +1555,10 @@ int h_inflate(z_streamp strm, int flush) #ifdef CONFIG_CIRCUMVENTION_FOR_Z_STREAM_END /* For MongoDB PoC */ /* FIXME Experimental check for Z_STREAM_END here */ if ((s->rc != Z_STREAM_END) && (strm->avail_out == 0)) { - rc = __check_stream_end(strm); - if (rc == Z_STREAM_END) { + int _rc; + + _rc = __check_stream_end(strm); + if (_rc == Z_STREAM_END) { hw_trace(" Suppress Z_STREAM_END %zd %zd (2)\n", s->obuf_avail, s->obuf_total); s->rc = Z_STREAM_END; @@ -1581,8 +1570,12 @@ int h_inflate(z_streamp strm, int flush) } #endif /* Hardware saw FEOB and output buffer is empty */ - if ((s->rc == Z_STREAM_END) && output_buffer_empty(s)) + if ((s->rc == Z_STREAM_END) && output_buffer_empty(s) && + (zedc_inflate_pending_output(h) == 0)) { + hw_trace("[%p] OBYTES_IN_DICT %d bytes (2) Must be 0!\n", + strm, h->obytes_in_dict); return Z_STREAM_END; /* nothing to do anymore */ + } if (strm->avail_out == 0) /* need more output space */ return Z_OK; diff --git a/lib/inflate.c b/lib/inflate.c index 9e2762c..3a69af2 100644 --- a/lib/inflate.c +++ b/lib/inflate.c @@ -768,6 +768,37 @@ int zedc_inflate_pending_output(struct zedc_stream_s *strm) return strm->obytes_in_dict; } +/** + * @brief Enable wrapper code to access internal buffer. + * If data is left from previous task due to insufficent + * output buffer space, this data must first be stored + * to the new output buffer. + * @param strm decompression job context + */ +int zedc_read_pending_output(struct zedc_stream_s *strm, + uint8_t *buf, unsigned int len) +{ + uint8_t *pdict; + unsigned int _len = 0; + + if (strm->obytes_in_dict == 0) + return ZEDC_OK; + if (strm->dict_len < strm->obytes_in_dict) + return ZEDC_ERR_DICT_OVERRUN; + + /* obytes at end of dict */ + pdict = strm->wsp->dict[strm->wsp_page] + + strm->out_dict_offs + strm->dict_len - strm->obytes_in_dict; + + while (len && strm->obytes_in_dict) { + *buf++ = *pdict++; + strm->obytes_in_dict--; + len--; + _len++; + } + return _len; +} + /** * @brief If data is left from previous task due to insufficent * output buffer space, this data must first be stored diff --git a/misc/zpipe_append.c b/misc/zpipe_append.c index 7f75a7b..0a6fa85 100644 --- a/misc/zpipe_append.c +++ b/misc/zpipe_append.c @@ -3,7 +3,7 @@ Version 1.4 11 December 2005 Mark Adler */ /* - * Copyright 2016, International Business Machines + * Copyright 2016, 2017 International Business Machines * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -375,13 +375,14 @@ static void usage(char *prog) fprintf(stderr, "%s usage: %s [-h] [-v]\n" " [-F, --format ]\n" - " [-e, --excact-input] input matches size of data\n" + " [-e, --excact-input] input matches size of data\n" " [-E, --excact-output] output matches size of data\n" " [-f, --fush ]\n" " [-i, --i_bufsize ]\n" " [-o, --o_bufsize ]\n" " [-p, --pattern ] pattern to generate test-data\n" - " [-s, --size ]\n", + " [-s, --size ]\n" + " [-k, --keep] do not delete resulting files\n", b, b); } @@ -400,6 +401,7 @@ int main(int argc, char **argv) int exact_input = 0, exact_output = 0; size_t size = 256 * 1024; int expect_z_stream_end = 0; + int keep = 0; _pattern = getpid(); @@ -419,12 +421,13 @@ int main(int argc, char **argv) { "o_bufsize", required_argument, NULL, 'o' }, { "size", required_argument, NULL, 's' }, { "pattern", required_argument, NULL, 'p' }, + { "keep", no_argument, NULL, 'k' }, { "verbose", no_argument, NULL, 'v' }, { "help", no_argument, NULL, 'h' }, { 0, no_argument, NULL, 0 }, }; - ch = getopt_long(argc, argv, "F:f:Eei:o:s:p:vh?", + ch = getopt_long(argc, argv, "kF:f:Eei:o:s:p:vh?", long_options, &option_index); if (ch == -1) /* all params processed ? */ break; @@ -450,6 +453,9 @@ int main(int argc, char **argv) case 'E': exact_output = 1; break; + case 'k': + keep = 1; + break; case 'v': verbose++; break; @@ -589,9 +595,11 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } - unlink(i_fname); - unlink(n_fname); - unlink(o_fname); + if (!keep) { + unlink(i_fname); + unlink(n_fname); + unlink(o_fname); + } exit(EXIT_SUCCESS); }