From 08bfc679ff51b319f4241f02069b44495153fd7a Mon Sep 17 00:00:00 2001 From: JasonJoosteCSIRO <120607685+JasonJoosteCSIRO@users.noreply.github.com> Date: Sat, 6 May 2023 05:23:32 +1000 Subject: [PATCH 1/9] Sequential cell ids (#184) * Add command line argument keep-id, which maintiains randomly generated cell ids. Otherwise cell ids are assigned incrementally (after the removal of cells), which should keep them consistent across runs in version control * Modify test_cell and test_exception in test_keep_output_tags.py to use the new strip_output signature * Fix failed test_end_to_end_nbstripout with test_max_size by passing --keep-id for keeping the existing ids * Add tests for notebooks with and without the --keep-id flag. A new extension expected_id was added for expected output with ordered ids * Modify the readme to include the --include-id flag * Add keyword arguments for None inputs in test_keep_output_tags.py * Rename expected output files to make desired sequential ids more explicit Co-authored-by: Florian Rathgeber --- README.rst | 4 + nbstripout/_nbstripout.py | 8 +- nbstripout/_utils.py | 8 +- ...test_max_size.ipynb.expected_sequential_id | 90 ++++++++++++++++++ tests/e2e_notebooks/test_nbformat45.ipynb | 93 +++++++++++++++++++ .../test_nbformat45.ipynb.expected | 61 ++++++++++++ ...st_nbformat45.ipynb.expected_sequential_id | 61 ++++++++++++ tests/test_end_to_end.py | 5 +- tests/test_keep_output_tags.py | 4 +- 9 files changed, 325 insertions(+), 9 deletions(-) create mode 100644 tests/e2e_notebooks/test_max_size.ipynb.expected_sequential_id create mode 100644 tests/e2e_notebooks/test_nbformat45.ipynb create mode 100644 tests/e2e_notebooks/test_nbformat45.ipynb.expected create mode 100644 tests/e2e_notebooks/test_nbformat45.ipynb.expected_sequential_id diff --git a/README.rst b/README.rst index a161c65..3d97290 100644 --- a/README.rst +++ b/README.rst @@ -274,6 +274,10 @@ Do not strip the output :: nbstripout --keep-output +Do not reassign the cell ids to be sequential :: + + nbstripout --keep-id + To mark special cells so that the output is not stripped, you can either: 1. Set the ``keep_output`` tag on the cell. To do this, enable the tags diff --git a/nbstripout/_nbstripout.py b/nbstripout/_nbstripout.py index 3957395..50f096a 100644 --- a/nbstripout/_nbstripout.py +++ b/nbstripout/_nbstripout.py @@ -373,6 +373,9 @@ def main(): help='Do not strip the execution count/prompt number') parser.add_argument('--keep-output', action='store_true', help='Do not strip output', default=None) + parser.add_argument('--keep-id', action='store_true', + help='Keep the randomly generated cell ids, ' + 'which will be different after each execution.') parser.add_argument('--extra-keys', default='', help='Space separated list of extra keys to strip ' 'from metadata, e.g. metadata.foo cell.metadata.bar') @@ -409,7 +412,6 @@ def main(): parser.add_argument('files', nargs='*', help='Files to strip output from') args = parser.parse_args() - git_config = ['git', 'config'] if args._system: @@ -487,7 +489,7 @@ def main(): warnings.simplefilter("ignore", category=UserWarning) nb = read(f, as_version=NO_CONVERT) - nb = strip_output(nb, args.keep_output, args.keep_count, extra_keys, args.drop_empty_cells, + nb = strip_output(nb, args.keep_output, args.keep_count, args.keep_id, extra_keys, args.drop_empty_cells, args.drop_tagged_cells.split(), args.strip_init_cells, _parse_size(args.max_size)) if args.dry_run: @@ -533,7 +535,7 @@ def main(): warnings.simplefilter("ignore", category=UserWarning) nb = read(input_stream, as_version=NO_CONVERT) - nb = strip_output(nb, args.keep_output, args.keep_count, extra_keys, args.drop_empty_cells, + nb = strip_output(nb, args.keep_output, args.keep_count, args.keep_id, extra_keys, args.drop_empty_cells, args.drop_tagged_cells.split(), args.strip_init_cells, _parse_size(args.max_size)) if args.dry_run: diff --git a/nbstripout/_utils.py b/nbstripout/_utils.py index d54ac91..322edbd 100644 --- a/nbstripout/_utils.py +++ b/nbstripout/_utils.py @@ -94,7 +94,7 @@ def strip_zeppelin_output(nb): return nb -def strip_output(nb, keep_output, keep_count, extra_keys=[], drop_empty_cells=False, drop_tagged_cells=[], +def strip_output(nb, keep_output, keep_count, keep_id, extra_keys=[], drop_empty_cells=False, drop_tagged_cells=[], strip_init_cells=False, max_size=0): """ Strip the outputs, execution count/prompt number and miscellaneous @@ -124,7 +124,7 @@ def strip_output(nb, keep_output, keep_count, extra_keys=[], drop_empty_cells=Fa for tag_to_drop in drop_tagged_cells: conditionals.append(lambda c: tag_to_drop not in c.get("metadata", {}).get("tags", [])) - for cell in _cells(nb, conditionals): + for i, cell in enumerate(_cells(nb, conditionals)): keep_output_this_cell = determine_keep_output(cell, keep_output, strip_init_cells) # Remove the outputs, unless directed otherwise @@ -148,7 +148,9 @@ def strip_output(nb, keep_output, keep_count, extra_keys=[], drop_empty_cells=Fa cell['prompt_number'] = None if 'execution_count' in cell and not keep_count: cell['execution_count'] = None - + # Replace the cell id with an incremental value that will be consistent across runs + if 'id' in cell and not keep_id: + cell['id'] = str(i) for field in keys['cell']: pop_recursive(cell, field) return nb diff --git a/tests/e2e_notebooks/test_max_size.ipynb.expected_sequential_id b/tests/e2e_notebooks/test_max_size.ipynb.expected_sequential_id new file mode 100644 index 0000000..44cddfb --- /dev/null +++ b/tests/e2e_notebooks/test_max_size.ipynb.expected_sequential_id @@ -0,0 +1,90 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "This notebook tests that outputs can be cleared based on size" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "aaaaaaaaaa\n" + ] + } + ], + "source": [ + "print(\"a\"*10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"a\"*100)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.4" + }, + "varInspector": { + "cols": { + "lenName": 16, + "lenType": 16, + "lenVar": 40 + }, + "kernels_config": { + "python": { + "delete_cmd_postfix": "", + "delete_cmd_prefix": "del ", + "library": "var_list.py", + "varRefreshCmd": "print(var_dic_list())" + }, + "r": { + "delete_cmd_postfix": ") ", + "delete_cmd_prefix": "rm(", + "library": "var_list.r", + "varRefreshCmd": "cat(var_dic_list()) " + } + }, + "types_to_exclude": [ + "module", + "function", + "builtin_function_or_method", + "instance", + "_Feature" + ], + "window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/e2e_notebooks/test_nbformat45.ipynb b/tests/e2e_notebooks/test_nbformat45.ipynb new file mode 100644 index 0000000..f6d3f16 --- /dev/null +++ b/tests/e2e_notebooks/test_nbformat45.ipynb @@ -0,0 +1,93 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "5c42035d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'This is the new Jupyter notebook'" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"This is the new Jupyter notebook\"" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "886205fa", + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'text2'" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"text2\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "a183d4e9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "f(3) = 4\n" + ] + } + ], + "source": [ + "def f(x):\n", + " \"\"\"My function\n", + " x : parameter\"\"\"\n", + " \n", + " return x+1\n", + "\n", + "print(\"f(3) = \", f(3))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/e2e_notebooks/test_nbformat45.ipynb.expected b/tests/e2e_notebooks/test_nbformat45.ipynb.expected new file mode 100644 index 0000000..fb78c3f --- /dev/null +++ b/tests/e2e_notebooks/test_nbformat45.ipynb.expected @@ -0,0 +1,61 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "5c42035d", + "metadata": {}, + "outputs": [], + "source": [ + "\"This is the new Jupyter notebook\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "886205fa", + "metadata": {}, + "outputs": [], + "source": [ + "\"text2\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a183d4e9", + "metadata": {}, + "outputs": [], + "source": [ + "def f(x):\n", + " \"\"\"My function\n", + " x : parameter\"\"\"\n", + " \n", + " return x+1\n", + "\n", + "print(\"f(3) = \", f(3))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/e2e_notebooks/test_nbformat45.ipynb.expected_sequential_id b/tests/e2e_notebooks/test_nbformat45.ipynb.expected_sequential_id new file mode 100644 index 0000000..8c499da --- /dev/null +++ b/tests/e2e_notebooks/test_nbformat45.ipynb.expected_sequential_id @@ -0,0 +1,61 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "\"This is the new Jupyter notebook\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "\"text2\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "def f(x):\n", + " \"\"\"My function\n", + " x : parameter\"\"\"\n", + " \n", + " return x+1\n", + "\n", + "print(\"f(3) = \", f(3))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 0b96923..575dd9b 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -15,7 +15,8 @@ ("test_drop_tagged_cells.ipynb", "test_drop_tagged_cells_dontdrop.ipynb.expected", []), ("test_drop_tagged_cells.ipynb", "test_drop_tagged_cells.ipynb.expected", ['--drop-tagged-cells=test']), ("test_execution_timing.ipynb", "test_execution_timing.ipynb.expected", []), - ("test_max_size.ipynb", "test_max_size.ipynb.expected", ["--max-size", "50"]), + ("test_max_size.ipynb", "test_max_size.ipynb.expected", ["--max-size", "50", "--keep-id"]), + ("test_max_size.ipynb", "test_max_size.ipynb.expected_sequential_id", ["--max-size", "50"]), ("test_metadata.ipynb", "test_metadata.ipynb.expected", []), ("test_metadata.ipynb", "test_metadata_extra_keys.ipynb.expected", ["--extra-keys", "metadata.kernelspec metadata.language_info"]), ("test_metadata.ipynb", "test_metadata_keep_count.ipynb.expected", ["--keep-count"]), @@ -26,6 +27,8 @@ ("test_metadata_period.ipynb", "test_metadata_period.ipynb.expected", ["--extra-keys", "cell.metadata.application/vnd.databricks.v1+cell metadata.application/vnd.databricks.v1+notebook"]), ("test_strip_init_cells.ipynb", "test_strip_init_cells.ipynb.expected", ["--strip-init-cells"]), ("test_nbformat2.ipynb", "test_nbformat2.ipynb.expected", []), + ("test_nbformat45.ipynb", "test_nbformat45.ipynb.expected", ["--keep-id"]), + ("test_nbformat45.ipynb", "test_nbformat45.ipynb.expected_sequential_id", []), ("test_unicode.ipynb", "test_unicode.ipynb.expected", []), ("test_widgets.ipynb", "test_widgets.ipynb.expected", []), ("test_zeppelin.zpln", "test_zeppelin.zpln.expected", ["--mode", "zeppelin"]), diff --git a/tests/test_keep_output_tags.py b/tests/test_keep_output_tags.py index 7ec8567..4243a82 100644 --- a/tests/test_keep_output_tags.py +++ b/tests/test_keep_output_tags.py @@ -24,7 +24,7 @@ def nb_with_exception(): def test_cells(orig_nb): nb_stripped = deepcopy(orig_nb) - nb_stripped = strip_output(nb_stripped, None, None) + nb_stripped = strip_output(nb_stripped, keep_output=None, keep_count=None, keep_id=None) for i, cell in enumerate(nb_stripped.cells): if cell.cell_type == 'code' and cell.source: match = re.match(r"\s*#\s*(output|no_output)", cell.source) @@ -41,4 +41,4 @@ def test_cells(orig_nb): def test_exception(nb_with_exception): with pytest.raises(MetadataError): - strip_output(nb_with_exception, None, None) + strip_output(nb_with_exception, keep_output=None, keep_count=None, keep_id=None) From 749f431366811f26114c15aefcb5149e014b9e97 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Sun, 22 Oct 2023 02:40:32 -0400 Subject: [PATCH 2/9] Add python3.11 classifier to setup.py, remove python3.6 (#186) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dede8c5..5982ec0 100644 --- a/setup.py +++ b/setup.py @@ -37,10 +37,10 @@ "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Version Control", ]) From 7b065ef1af9bac07b9cd1d6161491fd0ce0f8f45 Mon Sep 17 00:00:00 2001 From: Florian Rathgeber Date: Thu, 7 Dec 2023 21:51:48 +0100 Subject: [PATCH 3/9] Improve documentation for notebook and cell metadata stripping * Clarify that extra keys may _only_ specify notebook and cell metadata to be stripped. * Update metadata stripped by default. Addresses #187 --- README.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 3d97290..5e3161b 100644 --- a/README.rst +++ b/README.rst @@ -315,8 +315,9 @@ Stripping metadata The following metadata is stripped by default: -* Notebook metadata: ``signature``, ``widgets`` -* Cell metadata: ``ExecuteTime``, ``collapsed``, ``execution``, ``scrolled`` +* Notebook metadata: ``signature``, ``widgets`` +* Cell metadata: ``ExecuteTime``, ``collapsed``, ``execution``, + ``heading_collapsed``, ``hidden``, ``scrolled`` Additional metadata to be stripped can be configured via either @@ -341,6 +342,10 @@ Additional metadata to be stripped can be configured via either --extra-keys "metadata.celltoolbar cell.metadata.heading_collapsed" +Note: Only notebook and cell metadata is currently supported and every key +specified via ``filter.nbstripout.extrakeys`` or ``--extra-keys`` must start +with ``metadata.`` for notebook and ``cell.metadata.`` for cell metadata. + You can keep certain metadata with either * ``git config (--global/--system) filter.nbstripout.keepmetadatakeys``, e.g. :: From a58210dd6704c8ab8c46f02a0444c5d0126ef11f Mon Sep 17 00:00:00 2001 From: igor-sb <> Date: Wed, 24 Jan 2024 21:25:12 -0800 Subject: [PATCH 4/9] add option to keep output count or id during install --- nbstripout/_nbstripout.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nbstripout/_nbstripout.py b/nbstripout/_nbstripout.py index 50f096a..e7fbfcb 100644 --- a/nbstripout/_nbstripout.py +++ b/nbstripout/_nbstripout.py @@ -214,10 +214,12 @@ def _parse_size(num_str): raise ValueError(f"Unknown size identifier {num_str[-1]}") -def install(git_config, install_location=INSTALL_LOCATION_LOCAL, python=None, attrfile=None): +def install(git_config, install_location=INSTALL_LOCATION_LOCAL, python=None, attrfile=None, keep_args=None): """Install the git filter and set the git attributes.""" try: filepath = f'"{PureWindowsPath(python or sys.executable).as_posix()}" -m nbstripout' + if keep_args: + filepath = filepath + ' '.join(keep_args) check_call(git_config + ['filter.nbstripout.clean', filepath]) check_call(git_config + ['filter.nbstripout.smudge', 'cat']) check_call(git_config + ['diff.ipynb.textconv', filepath + ' -t']) @@ -414,6 +416,12 @@ def main(): args = parser.parse_args() git_config = ['git', 'config'] + keep_args = [ + f'--keep-{arg}' + for arg in ['output', 'count', 'id'] + if getattr(args, f'keep_{arg}', False) + ] + if args._system: git_config.append('--system') install_location = INSTALL_LOCATION_SYSTEM @@ -425,7 +433,7 @@ def main(): install_location = INSTALL_LOCATION_LOCAL if args.install: - raise SystemExit(install(git_config, install_location, python=args._python, attrfile=args.attributes)) + raise SystemExit(install(git_config, install_location, python=args._python, attrfile=args.attributes, keep_args=keep_args)) if args.uninstall: raise SystemExit(uninstall(git_config, install_location, attrfile=args.attributes)) if args.is_installed: From a850a81ccfe4fa63457d6f3006be256403b0a7b8 Mon Sep 17 00:00:00 2001 From: igor-sb <> Date: Wed, 24 Jan 2024 21:27:25 -0800 Subject: [PATCH 5/9] =?UTF-8?q?Bump=20version:=200.6.1=20=E2=86=92=200.6.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.rst | 4 ++-- nbstripout/_nbstripout.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 5e3161b..d2a9c62 100644 --- a/README.rst +++ b/README.rst @@ -34,7 +34,7 @@ Based on https://gist.github.com/minrk/6176788. Python 3 only ============= -As of version 0.6.1, nbstripout supports Python 3 *only*. If you need to use +As of version 0.6.2, nbstripout supports Python 3 *only*. If you need to use Python 2.7, install nbstripout 0.3.10 :: pip install nbstripout==0.3.10 @@ -444,7 +444,7 @@ Once you have `pre-commit`_ installed, add the following to the repos: - repo: https://github.com/kynan/nbstripout - rev: 0.6.1 + rev: 0.6.2 hooks: - id: nbstripout diff --git a/nbstripout/_nbstripout.py b/nbstripout/_nbstripout.py index e7fbfcb..4cfab40 100644 --- a/nbstripout/_nbstripout.py +++ b/nbstripout/_nbstripout.py @@ -145,7 +145,7 @@ def write(nb, f): return current.write(nb, f, 'json') __all__ = ["install", "uninstall", "status", "main"] -__version__ = '0.6.1' +__version__ = '0.6.2' INSTALL_LOCATION_LOCAL = 'local' diff --git a/setup.cfg b/setup.cfg index 61266f3..bdbda38 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.6.1 +current_version = 0.6.2 commit = True tag = True tag_name = {new_version} diff --git a/setup.py b/setup.py index 5982ec0..d90db8b 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ ] setup(name='nbstripout', - version='0.6.1', + version='0.6.2', author='Florian Rathgeber', author_email='florian.rathgeber@gmail.com', From 29dbb1c8e7b82e65fb297e843fdf1b1d4666770f Mon Sep 17 00:00:00 2001 From: igor-sb <> Date: Fri, 26 Jan 2024 16:59:16 -0800 Subject: [PATCH 6/9] reverting accidental auto-version-bump in readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d2a9c62..2f3417d 100644 --- a/README.rst +++ b/README.rst @@ -34,7 +34,7 @@ Based on https://gist.github.com/minrk/6176788. Python 3 only ============= -As of version 0.6.2, nbstripout supports Python 3 *only*. If you need to use +As of version 0.6.1, nbstripout supports Python 3 *only*. If you need to use Python 2.7, install nbstripout 0.3.10 :: pip install nbstripout==0.3.10 From ae7330c84020f6d755b6b9b184e38fd49e49c593 Mon Sep 17 00:00:00 2001 From: igor-sb <> Date: Fri, 26 Jan 2024 22:12:31 -0800 Subject: [PATCH 7/9] read and write extra flags from git config --- nbstripout/_nbstripout.py | 50 ++++++++++++++++++++++++++------------- nbstripout/_utils.py | 9 +++++++ 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/nbstripout/_nbstripout.py b/nbstripout/_nbstripout.py index 4cfab40..f94b6a6 100644 --- a/nbstripout/_nbstripout.py +++ b/nbstripout/_nbstripout.py @@ -120,7 +120,7 @@ import sys import warnings -from nbstripout._utils import strip_output, strip_zeppelin_output +from nbstripout._utils import strip_output, strip_zeppelin_output, snake_to_camel_case try: # Jupyter >= 4 from nbformat import read, write, NO_CONVERT @@ -200,6 +200,21 @@ def _get_attrfile(git_config, install_location=INSTALL_LOCATION_LOCAL, attrfile= return attrfile +def _get_default_extra_flag(git_config, flag_name): + camel_flag_name = snake_to_camel_case(flag_name) + try: + gitconfig_flag_value = check_output( + git_config + [f'filter.nbstripout.{camel_flag_name}'], + universal_newlines=True, + ).strip() + except CalledProcessError: + return None + if gitconfig_flag_value == 'true': + return True + elif gitconfig_flag_value == 'false': + return False + + def _parse_size(num_str): num_str = num_str.upper() if num_str[-1].isdigit(): @@ -214,15 +229,16 @@ def _parse_size(num_str): raise ValueError(f"Unknown size identifier {num_str[-1]}") -def install(git_config, install_location=INSTALL_LOCATION_LOCAL, python=None, attrfile=None, keep_args=None): +def install(git_config, install_location=INSTALL_LOCATION_LOCAL, python=None, attrfile=None, extra_flags=None): """Install the git filter and set the git attributes.""" try: filepath = f'"{PureWindowsPath(python or sys.executable).as_posix()}" -m nbstripout' - if keep_args: - filepath = filepath + ' '.join(keep_args) check_call(git_config + ['filter.nbstripout.clean', filepath]) check_call(git_config + ['filter.nbstripout.smudge', 'cat']) check_call(git_config + ['diff.ipynb.textconv', filepath + ' -t']) + for flag_name, flag_value in extra_flags.items(): + camel_flag_name = snake_to_camel_case(flag_name) + check_call(git_config + ['--type', 'bool', f'filter.nbstripout.{camel_flag_name}', str(flag_value)]) attrfile = _get_attrfile(git_config, install_location, attrfile) except FileNotFoundError: print('Installation failed: git is not on path!', file=sys.stderr) @@ -267,11 +283,14 @@ def install(git_config, install_location=INSTALL_LOCATION_LOCAL, python=None, at return 1 -def uninstall(git_config, install_location=INSTALL_LOCATION_LOCAL, attrfile=None): +def uninstall(git_config, install_location=INSTALL_LOCATION_LOCAL, attrfile=None, extra_flags=None): """Uninstall the git filter and unset the git attributes.""" try: call(git_config + ['--unset', 'filter.nbstripout.clean'], stdout=open(devnull, 'w'), stderr=STDOUT) call(git_config + ['--unset', 'filter.nbstripout.smudge'], stdout=open(devnull, 'w'), stderr=STDOUT) + for flag in extra_flags.keys(): + camel_flag = snake_to_camel_case(flag) + call(git_config + ['--unset', f'filter.nbstripout.{camel_flag}'], stdout=open(devnull, 'w'), stderr=STDOUT) call(git_config + ['--remove-section', 'diff.ipynb'], stdout=open(devnull, 'w'), stderr=STDOUT) attrfile = _get_attrfile(git_config, install_location, attrfile) except FileNotFoundError: @@ -374,7 +393,7 @@ def main(): parser.add_argument('--keep-count', action='store_true', help='Do not strip the execution count/prompt number') parser.add_argument('--keep-output', action='store_true', - help='Do not strip output', default=None) + help='Do not strip output') parser.add_argument('--keep-id', action='store_true', help='Keep the randomly generated cell ids, ' 'which will be different after each execution.') @@ -416,12 +435,6 @@ def main(): args = parser.parse_args() git_config = ['git', 'config'] - keep_args = [ - f'--keep-{arg}' - for arg in ['output', 'count', 'id'] - if getattr(args, f'keep_{arg}', False) - ] - if args._system: git_config.append('--system') install_location = INSTALL_LOCATION_SYSTEM @@ -432,10 +445,15 @@ def main(): git_config.append('--local') install_location = INSTALL_LOCATION_LOCAL + # Extra (bool) flags + extra_flags = {} + for flag_name in ('keep_count', 'keep_id', 'keep_output'): + extra_flags[flag_name] = getattr(args, flag_name) or _get_default_extra_flag(git_config, flag_name) + if args.install: - raise SystemExit(install(git_config, install_location, python=args._python, attrfile=args.attributes, keep_args=keep_args)) + raise SystemExit(install(git_config, install_location, python=args._python, attrfile=args.attributes, extra_flags=extra_flags)) if args.uninstall: - raise SystemExit(uninstall(git_config, install_location, attrfile=args.attributes)) + raise SystemExit(uninstall(git_config, install_location, attrfile=args.attributes, extra_flags=extra_flags)) if args.is_installed: raise SystemExit(status(git_config, install_location, verbose=False)) if args.status: @@ -497,7 +515,7 @@ def main(): warnings.simplefilter("ignore", category=UserWarning) nb = read(f, as_version=NO_CONVERT) - nb = strip_output(nb, args.keep_output, args.keep_count, args.keep_id, extra_keys, args.drop_empty_cells, + nb = strip_output(nb, extra_flags['keep_output'], extra_flags['keep_count'], extra_flags['keep_id'], extra_keys, args.drop_empty_cells, args.drop_tagged_cells.split(), args.strip_init_cells, _parse_size(args.max_size)) if args.dry_run: @@ -543,7 +561,7 @@ def main(): warnings.simplefilter("ignore", category=UserWarning) nb = read(input_stream, as_version=NO_CONVERT) - nb = strip_output(nb, args.keep_output, args.keep_count, args.keep_id, extra_keys, args.drop_empty_cells, + nb = strip_output(nb, extra_flags['keep_output'], extra_flags['keep_count'], extra_flags['keep_id'], extra_keys, args.drop_empty_cells, args.drop_tagged_cells.split(), args.strip_init_cells, _parse_size(args.max_size)) if args.dry_run: diff --git a/nbstripout/_utils.py b/nbstripout/_utils.py index 322edbd..c9e8ff9 100644 --- a/nbstripout/_utils.py +++ b/nbstripout/_utils.py @@ -154,3 +154,12 @@ def strip_output(nb, keep_output, keep_count, keep_id, extra_keys=[], drop_empty for field in keys['cell']: pop_recursive(cell, field) return nb + + +def snake_to_camel_case(string): + """Converts snake_case string to camelCase.""" + if '_' not in string: + return string + string_parts = string.split('_') + camel_parts = [part[0].upper() + part[1:] for part in string_parts[1:] if part != ''] + return string_parts[0] + ''.join(camel_parts) From 1e17e7d98db7a8a7a92c3450421074742dd08742 Mon Sep 17 00:00:00 2001 From: igor-sb <> Date: Fri, 26 Jan 2024 22:18:50 -0800 Subject: [PATCH 8/9] fix unit tests --- nbstripout/_nbstripout.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nbstripout/_nbstripout.py b/nbstripout/_nbstripout.py index f94b6a6..d1efe71 100644 --- a/nbstripout/_nbstripout.py +++ b/nbstripout/_nbstripout.py @@ -393,7 +393,7 @@ def main(): parser.add_argument('--keep-count', action='store_true', help='Do not strip the execution count/prompt number') parser.add_argument('--keep-output', action='store_true', - help='Do not strip output') + help='Do not strip output', default=None) parser.add_argument('--keep-id', action='store_true', help='Keep the randomly generated cell ids, ' 'which will be different after each execution.') @@ -448,7 +448,7 @@ def main(): # Extra (bool) flags extra_flags = {} for flag_name in ('keep_count', 'keep_id', 'keep_output'): - extra_flags[flag_name] = getattr(args, flag_name) or _get_default_extra_flag(git_config, flag_name) + extra_flags[flag_name] = getattr(args, flag_name) or _get_default_extra_flag(git_config, flag_name) or False if args.install: raise SystemExit(install(git_config, install_location, python=args._python, attrfile=args.attributes, extra_flags=extra_flags)) From bebb409a6cdeafd7d0bc2b446be1a4d20ec5116f Mon Sep 17 00:00:00 2001 From: igor-sb <> Date: Fri, 26 Jan 2024 22:27:08 -0800 Subject: [PATCH 9/9] fix unit tests --- nbstripout/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbstripout/_utils.py b/nbstripout/_utils.py index c9e8ff9..b2748bb 100644 --- a/nbstripout/_utils.py +++ b/nbstripout/_utils.py @@ -103,7 +103,7 @@ def strip_output(nb, keep_output, keep_count, keep_id, extra_keys=[], drop_empty `extra_keys` could be 'metadata.foo cell.metadata.bar metadata.baz' """ - if keep_output is None and 'keep_output' in nb.metadata: + if not keep_output and 'keep_output' in nb.metadata: keep_output = bool(nb.metadata['keep_output']) keys = defaultdict(list)