diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..38ebaf2a --- /dev/null +++ b/.flake8 @@ -0,0 +1,35 @@ +[flake8] +max-line-length = 120 +ignore = + # ============================================================== + # ========= Default list of things Flake8 would ignore ======== + # ============================================================== + # E121: Continuation line under-indented for hanging indent + # https://www.flake8rules.com/rules/E121.html + E121, + # E123: Closing bracket does not match indentation of opening bracket's line + # https://www.flake8rules.com/rules/E123.html + E123, + # E126: Continuation line over-indented for hanging indent + # https://www.flake8rules.com/rules/E126.html + E126, + # E226: Missing whitespace around arithmetic operator + # https://www.flake8rules.com/rules/E226.html + E226, + # E24: ??? Hard to find doc on why this undocumented code's in default list + # E24, + # E704: Multiple statements on one line (def) + # https://www.flake8rules.com/rules/E704.html + E704, + # W503: Line break occurred before a binary operator + # https://www.flake8rules.com/rules/W503.html + W503, + # W504: Line break occurred after a binary operator + # https://www.flake8rules.com/rules/W503.html + W504, + # ============================================================== + # ===== Other things we think it shouldn't complain about ===== + # ============================================================== + # F541: f-string is missing placeholders + # https://flake8.pycqa.org/en/latest/user/error-codes.html + F541 diff --git a/Makefile b/Makefile index 68fa9ce4..e937dab9 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,14 @@ .PHONY: test +clear-poetry-cache: # clear poetry/pypi cache. for user to do explicitly, never automatic + poetry cache clear pypi --all + configure: # does any pre-requisite installs pip install poetry==1.3.2 +lint: + flake8 wranglertools + build: # builds make configure poetry install @@ -13,6 +19,9 @@ test: update: # updates dependencies poetry update +tag-and-push: # tags the branch and pushes it + @scripts/tag-and-push + publish: scripts/publish @@ -22,7 +31,9 @@ help: info: @: $(info Here are some 'make' options:) $(info - Use 'make configure' to install poetry, though 'make build' will do it automatically.) + $(info - Use 'make lint' to check style with flake8.) $(info - Use 'make build' to install dependencies using poetry.) + $(info - Use 'make clear-poetry-cache' to clear the poetry pypi cache if in a bad state. (Safe, but later recaching can be slow.)) $(info - Use 'make publish' to publish this library, but only if auto-publishing has failed.) $(info - Use 'make test' to run tests with the normal options we use on travis) $(info - Use 'make update' to update dependencies (and the lock file)) diff --git a/poetry.lock b/poetry.lock index 47ea5f55..6b8a2c9e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -402,6 +402,24 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "flake8" +version = "5.0.4" +description = "the modular source code checker: pep8 pyflakes and co" +category = "dev" +optional = false +python-versions = ">=3.6.1" +files = [ + {file = "flake8-5.0.4-py2.py3-none-any.whl", hash = "sha256:7a1cf6b73744f5806ab95e526f6f0d8c01c66d7bbe349562d22dfca20610b248"}, + {file = "flake8-5.0.4.tar.gz", hash = "sha256:6fbe320aad8d6b95cec8b8e47bc933004678dc63095be98528b7bdd2a9f510db"}, +] + +[package.dependencies] +importlib-metadata = {version = ">=1.1.0,<4.3", markers = "python_version < \"3.8\""} +mccabe = ">=0.7.0,<0.8.0" +pycodestyle = ">=2.9.0,<2.10.0" +pyflakes = ">=2.5.0,<2.6.0" + [[package]] name = "gitdb" version = "4.0.10" @@ -447,14 +465,14 @@ files = [ [[package]] name = "importlib-metadata" -version = "6.1.0" +version = "4.2.0" description = "Read metadata from Python packages" category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.6" files = [ - {file = "importlib_metadata-6.1.0-py3-none-any.whl", hash = "sha256:ff80f3b5394912eb1b108fcfd444dc78b7f1f3e16b16188054bd01cb9cb86f09"}, - {file = "importlib_metadata-6.1.0.tar.gz", hash = "sha256:43ce9281e097583d758c2c708c4376371261a02c34682491a8e98352365aad20"}, + {file = "importlib_metadata-4.2.0-py3-none-any.whl", hash = "sha256:057e92c15bc8d9e8109738a48db0ccb31b4d9d5cfbee5a8670879a30be66304b"}, + {file = "importlib_metadata-4.2.0.tar.gz", hash = "sha256:b7e52a1f8dec14a75ea73e0891f3060099ca1d8e6a462a4dff11c3e119ea1b31"}, ] [package.dependencies] @@ -462,9 +480,8 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pep517", "pyfakefs", "pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"] [[package]] name = "iniconfig" @@ -490,6 +507,18 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "mccabe" +version = "0.7.0" +description = "McCabe checker, plugin for flake8" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] + [[package]] name = "openpyxl" version = "3.1.2" @@ -573,6 +602,30 @@ files = [ {file = "pyasn1-0.4.8.tar.gz", hash = "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba"}, ] +[[package]] +name = "pycodestyle" +version = "2.9.1" +description = "Python style guide checker" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pycodestyle-2.9.1-py2.py3-none-any.whl", hash = "sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b"}, + {file = "pycodestyle-2.9.1.tar.gz", hash = "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785"}, +] + +[[package]] +name = "pyflakes" +version = "2.5.0" +description = "passive checker of Python programs" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pyflakes-2.5.0-py2.py3-none-any.whl", hash = "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2"}, + {file = "pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"}, +] + [[package]] name = "pytest" version = "7.2.2" @@ -1005,4 +1058,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.7.0,<3.10" -content-hash = "3caaba70c5613e1f6bdce07daa8f7f492b771a2f30d0af67a079cd6b5302dd1e" +content-hash = "5c7ea0a410c2164647d29b5768a67c5d0845f72057ae73e4a38ffeb7d8aed49b" diff --git a/pyproject.toml b/pyproject.toml index f307f4e3..a78eee6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dcicutils = "^6.10.1" awscli = "^1.27" [tool.poetry.dev-dependencies] +flake8 = ">=3.9.2" pytest = ">=7.2.2" pytest-cov = ">=4.0.0" pytest-mock = ">=3.10.0" diff --git a/scripts/tag-and-push b/scripts/tag-and-push new file mode 100755 index 00000000..4338c1cc --- /dev/null +++ b/scripts/tag-and-push @@ -0,0 +1,39 @@ +#!/bin/bash + +dirty=`git describe --all --dirty | rev | cut -d '-' -f 1 | rev` +if [ "$dirty" = "dirty" ]; then + echo "This branch is dirty. That would create an ambiguity in what to tag." + exit 1 +fi + +if [ ! -e "pyproject.toml" ]; then + echo "There is no pyproject.toml" + exit 1 +fi + +version=`grep '^version = ' pyproject.toml | sed -E "s/^version = ['\"]([^'\"]+)['\"].*/\1/g"` + +if [ -z "${version}" ]; then + echo "pyproject.toml has no 'version = \"...\"' line." + exit 1 +fi + +new_tag=v${version} + +git tag ${new_tag} + +if [ $? -ne 0 ]; then + echo "Adding tag ${new_tag} failed." + exit 1 +else + echo "Added tag ${new_tag}." +fi + +git push origin ${new_tag} + +if [ $? -ne 0 ]; then + echo "Pushing tag ${new_tag} failed." + exit 1 +else + echo "Pushed tag ${new_tag}." +fi diff --git a/wranglertools/get_field_info.py b/wranglertools/get_field_info.py index 9995478c..3b959bd6 100755 --- a/wranglertools/get_field_info.py +++ b/wranglertools/get_field_info.py @@ -12,15 +12,15 @@ EPILOG = ''' To create an excel workbook file with sheets to be filled use the examples below and modify to your needs. It will accept the following optional parameters. - --keyfile the path to the file where you have stored your access key info (default ~/keypairs.json) - --key the name of the key identifier for the access key and secret in your keys file (default=default) - --type use for each sheet that you want to add to the excel workbook - --nodesc do not add the descriptions in the second line (by default they are added) - --noenums do not add the list of options for a field if they are specified (by default they are added) - --comments adds any (usually internal) comments together with enums (by default False) - --outfile change the default file name "fields.xlsx" to a specified one - --debug to add more debugging output - --noadmin if you have admin access to 4DN this option lets you generate the sheet as a non-admin user + --keyfile the path to the file where you have stored your access key info (default ~/keypairs.json) + --key the name of the key identifier for the access key and secret in your keys file (default=default) + --type use for each sheet that you want to add to the excel workbook + --nodesc do not add the descriptions in the second line (by default they are added) + --noenums do not add the list of options for a field if they are specified (by default they are added) + --comments adds any (usually internal) comments together with enums (by default False) + --outfile change the default file name "fields.xlsx" to a specified one + --debug to add more debugging output + --noadmin if you have admin access to 4DN this option lets you generate the sheet as a non-admin user This program graphs uploadable fields (i.e. not calculated properties) @@ -44,9 +44,9 @@ def _remove_all_from_types(args): - ''' helper method to remove the default 'all' argument that is automatically + """ helper method to remove the default 'all' argument that is automatically add by having a default with the append action for types option - ''' + """ if len(args.type) > 1: types = args.type types.remove('all') @@ -144,7 +144,7 @@ def __init__(self, key4dn): self.email = me_page['email'] self.check = True self.admin = True if 'admin' in me_page.get('groups', []) else False - except: + except Exception: print('Can not establish connection, please check your keys') me_page = {} if not me_page: @@ -162,11 +162,11 @@ def __init__(self, key4dn): self.award = None def set_award(self, lab, dontPrompt=False): - '''Sets the award for the connection for use in import_data + """Sets the award for the connection for use in import_data if dontPrompt is False will ask the User to choose if there are more than one award for the connection.lab otherwise the first award for the lab will be used - ''' + """ self.award = None if lab is not None: labjson = ff_utils.get_metadata(lab, key=self.key) @@ -195,10 +195,10 @@ def set_award(self, lab, dontPrompt=False): return def prompt_for_lab_award(self, lab=None, award=None): - '''Check to see if user submits_for multiple labs or the lab + """Check to see if user submits_for multiple labs or the lab has multiple awards and if so prompts for the one to set for the connection - ''' + """ if lab: if not award: self.set_award(self.lab) @@ -283,7 +283,7 @@ def is_subobject(field): return True try: return field['items']['type'] == 'object' - except: + except Exception: return False @@ -303,7 +303,7 @@ def build_field_list(properties, required_fields=None, no_description=False, continue if 'submit4dn' in props.get('exclude_from', []): continue - if ('import_items' in props.get('permission', []) and not admin): + if 'import_items' in props.get('permission', []) and not admin: continue if is_subobject(props) and name != 'attachment': if get_field_type(props).startswith('array'): @@ -391,12 +391,12 @@ def get_uploadable_fields(connection, types, no_description=False, def create_excel(all_fields, filename): - ''' + """ all_fields being a dictionary of sheet/Item names -> list of FieldInfo(objects) create one sheet per dictionary item, that inserts 4 commented header rows for each column that corresponds to one of the FieldInfo objects in the list header rows are for fieldname, fieldtype, description and comments/enums - ''' + """ wb = openpyxl.Workbook() wb.remove(wb.active) # removes the by default created empty sheet named Sheet # order sheets diff --git a/wranglertools/import_data.py b/wranglertools/import_data.py index 442d3575..1ed1628c 100755 --- a/wranglertools/import_data.py +++ b/wranglertools/import_data.py @@ -1087,7 +1087,7 @@ def workbook_reader(workbook, sheet, update, connection, patchall, aliases_by_ty on the options passed in. """ # determine right from the top if dry run - dryrun = not(update or patchall) + dryrun = not (update or patchall) all_aliases = [k for k in aliases_by_type] # dict for acumulating cycle patch data patch_loadxl = [] @@ -1420,7 +1420,7 @@ def user_workflow_reader(workbook, sheet, connection): def get_upload_creds(file_id, connection): # pragma: no cover - url = "%s/upload/" % (file_id) + url = f"{file_id}/upload/" req = ff_utils.post_metadata({}, url, key=connection.key) return req['@graph'][0]['upload_credentials'] @@ -1451,7 +1451,7 @@ def upload_file(creds, path): # pragma: no cover 'AWS_SECURITY_TOKEN': creds['SessionToken'], }) except Exception as e: - raise("Didn't get back s3 access keys from file/upload endpoint. Error was %s" % str(e)) + raise Exception(f"Didn't get back s3 access keys from file/upload endpoint. Error was {e}") # ~10s/GB from Stanford - AWS Oregon # ~12-15s/GB from AWS Ireland - AWS Oregon print("Uploading file.") @@ -1490,8 +1490,9 @@ def order_sorter(list_of_names): # expected list if multiple; ['user_workflow_1', 'user_workflow_2'] user_workflows = sorted([sh for sh in list_of_names if sh.startswith('user_workflow')]) ret_list.extend(user_workflows) - if list(set(list_of_names)-set(ret_list)) != []: - missing_items = ", ".join(list(set(list_of_names)-set(ret_list))) + missing = set(list_of_names) - set(ret_list) + if missing: + missing_items = ", ".join(missing) print("WARNING!", missing_items, "sheet(s) are not loaded") print("WARNING! Check the sheet names and the reference list \"sheet_order\"") return ret_list