Various updates.

pyproject.toml

@@ -13,17 +13,17 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.7.0,<3.10"
-python-magic = ">=0.4.12"
-attrs = ">=21.4"
-openpyxl = "^3.0.9"
-dcicutils = ">=4.0"
+python-magic = ">=0.4.12,<1"
+attrs = "^22.2"
+openpyxl = "^3.1.2"
+dcicutils = "^6.10.1"
 # awscli is not directly imported but is required for aws cp operation
-awscli = "^1.22.88"
+awscli = "^1.27"
 
 [tool.poetry.dev-dependencies]
-pytest = "3.0.1"
-pytest-cov = "2.3.1"
-pytest-mock = ">=1.11.2, <=2.0.0"
+pytest = ">=7.2.2"
+pytest-cov = ">=4.0.0"
+pytest-mock = ">=3.10.0"
 
 [tool.poetry.scripts]
 import_data = "wranglertools.import_data:main"

tests/test_import_data.py

@@ -1,3 +1,4 @@
+from dcicutils.misc_utils import get_error_message
 import wranglertools.import_data as imp
 import pytest
 import pathlib as pp
@@ -7,7 +8,13 @@
 # @pytest.mark.file_operation
 @pytest.mark.ftp
 def test_attachment_from_ftp():
-    attach = imp.attachment("ftp://speedtest.tele2.net/1KB.zip")
+    try:
+        attach = imp.attachment("ftp://speedtest.tele2.net/1KB.zip")
+    except Exception as e:
+        msg = get_error_message(e)
+        if 'timeout' in msg.lower():
+            pytest.xfail(f"Integration test failure. {msg}")
+        raise
     assert attach
 
 

wranglertools/import_data.py

@@ -350,7 +350,7 @@ def get_sub_field(field_name):
     """Construct embeded field names."""
     try:
         return field_name.split(".")[1].rstrip('-0123456789')
-    except:  # pragma: no cover
+    except Exception:  # pragma: no cover
         return ''
 
 
@@ -373,7 +373,7 @@ def get_sub_field_number(field_name):
     field = field_name.split(":")[0]
     try:
         return int(field.split("-")[1])
-    except:
+    except Exception:
         return 0
 
 
@@ -418,7 +418,7 @@ def parse_exception(e):
         resp_dict = ast.literal_eval(resp_text)
         return resp_dict
     # if not re-raise
-    except:  # pragma: no cover
+    except Exception:  # pragma: no cover
         raise e
 
 
@@ -802,13 +802,13 @@ def error_report(error_dic, sheet, all_aliases, connection, error_id=''):
         try:
             report.append("{sheet:<30}{eid}: {des}"
                           .format(des=error_description, eid=error_id, sheet="ERROR " + sheet.lower()))
-        except:
+        except Exception:
             return error_dic
     # if there is a conflict
     elif error_dic.get('title') == "Conflict":
         try:
             report.extend(conflict_error_report(error_dic, sheet, connection))
-        except:
+        except Exception:
             return error_dic
     # if nothing works, give the full error, we should add that case to our reporting
     else:
@@ -840,15 +840,15 @@ def conflict_error_report(error_dic, sheet, connection):
                 existing_item = ff_utils.search_metadata(search, key=connection.key)
                 at_id = existing_item.get('@id')
                 add_text = "please use " + at_id
-            except:
+            except Exception:
                 # if there is a conflicting item, but it is not viewable by the user,
                 # we should release the item to the project/public
                 add_text = "please contact DCIC"
             conflict_rep = ("{sheet:<30}Field '{er}': '{des}' already exists, {at}"
                             .format(er=error_field, des=error_value, sheet="ERROR " + sheet.lower(), at=add_text))
         all_conflicts.append(conflict_rep)
         return all_conflicts
-    except:
+    except Exception:
         return
 
 
@@ -860,7 +860,7 @@ def update_item(verb, file_to_upload, post_json, filename_to_post, extrafiles, c
         file_to_upload, post_json, filename_to_post = ftp_copy(filename_to_post, post_json)
     # add the md5
     if file_to_upload and not post_json.get('md5sum'):
-        print("calculating md5 sum for file %s " % (filename_to_post))
+        print(f"calculating md5 sum for file {filename_to_post} ")
         post_json['md5sum'] = md5(filename_to_post)
     try:
         if verb == 'PATCH':
@@ -888,7 +888,7 @@ def update_item(verb, file_to_upload, post_json, filename_to_post, extrafiles, c
             try:
                 file_format = ff_utils.get_metadata(fformat, key=connection.key)
                 ff_uuid = file_format.get('uuid')
-            except:
+            except Exception:
                 raise "Can't find file_format item for %s" % fformat
             for ecred in extcreds:
                 if ff_uuid == ecred.get('file_format'):
@@ -922,7 +922,7 @@ def ftp_copy(filename_to_post, post_json):
             with open(new_file, 'wb') as f:
                 shutil.copyfileobj(r, f)
         return True, post_json, new_file
-    except:
+    except Exception:
         # if download did not work, delete the filename from the post json
         print("WARNING: Download failed")
         post_json.pop('filename')
@@ -1596,10 +1596,7 @@ def cabin_cross_check(connection, patchall, update, infile, remote, lab=None, aw
         print("##############   DRY-RUN MODE   ################\n")
     else:
         if not remote:
-            try:
-                response = raw_input("Do you want to continue with these credentials? (Y/N): ") or "N"
-            except NameError:
-                response = input("Do you want to continue with these credentials? (Y/N): ") or "N"
+            response = input("Do you want to continue with these credentials? (Y/N): ") or "N"
             if response.lower() not in ["y", "yes"]:
                 sys.exit(1)
 
@@ -1636,7 +1633,7 @@ def get_all_aliases(workbook, sheets):
         keys = next(rows)  # grab the first row of headers
         try:
             alias_col = keys.index("aliases")
-        except:
+        except Exception:
             continue
         for row in rows:
             my_aliases = []