Config file

nb2workflow/__init__.py

@@ -1,4 +1,5 @@
 import pkg_resources
+import os
 
 name = "nb2workflow"
 

nb2workflow/deploy.py

@@ -19,6 +19,7 @@
 import rdflib
 from oda_api.ontology_helper import Ontology
 from nb2workflow.nbadapter import NotebookAdapter
+from dynaconf import Dynaconf
 
 logger = logging.getLogger(__name__)
 
@@ -30,7 +31,9 @@
     "filename_pattern": '.*', 
 }
 
-default_ontology_path = "https://odahub.io/ontology/ontology.ttl"
+local_config = Dynaconf(settings_files=['settings.toml'])
+config_ontology_path = local_config.get('default.service.ontology_path', 'http://odahub.io/ontology/ontology.ttl')
+
 
 default_python_version = '3.10'
 
@@ -69,8 +72,9 @@ def build_container(git_origin,
                     engine="docker",
                     cleanup=False,
                     nb2wversion=version(),
-                    ontology_path=default_ontology_path,
+                    ontology_path=config_ontology_path,
                     **kwargs):
+
     if engine == "docker":
         return _build_with_docker(git_origin=git_origin,
                                  local=local,
@@ -200,7 +204,10 @@ def _build_with_kaniko(git_origin,
                       namespace="oda-staging",
                       cleanup=True,
                       nb2wversion=version(),
-                      ontology_path=default_ontology_path):
+                      ontology_path=None):
+
+    if ontology_path is None:
+        ontology_path = local_config.get('default.service.ontology_path', config_ontology_path)
 
     #secret should be created beforehand https://github.com/GoogleContainerTools/kaniko#pushing-to-docker-hub
 
@@ -316,7 +323,8 @@ def _build_with_kaniko(git_origin,
         return container_metadata
 
 
-def _extract_resource_requirements(local_repo_path, ontology_path=default_ontology_path):
+def _extract_resource_requirements(local_repo_path, ontology_path=config_ontology_path):
+
     ontology = Ontology(ontology_path)
     resources = {}
 
@@ -345,7 +353,8 @@ def _build_with_docker(git_origin,
                     source_from='localdir',
                     cleanup=False,
                     nb2wversion=version(),
-                    ontology_path=default_ontology_path):
+                    ontology_path=config_ontology_path):
+
     if cleanup:
         logger.warning('Post-build cleanup is not implemented for docker builds')
 
@@ -575,8 +584,8 @@ def deploy(git_origin,
            build_timestamp=False,
            cleanup=False,
            nb2wversion=version(),
-           ontology_path=default_ontology_path):
-    
+           ontology_path=config_ontology_path):
+
     container = build_container(git_origin,
                                 local=local, 
                                 run_tests=run_tests, 
@@ -617,10 +626,14 @@ def main():
     parser.add_argument('--local', action="store_true", default=False)
     parser.add_argument('--build-engine', metavar="build_engine", default="docker")
     parser.add_argument('--nb2wversion', metavar="nb2wversion", default=version())
-    parser.add_argument('--ontology-path', metavar="ontology_path", default=default_ontology_path)
-    
+    parser.add_argument('--ontology-path', metavar="ontology_path")
+
     args = parser.parse_args()
 
+    deploy_ontology_path = args.ontology_path
+    if deploy_ontology_path is None:
+        deploy_ontology_path = config_ontology_path
+
     setup_logging()
 
     deploy(args.repository, 
@@ -629,7 +642,7 @@ def main():
            local=args.local, 
            build_engine=args.build_engine, 
            nb2wversion=args.nb2wversion,
-           ontology_path=args.ontology_path)
+           ontology_path=deploy_ontology_path)
 
 
 if __name__ == "__main__":

nb2workflow/galaxy.py

@@ -15,6 +15,7 @@
 
 import nbformat
 from nbconvert.exporters import ScriptExporter
+from dynaconf import Dynaconf
 
 from ensureconda.api import ensureconda
 import subprocess as sp
@@ -29,8 +30,8 @@
 
 logger = logging.getLogger()
 
-               
-default_ontology_path = 'http://odahub.io/ontology/ontology.ttl'
+local_config = Dynaconf(settings_files=['settings.toml'])
+config_ontology_path = local_config.get('default.service.ontology_path', 'http://odahub.io/ontology/ontology.ttl')
 
 global_req = []
 
@@ -539,7 +540,7 @@ def to_galaxy(input_path,
               citations_bibfile = None,
               help_file = None,
               available_channels = ['default', 'conda-forge'],
-              ontology_path = default_ontology_path,
+              ontology_path = config_ontology_path,
               test_data_baseurl = None
               ):
 
@@ -690,7 +691,7 @@ def main():
     tool_version = args.tool_version
     ontology_path = args.ontology_path
     if ontology_path is None:
-        ontology_path = default_ontology_path
+        ontology_path = config_ontology_path
     bibfile = args.citations_bibfile
     help_file = args.help_file
     test_data_baseurl = args.test_data_baseurl

nb2workflow/nbadapter.py

@@ -28,9 +28,10 @@
 import nbformat
 from nbconvert import HTMLExporter
 from urllib.parse import urlencode, urlparse
+from urllib import request
+from dynaconf import Dynaconf
 
 from . import logstash
-
 from nb2workflow.sentry import sentry
 from nb2workflow.health import current_health
 from nb2workflow import workflows
@@ -249,14 +250,20 @@ class NotebookAdapter:
     limit_output_attachment_file = None
 
 
-    def __init__(self, notebook_fn, tempdir_cache=None, n_download_max_tries=10, download_retry_sleep=.5):
+    def __init__(self, notebook_fn, tempdir_cache=None, config=None):
         self.notebook_fn = os.path.abspath(notebook_fn)
         self.name = notebook_short_name(notebook_fn)
         self.tempdir_cache = tempdir_cache
         logger.debug("notebook adapter for %s", self.notebook_fn)
         logger.debug(self.extract_parameters())
-        self.n_download_max_tries = n_download_max_tries
-        self.download_retry_sleep_s = download_retry_sleep
+
+        if config is None:
+            config = dict()
+
+        self.n_download_max_tries = config.get('SERVICE.N_DOWNLOAD_MAX_TRIES', 10)
+        self.download_retry_sleep_s = config.get('SERVICE.DOWNLOAD_RETRY_SLEEP', .5)
+        self.max_download_size = config.get('SERVICE.MAX_DOWNLOAD_SIZE', 1e6)
+        sentry.sentry_url = config.get('SERVICE.SENTRY_URL', None)
 
     @staticmethod
     def get_unique_filename_from_url(file_url):
@@ -567,26 +574,51 @@ def extract_output(self):
 
     def download_file(self, file_url, tmpdir):
         n_download_tries_left = self.n_download_max_tries
+        size_ok = False
+        file_downloaded = False
         file_name = NotebookAdapter.get_unique_filename_from_url(file_url)
-        while True:
+        file_path = os.path.join(tmpdir, file_name)
+        for _ in range(n_download_tries_left):
+            step = 'getting the file size'
+            if not size_ok:
+                response = requests.head(file_url)
+                if response.status_code == 200:
+                    file_size = int(response.headers.get('Content-Length', 0))
+                    if file_size > self.max_download_size:
+                        msg = ("The file appears to be too large to download, "
+                               f"and the download limit is set to {self.max_download_size} bytes.")
+                        logger.warning(msg)
+                        sentry.capture_message(msg)
+                        raise Exception(msg)
+                else:
+                    logger.warning(
+                        (f"An issue occurred when attempting to {step} of the file at the url {file_url}. "
+                         f"Sleeping {self.download_retry_sleep_s} seconds until retry")
+                    )
+                    time.sleep(self.download_retry_sleep_s)
+                    continue
+            size_ok = True
+            step = 'downloading file'
             response = requests.get(file_url)
             if response.status_code == 200:
-                with open(os.path.join(tmpdir, file_name), 'wb') as file:
+                with open(file_path, 'wb') as file:
                     file.write(response.content)
+                file_downloaded = True
                 break
             else:
-                n_download_tries_left -= 1
-                if n_download_tries_left > 0:
-                    logger.warning(
-                        f"An issue occurred when attempting to download the file at the url {file_url}, "
-                        f"sleeping {self.download_retry_sleep_s} seconds until retry")
-                    time.sleep(self.download_retry_sleep_s)
-                else:
-                    msg = (f"An issue occurred when attempting to download the url {file_url}, "
-                           "this might be related to an invalid url, please check the input provided")
-                    logger.warning(msg)
-                    sentry.capture_message(msg)
-                    raise Exception(msg)
+                logger.warning(
+                    (f"An issue occurred when attempting to {step} the file at the url {file_url}. "
+                     f"Sleeping {self.download_retry_sleep_s} seconds until retry")
+                )
+                time.sleep(self.download_retry_sleep_s)
+                continue
+
+        if not (file_downloaded and size_ok):
+            msg = (f"An issue occurred when attempting to {step} at the url {file_url}. "
+                   "This might be related to an invalid url, please check the input provided")
+            logger.warning(msg)
+            sentry.capture_message(msg)
+            raise Exception(msg)
 
         return file_name
 
@@ -712,13 +744,12 @@ def remove_tmpdir(self):
 def notebook_short_name(ipynb_fn):
     return os.path.basename(ipynb_fn).replace(".ipynb","")
 
-def find_notebooks(source, tests=False, pattern = r'.*') -> Dict[str, NotebookAdapter]:
+def find_notebooks(source, tests=False, pattern = r'.*', config=None) -> Dict[str, NotebookAdapter]:
 
     def base_filter(fn): 
         good = "output" not in fn and "preproc" not in fn
         good = good and re.match(pattern, os.path.basename(fn)) 
         return good
-
 
     if tests:
         filt = lambda fn: base_filter(fn) and "/test_" in fn
@@ -734,23 +765,26 @@ def base_filter(fn):
             raise Exception("no notebooks found in the directory:",source)
 
         notebook_adapters=dict([
-                (notebook_short_name(notebook),NotebookAdapter(notebook)) for notebook in notebooks
+                (
+                    notebook_short_name(notebook),
+                    NotebookAdapter(notebook, config=config)
+                 ) for notebook in notebooks
             ])
         logger.debug("notebook adapters: %s",notebook_adapters)
 
 
     elif os.path.isfile(source):
         if pattern != r'.*':
             logger.warning('Filename pattern is set but source %s is a single file. Ignoring pattern.')
-        notebook_adapters={notebook_short_name(source): NotebookAdapter(source)}
+        notebook_adapters={notebook_short_name(source): NotebookAdapter(source, config=config)}
 
     else:
         raise Exception("requested notebook not found:",source)
 
     return notebook_adapters
 
-def nbinspect(nb_source, out=True, machine_readable=False):
-    nbas = find_notebooks(nb_source)
+def nbinspect(nb_source, out=True, machine_readable=False, config=None):
+    nbas = find_notebooks(nb_source, config=config)
 
     # class CustomEncoder(json.JSONEncoder):
     #     def default(self, obj):
@@ -885,9 +919,9 @@ def json():
 
 
 
-def nbrun(nb_source, inp, inplace=False, optional_dispather=True, machine_readable=False):
+def nbrun(nb_source, inp, inplace=False, optional_dispather=True, machine_readable=False, config=None):
 
-    nbas = find_notebooks(nb_source)
+    nbas = find_notebooks(nb_source, config=config)
 
     if len(nbas) > 1:
         nba = nbas[inp.pop('notebook')]
@@ -1011,7 +1045,9 @@ def main_inspect():
 
     setup_logging(args.debug)
 
-    nbinspect(args.notebook, machine_readable=args.machine_readable)
+    config = Dynaconf(settings_files=['settings.toml'])
+
+    nbinspect(args.notebook, machine_readable=args.machine_readable, config=config)
 
 
 def main():
@@ -1034,7 +1070,13 @@ def main():
 
     setup_logging(args.debug)
 
-    nbrun(args.notebook, inputs, inplace=args.inplace, optional_dispather=not args.mmoda_validation, machine_readable=args.machine_readable)
+    config = Dynaconf(settings_files=['settings.toml'])
+
+    nbrun(args.notebook, inputs,
+          inplace=args.inplace,
+          optional_dispather=not args.mmoda_validation,
+          machine_readable=args.machine_readable,
+          config=config)
 
 
 if __name__ == "__main__":

nb2workflow/sentry.py

@@ -20,6 +20,10 @@ def sentry_url(self):
             self._sentry_url = os.environ.get('SENTRY_URL', "https://[email protected]/4506186624335872")
         return self._sentry_url
 
+    @sentry_url.setter
+    def sentry_url(self, url):
+        self._sentry_url = url
+
     @property
     def have_sentry(self):
         if self.sentry_url is None or self.sentry_url == '' or sentry_sdk is None: