Merge pull request #128 from Michaelliv/function_to_item

Function yamls to item yamls

Author: aviaIguazio

aggregate/item.yaml

@@ -4,21 +4,20 @@ categories:
 description: Rolling aggregation over Metrics and Lables according to specifications
 doc: ''
 example: aggregate.ipynb
-generationDate: '2021-02-13 11:28:47.287600'
+generationDate: 2021-05-19:22-31
 icon: ''
 labels:
   author: avia
 maintainers: []
 marketplaceType: ''
 mlrunVersion: 0.6.2
 name: aggregate
-org: Iguazio
 platformVersion: 3.0.0
 spec:
   filename: aggregate.py
   handler: aggregate
   image: mlrun/ml-models
   kind: job
-  requirements: ['mlrun','pandas']
+  requirements: []
 url: ''
-version: 0.0.1
+version: 0.0.1

aggregate/requirements.txt

@@ -0,0 +1,2 @@
+mlrun
+pandas

arc_to_parquet/item.yaml

@@ -5,21 +5,20 @@ categories:
 description: retrieve remote archive, open and save as parquet
 doc: ''
 example: arc_to_parquet.ipynb
-generationDate: '2021-02-13 11:28:47.396240'
+generationDate: 2021-05-19:22-04
 icon: ''
 labels:
   author: yjb
 maintainers: []
 marketplaceType: ''
 mlrunVersion: 0.5.4
 name: arc-to-parquet
-org: Iguazio
 platformVersion: 2.10.0
 spec:
   filename: arc_to_parquet.py
   handler: arc_to_parquet
   image: mlrun/ml-base
   kind: job
-  requirements: ['mlrun','pandas','pyarrow']
+  requirements: []
 url: ''
-version: 0.0.1
+version: 0.0.1

arc_to_parquet/requirements.txt

@@ -0,0 +1,3 @@
+mlrun
+pyarrow
+pandas

bert_embeddings/bert_embeddings.py

@@ -1,9 +1,9 @@
-from transformers import BertModel, BertTokenizer
-import torch
-from typing import Union, List
 import json
 import pickle
 
+import torch
+from transformers import BertModel, BertTokenizer
+
 
 def init_context(context):
     tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
@@ -24,4 +24,4 @@ def handler(context, event):
     with torch.no_grad():
         embeddings = context.user_data.model(**docs)
     embeddings = [embeddings[0].numpy(), embeddings[1].numpy()]
-    return pickle.dumps(embeddings)
+    return pickle.dumps(embeddings)

bert_embeddings/item.yaml

@@ -6,21 +6,22 @@ categories:
 description: Get BERT based embeddings for given text
 doc: ''
 example: bert_embeddings.ipynb
-generationDate: '2021-02-13 11:28:47.759006'
+generationDate: 2021-05-19:22-04
 icon: ''
 labels:
   framework: pytorch
 maintainers: []
 marketplaceType: ''
 mlrunVersion: 0.5.4
 name: bert-embeddings
-org: Iguazio
 platformVersion: 2.10.0
 spec:
   filename: bert_embeddings.py
-  handler: ''
+  handler: handler
   image: ''
   kind: nuclio
-  requirements: ['torch==1.6.0','transformers==3.0.1','nuclio']
+  requirements:
+  - torch==1.6.0
+  - transformers==3.0.1
 url: ''
-version: 0.0.1
+version: 0.0.1

catalog.yaml

@@ -153,14 +153,14 @@ model-monitoring-batch:
   docfile: /home/michaell/projects/functions/model_monitoring_batch/model_monitoring_batch.ipynb
   kind: job
   versions:
-    latest: /home/michaell/projects/functions/model_monitoring_batch/model_monitoring_batch.yaml
+    latest: /home/michaell/projects/functions/model_monitoring_batch/function.yaml
 model-monitoring-stream:
   categories: []
   description: ''
   docfile: /home/michaell/projects/functions/model_monitoring_stream/model_monitoring_stream.ipynb
   kind: remote
   versions:
-    latest: /home/michaell/projects/functions/model_monitoring_stream/model_monitoring_stream.yaml
+    latest: /home/michaell/projects/functions/model_monitoring_stream/function.yaml
 model-server:
   categories:
   - serving

churn_server/item.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+categories:
+- serving
+- ml
+description: churn classification and predictor
+doc: ''
+example: churn_server.ipynb
+generationDate: 2021-05-19:22-04
+icon: ''
+labels:
+  framework: churn
+maintainers: []
+marketplaceType: ''
+mlrunVersion: ''
+name: churn-server
+platformVersion: ''
+spec:
+  filename: churn_server.py
+  handler: handler
+  image: mlrun/ml-models
+  kind: remote
+  requirements: []
+url: ''
+version: 0.0.1

cli/function_to_item.py

@@ -0,0 +1,198 @@
+from datetime import datetime
+from pathlib import Path
+from typing import Union
+
+import click
+import yaml
+
+from cli.helpers import is_function_dir
+from cli.path_iterator import PathIterator
+
+
+@click.command()
+@click.option(
+    "-p",
+    "-path",
+    help="Path to one of: specific function.yaml, directory containing function.yaml or a root directory to search function.yamls in",
+)
+def function_to_item_cli(path: str):
+    function_to_item(path)
+
+
+def function_to_item(path: str):
+    path = Path(path)
+
+    if not path.exists():
+        click.echo(f"{path} not found")
+        exit(1)
+
+    # If this is a directory, its either project root or a specific directory in the project
+    if path.is_dir():
+        function_path = path / "function.yaml"
+        # If its a function directory
+        if function_path.exists():
+            item_path = path / "item.yaml"
+            item = function_yaml_to_item(function_path)
+            with open(item_path, "w") as f:
+                yaml.dump(item, f)
+        # Otherwise its a root directory to be iterated
+        else:
+            function_iterator = PathIterator(
+                root=path, rule=is_function_dir, as_path=True
+            )
+            for function in function_iterator:
+                function_path = function / "function.yaml"
+                item_path = function / "item.yaml"
+                item = function_yaml_to_item(function_path)
+                with open(item_path, "w") as f:
+                    yaml.dump(item, f)
+    # Otherwise its a path to function.yaml
+    else:
+        path_dir = path.parent
+        item_path = path_dir / "item.yaml"
+        item = function_yaml_to_item(path)
+        with open(item_path, "w") as f:
+            yaml.dump(item, f)
+    exit(0)
+
+
+def function_yaml_to_item(function_path: Union[str, Path]) -> dict:
+
+    function_path = Path(function_path)
+    function_yaml = yaml.full_load(open(function_path))
+
+    metadata = function_yaml.get("metadata", {})
+    spec = function_yaml.get("spec", {})
+
+    item = {
+        "apiVersion": "v1",
+        "categories": metadata.get("categories") or [],
+        "description": spec.get("description") or "",
+        "doc": "",
+        "example": get_ipynb_file(function_path.parent),
+        "generationDate": datetime.utcnow().strftime("%Y-%m-%d:%H-%M"),
+        "icon": "",
+        "labels": metadata.get("labels") or {},
+        "maintainers": [],
+        "mlrunVersion": "",
+        "name": metadata.get("name") or "",
+        "platformVersion": "",
+        "spec": {
+            "filename": get_py_file(function_path.parent),
+            "handler": get_handler(function_yaml),
+            "image": get_image(function_yaml),
+            "kind": function_yaml.get("kind") or "",
+            "requirements": get_requirements(function_yaml),
+        },
+        "url": "",
+        "version": metadata.get("tag") or "0.0.1",
+        "marketplaceType": "",
+    }
+
+    return item
+
+
+def get_ipynb_file(path: Path) -> str:
+    default = path / f"{path.name}.ipynb"
+
+    if default.exists():
+        return f"{path.name}.ipynb"
+
+    ipynbs = list(filter(lambda d: d.suffix == ".ipynb", path.iterdir()))
+    ipynbs = list(filter(lambda d: not d.name.startswith("test"), ipynbs))
+
+    if len(ipynbs) > 1:
+        click.echo(f"{path.name}: Notebook not found")
+        return ""
+    elif len(ipynbs) == 1:
+        return ipynbs[0].name
+
+    click.echo(f"{path.name}: Notebook not found")
+    return ""
+
+
+def get_py_file(path: Path) -> str:
+    default_py_file = path / "function.py"
+
+    if default_py_file.exists():
+        return "function.py"
+
+    py_file = filter(lambda d: d.suffix == ".py", path.iterdir())
+    py_file = list(filter(lambda d: not d.name.startswith("test"), py_file))
+
+    if len(py_file) > 1:
+        click.echo(f"{path.name}: Python file not found")
+        return ""
+    elif len(py_file) == 1:
+        return py_file[0].name
+
+    click.echo(f"{path.name}: Python file not found")
+    return ""
+
+
+def get_handler(function_yaml: dict) -> str:
+    spec = function_yaml["spec"]
+    handler = spec.get("default_handler", "handler")
+    return handler
+
+
+def get_image(function_yaml: dict):
+    spec = function_yaml.get("spec", {})
+    spec_image = spec.get("image")
+
+    build = spec.get("build", {})
+    build_image = build.get("base_image")
+
+    base_spec = spec.get("base_spec", {}).get("spec", {})
+    base_spec_image = base_spec.get("build", {}).get("baseImage")
+
+    return spec_image or build_image or base_spec_image or ""
+
+
+def get_requirements(function_yaml: dict):
+    spec = function_yaml.get("spec", {})
+    base_spec = spec.get("base_spec", {}).get("spec", {})
+
+    spec_commands = spec.get("build", {}).get("commands", [])
+    base_spec_commands = base_spec.get("build", {}).get("commands", [])
+
+    commands = set()
+
+    for command in spec_commands:
+        commands.add(command)
+
+    for command in base_spec_commands:
+        commands.add(command)
+
+    requirements = []
+    for command in commands:
+        if "uninstall" in command:
+            click.echo(f"{function_yaml['metadata']['name']}: Unsupported requirements")
+            return []
+        if "python -m pip install " in command:
+            command = command.split("python -m pip install ")[-1]
+        elif "pip install " in command:
+            command = command.split("pip install ")[-1]
+        else:
+            click.echo(f"{function_yaml['metadata']['name']}: Unsupported requirements")
+            return []
+
+        allowed_chars = {".", "_", "-", "="}
+
+        if " " in command:
+            sub_commands = command.split(" ")
+            for sub_command in sub_commands:
+                for char in sub_command:
+                    if not char.isalnum() and char not in allowed_chars:
+                        click.echo(
+                            f"{function_yaml['metadata']['name']}: Unsupported requirements"
+                        )
+                        return []
+                requirements.append(sub_command)
+        else:
+            requirements.append(command)
+    return requirements
+
+
+if __name__ == "__main__":
+    function_to_item("/home/michaell/projects/functions")

cli/helpers.py

@@ -12,6 +12,15 @@ def is_item_dir(path: Path) -> bool:
     return path.is_dir() and (path / "item.yaml").exists()
 
 
+def is_function_dir(path: Path) -> bool:
+    if path.is_file():
+        return False
+    # dir_name = path.name
+    # ipynb_found = any((f.name.endswith(".ipynb") for f in path.iterdir()))
+    # py_found = any((f.name.endswith(".py") for f in path.iterdir()))
+    return any((f.name == "function.yaml" for f in path.iterdir()))
+
+
 def render_jinja_file(
     template_path: Union[str, Path], output_path: Union[str, Path], data: dict
 ):

cli/item_to_function.py

@@ -1,4 +1,3 @@
-from optparse import OptionParser
 from pathlib import Path
 from typing import Optional
 
@@ -42,7 +41,7 @@ def item_to_function(item_path: str, output_path: Optional[str] = None):
         description=item_yaml.get("description", ""),
         requirements=item_yaml.get("spec", {}).get("requirements"),
         categories=item_yaml.get("categories", []),
-        labels=item_yaml.get("labels", {})
+        labels=item_yaml.get("labels", {}),
     )
 
     if output_path is None: