Skip to content

Commit a19e782

Browse files
authored
add extract method to MetaflowCode (Netflix#2246)
* add extract method to MetaflowCode * add script property to metaflow code * add script property to metaflow code * add script property to metaflow code * changes * Address comments
1 parent b6b0dc8 commit a19e782

File tree

1 file changed

+68
-0
lines changed

1 file changed

+68
-0
lines changed

metaflow/client/core.py

+68
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import tarfile
66
from collections import namedtuple
77
from datetime import datetime
8+
from tempfile import TemporaryDirectory
89
from io import BytesIO
910
from itertools import chain
1011
from typing import (
@@ -878,6 +879,73 @@ def tarball(self) -> tarfile.TarFile:
878879
"""
879880
return self._tar
880881

882+
def extract(self) -> TemporaryDirectory:
883+
"""
884+
Extracts the code package to a temporary directory.
885+
886+
This creates a temporary directory containing all user code
887+
files from the code package. The temporary directory is
888+
automatically deleted when the returned TemporaryDirectory
889+
object is garbage collected or when its cleanup() is called.
890+
891+
To preserve the contents to a permanent location, use
892+
os.replace() which performs a zero-copy move on the same
893+
filesystem:
894+
895+
```python
896+
with task.code.extract() as tmp_dir:
897+
# Move contents to permanent location
898+
for item in os.listdir(tmp_dir):
899+
src = os.path.join(tmp_dir, item)
900+
dst = os.path.join('/path/to/permanent/dir', item)
901+
os.makedirs(os.path.dirname(dst), exist_ok=True)
902+
os.replace(src, dst) # Atomic move operation
903+
```
904+
Returns
905+
-------
906+
TemporaryDirectory
907+
A temporary directory containing the extracted code files.
908+
The directory and its contents are automatically deleted when
909+
this object is garbage collected.
910+
"""
911+
exclusions = [
912+
"metaflow/",
913+
"metaflow_extensions/",
914+
"INFO",
915+
"CONFIG_PARAMETERS",
916+
"conda.manifest",
917+
# This file is created when using the conda/pypi features available in
918+
# nflx-metaflow-extensions: https://github.com/Netflix/metaflow-nflx-extensions
919+
"condav2-1.cnd",
920+
]
921+
members = [
922+
m
923+
for m in self.tarball.getmembers()
924+
if not any(
925+
(x.endswith("/") and m.name.startswith(x)) or (m.name == x)
926+
for x in exclusions
927+
)
928+
]
929+
930+
tmp = TemporaryDirectory()
931+
self.tarball.extractall(tmp.name, members)
932+
return tmp
933+
934+
@property
935+
def script_name(self) -> str:
936+
"""
937+
Returns the filename of the Python script containing the FlowSpec.
938+
939+
This is the main Python file that was used to execute the flow. For example,
940+
if your flow is defined in 'myflow.py', this property will return 'myflow.py'.
941+
942+
Returns
943+
-------
944+
str
945+
Name of the Python file containing the FlowSpec
946+
"""
947+
return self._info["script"]
948+
881949
def __str__(self):
882950
return "<MetaflowCode: %s>" % self._info["script"]
883951

0 commit comments

Comments
 (0)