Skip to content

Commit

Permalink
feat: add compound file ordinal endpoints update (#56)
Browse files Browse the repository at this point in the history
  • Loading branch information
@PJEstrada
PJEstrada authored Feb 7, 2023
1 parent facca7b commit 74eeef3
Show file tree
Hide file tree
Showing 2 changed files with 131 additions and 28 deletions.
122 changes: 104 additions & 18 deletions sdk/diffgram/file/compound_file.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@


class CompoundChildFile:
id: int
path: str
url: str
blob_path: str
Expand All @@ -24,8 +25,10 @@ class CompoundChildFile:
video_split_duration: int
local_id: str
ordinal: int
root_file: 'CompoundFile'

def __init__(self,
root_file: 'CompoundFile',
child_file_type: str,
path: str = None,
url: str = None,
Expand All @@ -42,7 +45,9 @@ def __init__(self,
assume_new_instances_machine_made: bool = None,
convert_names_to_label_files: bool = None,
video_split_duration: int = None,
ordinal: int = 0):
ordinal: int = 0,
id: int = None):
self.root_file = root_file
self.child_file_type = child_file_type
self.path = path
self.url = url
Expand All @@ -61,6 +66,38 @@ def __init__(self,
self.video_split_duration = video_split_duration
self.local_id = str(uuid4())
self.ordinal = ordinal
self.id = None

def __str__(self):
return f'<CompoundChildFile id={self.id} ordinal={self.ordinal} file_name={self.file_name} child_file_type={self.child_file_type}>'
def refresh_from_data_dict(self, data: dict):
if not data:
return
for key in data:
setattr(self, key, data[key])

def update(self):
"""
Syncs child file data with backend API
:return:
"""
payload = {
'ordinal': self.ordinal,
}
client = self.root_file.project
endpoint = f"/api/v1/project/{self.root_file.project.project_string_id}/file/{self.id}/update-metadata"

response = client.session.put(
client.host + endpoint,
json = payload)

client.handle_errors(response)

data = response.json()
new_file_data = data['file']

self.refresh_from_data_dict(data = new_file_data)
return self

def set_ordinal(self, value: int):
self.ordinal = value
Expand All @@ -69,23 +106,57 @@ def set_ordinal(self, value: int):
class CompoundFile:
project: Project
parent_file_data: dict
child_files_to_upload: List[CompoundChildFile]
child_files: List[CompoundChildFile]

def __init__(self, project: Project, name: str, directory_id: int):
self.project = project
self.name = name
self.directory_id = directory_id
self.child_files_to_upload = []
self.child_files = []

@staticmethod
def from_dict(project: Project, dir_id: int, dict_data: dict):
result = CompoundFile(project = project, name = dict_data.get('original_filename'), directory_id = dir_id)
result.__refresh_compound_file_from_data_dict(data = dict_data)
child_files = result.__fetch_child_files()
result.child_files = child_files
return result

def __fetch_child_files(self) -> List[CompoundChildFile]:
client = self.project
endpoint = f"/api/v1/project/{self.project.project_string_id}/file/{self.id}/child-files"

response = client.session.get(client.host + endpoint)

client.handle_errors(response)

data = response.json()
child_files_data = data['child_files']
result = []
for elm in child_files_data:
child_file = CompoundChildFile(root_file = self, child_file_type = elm.get('type'))
child_file.refresh_from_data_dict(data = elm)
result.append(child_file)
return result

def update_all(self) -> bool:
"""
Syncs parent and child metadata with backend API.
:return: True/False
"""
for child in self.child_files:
child.update()
return True

def __refresh_compound_file_from_data_dict(self, data: dict):
if not data:
return
for key in data:
setattr(self, key, data[key])

def remove_compound_file(self, child_file: CompoundChildFile) -> List[CompoundChildFile]:
self.child_files_to_upload.remove(child_file)
return self.child_files_to_upload
def remove_child_file(self, child_file: CompoundChildFile) -> List[CompoundChildFile]:
self.child_files.remove(child_file)
return self.child_files

def __create_compound_parent_file(self):
url = f'/api/v1/project/{self.project.project_string_id}/file/new-compound'
Expand All @@ -110,7 +181,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
frame_packet_map = child_file.frame_packet_map,
assume_new_instances_machine_made = child_file.assume_new_instances_machine_made,
convert_names_to_label_files = child_file.convert_names_to_label_files,
parent_file_id = self.parent_file_data.get('id')
parent_file_id = self.parent_file_data.get('id'),
ordinal = child_file.ordinal,
)
elif child_file.child_file_type == 'from_url':
return self.project.file.from_url(
Expand All @@ -121,7 +193,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
video_split_duration = child_file.video_split_duration,
instance_list = child_file.instance_list,
frame_packet_map = child_file.frame_packet_map,
parent_file_id = self.parent_file_data.get('id')
parent_file_id = self.parent_file_data.get('id'),
ordinal = child_file.ordinal,
)
elif child_file.child_file_type == 'from_blob_path':
return self.project.file.from_blob_path(
Expand All @@ -132,7 +205,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
instance_list = child_file.instance_list,
file_name = child_file.file_name,
frame_packet_map = child_file.frame_packet_map,
parent_file_id = self.parent_file_data.get('id')
parent_file_id = self.parent_file_data.get('id'),
ordinal = child_file.ordinal,
)

def add_child_from_local(self,
Expand All @@ -141,17 +215,21 @@ def add_child_from_local(self,
frame_packet_map: dict = None,
assume_new_instances_machine_made: bool = True,
convert_names_to_label_files: bool = True,
ordinal: int = 0):
ordinal: int = None):
if ordinal is None:
ordinal = len(self.child_files)
new_child_file = CompoundChildFile(
root_file = self,
child_file_type = "from_local",
path = path,
directory_id = self.directory_id,
instance_list = instance_list,
frame_packet_map = frame_packet_map,
assume_new_instances_machine_made = assume_new_instances_machine_made,
convert_names_to_label_files = convert_names_to_label_files
convert_names_to_label_files = convert_names_to_label_files,
ordinal = ordinal
)
self.child_files_to_upload.append(new_child_file)
self.child_files.append(new_child_file)
return new_child_file

def add_child_file_from_url(self,
Expand All @@ -162,8 +240,11 @@ def add_child_file_from_url(self,
video_split_duration: int = None,
instance_list: list = None,
frame_packet_map: dict = None,
ordinal: int = 0):
ordinal: int = None):
if ordinal is None:
ordinal = len(self.child_files)
new_child_file = CompoundChildFile(
root_file = self,
child_file_type = "from_url",
url = url,
media_type = media_type,
Expand All @@ -173,8 +254,9 @@ def add_child_file_from_url(self,
video_split_duration = video_split_duration,
instance_list = instance_list,
frame_packet_map = frame_packet_map,
ordinal = ordinal
)
self.child_files_to_upload.append(new_child_file)
self.child_files.append(new_child_file)
return new_child_file

def add_child_from_blob_path(self,
Expand All @@ -185,9 +267,12 @@ def add_child_from_blob_path(self,
instance_list: list = None,
file_name: str = None,
frame_packet_map: dict = None,
ordinal: int = 0
ordinal: int = None
):
if ordinal is None:
ordinal = len(self.child_files)
new_child_file = CompoundChildFile(
root_file = self,
child_file_type = "from_blob_path",
blob_path = blob_path,
bucket_name = bucket_name,
Expand All @@ -197,15 +282,16 @@ def add_child_from_blob_path(self,
instance_list = instance_list,
file_name = file_name,
frame_packet_map = frame_packet_map,
ordinal = ordinal
)
self.child_files_to_upload.append(new_child_file)
self.child_files.append(new_child_file)
return new_child_file

def upload(self):
if len(self.child_files_to_upload) == 0:
if len(self.child_files) == 0:
raise AssertionError('Need to add at least one child file to the compound file before calling upload()')
parent_file_data: dict = self.__create_compound_parent_file()

for child_file in self.child_files_to_upload:
for child_file in self.child_files:
self.__create_child_file(child_file)
return parent_file_data
37 changes: 27 additions & 10 deletions sdk/diffgram/file/file_constructor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ def from_local(
frame_packet_map: dict = None,
assume_new_instances_machine_made: bool = True,
convert_names_to_label_files: bool = True,
parent_file_id: int = None
parent_file_id: int = None,
ordinal: int = 0
):
"""
Create a Project file from local path
Expand All @@ -56,7 +57,9 @@ def from_local(

files = {'file': (os.path.basename(path), open(path, 'rb'), 'application/octet-stream')}

json_payload = {}
json_payload = {
'ordinal': ordinal
}

if directory_id is None:
directory_id = self.client.directory_id
Expand Down Expand Up @@ -107,13 +110,15 @@ def __build_packet_payload(self,
bucket_name: str = None,
file_name: str = None,
blob_path: str = None,
parent_file_id: int = None):
parent_file_id: int = None,
ordinal: int = 0):
packet = {'media': {}}
packet['media']['url'] = url
packet['media']['type'] = media_type

# Existing Instances
packet['frame_packet_map'] = frame_packet_map
packet['ordinal'] = ordinal
packet['type'] = type
packet['connection_id'] = connection_id
packet['parent_file_id'] = parent_file_id
Expand All @@ -140,7 +145,8 @@ def from_blob_path(self,
instance_list: list = None,
file_name: str = None,
frame_packet_map: dict = None,
parent_file_id: int = None):
parent_file_id: int = None,
ordinal: int = 0):
"""
Bind a blob path in the given connection ID into Diffgram
:param blob_path:
Expand All @@ -167,7 +173,8 @@ def from_blob_path(self,
file_name = name,
directory_id = directory_id,
type = "from_blob_path",
parent_file_id=parent_file_id
parent_file_id=parent_file_id,
ordinal = ordinal
)
self.from_packet(packet = packet)
return True
Expand All @@ -181,7 +188,8 @@ def from_url(
video_split_duration: int = None,
instance_list: list = None, # for Images
frame_packet_map: dict = None, # for Video
parent_file_id: int = None
parent_file_id: int = None,
ordinal: int = 0
):
"""
Expand Down Expand Up @@ -214,7 +222,8 @@ def from_url(
video_split_duration = video_split_duration,
instance_list = instance_list,
frame_packet_map = frame_packet_map,
parent_file_id = parent_file_id
parent_file_id = parent_file_id,
ordinal = ordinal
)
self.from_packet(packet = packet)

Expand Down Expand Up @@ -554,7 +563,15 @@ def get_by_id(self,

response_json = response.json()
file_data = response_json.get(file_response_key)
if file_data.get('type') == 'compound':
from diffgram.file.compound_file import CompoundFile
return CompoundFile.from_dict(
project = self.client,
dir_id = self.client.directory_id,
dict_data = file_data
)

return File.new(
client = self.client,
file_json = file_data)
else:
return File.new(
client = self.client,
file_json = file_data)

0 comments on commit 74eeef3

Please sign in to comment.