diff --git a/recipe/meta.yaml b/recipe/meta.yaml index c96c5f83..a197ebda 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -48,6 +48,7 @@ requirements: - matplotlib - openmp # [osx] - qdarkstyle + - brem - vtk=8.1.2 about: diff --git a/setup.py b/setup.py index 0d6be3bb..6277212f 100644 --- a/setup.py +++ b/setup.py @@ -57,8 +57,8 @@ def version2pep440(version): setup( name = "idvc", description = 'CCPi DVC Configurator', - version = dversion, - packages = {'idvc'}, + version = dversion, + packages = {'idvc'}, package_dir = {'idvc': os.path.join('src','idvc')}, package_data = {'idvc':['DVCIconSquare.png']}, # metadata for upload to PyPI diff --git a/src/idvc/dialogs.py b/src/idvc/dialogs.py new file mode 100644 index 00000000..2a9877b2 --- /dev/null +++ b/src/idvc/dialogs.py @@ -0,0 +1,211 @@ +from PySide2 import QtCore, QtGui, QtWidgets +from PySide2.QtWidgets import QCheckBox, QLabel, QDoubleSpinBox, QFrame, QVBoxLayout,\ + QDialogButtonBox, QPushButton, QDialog, QLineEdit +from PySide2.QtCore import Qt +import vtk +from brem.ui import RemoteServerSettingDialog, RemoteFileDialog +from eqt.ui import UIFormFactory + +import os, posixpath + +class SettingsWindow(QDialog): + + def __init__(self, parent): + super(SettingsWindow, self).__init__(parent) + + self.parent = parent + + self.setWindowTitle("Settings") + + self.dark_checkbox = QCheckBox("Dark Mode") + + self.copy_files_checkbox = QCheckBox("Allow a copy of the image files to be stored. ") + self.vis_size_label = QLabel("Maximum downsampled image size (GB): ") + self.vis_size_entry = QDoubleSpinBox() + + self.vis_size_entry.setMaximum(64.0) + self.vis_size_entry.setMinimum(0.01) + self.vis_size_entry.setSingleStep(0.01) + + if self.parent.settings.value("vis_size") is not None: + self.vis_size_entry.setValue(float(self.parent.settings.value("vis_size"))) + + else: + self.vis_size_entry.setValue(1.0) + + + if self.parent.settings.value("dark_mode") is not None: + if self.parent.settings.value("dark_mode") == "true": + self.dark_checkbox.setChecked(True) + else: + self.dark_checkbox.setChecked(False) + else: + self.dark_checkbox.setChecked(True) + + separator = QFrame() + separator.setFrameShape(QFrame.HLine) + separator.setFrameShadow(QFrame.Raised) + self.adv_settings_label = QLabel("Advanced") + + + self.gpu_label = QLabel("Please set the size of your GPU memory.") + self.gpu_size_label = QLabel("GPU Memory (GB): ") + self.gpu_size_entry = QDoubleSpinBox() + + + if self.parent.settings.value("gpu_size") is not None: + self.gpu_size_entry.setValue(float(self.parent.settings.value("gpu_size"))) + + else: + self.gpu_size_entry.setValue(1.0) + + self.gpu_size_entry.setMaximum(64.0) + self.gpu_size_entry.setMinimum(0.00) + self.gpu_size_entry.setSingleStep(0.01) + self.gpu_checkbox = QCheckBox("Use GPU for volume render. (Recommended) ") + self.gpu_checkbox.setChecked(True) #gpu is default + if self.parent.settings.value("volume_mapper") == "cpu": + self.gpu_checkbox.setChecked(False) + + if hasattr(self.parent, 'copy_files'): + self.copy_files_checkbox.setChecked(self.parent.copy_files) + + self.layout = QVBoxLayout(self) + self.layout.addWidget(self.dark_checkbox) + self.layout.addWidget(self.copy_files_checkbox) + self.layout.addWidget(self.vis_size_label) + self.layout.addWidget(self.vis_size_entry) + self.layout.addWidget(separator) + self.layout.addWidget(self.adv_settings_label) + self.layout.addWidget(self.gpu_checkbox) + self.layout.addWidget(self.gpu_label) + self.layout.addWidget(self.gpu_size_label) + self.layout.addWidget(self.gpu_size_entry) + + # configure remote server settings + remote_separator = QFrame() + remote_separator.setFrameShape(QFrame.HLine) + remote_separator.setFrameShadow(QFrame.Raised) + fw = UIFormFactory.getQWidget(parent=self) + + self.remote_button_entry = QPushButton(self) + self.remote_button_entry.setText("Open Preferences") + self.remote_button_entry.clicked.connect(self.openConfigRemote) + fw.addWidget(self.remote_button_entry, 'Configure remote settings', 'remote_preferences') + cb = QCheckBox(self) + cb.setChecked(self.parent.connection_details is not None) + fw.addWidget(cb, 'Connect to remote server', 'connect_to_remote') + select_remote_workdir = QPushButton(self) + select_remote_workdir.setText('Browse') + select_remote_workdir.clicked.connect(self.browseRemote) + fw.addWidget(select_remote_workdir, 'Select remote workdir', 'select_remote_workdir') + remote_workdir = QLineEdit(self) + fw.addWidget(remote_workdir, 'Remote workdir', 'remote_workdir') + + self.fw = fw + for k,v in fw.widgets.items(): + print ("fw", k) + # add to layout + self.layout.addWidget(remote_separator) + self.layout.addWidget(fw) + + self.buttons = QDialogButtonBox( + QDialogButtonBox.Save | QDialogButtonBox.Cancel, + Qt.Horizontal, self) + self.layout.addWidget(self.buttons) + self.buttons.accepted.connect(self.accept) + self.buttons.rejected.connect(self.quit) + + def accept(self): + #self.parent.settings.setValue("settings_chosen", 1) + if self.dark_checkbox.isChecked(): + self.parent.settings.setValue("dark_mode", True) + else: + self.parent.settings.setValue("dark_mode", False) + self.parent.SetAppStyle() + + if self.copy_files_checkbox.isChecked(): + self.parent.copy_files = 1 # save for this session + self.parent.settings.setValue("copy_files", 1) #save for next time we open app + else: + self.parent.copy_files = 0 + self.parent.settings.setValue("copy_files", 0) + + if self.gpu_checkbox.isChecked(): + self.parent.settings.setValue("volume_mapper", "gpu") + self.parent.vis_widget_3D.volume_mapper = vtk.vtkSmartVolumeMapper() + else: + self.parent.settings.setValue("volume_mapper", "cpu") + + self.parent.settings.setValue("gpu_size", float(self.gpu_size_entry.value())) + self.parent.settings.setValue("vis_size", float(self.vis_size_entry.value())) + + if self.parent.settings.value("first_app_load") != "False": + self.parent.CreateSessionSelector("new window") + self.parent.settings.setValue("first_app_load", "False") + + # if remote is checked + statusBar = self.parent.statusBar() + if self.fw.widgets['connect_to_remote_field'].isChecked(): + self.parent.connection_details = self.connection_details + statusBar.showMessage("Connected to {}@{}:{}".format( + self.connection_details['username'], + self.connection_details['server_name'], + self.connection_details['server_port']) + ) + else: + statusBar.clearMessage() + self.parent.connection_details = None + self.close() + + + #print(self.parent.settings.value("copy_files")) + def quit(self): + if self.parent.settings.value("first_app_load") != "False": + self.parent.CreateSessionSelector("new window") + self.parent.settings.setValue("first_app_load", "False") + self.close() + + def openConfigRemote(self): + + dialog = RemoteServerSettingDialog(self,port=None, + host=None, + username=None, + private_key=None) + dialog.Ok.clicked.connect(lambda: self.getConnectionDetails(dialog)) + dialog.exec() + + def getConnectionDetails(self, dialog): + for k,v in dialog.connection_details.items(): + print (k,v) + self.connection_details = dialog.connection_details + + + def browseRemote(self): + # start the RemoteFileBrowser + logfile = os.path.join(os.getcwd(), '..','..',"RemoteFileDialog.log") + # logfile = None + dialog = RemoteFileDialog(self, logfile=logfile, port=self.connection_details['server_port'], + host=self.connection_details['server_name'], + username=self.connection_details['username'], + private_key=self.connection_details['private_key'], + remote_os=self.connection_details['remote_os']) + dialog.Ok.clicked.connect( + lambda: self.getSelectedRemoteWorkdir(dialog) + ) + if hasattr(self, 'files_to_get'): + try: + dialog.widgets['lineEdit'].setText(self.files_to_get[0][0]) + except: + pass + dialog.exec() + + + def getSelectedRemoteWorkdir(self, dialog): + if hasattr(dialog, 'selected'): + print (type(dialog.selected)) + for el in dialog.selected: + print ("Return from dialogue", el) + self.files_to_get = list (dialog.selected) + remote_workdir = posixpath.join(self.files_to_get[0][0], self.files_to_get[0][1]) + self.fw.widgets['remote_workdir_field'].setText(remote_workdir) diff --git a/src/idvc/dvc_interface.py b/src/idvc/dvc_interface.py index 805a73ca..400bd295 100644 --- a/src/idvc/dvc_interface.py +++ b/src/idvc/dvc_interface.py @@ -1,3 +1,4 @@ +import pysnooper # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -70,7 +71,7 @@ import json import shutil import zipfile - +from time import sleep from functools import reduce import copy @@ -88,6 +89,11 @@ from qdarkstyle.light.palette import LightPalette from idvc import version as gui_version +from idvc.dialogs import SettingsWindow + +from brem.ui import RemoteFileDialog +from brem import AsyncCopyOverSSH +from idvc.dvc_remote import DVCRemoteRunControl __version__ = gui_version.version @@ -256,7 +262,8 @@ def CreateWorkingTempFolder(self): os.mkdir("Results") def OpenSettings(self): - self.settings_window = SettingsWindow(self) + if not hasattr(self, 'settings_window'): + self.settings_window = SettingsWindow(self) self.settings_window.show() def InitialiseSessionVars(self): @@ -284,6 +291,7 @@ def InitialiseSessionVars(self): self.dvc_input_image_in_session_folder = False if hasattr(self, 'ref_image_data'): del self.ref_image_data + self.connection_details = None #Loading the DockWidgets: @@ -570,11 +578,133 @@ def view_and_load_images(self): self.view_image() self.resetRegistration() + def SelectImage(self, image_var, image, label=None, next_button=None): + if self.connection_details is None: + return self.SelectImageLocal(image_var, image, label, next_button) + else: + return self.SelectImageRemote(image_var, image, label, next_button) + + def SelectImageRemote(self, image_var, image, label=None, next_button=None): + # start the RemoteFileBrowser + logfile = os.path.join(os.getcwd(), '..','..',"RemoteFileDialog.log") + dialog = RemoteFileDialog(self, logfile=logfile, port=self.connection_details['server_port'], + host=self.connection_details['server_name'], + username=self.connection_details['username'], + private_key=self.connection_details['private_key'], + remote_os=self.connection_details['remote_os']) + dialog.Ok.clicked.connect( + lambda: self.getSelectedDownloadAndUpdateUI(dialog, image_var, image, label, next_button) + ) + if hasattr(self, 'files_to_get'): + try: + dialog.widgets['lineEdit'].setText(self.files_to_get[0][0]) + except: + pass + dialog.exec() + def getSelectedDownloadAndUpdateUI(self, dialog, image_var, image, label, next_button): + if hasattr(dialog, 'selected'): + print (type(dialog.selected)) + for el in dialog.selected: + print ("Return from dialogue", el) + self.files_to_get = list (dialog.selected) + if len(self.files_to_get) == 1: + self.GetFileFromRemote(image_var, image, label, next_button) + else: + self.warningDialog("Sorry, currently we can only get one file.", + "Error: cannot handle multiple files") + + + + + def GetFileFromRemote(self, image_var, image, label, next_button): + '''Downloads a file from remote''' + # 1 download self.files_to_get + if image_var == 1: + # let's not download the correlate image + if next_button is not None: + try: + for el in next_button: + el.setEnabled(True) + except: + next_button.setEnabled(True) + + if len(self.files_to_get) == 1: + self.asyncCopy = AsyncCopyOverSSH() + if not hasattr(self, 'connection_details'): + self.statusBar().showMessage("define the connection") + return + username = self.connection_details['username'] + port = self.connection_details['server_port'] + host = self.connection_details['server_name'] + private_key = self.connection_details['private_key'] + + self.asyncCopy.setRemoteConnectionSettings(username=username, + port=port, host=host, private_key=private_key) + + + remotepath = self.asyncCopy.remotepath.join(self.files_to_get[0][0], self.files_to_get[0][1]) + if image_var == 1: + self.remote_correlate_image_fname = remotepath + return + else: + self.remote_reference_image_fname = remotepath + + files = [os.path.join(tempfile.tempdir, self.files_to_get[0][1])] + + + # this shouldn't be necessary, however the signals and the workers are created before the async copy + # object is created and then the local dir is not set in the worker. + self.asyncCopy.SetCopyFromRemote() + self.asyncCopy.SetLocalDir(tempfile.tempdir) + self.asyncCopy.SetRemoteDir(self.asyncCopy.remotepath.dirname(remotepath)) + self.asyncCopy.SetFileName(self.asyncCopy.remotepath.basename(remotepath)) + + self.asyncCopy.signals.finished.connect( + lambda: self._UpdateSelectFileUI(files, image_var, image, label, next_button) + ) + # this should also not be done like this. + self.asyncCopy.threadpool.start(self.asyncCopy.worker) + + # save into these variables for the remote run in create_run_config + if image_var == 0: + self.reference_file = remotepath + elif image_var == 1: + self.correlate_file = remotepath + + sleep(1) + + self.create_progress_window("Getting files from remote", "", 0, None, False, 0) + self.updateUnknownProgressDialog = Worker(self.UnknownProgressUpdateDialog) + self.updateUnknownProgressDialog.signals.finished.connect(self.StopUnknownProgressUpdate) + self.threadpool.start(self.updateUnknownProgressDialog) + + + def UnknownProgressUpdateDialog(self, **kwargs): + '''Update the progress dialog where we don't know at what stage we are''' + t0 = time.time() + while True: + tc = self.asyncCopy.threadpool.activeThreadCount() + if tc == 0: + break + # print (tc) + sleep(0.25) + self.progress_window.setLabelText( + "Copying {} ... {:.1f} s".format(self.files_to_get[0][1], time.time()-t0) + ) + + + def StopUnknownProgressUpdate(self): + self.progress_window.close() + + + def SelectImageLocal(self, image_var, image, label=None, next_button=None): #print("In select image") dialogue = QFileDialog() files = dialogue.getOpenFileNames(self,"Load Images")[0] + self._UpdateSelectFileUI(files, image_var, image, label, next_button) + def _UpdateSelectFileUI(self, files, image_var, image, label=None, next_button=None): if len(files) > 0: if self.copy_files: self.image_copied[image_var] = True @@ -604,7 +734,7 @@ def SelectImage(self, image_var, image, label=None, next_button=None): else: image[image_var].append(files[0]) if label is not None: - label.setText(os.path.basename(files[0])) + label.setText(files[0]) else: # Make sure that the files are sorted 0 - end @@ -627,7 +757,12 @@ def SelectImage(self, image_var, image, label=None, next_button=None): os.path.basename(self.image[image_var][0]) + " + " + str(len(files)) + " more files.") if next_button is not None: - next_button.setEnabled(True) + try: + for el in next_button: + el.setEnabled(True) + except: + next_button.setEnabled(True) + def copy_file(self, **kwargs): @@ -858,19 +993,29 @@ def visualise(self): #bring image loading panel to front if it isnt already: self.select_image_dock.raise_() - def create_progress_window(self, title, text, max = 100, cancel = None): - self.progress_window = QProgressDialog(text, "Cancel", 0,max, self, QtCore.Qt.Dialog) + def create_progress_window(self, title, text, max = 100, cancel = None, autoClose = True, minimumDuration=4000): + '''Creates a QProgressDialog + + :param title: title + :param text: text in the dialog + :param max: max value of the progress, default 100. Minimum is set to 0. + :param cancel: if to show a cancel button, default None hence Cancel not shown. + :autoClose: if the dialog should close when max is reached + :minimumDuration: This property holds the time that must pass before the dialog appears, default 4000 ms + ''' + self.progress_window = QProgressDialog(text, "Cancel", 0,max, self, QtCore.Qt.Window) self.progress_window.setWindowTitle(title) self.progress_window.setWindowModality(QtCore.Qt.ApplicationModal) #This means the other windows can't be used while this is open - self.progress_window.setMinimumDuration(0.01) + self.progress_window.setMinimumDuration(int(minimumDuration)) self.progress_window.setWindowFlag(QtCore.Qt.WindowCloseButtonHint, True) self.progress_window.setWindowFlag(QtCore.Qt.WindowMaximizeButtonHint, False) - self.progress_window.setAutoClose(True) + self.progress_window.setAutoClose(autoClose) if cancel is None: self.progress_window.setCancelButton(None) else: self.progress_window.canceled.connect(cancel) + self.progress_window.show() def setup2DPointCloudPipeline(self): @@ -1126,7 +1271,7 @@ def CreateRegistrationPanel(self): rp['translate_X_entry'].setValidator(validatorint) rp['translate_X_entry'].setText("0") rp['translate_X_entry'].setToolTip(translation_tooltip_text) - #rp['translate_X_entry'].setEnabled(False) + rp['translate_X_entry'].textEdited.connect(self._updateTranslateObject) formLayout.setWidget(widgetno, QFormLayout.FieldRole, rp['translate_X_entry']) widgetno += 1 # Translate Y field @@ -1138,7 +1283,7 @@ def CreateRegistrationPanel(self): rp['translate_Y_entry'].setValidator(validatorint) rp['translate_Y_entry'].setText("0") rp['translate_Y_entry'].setToolTip(translation_tooltip_text) - #rp['translate_Y_entry'].setEnabled(False) + rp['translate_Y_entry'].textEdited.connect(self._updateTranslateObject) formLayout.setWidget(widgetno, QFormLayout.FieldRole, rp['translate_Y_entry']) widgetno += 1 # Translate Z field @@ -1150,10 +1295,12 @@ def CreateRegistrationPanel(self): rp['translate_Z_entry'].setValidator(validatorint) rp['translate_Z_entry'].setText("0") rp['translate_Z_entry'].setToolTip(translation_tooltip_text) - #rp['translate_Z_entry'].setEnabled(False) + rp['translate_Y_entry'].textEdited.connect(self._updateTranslateObject) formLayout.setWidget(widgetno, QFormLayout.FieldRole, rp['translate_Z_entry']) widgetno += 1 + # self.translate.SetTranslation(-int(rp['translate_X_entry'].text()),-int(rp['translate_Y_entry'].text()),-int(rp['translate_Z_entry'].text())) + # Add submit button rp['start_registration_button'] = QPushButton(groupBox) rp['start_registration_button'].setText("Start Registration") @@ -1168,6 +1315,24 @@ def CreateRegistrationPanel(self): # save to instance self.registration_parameters = rp + + def _updateTranslateObject(self, text, **kwargs): + rp = self.registration_parameters + + + # setup the appropriate stuff to run the registration + if not hasattr(self, 'translate'): + self.translate = vtk.vtkImageTranslateExtent() + elif self.translate is None: + self.translate = vtk.vtkImageTranslateExtent() + + self.translate.SetTranslation(-int(rp['translate_X_entry'].text()), + -int(rp['translate_Y_entry'].text()), + -int(rp['translate_Z_entry'].text()) + ) + + + def createRegistrationViewer(self): # print("Create reg viewer") #Get current orientation and slice of 2D viewer, registration viewer will be set up to have these @@ -4066,6 +4231,7 @@ def CreateRunDVCPanel(self): #Add button functionality: rdvc_widgets['run_type_entry'].currentIndexChanged.connect(self.show_run_groupbox) + # if connected to remote do something else. rdvc_widgets['run_button'].clicked.connect(self.create_config_worker) self.addDockWidget(QtCore.Qt.LeftDockWidgetArea, dockWidget) @@ -4111,6 +4277,7 @@ def select_roi(self, label, next_button): if self.roi: next_button.setEnabled(True) + def create_config_worker(self): if hasattr(self, 'translate'): if self.translate is None: @@ -4149,26 +4316,101 @@ def create_config_worker(self): message="Please set a run name not in the following list: {}".format(saved_run_names)) return - folder_name = "_" + self.rdvc_widgets['name_entry'].text() - - results_folder = os.path.join(tempfile.tempdir, "Results") - - new_folder = os.path.join(results_folder, folder_name) - - if os.path.exists(new_folder): - self.warningDialog(window_title="Error", - message="This directory already exists. Please choose a different name." ) - return - + self.config_worker = Worker(self.create_run_config) self.create_progress_window("Loading", "Generating Run Config") self.config_worker.signals.progress.connect(self.progress) # if single or bulk use the line below, if remote develop new functionality - self.config_worker.signals.result.connect(partial (self.run_external_code)) - self.config_worker.signals.message.connect(self.updateProgressDialogMessage) + run_local = True + if hasattr(self, 'settings_window'): + if not self.settings_window.fw.widgets['connect_to_remote_field'].isChecked(): + run_local = False + + if run_local: + self.config_worker.signals.result.connect(partial (self.run_external_code)) + self.config_worker.signals.message.connect(self.updateProgressDialogMessage) + else: + # do not run the dvc locally but + # 1 zip and + # 2 upload the config to remote and then + # 3 run the code on the remote + self.config_worker.signals.finished.connect(self.ZipAndUploadConfigToRemote) + + self.threadpool.start(self.config_worker) self.progress_window.setValue(10) - + + @pysnooper.snoop() + def ZipAndUploadConfigToRemote(self): + # this command will call DVC_runner to create the directories + self.run_succeeded = True + self.dvc_runner = DVC_runner(self, os.path.abspath(self.run_config_file), + self.finished_run, self.run_succeeded, tempfile.tempdir, remote_os=self.connection_details['remote_os']) + self.config_worker = Worker(self.dvc_runner.zip_workdir_and_upload) + # self.create_progress_window("Connecting with remote", "Zipping and uploading", 0, None, False) + self.config_worker.signals.finished.connect( self.unzip_on_remote ) + self.threadpool.start(self.config_worker) + + + def unzip_on_remote(self): + # TODO: keep this here and remove the async copy in dvc_runner + print ("run_code_remote") + while True: + tc = self.dvc_runner.asyncCopy.threadpool.activeThreadCount() + if tc == 0: + break + # print (tc) + sleep(0.25) + self.unzip_worker = Worker(self.dvc_runner._unzip_on_remote, self.dvc_runner.asyncCopy.remotedir, self.dvc_runner.asyncCopy.filename) + self.create_progress_window("Connecting with remote", "Unzipping", 0, None, False) + self.unzip_worker.signals.finished.connect( self.remote_run_code ) + self.unzip_worker.signals.status.connect( self.update_status) + self.unzip_worker.signals.error.connect( self.update_on_error) + + self.threadpool.start(self.unzip_worker) + + + def update_status(self, data): + print ("STDOUT", data[0]) + print ("STDERR", data[1]) + + + def update_on_error(self, data): + # traceback.print_exc() + # exctype, value = sys.exc_info()[:2] + # self.signals.error.emit((exctype, value, traceback.format_exc())) + print(data) + msg = QMessageBox(self) + msg.setIcon(QMessageBox.Critical) + msg.setWindowTitle("Remote execution Error") + msg.setText("exectype {}, value {}".format(data[0], data[1])) + msg.setDetailedText(data[2]) + msg.exec_() + + + def remote_run_code(self): + self.progress_window.close() + + self.dvc_remote_controller = DVCRemoteRunControl(self.connection_details) + self.dvc_remote_controller.set_workdir(self.dvc_runner.asyncCopy.remotedir) + self.dvc_remote_controller.set_num_runs(len(self.dvc_runner.processes)) + + # self.dvc_worker = Worker(self.dvc_runner.run_dvc_on_remote, self.dvc_runner.asyncCopy.remotedir) + self.dvc_worker = Worker(self.dvc_remote_controller.run_dvc_on_remote) + self.create_progress_window("Connecting with remote", "Running DVC remote", 0, None, False) + self.dvc_worker.signals.finished.connect( self.remote_retrieve_results ) + self.threadpool.start(self.dvc_worker) + + def remote_retrieve_results(self): + self.dvc_worker = Worker(self.dvc_remote_controller.retrieve_results, os.path.abspath(self.run_config_file)) + self.dvc_worker.signals.finished.connect( self.remote_update_result_panel ) + self.threadpool.start(self.dvc_worker) + + def remote_update_result_panel(self): + self.run_succeeded = True + self.progress_window.close() + self.finished_run() + def create_run_config(self, **kwargs): os.chdir(tempfile.tempdir) @@ -4247,16 +4489,18 @@ def create_run_config(self, **kwargs): progress_callback.emit(subvol_size_count/len(self.subvol_sizes)*90) #print("finished making pointclouds") - #print(self.roi_files) - - #print("DVC in: ", self.dvc_input_image) - - self.reference_file = self.dvc_input_image[0][0] - if len(self.dvc_input_image[0]) > 1: - self.reference_file = self.dvc_input_image[0] - self.correlate_file = self.dvc_input_image[1][0] - if len(self.dvc_input_image[1]) > 1: - self.correlate_file = self.dvc_input_image[1] + # if remote mode this should not be the local copy + if hasattr(self, 'settings_window') and self.settings_window.fw.widgets['connect_to_remote_field'].isChecked(): + # this should point to the remote files set at the time of download + self.reference_file = self.remote_reference_image_fname + self.correlate_file = self.remote_correlate_image_fname + else: + self.reference_file = self.dvc_input_image[0][0] + if len(self.dvc_input_image[0]) > 1: + self.reference_file = self.dvc_input_image[0] + self.correlate_file = self.dvc_input_image[1][0] + if len(self.dvc_input_image[1]) > 1: + self.correlate_file = self.dvc_input_image[1] #print("REF: ", self.reference_file) @@ -4287,7 +4531,7 @@ def create_run_config(self, **kwargs): else: run_config['rigid_trans']= "0.0 0.0 0.0" - self.run_folder = "Results/" + folder_name + self.run_folder = os.path.join("Results", folder_name) run_config['run_folder'] = self.run_folder #where is point0 @@ -4333,8 +4577,6 @@ def run_external_code(self, error = None): self.cancelled = True return - - self.run_succeeded = True # this command will call DVC_runner to create the directories @@ -4343,6 +4585,7 @@ def run_external_code(self, error = None): self.dvc_runner.run_dvc() + def update_progress(self, exe = None): if exe: line_b = self.process.readLine() @@ -4513,7 +4756,7 @@ def CreateViewDVCResultsPanel(self): self.addDockWidget(QtCore.Qt.LeftDockWidgetArea, dockWidget) self.result_widgets = result_widgets - + @pysnooper.snoop() def show_run_pcs(self): #show pointcloud files in list self.result_widgets['pc_entry'].clear() @@ -5421,123 +5664,7 @@ def progress(self, value): -class SettingsWindow(QDialog): - - def __init__(self, parent): - super(SettingsWindow, self).__init__(parent) - - self.parent = parent - - self.setWindowTitle("Settings") - self.dark_checkbox = QCheckBox("Dark Mode") - - self.copy_files_checkbox = QCheckBox("Allow a copy of the image files to be stored. ") - self.vis_size_label = QLabel("Maximum downsampled image size (GB): ") - self.vis_size_entry = QDoubleSpinBox() - - self.vis_size_entry.setMaximum(64.0) - self.vis_size_entry.setMinimum(0.01) - self.vis_size_entry.setSingleStep(0.01) - - if self.parent.settings.value("vis_size") is not None: - self.vis_size_entry.setValue(float(self.parent.settings.value("vis_size"))) - - else: - self.vis_size_entry.setValue(1.0) - - - if self.parent.settings.value("dark_mode") is not None: - if self.parent.settings.value("dark_mode") == "true": - self.dark_checkbox.setChecked(True) - else: - self.dark_checkbox.setChecked(False) - else: - self.dark_checkbox.setChecked(True) - - separator = QFrame() - separator.setFrameShape(QFrame.HLine) - separator.setFrameShadow(QFrame.Raised) - self.adv_settings_label = QLabel("Advanced") - - - self.gpu_label = QLabel("Please set the size of your GPU memory.") - self.gpu_size_label = QLabel("GPU Memory (GB): ") - self.gpu_size_entry = QDoubleSpinBox() - - - if self.parent.settings.value("gpu_size") is not None: - self.gpu_size_entry.setValue(float(self.parent.settings.value("gpu_size"))) - - else: - self.gpu_size_entry.setValue(1.0) - - self.gpu_size_entry.setMaximum(64.0) - self.gpu_size_entry.setMinimum(0.00) - self.gpu_size_entry.setSingleStep(0.01) - self.gpu_checkbox = QCheckBox("Use GPU for volume render. (Recommended) ") - self.gpu_checkbox.setChecked(True) #gpu is default - if self.parent.settings.value("volume_mapper") == "cpu": - self.gpu_checkbox.setChecked(False) - - if hasattr(self.parent, 'copy_files'): - self.copy_files_checkbox.setChecked(self.parent.copy_files) - - self.layout = QVBoxLayout(self) - self.layout.addWidget(self.dark_checkbox) - self.layout.addWidget(self.copy_files_checkbox) - self.layout.addWidget(self.vis_size_label) - self.layout.addWidget(self.vis_size_entry) - self.layout.addWidget(separator) - self.layout.addWidget(self.adv_settings_label) - self.layout.addWidget(self.gpu_checkbox) - self.layout.addWidget(self.gpu_label) - self.layout.addWidget(self.gpu_size_label) - self.layout.addWidget(self.gpu_size_entry) - self.buttons = QDialogButtonBox( - QDialogButtonBox.Save | QDialogButtonBox.Cancel, - Qt.Horizontal, self) - self.layout.addWidget(self.buttons) - self.buttons.accepted.connect(self.accept) - self.buttons.rejected.connect(self.quit) - - def accept(self): - #self.parent.settings.setValue("settings_chosen", 1) - if self.dark_checkbox.isChecked(): - self.parent.settings.setValue("dark_mode", True) - else: - self.parent.settings.setValue("dark_mode", False) - self.parent.SetAppStyle() - - if self.copy_files_checkbox.isChecked(): - self.parent.copy_files = 1 # save for this session - self.parent.settings.setValue("copy_files", 1) #save for next time we open app - else: - self.parent.copy_files = 0 - self.parent.settings.setValue("copy_files", 0) - - if self.gpu_checkbox.isChecked(): - self.parent.settings.setValue("volume_mapper", "gpu") - self.parent.vis_widget_3D.volume_mapper = vtk.vtkSmartVolumeMapper() - else: - self.parent.settings.setValue("volume_mapper", "cpu") - - self.parent.settings.setValue("gpu_size", float(self.gpu_size_entry.value())) - self.parent.settings.setValue("vis_size", float(self.vis_size_entry.value())) - - if self.parent.settings.value("first_app_load") != "False": - self.parent.CreateSessionSelector("new window") - self.parent.settings.setValue("first_app_load", "False") - - self.close() - - - #print(self.parent.settings.value("copy_files")) - def quit(self): - if self.parent.settings.value("first_app_load") != "False": - self.parent.CreateSessionSelector("new window") - self.parent.settings.setValue("first_app_load", "False") - self.close() diff --git a/src/idvc/dvc_remote.py b/src/idvc/dvc_remote.py new file mode 100644 index 00000000..f4aea974 --- /dev/null +++ b/src/idvc/dvc_remote.py @@ -0,0 +1,349 @@ +import os +from brem import RemoteRunControl +import brem +from PySide2 import QtCore +import ntpath, posixpath +import pysnooper +import json + +class PrepareDVCRemote(object): + def __init__(self, parent): + self._config_json = None + self._remote_workdir = None + self.parent = parent + + + @property + def config_json(self): + return self._config_json + + + def set_config_json(self, value): + self._config_json = os.path.abspath(value) + + + @property + def remote_workdir(self): + return self._remote_workdir + + + def set_remote_workdir(self): + self._remote_workdir = self.parent.settings_window + + +class DVCRemoteRunControlSignals(QtCore.QObject): + status = QtCore.Signal(tuple) + progress = QtCore.Signal(int) + +class DVCRemoteRunControl(object): + def __init__(self, connection_details): + + super(DVCRemoteRunControl, self).__init__() + # self.signals is a property of RemoteRunControl + self.signals = DVCRemoteRunControlSignals() + self._num_runs = 0 + self._workdir = None + self.connection_details = connection_details + + + def set_num_runs(self, value): + self._num_runs = value + + + def set_workdir(self, value): + self._workdir = value + + + @property + def num_runs(self): + return self._num_runs + + + @property + def workdir(self): + return self._workdir + + def _create_connection(self): + # 1 create a BasicRemoteExecutionManager + username = self.connection_details['username'] + port = self.connection_details['server_port'] + host = self.connection_details['server_name'] + private_key = self.connection_details['private_key'] + remote_os = self.connection_details['remote_os'] + logfile = os.path.join('ssh.log') + + conn = brem.BasicRemoteExecutionManager(port, host, username, private_key, remote_os, logfile=logfile) + conn.login(passphrase=False) + # 2 go to workdir + conn.changedir(self.workdir) + + return conn + + @pysnooper.snoop() + def run_dvc_on_remote(self, **kwargs): + # 1 create a BasicRemoteExecutionManager + + conn = self._create_connection() + remote_os = conn.remote_os + + progress_callback = kwargs.get('progress_callback', None) + + if remote_os == 'POSIX': + dpath = posixpath + else: + dpath = ntpath + + for i in range(self.num_runs): + if progress_callback is not None: + progress_callback.emit(i) + + wdir = dpath.join(self.workdir, 'dvc_result_{}'.format(i)) + # 2 run 'unzip filename' + stdout, stderr = conn.run('cd {} && . ~/condarc && conda activate dvc && dvc dvc_config.txt'.format(wdir)) + + @pysnooper.snoop() + def retrieve_results(self, config_file, **kwargs): + + # created in dvc_interface create_run_config + with open(config_file) as tmp: + config = json.load(tmp) + run_folder = config['run_folder'] + + conn = self._create_connection() + remote_os = conn.remote_os + + progress_callback = kwargs.get('progress_callback', None) + + if remote_os == 'POSIX': + dpath = posixpath + else: + dpath = ntpath + + # retrieve the results in each directory and store it locally + for i in range(self.num_runs): + if progress_callback is not None: + progress_callback.emit(i) + fname = 'dvc_result_{}'.format(i) + + localdir = os.path.join(run_folder, "dvc_result_{}".format(i)) + + for extension in ['stat', 'disp']: + path_to_file = dpath.join(self.workdir, fname) + file_to_get = '{}.{}'.format(fname, extension) + conn.changedir(path_to_file) + conn.get_file(file_to_get, localdir) + + +class DVCSLURMRemoteRunControl(RemoteRunControl): + def __init__(self, connection_details=None, + reference_filename=None, correlate_filename=None, + dvclog_filename=None, + dev_config=None): + + super(DVCSLURMRemoteRunControl, self).__init__(connection_details=connection_details) + self.reference_fname = reference_filename + self.correlate_fname = correlate_filename + self.dvclog_fname = dvclog_filename + + # try to create a worker + self.create_job(self.run_worker, + reference_fname=self.reference_fname, + correlate_fname=self.correlate_fname, + update_delay=10, logfile=self.dvclog_fname) + + # Not required for base class + @property + def reference_fname(self): + return self._reference_fname + @reference_fname.setter + def reference_fname(self, value): + '''setter for reference file name.''' + self._reference_fname = value + + @property + def correlate_fname(self): + return self._correlate_fname + @correlate_fname.setter + def correlate_fname(self, value): + '''setter for correlate file name.''' + self._correlate_fname = value + + @property + def dvclog_fname(self): + return self._dvclog_fname + @dvclog_fname.setter + def dvclog_fname(self, value): + '''setter for dvclog file name.''' + self._dvclog_fname = value + + + + + # @pysnooper.snoop() + def run_worker(self, **kwargs): + # retrieve the appropriate parameters from the kwargs + host = kwargs.get('host', None) + username = kwargs.get('username', None) + port = kwargs.get('port', None) + private_key = kwargs.get('private_key', None) + logfile = kwargs.get('logfile', None) + update_delay = kwargs.get('update_delay', None) + # get the callbacks + message_callback = kwargs.get('message_callback', None) + progress_callback = kwargs.get('progress_callback', None) + status_callback = kwargs.get('status_callback', None) + + reference_fname = kwargs.get('reference_fname', None) + correlate_fname = kwargs.get('correlate_fname', None) + + + from time import sleep + + a = brem.BasicRemoteExecutionManager( host=host, + username=username, + port=22, + private_key=private_key) + + a.login(passphrase=False) + + inp="input.dvc" + # folder="/work3/cse/dvc/test-edo" + folder = dpath.dirname(logfile) + datafolder="/work3/cse/dvc/test_data" + + with open(inp,'w', newline='\n') as f: + print("""############################################################################### +# +# +# example dvc process control file +# +# +############################################################################### + +# all lines beginning with a # character are ignored +# some parameters are conditionally required, depending on the setting of other parameters +# for example, if subvol_thresh is off, the threshold description parameters are not required + +### file names + +reference_filename\t{0}/frame_000_f.npy\t### reference tomography image volume +correlate_filename\t{0}/frame_010_f.npy\t### correlation tomography image volume + +point_cloud_filename\t{1}/medium_grid.roi\t### file of search point locations +output_filename\t{1}/medium_grid\t### base name for output files + +### description of the image data files, all must be the same size and structure + +vol_bit_depth 8 ### 8 or 16 +vol_hdr_lngth 96 ### fixed-length header size, may be zero +vol_wide 1520 ### width in pixels of each slice +vol_high 1257 ### height in pixels of each slice +vol_tall 1260 ### number of slices in the stack + +### parameters defining the subvolumes that will be created at each search point + +subvol_geom sphere ### cube, sphere +subvol_size 80 ### side length or diameter, in voxels +subvol_npts 8000 ### number of points to distribute within the subvol + +subvol_thresh off ### on or off, evaluate subvolumes based on threshold +# gray_thresh_min 27 ### lower limit of a gray threshold range if subvol_thresh is on +# gray_thresh_max 127 ### upper limit of a gray threshold range if subvol_thresh is on +# min_vol_fract 0.2 ### only search if subvol fraction is greater than + +### required parameters defining the basic the search process + +disp_max 38 ### in voxels, used for range checking and global search limits +num_srch_dof 6 ### 3, 6, or 12 +obj_function znssd ### sad, ssd, zssd, nssd, znssd +interp_type tricubic ### trilinear, tricubic + +### optional parameters tuning and refining the search process + +rigid_trans 34.0 4.0 0.0 ### rigid body offset of target volume, in voxels +basin_radius 0.0 ### coarse-search resolution, in voxels, 0.0 = none +subvol_aspect 1.0 1.0 1.0 ### subvolume aspect ratio + + + +""".format(datafolder,folder),file=f) + + a.put_file(inp, remote_filename=dpath.join(folder, inp)) + + + job=""" + +module purge +module load AMDmodules foss/2019b + +/work3/cse/dvc/codes/CCPi-DVC/build-amd/Core/dvc {0} > {1} 2>&1 +#{0} + """.format(inp, logfile) + + + + jobid = a.submit_job(folder,job) + self.job_id = jobid + print(jobid) + status = a.job_status(jobid) + print(status) + i = 0 + start_at = 0 + while status in [b'PENDING',b'RUNNING']: + i+=1 + # widgets['jobid'].setText("Job id: {} {}".format(jobid, str(status))) + status_callback.emit((jobid, status.decode('utf-8'))) + self.internalsignals.status.emit((jobid, status.decode('utf-8'))) + if status == b'PENDING': + print("job is queueing") + # message_callback.emit("Job {} queueing".format(jobid)) + else: + print("job is running") + # widgets['buttonBox'].button(QtWidgets.QDialogButtonBox.Apply).setText('Running') + + # tails the output of dvc + tail = self.pytail(a, logfile, start_at) + # count the number of newlines + for i in tail: + if i == "\n": + start_at+=1 + message_callback.emit("{}".format(tail.decode('utf-8'))) + # try to infer the progress + def progress(line): + import re + try: + match = re.search('^([0-9]*)/([0-9]*)', line.decode('utf-8')) + if match is not None: + return eval(match.group(0)) + except Exception as err: + print (err) + + line = tail.splitlines() + if len(line) >= 2: + line = line[-2] + + curr_progress = progress(line) + if curr_progress is not None: + # widgets['progressBar'].setValue(int(progress(line)*100)) + progress_callback.emit(int(progress(line)*100)) + print ("attempt evaluate progress ", progress(line)) + + sleep(update_delay) + status = a.job_status(jobid) + + + # dvc computation is finished, we get the last part of the output + tail = self.pytail(a, logfile, start_at) + message_callback.emit("{}".format(tail.decode('utf-8'))) + # set the progress to 100 + progress_callback.emit(100) + + a.changedir(folder) + a.get_file("slurm-{}.out".format(jobid)) + a.get_file("dvc.out".format(jobid)) + # here we should fetch also all the output files defined at + # output_filename\t{1}/small_grid\t### base name for output files + + a.logout() + self.internalsignals.status.emit((jobid, 'FINISHED')) + diff --git a/src/idvc/dvc_runner.py b/src/idvc/dvc_runner.py index bf33f2bf..49a6a993 100644 --- a/src/idvc/dvc_runner.py +++ b/src/idvc/dvc_runner.py @@ -17,11 +17,15 @@ import numpy as np from PySide2 import QtCore from datetime import datetime -from PySide2.QtWidgets import QMessageBox +from PySide2.QtWidgets import QMessageBox, QProgressDialog import json import time import shutil import platform +import pysnooper +from brem import AsyncCopyOverSSH, BasicRemoteExecutionManager +import tempfile +import ntpath, posixpath from .io import save_tiff_stack_as_raw count = 0 @@ -136,7 +140,7 @@ def create_progress_window(main_window, title, text, max = 100, cancel = None): main_window.progress_window = QProgressDialog(text, "Cancel", 0,max, main_window, QtCore.Qt.Window) main_window.progress_window.setWindowTitle(title) main_window.progress_window.setWindowModality(QtCore.Qt.ApplicationModal) #This means the other windows can't be used while this is open - main_window.progress_window.setMinimumDuration(0.1) + main_window.progress_window.setMinimumDuration(100) main_window.progress_window.setWindowFlag(QtCore.Qt.WindowCloseButtonHint, False) main_window.progress_window.setWindowFlag(QtCore.Qt.WindowMaximizeButtonHint, False) if cancel is None: @@ -189,7 +193,8 @@ def finished_run(main_window, exitCode, exitStatus, process = None, required_run # print("did") class DVC_runner(object): - def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_folder): + + def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_folder, remote_os=None): # print("The session folder is", session_folder) self.main_window = main_window self.input_file = input_file @@ -314,6 +319,18 @@ def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_fo # this is not really a nice way to open an error message! mainwindow.displayFileErrorDialog(message=str(err), title="Error creating config files") return + + newline = None + if remote_os is not None: + if remote_os in ['Windows', 'POSIX'] : + # on remote we aim at running in the directory + grid_roi_fname = os.path.basename(grid_roi_fname) + output_filename = os.path.basename(output_filename) + if remote_os == 'POSIX': + newline = "\n" + + + config = blank_config.format( @@ -344,7 +361,9 @@ def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_fo num_points_to_process=num_points_to_process, starting_point='{} {} {}'.format(*starting_point)) time.sleep(1) - with open(config_filename,"w") as config_file: + + + with open(config_filename,"w", newline=newline) as config_file: config_file.write(config) #if run_count == len( subvolume_points): @@ -361,7 +380,105 @@ def __init__(self, main_window, input_file, finish_fn, run_succeeded, session_fo self.processes.append( (exe_file, [ config_filename ], required_runs, total_points, num_points_to_process) ) + + def zip_workdir_and_upload(self, **kwargs): + # TODO this is already in a worker and does not need to run the AsyncCopyOverSSH + + #param_file is a list with at least 1 item but we are interested in the first + # because we want to know the path to it and all files will be in the same directory + exe_file, param_file, required_runs,\ + total_points = self.processes[0] + # config is in the directory + + configdir = os.path.join(os.path.dirname(param_file[0]),'..') + zipped = shutil.make_archive(os.path.join(configdir, '..', 'remote_run'), 'zip', configdir) + + self.asyncCopy = AsyncCopyOverSSH() + if not hasattr(self.main_window, 'connection_details'): + self.main_window.statusBar().showMessage("define the connection") + return + username = self.main_window.connection_details['username'] + port = self.main_window.connection_details['server_port'] + host = self.main_window.connection_details['server_name'] + private_key = self.main_window.connection_details['private_key'] + self.asyncCopy.setRemoteConnectionSettings(username=username, + port=port, host=host, private_key=private_key) + + + + remote_dir = self.main_window.settings_window.fw.widgets['remote_workdir_field'].text() + # this shouldn't be necessary, however the signals and the workers are created before the async copy + # object is created and then the local dir is not set in the worker. + self.asyncCopy.SetRemoteDir(remote_dir) + self.asyncCopy.SetCopyToRemote() + self.asyncCopy.SetLocalDir(os.path.dirname(zipped)) + self.asyncCopy.SetFileName(os.path.basename(zipped)) + + self.asyncCopy.signals.status.connect(print) + self.asyncCopy.signals.finished.connect( + lambda: self._unzip_on_remote(remote_dir, os.path.basename(zipped)) + ) + + self.asyncCopy.threadpool.start(self.asyncCopy.worker) + + @pysnooper.snoop() + def _unzip_on_remote(self, workdir, filename, **kwargs): + + # 1 create a BasicRemoteExecutionManager + username = self.main_window.connection_details['username'] + port = self.main_window.connection_details['server_port'] + host = self.main_window.connection_details['server_name'] + private_key = self.main_window.connection_details['private_key'] + remote_os = self.main_window.connection_details['remote_os'] + logfile = os.path.join(tempfile.tempdir, 'ssh.log') + conn = BasicRemoteExecutionManager(port, host, username, private_key, remote_os, logfile=logfile) + conn.login(passphrase=False) + # 2 go to workdir + conn.changedir(workdir) + # 2 run 'unzip filename' + # -o forces to overwrite the files being unzipped + # unzip should be substituted with some python like + # import zipfile + # with zipfile.ZipFile("remote_run.zip", 'r') as mz: + # mz.extractall() + stdout, stderr = conn.run('cd {} && unzip -o {}'.format(workdir, filename)) + status_callback = kwargs.get('status_callback', None) + if status_callback is not None: + status_callback.emit((stdout, stderr)) + + @pysnooper.snoop() + def run_dvc_on_remote(self, workdir, **kwargs): + # 1 create a BasicRemoteExecutionManager + username = self.main_window.connection_details['username'] + port = self.main_window.connection_details['server_port'] + host = self.main_window.connection_details['server_name'] + private_key = self.main_window.connection_details['private_key'] + remote_os = self.main_window.connection_details['remote_os'] + logfile = os.path.join(tempfile.tempdir, 'ssh.log') + + progress_callback = kwargs.get('progress_callback', None) + + if remote_os == 'POSIX': + dpath = posixpath + else: + dpath = ntpath + + conn = BasicRemoteExecutionManager(port, host, username, private_key, remote_os, logfile=logfile) + conn.login(passphrase=False) + # 2 go to workdir + conn.changedir(workdir) + for i,el in enumerate(self.processes): + if progress_callback is not None: + progress_callback.emit(i) + + param_file = el[1][0] + + wdir = dpath.join(workdir, 'dvc_result_{}'.format(i)) + # 2 run 'unzip filename' + stdout, stderr = conn.run('cd {} && . ~/condarc && conda activate dvc && dvc dvc_config.txt'.format(wdir)) + + def run_dvc(self): main_window = self.main_window input_file = self.input_file