Merge pull request #518 from MEDomicsLab/fix_supersetVenvMacOS

Fix superset venv mac os

Author: MahdiAll99

go_server/main

@@ -1,25 +1,34 @@
 import json
 import subprocess
 import sys
+import os
 import time
 from pathlib import Path
+import venv
 
 SUPERSET_PACKAGES = [
     "apache-superset==4.1.1",
     "flask-cors==5.0.0",
     "marshmallow==3.26.1",
-    "psycopg2-binary==2.9.9"
+    "psycopg2-binary==2.9.9",
+    "wtforms==2.3.3", # https://github.com/apache/superset/issues/29289#issuecomment-2341321222
 ]
 
 class SupersetEnvManager:
-    def __init__(self, python_path):
+    def __init__(self, python_path, app_data_path):
         self.python_path = python_path
         self.env_path = None
+        
+        # Using app_data_path to define environment path (defined by userData https://www.electronjs.org/docs/latest/api/app#appgetapppath)
+        # This is a writeable path where we can create our virtual environment (Fix for macOS venv creation issues)
+        base_dir = Path(app_data_path)
         if sys.platform == "win32":
-            self.env_path = Path(python_path).parent / "superset_env/Scripts/python.exe"
-        else:
+            self.env_path = base_dir / "superset_env/Scripts/python.exe"
+        elif sys.platform == "linux":
             self.env_path = python_path.replace("bin", "bin/superset_env/bin")
-    
+        else:
+            self.env_path = base_dir / "superset_env/bin/python"
+
     def check_env_exists(self):
         """Check if the virtual environment exists"""
         if sys.platform == "win32":
@@ -34,17 +43,48 @@ def create_env(self):
             env_name = str(self.env_path)[:env_name+len("superset_env")]
         else:
             return False
+        
+        print(f"Creating virtual environment at: {env_name}")
+
+        try:
+            # Fix for macOS standalone python: symlink libpython
+            if sys.platform == "darwin":
+                # Create venv without pip first to avoid crash due to missing lib on macOS standalone builds
+                venv.create(env_name, with_pip=False, clear=True)
+
+                base_python_lib = Path(self.python_path).parent.parent / "lib"
+                venv_lib = Path(env_name) / "lib"
+                
+                # Find libpython dylib
+                lib_files = list(base_python_lib.glob("libpython*.dylib"))
+                if lib_files:
+                    target_lib = lib_files[0]
+                    link_name = venv_lib / target_lib.name
+                    if not link_name.exists():
+                        try:
+                            link_name.symlink_to(target_lib)
+                            print(f"Symlinked {target_lib} to {link_name}")
+                        except Exception as e:
+                            print(f"Failed to symlink libpython: {e}")
+
+                # Now install pip
+                print("Installing pip...")
+                subprocess.run([str(self.env_path), "-m", "ensurepip"], check=True)
 
-        # Create virtual environment
-        process = subprocess.Popen([
-            self.python_path, "-m", "venv", env_name
-        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-        process.wait()
-        if process.returncode == 0:
+            else:
+                # Create virtual environment
+                process = subprocess.Popen([
+                    self.python_path, "-m", "venv", env_name
+                ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+                process.wait()
+                if process.returncode != 0:
+                    print(f"Error creating environment: {process.stderr}")
+                    raise Exception(f"Failed to create virtual environment. Error: {process.stderr}")
+            
             print(f"Environment created at: {self.env_path}")
             return True
-        else:
-            print(f"Error creating environment: {process.stderr}")
+        except Exception as e:
+            print(f"Error creating environment: {e}")
             return False
 
     def check_requirements(self):
@@ -82,11 +122,11 @@ def get_installed_packages(self):
                 str(self.python_path), "-m", "pip", "freeze"
             ], capture_output=True, text=True, check=False)
 
-            packages = {}
+            packages = []
             for line in result.stdout.split('\n'):
                 if '==' in line:
                     name, version = line.split('==', 1)
-                    packages[name.lower()] = version.strip()
+                    packages.append({'name': name, 'version': version})
             return packages
 
     def is_package_installed(self, package_name, installed_packages=None):
@@ -102,8 +142,20 @@ def is_package_installed(self, package_name, installed_packages=None):
 
     def install_requirements(self):
         """Install packages in the environment"""
+        # Upgrade pip, setuptools and wheel first to ensure we can install binary wheels
+        try:
+            subprocess.run(
+                [str(self.env_path), "-m", "pip", "install", "--upgrade", "--prefer-binary", "pip", "setuptools", "wheel"],
+                check=True,
+                capture_output=True,
+                text=True
+            )
+        except subprocess.CalledProcessError as e:
+            print(f"Warning: Failed to upgrade pip/setuptools/wheel: {e.stderr}")
+
         # Build install command
-        install_cmd = [str(self.env_path), "-m", "pip", "install"]
+        # Use --prefer-binary to avoid compiling from source if possible (fixes issues on machines without build tools)
+        install_cmd = [str(self.env_path), "-m", "pip", "install", "--prefer-binary"]
 
         for requirement in SUPERSET_PACKAGES:
             if isinstance(requirement, dict):
@@ -153,11 +205,23 @@ def install_packages(self, set_progress: callable, current_progress: int, step:
             print(f"Failed to ensure pip: {result.stderr}")
             return False
 
+        # Upgrade pip, setuptools and wheel first
+        try:
+            subprocess.run(
+                [str(self.env_path), "-m", "pip", "install", "--upgrade", "--prefer-binary", "pip", "setuptools", "wheel"],
+                check=True,
+                capture_output=True,
+                text=True
+            )
+        except subprocess.CalledProcessError as e:
+            print(f"Warning: Failed to upgrade pip/setuptools/wheel: {e.stderr}")
+        
         success = True
         for package in SUPERSET_PACKAGES:
             set_progress(label=f"Installing {package.split('==')[0]}...")
+            # Use --prefer-binary to avoid compiling from source
             result = subprocess.run([
-                str(self.env_path), "-m", "pip", "install", package
+                str(self.env_path), "-m", "pip", "install", "--prefer-binary", package
             ], check=True, capture_output=True, text=True)
 
             if result.returncode == 0:

pythonCode/modules/superset/SupersetEnvManager.py

@@ -34,14 +34,25 @@ def _custom_process(self, json_config: dict) -> dict:
 
         # Map settings
         python_path = json_config["pythonPath"]
+        app_data_path = json_config.get("appDataPath")
+        
+        if not app_data_path:
+            # Fallback to default locations if not provided (for backward compatibility or testing)
+            if sys.platform == "win32":
+                app_data_path = str(Path(os.getenv('APPDATA')) / "medomics-platform")
+            elif sys.platform == "darwin":
+                app_data_path = str(Path.home() / "Library/Application Support/medomics-platform")
+            else:
+                app_data_path = str(Path.home() / ".config/medomics-platform")
+
         username = json_config["username"]
         firstname = json_config["firstname"]
         lastname = json_config["lastname"]
         email = json_config["email"]
         password = json_config["password"]
 
         # Set up Superset
-        output = self.create_user(python_path, username, firstname, lastname, email, password)
+        output = self.create_user(python_path, app_data_path, username, firstname, lastname, email, password)
 
         return output
 
@@ -59,12 +70,13 @@ def run_command(self, command, env=None, capture_output=True, timeout=None):
 
         return {}
 
-    def create_user(self, python_path: str, username: str, firstname: str, lastname: str, email: str, password: str):
+    def create_user(self, python_path: str, app_data_path: str, username: str, firstname: str, lastname: str, email: str, password: str):
         """
         Creates a new Superset user.
 
         Args:
             python_path: The path to the Python installation.
+            app_data_path: The path to the application data directory.
             username: The username of the new user.
             firstname: The first name of the new user.
             lastname: The last name of the new user.
@@ -76,7 +88,7 @@ def create_user(self, python_path: str, username: str, firstname: str, lastname:
         """
         # Prepare environment variables
         env = os.environ.copy()
-        manager = SupersetEnvManager(python_path)
+        manager = SupersetEnvManager(python_path, app_data_path)
         path_superset = manager.get_superset_path()
         env["FLASK_APP"] = "superset"
 
@@ -90,7 +102,7 @@ def create_user(self, python_path: str, username: str, firstname: str, lastname:
             "password": password,
         }
         output = self.run_command(
-            f"{path_superset} fab create-admin "
+            f'"{path_superset}" fab create-admin '
             f"--username {admin_user['username']} "
             f"--firstname {admin_user['firstname']} "
             f"--lastname {admin_user['lastname']} "

pythonCode/modules/superset/create_user.py

@@ -28,14 +28,24 @@ def __init__(self, json_params: dict, _id: str = None):
 
     def _custom_process(self, json_config: dict) -> dict:
         """
-            This function predicts from a model, a dataset, and a new dataset
+            This function is called to execute the custom process which sets up Superset.
         """
         # Map settings
         port = json_config["port"]
         python_path = json_config["pythonPath"]
+        app_data_path = json_config.get("appDataPath")
+        
+        if not app_data_path:
+            # Fallback to default locations if not provided (for backward compatibility or testing)
+            if sys.platform == "win32":
+                app_data_path = str(Path(os.getenv('APPDATA')) / "medomics-platform")
+            elif sys.platform == "darwin":
+                app_data_path = str(Path.home() / "Library/Application Support/medomics-platform")
+            else:
+                app_data_path = str(Path.home() / ".config/medomics-platform")
 
         # Set up Superset
-        result = self.setup_superset(port, python_path)
+        result = self.setup_superset(port, python_path, app_data_path)
 
         return result
 
@@ -53,13 +63,14 @@ def run_command(self, command, env=None, capture_output=True, timeout=None):
             return {"error": f"Error while running command: {command}. Full error log:" + e.stderr}
 
 
-    def setup_superset(self, port, python_path):
+    def setup_superset(self, port, python_path, app_data_path):
         """
         Set up Superset with the provided settings.
         
         Args:
             scripts_path (path): The path to the Python scripts directory.
             port (path): The port on which to run Superset.
+            app_data_path (path): The path to the application data directory.
             
         Returns:
             A dictionary containing the error message, if any.
@@ -70,7 +81,7 @@ def setup_superset(self, port, python_path):
 
         # Check if the virtual environment exists
         self.set_progress(now=progress, label="Checking the Superset virtual environment...")
-        manager = SupersetEnvManager(python_path)
+        manager = SupersetEnvManager(python_path, app_data_path)
         if not manager.check_env_exists():
             print("Creating Superset virtual environment...")
             self.set_progress(now=self._progress["now"]+step, label="Creating Superset virtual environment...")
@@ -96,7 +107,13 @@ def setup_superset(self, port, python_path):
         # Generate a private key
         print("Generating a private key...")
         self.set_progress(label="Checking the private key...")
-        private_key = subprocess.check_output("openssl rand -base64 42", shell=True, text=True).strip()
+        try:
+            import secrets
+            private_key = secrets.token_urlsafe(42)
+        except ImportError:
+            import base64
+            private_key = base64.b64encode(os.urandom(42)).decode('utf-8')
+            
         if private_key is None:
             print("Error while generating a private key.")
             return {"error": "Error while generating a private key."}
@@ -151,7 +168,7 @@ def setup_superset(self, port, python_path):
         # Initialize the database
         print("Initializing the Superset database...")
         self.set_progress(now=self._progress["now"]+step, label="Initializing the Superset database...")
-        result = self.run_command(f"{superset_path} db upgrade", env)
+        result = self.run_command(f'"{superset_path}" db upgrade', env)
         if "error" in result:
             return result
 
@@ -166,7 +183,7 @@ def setup_superset(self, port, python_path):
             "password": "admin",
         }
         result = self.run_command(
-            f"{superset_path} fab create-admin "
+            f'"{superset_path}" fab create-admin '
             f"--username {admin_user['username']} "
             f"--firstname {admin_user['firstname']} "
             f"--lastname {admin_user['lastname']} "
@@ -180,10 +197,18 @@ def setup_superset(self, port, python_path):
         # Initialize Superset
         print("Initializing Superset...")
         self.set_progress(now=self._progress["now"]+2*step, label="Initializing Superset...")
-        result = self.run_command(f"{superset_path} init", env)
+        result = self.run_command(f'"{superset_path}" init', env)
         if "error" in result:
             return result
 
+        # Load examples (optional)
+        print("Loading example data...")
+        self.set_progress(now=self._progress["now"]+step, label="Loading default example data (this may take several minutes)...")
+        result = self.run_command(f'"{superset_path}" load_examples', env)
+        if "error" in result:
+            print(f"Warning: Failed to load examples: {result.get('error')}")
+            # We continue even if examples fail to load, as it is optional
+
         # Check if port is available
         print(f"Checking if port {port} is available...")
         self.set_progress(now=self._progress["now"]+step, label="Checking if port is available...")
@@ -202,7 +227,7 @@ def setup_superset(self, port, python_path):
         print(f"Launching Superset on port {port}...")
         self.set_progress(now=self._progress["now"]+step, label="Launching Superset...")
         try:
-            subprocess.Popen(f"{superset_path} run -p {port}", shell=True, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            subprocess.Popen(f'"{superset_path}" run -p {port}', shell=True, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
         except subprocess.CalledProcessError as e:
             print(f"Error while running command: {f'{superset_path} run -p {port}'}")
             print(e.stderr)

pythonCode/modules/superset/launch.py

@@ -1,3 +1,4 @@
+/* eslint-disable no-unreachable */
 import { randomUUID } from "crypto"
 import { DataFrame } from "../../../utilities/danfo.js"
 import { Button } from 'primereact/button'
@@ -54,7 +55,7 @@ const MEDprofilesPrepareData = () => {
         className="mx-3"
       />
     </div>
-  )
+    )
   const [binaryFileList, setBinaryFileList] = useState([]) // list of available binary files
   const [binaryFilename, setBinaryFilename] = useState("MEDprofiles_bin.pkl") // name under which the MEDprofiles binary file will be saved
   const [generatedClassesFolder, setGeneratedClassesFolder] = useState(null) // folder containing the generated MEDclasses