Fixes, clean up, and an attempt at Linux support.

CMakeLists.txt

@@ -37,7 +37,7 @@ elseif(WIN32)
     list(APPEND obs-vst_SOURCES
             win/VSTPlugin-win.cpp
             win/EditorWidget-win.cpp)
-elseif(LINUX)
+elseif("${CMAKE_SYSTEM_NAME}" MATCHES "Linux")
 	list (APPEND obs-vst_SOURCES
 			linux/VSTPlugin-linux.cpp
 			linux/EditorWidget-linux.cpp)
@@ -54,8 +54,8 @@ add_library(obs-vst MODULE
 		${vst-HEADER})
 
 target_link_libraries(obs-vst
-        Qt5::Widgets
-        libobs)
+		libobs
+        Qt5::Widgets)
 
 if(APPLE)
     target_link_libraries(obs-vst

VSTPlugin.cpp

@@ -1,5 +1,6 @@
 /*****************************************************************************
 Copyright (C) 2016-2017 by Colin Edwards.
+Additional Code Copyright (C) 2016-2017 by c3r1c3 <[email protected]>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -75,6 +76,7 @@ void silenceChannel(float **channelData, int numChannels, long numFrames) {
 }
 
 obs_audio_data *VSTPlugin::process(struct obs_audio_data *audio) {
+
 	if (effect && effectReady) {
 		silenceChannel(outputs, VST_MAX_CHANNELS, audio->frames);
 

data/locale/en-US.ini

@@ -1,3 +1,3 @@
 OpenPluginInterface="Open Plug-in Interface"
 ClosePluginInterface="Close Plug-in Interface"
-VstPlugin="VST Plug-in"
+VstPlugin="VST 2.x Plug-in"

headers/VSTPlugin.h

@@ -1,5 +1,6 @@
 /*****************************************************************************
 Copyright (C) 2016-2017 by Colin Edwards.
+Additional Code Copyright (C) 2016-2017 by c3r1c3 <[email protected]>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

linux/EditorWidget-linux.cpp

@@ -1,5 +1,10 @@
 /*****************************************************************************
 Copyright (C) 2016-2017 by Colin Edwards.
+Additional Code Copyright (C) 2016-2017 by c3r1c3 <[email protected]>
+
+Special thanks to Nik Reiman for sharing his awesome code with the world.
+Some of the original code can be found here:
+https://github.com/teragonaudio/MrsWatson/blob/master/source/plugin/PluginVst2xLinux.cpp
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -17,12 +22,13 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 #include "../headers/EditorWidget.h"
 
-#include <QWindow>
 #include <X11/Xlib.h>
+#include <QWindow>
+
 
 void EditorWidget::buildEffectContainer(AEffect *effect) {
-	WNDCLASSEX wcex{ sizeof(wcex) };
-	wcex.lpfnWndProc = DefWindowProc;
+	//WNDCLASSEX wcex{sizeof(wcex)};
+	/*wcex.lpfnWndProc = DefWindowProc;
 	wcex.hInstance = GetModuleHandle(0);
 	wcex.lpszClassName = L"Minimal VST host - Guest VST Window Frame";
 	RegisterClassEx(&wcex);
@@ -31,23 +37,82 @@ void EditorWidget::buildEffectContainer(AEffect *effect) {
 	HWND hwnd = CreateWindow(
 		wcex.lpszClassName, TEXT(""), style
 		, 0, 0, 0, 0, 0, 0, 0, 0
-	);
+	);*/
+
+
+	Display *display;
+	Window window;
+	XEvent event;
+	int screenNumber;
+	printf("Opening X display");
+	display = XOpenDisplay(NULL);
+	if (display == NULL) {
+		printf("Can't open default display");
+		return;
+	}
+
+	printf("Acquiring default screen for X display");
+	screenNumber = DefaultScreen(display);
+	Screen *screen = DefaultScreenOfDisplay(display);
+	int screenWidth = WidthOfScreen(screen);
+	int screenHeight = HeightOfScreen(screen);
+	printf("Screen dimensions: %dx%d", screenWidth, screenHeight);
+
+	// Default size is 300x300 pixels
+	int windowX = (screenWidth - 300) / 2;
+	int windowY = (screenHeight - 300) / 2;
+
+	printf("Creating window at %dx%d", windowX, windowY);
+	window = XCreateSimpleWindow(display, RootWindow(display, screenNumber), 0, 0,
+			300, 300, 1,
+				     BlackPixel(display, screenNumber),
+				     BlackPixel(display, screenNumber));
+
+	//XStoreName(display, window, pluginName->data);
+/*
+	XSelectInput(display, window, ExposureMask | KeyPressMask);
+	XMapWindow(display, window);
+	XMoveWindow(display, window, windowX, windowY);
+
+
+	printf("Opening plugin editor window");
+	effect->dispatcher(effect, effEditOpen, 0, 0, (void *) window, 0);
+
+	while (true) {
+		XNextEvent(display, &event);
+
+		if (event.type == Expose) {
+		}
+
+		if (event.type == KeyPress) {
+			break;
+		}
+	}
 
-	QWidget *widget = QWidget::createWindowContainer(QWindow::fromWinId((WId)hwnd), this);
+	printf("Closing plugin editor window");
+	effect->dispatcher(effect, effEditClose, 0, 0, 0, 0);
+	XDestroyWindow(display, window);
+	XCloseDisplay(display);
+*/
+	/*
+	QWidget *widget = QWidget::createWindowContainer(QWindow::window);
 	widget->move(0, 0);
 	widget->resize(300, 300);
 
-	effect->dispatcher(effect, effEditOpen, 0, 0, hwnd, 0);
+	effect->dispatcher(effect, effEditOpen, 0, 0, window, 0);
 
 	VstRect* vstRect = nullptr;
 	effect->dispatcher(effect, effEditGetRect, 0, 0, &vstRect, 0);
 	if (vstRect)
 	{
-		widget->resize(vstRect->right - vstRect->left, vstRect->bottom - vstRect->top);
+		widget->resize(vstRect->right - vstRect->left,
+			       vstRect->bottom - vstRect->top);
 	}
+	 */
 }
 
 void EditorWidget::handleResizeRequest(int width, int height) {
-	// We don't have to do anything here as far as I can tell. The widget will resize the HWIND itself and then
+	// We don't have to do anything here as far as I can tell.
+	// The widget will resize the HWIND itself and then
 	// this widget will automatically size depending on that.
 }

linux/VSTPlugin-linux.cpp

@@ -1,5 +1,6 @@
 /*****************************************************************************
 Copyright (C) 2016-2017 by Colin Edwards.
+Additional Code Copyright (C) 2016-2017 by c3r1c3 <[email protected]>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -15,28 +16,56 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *****************************************************************************/
 #include "../headers/VSTPlugin.h"
-#include "../headers/vst-plugin-callbacks.hpp"
 
 #include <util/platform.h>
 #include <X11/Xlib.h>
 
 AEffect* VSTPlugin::loadEffect() {
 	AEffect *plugin = nullptr;
 
-	wchar_t *wpath;
-	os_utf8_to_wcs_ptr(pluginPath.c_str(), 0, &wpath);
-	soHandle = LoadLibraryW(wpath);
+	// Why is this a (non-portable) wide char?
+	wchar_t *wpath = NULL;
+
+	// We need to convert the above wide char to a 'normal' char
+	// 4096 elements should be enough to contain it... I hope.
+	// We also initialize the array with NULLs to prevent issues
+	// down the road.
+	char charPath[4096] = {NULL};
+	uint32_t wcs_returnValue;
+	//os_utf8_to_wcs_ptr(pluginPath.c_str(), 0, &wpath);
+	wcs_returnValue = wcstombs(charPath, wpath, sizeof(charPath));
+
+	// Now check for errors in the conversion before trying to open
+	// the file/path.
+	if (wcs_returnValue < 4) {
+		wprintf(L"Path conversion error from '%s'", wpath);
+		printf(", to '%s'\n", charPath);
+		return nullptr;
+	}
+
+	soHandle = os_dlopen(charPath);
 	bfree(wpath);
+	bfree(charPath);
 	if(soHandle == nullptr) {
 		printf("Failed trying to load VST from '%s', error %d\n",
-			pluginPath, GetLastError());
+		       pluginPath.c_str(), errno);
 		return nullptr;
 	}
 
 	vstPluginMain mainEntryPoint =
-		(vstPluginMain)GetProcAddress(soHandle, "VSTPluginMain");
+			(vstPluginMain)(soHandle, "VSTPluginMain");
+
+	if (mainEntryPoint == nullptr) {
+		mainEntryPoint =
+				(vstPluginMain)os_dlsym(soHandle, "VstPluginMain()");
+	}
+
+	if (mainEntryPoint == nullptr) {
+		mainEntryPoint = (vstPluginMain)os_dlsym(soHandle, "main");
+	}
 
-	if (!mainEntryPoint) {
+	if (mainEntryPoint == nullptr) {
+		printf("Couldn't get a pointer to plugin's main()");
 		return nullptr;
 	}
 
@@ -48,7 +77,7 @@ AEffect* VSTPlugin::loadEffect() {
 
 void VSTPlugin::unloadLibrary() {
 	if (soHandle) {
-		FreeLibrary(soHandle);
+		os_dlclose(soHandle);
 		soHandle = nullptr;
 	}
 }

mac/EditorWidget-osx.mm

@@ -30,11 +30,13 @@
     effect->dispatcher (effect, effEditGetRect, 0, 0, &vstRect, 0);
     if (vstRect)
     {
-        NSRect frame = NSMakeRect(vstRect->left, vstRect->top, vstRect->right, vstRect->bottom);
+        NSRect frame = NSMakeRect(vstRect->left, vstRect->top, vstRect->right,
+                                  vstRect->bottom);
 
         [view setFrame:frame];
 
-        cocoaViewContainer->resize(vstRect->right - vstRect->left, vstRect->bottom- vstRect->top);
+        cocoaViewContainer->resize(vstRect->right - vstRect->left,
+                                   vstRect->bottom- vstRect->top);
     }
 
     effect->dispatcher (effect, effEditOpen, 0, 0, view, 0);

obs-vst.cpp

@@ -16,12 +16,10 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *****************************************************************************/
 
-#include <obs-module.h>
+#include "headers/VSTPlugin.h"
 #include <QDir>
 #include <QDirIterator>
 
-#include "headers/VSTPlugin.h"
-
 #define OPEN_VST_SETTINGS      "open_vst_settings"
 #define CLOSE_VST_SETTINGS     "close_vst_settings"
 
@@ -139,19 +137,43 @@ static void fill_out_plugins(obs_property_t *list)
 		<< "C:/Program Files/Common Files/VST2"
 		<< "C:/Program Files/Common Files/VSTPlugins/"
 		<< "C:/Program Files/VSTPlugins/";
-		// If VST3 support is added....
-		// << "C:/Program Files/Common Files/VST3/";
 	#elif __linux__
+	// If the user has set the VST_PATH environmental
+	// variable, then use it. Else default to a list
+	// of common locations.
+	char *vstPathEnv;
+	vstPathEnv = getenv("VST_PATH");
+	if (vstPathEnv != nullptr)
+	{
+		dir_list << vstPathEnv;
+	} else {
+		// Choose the most common locations
 		dir_list << "/usr/lib/vst/"
-		<< "/usr/lib/lxvst/"
-		<< "/usr/local/lib/vst/"
-		<< "/usr/local/lib/lxvst/"
-		<< "~/.vst/"
-		<< "~/.lxvst/";
+			 << "/usr/lib/lxvst/"
+			 << "/usr/lib/linux_vst/"
+			 << "/usr/lib64/vst/"
+			 << "/usr/lib64/lxvst/"
+			 << "/usr/lib64/linux_vst/"
+			 << "/usr/local/lib/vst/"
+			 << "/usr/local/lib/lxvst/"
+			 << "/usr/local/lib/linux_vst/"
+			 << "/usr/local/lib64/vst/"
+			 << "/usr/local/lib64/lxvst/"
+			 << "/usr/local/lib64/linux_vst/"
+			 << "~/.vst/"
+			 << "~/.lxvst/";
+	}
 	#endif
 
 	QStringList filters;
-	filters << "*.vst" << "*.dll" << "*.so" << "*.o";
+
+#ifdef __APPLE__
+	filters << "*.vst";
+	#elif WIN32
+	filters << "*.dll";
+		#elif __linux__
+	filters << "*.so" << "*.o";
+		#endif
 
 	QStringList vst_list;
 
@@ -164,13 +186,21 @@ static void fill_out_plugins(obs_property_t *list)
 		while (it.hasNext()) {
 			QString path = it.next();
 			QString name = it.fileName();
-			name.remove(QRegExp("(\\.dll|\\.vst|\\.so|\\.o)"));
+
+#ifdef __APPLE__
+			name.remove(QRegExp("(\\.vst)"));
+#elif WIN32
+			name.remove(QRegExp("(\\.dll)"));
+#elif __linux__
+			name.remove(QRegExp("(\\.so|\\.o)"));
+#endif
+
 			name.append("=").append(path);
 			vst_list << name;
 		}
 	}
 
-	// Now sort list alphabetically (but still case-sensitive).
+	// Now sort list alphabetically (still case-sensitive though).
 	std::stable_sort(vst_list.begin(), vst_list.end(),
 			std::less<QString>());
 

win/VSTPlugin-win.cpp

@@ -1,5 +1,6 @@
 /*****************************************************************************
 Copyright (C) 2016-2017 by Colin Edwards.
+Additional Code Copyright (C) 2016-2017 by c3r1c3 <[email protected]>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -28,15 +29,33 @@ AEffect* VSTPlugin::loadEffect() {
 	dllHandle = LoadLibraryW(wpath);
 	bfree(wpath);
 	if(dllHandle == nullptr) {
-		printf("Failed trying to load VST from '%s', error %d\n",
-			pluginPath, GetLastError());
+
+		DWORD errorCode = GetLastError();
+
+		// Display the error message and exit the process
+		if (errorCode == ERROR_BAD_EXE_FORMAT) {
+			printf("Could not open library, wrong architecture.");
+		} else {
+			printf("Failed trying to load VST from '%s', error %d\n",
+			       pluginPath, GetLastError());
+		}
 		return nullptr;
 	}
 
 	vstPluginMain mainEntryPoint =
 		(vstPluginMain)GetProcAddress(dllHandle, "VSTPluginMain");
 
-	if (!mainEntryPoint) {
+	if (mainEntryPoint == nullptr) {
+		mainEntryPoint =
+				(vstPluginMain)GetProcAddress(dllHandle, "VstPluginMain()");
+	}
+
+	if (mainEntryPoint == nullptr) {
+		mainEntryPoint = (vstPluginMain)GetProcAddress(dllHandle, "main");
+	}
+
+	if (mainEntryPoint == nullptr) {
+		printf("Couldn't get a pointer to plug-in's main()");
 		return nullptr;
 	}