diff --git a/ai.py b/ai.py index c99cde1..6f23e98 100755 --- a/ai.py +++ b/ai.py @@ -9,15 +9,16 @@ from game_control import * from keras.models import model_from_json + def main(): # Get Model: - model_file = open('Data/Model/model.json', 'r') + model_file = open('data/model/model.json', 'r') model = model_file.read() model_file.close() model = model_from_json(model) - model.load_weights("Data/Model/weights.h5") + model.load_weights('data/model/weights.h5') - print('AI start now!') + print('AI training has begun') while 1: # Get screenshot: @@ -25,34 +26,36 @@ def main(): # Image to numpy array: screen = np.array(screen) # 4 channel(PNG) to 3 channel(JPG) - Y = predict(model, screen) - if Y == [0,0,0,0]: + y = predict(model, screen) + if y == [0, 0, 0, 0]: # Not action continue - elif Y[0] == -1 and Y[1] == -1: + elif y[0] == -1 and y[1] == -1: # Only keyboard action. - key = get_key(Y[3]) - if Y[2] == 1: + key = get_key(y[3]) + if y[2] == 1: # Press: press(key) else: # Release: release(key) - elif Y[2] == 0 and Y[3] == 0: + elif y[2] == 0 and y[3] == 0: # Only mouse action. - click(Y[0], Y[1]) + pass + # click(y[0], y[1]) else: # Mouse and keyboard action. # Mouse: - click(Y[0], Y[1]) + # click(y[0], y[1]) # Keyboard: - key = get_key(Y[3]) - if Y[2] == 1: + key = get_key(y[3]) + if y[2] == 1: # Press: press(key) else: # Release: release(key) + if __name__ == '__main__': main() diff --git a/create_dataset.py b/create_dataset.py index 1142edb..af30e2b 100644 --- a/create_dataset.py +++ b/create_dataset.py @@ -1,5 +1,6 @@ # Arda Mavi import os +import cv2 import sys import platform import numpy as np @@ -7,20 +8,33 @@ from PIL import ImageGrab from game_control import * from predict import predict -from scipy.misc import imresize from game_control import get_id from get_dataset import save_img from multiprocessing import Process from keras.models import model_from_json -from pynput.mouse import Listener as mouse_listener -from pynput.keyboard import Listener as key_listener +from pynput.mouse import Listener as Mouse_Listener +from pynput.keyboard import Listener as Key_Listener + def get_screenshot(): img = ImageGrab.grab() - img = np.array(img)[:,:,:3] # Get first 3 channel from image as numpy array. - img = imresize(img, (150, 150, 3)).astype('float32')/255. + img = np.array(img)[:, :, :3] # Get first 3 channel from image as numpy array. + + """ + The ratio is r. The new image will + have a height of 150 pixels. To determine the ratio of the new + height to the old height, we divide 150 by the old height. + """ + + r = 150.0 / img.shape[0] + dim = (int(img.shape[1] * r), 150) + + img = cv2.resize(img, dim, interpolation=cv2.INTER_AREA) + print(img) + return img + def save_event_keyboard(data_path, event, key): key = get_id(key) data_path = data_path + '/-1,-1,{0},{1}'.format(event, key) @@ -28,14 +42,18 @@ def save_event_keyboard(data_path, event, key): save_img(data_path, screenshot) return + +""" + def save_event_mouse(data_path, x, y): data_path = data_path + '/{0},{1},0,0'.format(x, y) screenshot = get_screenshot() save_img(data_path, screenshot) return + def listen_mouse(): - data_path = 'Data/Train_Data/Mouse' + data_path = 'data/train_data/mouse' if not os.path.exists(data_path): os.makedirs(data_path) @@ -44,15 +62,17 @@ def on_click(x, y, button, pressed): def on_scroll(x, y, dx, dy): pass - + def on_move(x, y): pass - with mouse_listener(on_move=on_move, on_click=on_click, on_scroll=on_scroll) as listener: + with Mouse_Listener(on_move=on_move, on_click=on_click, on_scroll=on_scroll) as listener: listener.join() +""" + def listen_keyboard(): - data_path = 'Data/Train_Data/Keyboard' + data_path = 'data/train_data/keyboard' if not os.path.exists(data_path): os.makedirs(data_path) @@ -62,18 +82,20 @@ def on_press(key): def on_release(key): save_event_keyboard(data_path, 2, key) - with key_listener(on_press=on_press, on_release=on_release) as listener: + with Key_Listener(on_press=on_press, on_release=on_release) as listener: listener.join() + def main(): - dataset_path = 'Data/Train_Data/' + dataset_path = 'data/train_data/' if not os.path.exists(dataset_path): os.makedirs(dataset_path) # Start to listening mouse with new process: - Process(target=listen_mouse, args=()).start() + # Process(target=listen_mouse, args=()).start() listen_keyboard() return + if __name__ == '__main__': main() diff --git a/database_process.py b/database_process.py index 73ec22b..6fa1bb3 100644 --- a/database_process.py +++ b/database_process.py @@ -2,26 +2,33 @@ import os import sqlite3 + def set_sql_connect(database_name): return sqlite3.connect(database_name) + + def set_sql_cursor(database_connect): return database_connect.cursor() + def close_connect(vt): if vt: vt.commit() vt.close + def set_connect_and_cursor(path='Data/database.sqlite'): vt = set_sql_connect(path) db = set_sql_cursor(vt) return vt, db + def create_table(table_name, columns): vt, db = set_connect_and_cursor() db.execute("CREATE TABLE IF NOT EXISTS {0} ({1})".format(table_name, columns)) close_connect(vt) + def get_data(sql_command): vt, db = set_connect_and_cursor() db.execute(sql_command) @@ -29,6 +36,7 @@ def get_data(sql_command): close_connect(vt) return gelen_veri + def add_data(table, adding): vt, db = set_connect_and_cursor() db.execute("INSERT INTO '{0}' VALUES ({1})".format(table, adding)) diff --git a/game_control.py b/game_control.py index a37b8a9..dadda09 100755 --- a/game_control.py +++ b/game_control.py @@ -2,37 +2,52 @@ from pynput.mouse import Button, Controller as Mouse from pynput.keyboard import Controller as Keyboard +keyboard = Keyboard() + +mouse = Mouse() + + # For encoding keyboard keys: def get_keys(): - return ['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e','f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 'accept', 'add', 'alt', 'altleft', 'altright', 'apps', 'backspace', 'browserback', 'browserfavorites', 'browserforward', 'browserhome', 'browserrefresh', 'browsersearch', 'browserstop', 'capslock', 'clear', 'convert', 'ctrl', 'ctrlleft', 'ctrlright', 'decimal', 'del', 'delete', 'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20', 'f21', 'f22', 'f23', 'f24', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja', 'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail', 'launchmediaselect', 'left', 'modechange', 'multiply', 'nexttrack', 'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6', 'num7', 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn', 'pgup', 'playpause', 'prevtrack', 'print', 'printscreen', 'prntscrn', 'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator', 'shift', 'shiftleft', 'shiftright', 'sleep', 'space', 'stop', 'subtract', 'tab', 'up', 'volumedown', 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen', 'command', 'option', 'optionleft', 'optionright'] + return ['a', 'd', 's', 'enter', 'return', 'space', 'right', 'left', 'up', 'down'] + + +def get_key(key_id): + return get_keys()[key_id] -def get_key(id): - return get_keys()[id] def get_id(key): - return get_keys().index(key) + print('key', key) + print(get_keys(), 'index') + return + # return get_keys().index(key) + + +# Keyboard: +def press(key): + keyboard.press(key) + return + + +def release(key): + keyboard.release(key) + return -keyboard = Keyboard() -mouse = Mouse() # Mouse: +""" + def move(x, y): mouse.position = (x, y) return + def scroll(x, y): mouse.scroll(x, y) return + def click(x, y): mouse.press(Button.left) return - -# Keyboard: -def press(key): - keyboard.press(key) - return - -def release(key): - keyboard.release(key) - return +""" diff --git a/get_dataset.py b/get_dataset.py index 0f1ca04..7ddf20a 100755 --- a/get_dataset.py +++ b/get_dataset.py @@ -1,47 +1,69 @@ # Arda Mavi import os import numpy as np +import cv2 from keras.utils import to_categorical -from scipy.misc import imread, imresize, imsave +import imageio +from skimage import * +import skimage.io from sklearn.model_selection import train_test_split + def get_img(data_path): # Getting image array from path: - img = imread(data_path) - img = imresize(img, (150, 150, 3)) + img = skimage.io.imread(data_path) + """ + The ratio is r. The new image will + have a height of 150 pixels. To determine the ratio of the new + height to the old height, we divide 150 by the old height. + """ + + r = 150.0 / img.shape[0] + dim = (int(img.shape[1] * r), 150) + + img = cv2.resize(img, dim, interpolation=cv2.INTER_AREA) + # img = imresize(img, (150, 150, 3)).astype('float32') / 255. + return img + def save_img(img, path): - imsave(path, img) + sk_img = skimage.io.imread(path) + sk_img = sk_img.transpose(1, 0, 2).reshape(130, -1) + image = img_as_ubyte(sk_img) + skimage.io.imsave(path, image) + skimage.io.imsave(path, img) + # skimage.io.imsave(path, img_as_uint(img)) return -def get_dataset(dataset_path='Data/Train_Data'): + +def get_dataset(dataset_path='data\\train_data'): # Getting all data from data path: try: - X = np.load('Data/npy_train_data/X.npy') - Y = np.load('Data/npy_train_data/Y.npy') - except: - labels = os.listdir(dataset_path) # Geting labels - X = [] - Y = [] - count_categori = [-1,''] # For encode labels + x = np.load('data\\npy_train_data\\x.npy') + y = np.load('data\\npy_train_data\\y.npy') + except AttributeError: + labels = os.listdir(dataset_path) # getting labels + x = [] + y = [] + count_category = [-1, ''] # For encode labels for label in labels: - datas_path = dataset_path+'/'+label - for data in os.listdir(datas_path): - img = get_img(datas_path+'/'+data) - X.append(img) + datas_path = dataset_path + '/' + label + for d_data in os.listdir(datas_path): + img = get_img(datas_path + '/' + d_data) + x.append(img) # For encode labels: - if data != count_categori[1]: - count_categori[0] += 1 - count_categori[1] = data.split(',') - Y.append(count_categori[0]) + if d_data != count_category[1]: + count_category[0] += 1 + count_category[1] = d_data.split(',') + y.append(count_category[0]) # Create dateset: - X = np.array(X).astype('float32')/255. - Y = np.array(Y).astype('float32') - Y = to_categorical(Y, count_categori[0]+1) - if not os.path.exists('Data/npy_train_data/'): - os.makedirs('Data/npy_train_data/') - np.save('Data/npy_train_data/X.npy', X) - np.save('Data/npy_train_data/Y.npy', Y) - X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.1, random_state=42) - return X, X_test, Y, Y_test + x = np.array(x).astype('float32') / 255. + y = np.array(y).astype('float32') + y = to_categorical(y, count_category[0] + 1) + if not os.path.exists('data\\npy_train_data/'): + os.makedirs('data\\npy_train_data/') + np.save('data\\npy_train_data/x.npy', x) + np.save('data\\npy_train_data/y.npy', y) + x, x_test, y, y_test = train_test_split(x, y, test_size=0.1, random_state=42) + return x, x_test, y, y_test diff --git a/get_model.py b/get_model.py index e50271f..de23284 100755 --- a/get_model.py +++ b/get_model.py @@ -4,6 +4,7 @@ from keras.optimizers import Adadelta from keras.layers import Input, Conv2D, Activation, MaxPooling2D, Flatten, Dense, Dropout + def save_model(model): if not os.path.exists('Data/Model/'): os.makedirs('Data/Model/') @@ -19,18 +20,18 @@ def save_model(model): def get_model(): inputs = Input(shape=(150, 150, 3)) - conv_1 = Conv2D(32, (3,3), strides=(1,1))(inputs) + conv_1 = Conv2D(32, (3, 3), strides=(1, 1))(inputs) act_1 = Activation('relu')(conv_1) - conv_2 = Conv2D(64, (3,3), strides=(1,1))(act_1) + conv_2 = Conv2D(64, (3, 3), strides=(1, 1))(act_1) act_2 = Activation('relu')(conv_2) - conv_3 = Conv2D(64, (3,3), strides=(1,1))(act_2) + conv_3 = Conv2D(64, (3, 3), strides=(1, 1))(act_2) act_3 = Activation('relu')(conv_3) pooling_1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(act_3) - conv_4 = Conv2D(128, (3,3), strides=(1,1))(pooling_1) + conv_4 = Conv2D(128, (3, 3), strides=(1, 1))(pooling_1) act_4 = Activation('relu')(conv_4) pooling_2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(act_4) @@ -50,5 +51,6 @@ def get_model(): return model + if __name__ == '__main__': save_model(get_model()) diff --git a/predict.py b/predict.py index 1e2f4a4..ea9e6c1 100755 --- a/predict.py +++ b/predict.py @@ -1,8 +1,12 @@ # Arda Mavi -import numpy as np -from scipy.misc import imresize +import cv2 -def predict(model, X): - X = imresize(X, (150, 150, 3)).astype('float32')/255. - Y = model.predict(X.reshape(1,150,150,3)) - return Y + +def predict(model, x): + r = 50.0 / x.shape[0] + dim = (int(x.shape[1] * r), 50) + + x = cv2.resize(x, dim, interpolation=cv2.INTER_AREA) + # x = skimage.io.imresize(x, (150, 150, 3)).astype('float32') / 255. + y = model.predict(x.reshape(1, 150, 150, 3)) + return y diff --git a/requirements.txt b/requirements.txt index 508a7f3..360c81e 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,12 @@ -numpy -scikit-learn +numpy~=1.19.5 +scikit-learn~=0.24.0 scikit-image -pillow +pillow~=8.1.0 tensorflow -keras -pynput +keras~=2.4.3 +pynput~=1.7.2 h5py + +scipy~=1.6.0 +imageio~=2.9.0 +opencv-python~=4.5.1.48 \ No newline at end of file diff --git a/return get keys.txt b/return get keys.txt new file mode 100644 index 0000000..4f719dd --- /dev/null +++ b/return get keys.txt @@ -0,0 +1,20 @@ + return ['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', '0', + '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', + '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 'accept', 'add', 'alt', 'altleft', 'altright', 'apps', + 'backspace', 'browserback', 'browserfavorites', 'browserforward', 'browserhome', 'browserrefresh', + 'browsersearch', 'browserstop', 'capslock', 'clear', 'convert', 'ctrl', 'ctrlleft', 'ctrlright', + 'decimal', + 'del', 'delete', 'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10', 'f11', 'f12', + 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20', 'f21', 'f22', 'f23', 'f24', 'f3', 'f4', + 'f5', + 'f6', 'f7', 'f8', 'f9', 'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja', + 'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail', 'launchmediaselect', 'left', 'modechange', + 'multiply', 'nexttrack', 'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6', 'num7', + 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn', 'pgup', 'playpause', 'prevtrack', + 'print', + 'printscreen', 'prntscrn', 'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator', + 'shift', 'shiftleft', 'shiftright', 'sleep', 'space', 'stop', 'subtract', 'tab', 'up', 'volumedown', + 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen', 'command', 'option', 'optionleft', + 'optionright', 'right', 'left', 'up', 'down'] + diff --git a/train.py b/train.py index f97d886..6334d3b 100755 --- a/train.py +++ b/train.py @@ -8,24 +8,32 @@ epochs = 100 batch_size = 5 -def train_model(model, X, X_test, Y, Y_test): + +def train_model(model, x, x_test, y, y_test): checkpoints = [] if not os.path.exists('Data/Checkpoints/'): os.makedirs('Data/Checkpoints/') - checkpoints.append(ModelCheckpoint('Data/Checkpoints/best_weights.h5', monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=True, mode='auto', period=1)) - checkpoints.append(TensorBoard(log_dir='Data/Checkpoints/./logs', histogram_freq=0, write_graph=True, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None)) + checkpoints.append( + ModelCheckpoint('Data/Checkpoints/best_weights.h5', monitor='val_loss', verbose=0, save_best_only=True, + save_weights_only=True, mode='auto', period=1)) + checkpoints.append( + TensorBoard(log_dir='Data/Checkpoints/./logs', histogram_freq=0, write_graph=True, write_images=False, + embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None)) - model.fit(X, Y, batch_size=batch_size, epochs=epochs, validation_data=(X_test, Y_test), shuffle=True, callbacks=checkpoints) + model.fit(x, y, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test), shuffle=True, + callbacks=checkpoints) return model + def main(): - X, X_test, Y, Y_test = get_dataset() + x, x_test, y, y_test = get_dataset() model = get_model() - model = train_model(model, X, X_test, Y, Y_test) + model = train_model(model, x, x_test, y, y_test) save_model(model) return model + if __name__ == '__main__': main()