diff --git a/.genius b/.genius new file mode 100755 index 0000000..bad8f4f Binary files /dev/null and b/.genius differ diff --git a/.heroku_data b/.heroku_data new file mode 100644 index 0000000..3ab8941 Binary files /dev/null and b/.heroku_data differ diff --git a/.spotify b/.spotify new file mode 100755 index 0000000..6e1bd21 Binary files /dev/null and b/.spotify differ diff --git a/.status b/.status new file mode 100644 index 0000000..66e679d Binary files /dev/null and b/.status differ diff --git a/.telegram b/.telegram new file mode 100755 index 0000000..daf4ca4 Binary files /dev/null and b/.telegram differ diff --git a/.youtube b/.youtube new file mode 100644 index 0000000..4eb8b89 Binary files /dev/null and b/.youtube differ diff --git a/Data/9.png b/Data/9.png new file mode 100755 index 0000000..a0d9f56 Binary files /dev/null and b/Data/9.png differ diff --git a/Data/Heavy.ttf b/Data/Heavy.ttf new file mode 100755 index 0000000..81012b4 Binary files /dev/null and b/Data/Heavy.ttf differ diff --git a/Data/Ultralight.ttf b/Data/Ultralight.ttf new file mode 100755 index 0000000..308643e Binary files /dev/null and b/Data/Ultralight.ttf differ diff --git a/Data/header1.png b/Data/header1.png index 9b7d64b..968cc1f 100755 Binary files a/Data/header1.png and b/Data/header1.png differ diff --git a/Data/header3.png b/Data/header3.png old mode 100644 new mode 100755 diff --git a/Data/header5.png b/Data/header5.png old mode 100644 new mode 100755 diff --git a/Data/logo-w.png b/Data/logo-w.png new file mode 100755 index 0000000..6e5064e Binary files /dev/null and b/Data/logo-w.png differ diff --git a/Data/s1.webp b/Data/s1.webp old mode 100644 new mode 100755 diff --git a/Data/s2.webp b/Data/s2.webp old mode 100644 new mode 100755 diff --git a/Data/s3.webp b/Data/s3.webp old mode 100644 new mode 100755 diff --git a/Data/s4.webp b/Data/s4.webp old mode 100644 new mode 100755 diff --git a/Data/s5.webp b/Data/s5.webp old mode 100644 new mode 100755 diff --git a/Data/s6.webp b/Data/s6.webp new file mode 100755 index 0000000..6c09a45 Binary files /dev/null and b/Data/s6.webp differ diff --git a/Data/temp.png b/Data/temp.png old mode 100644 new mode 100755 index 8c78609..3301163 Binary files a/Data/temp.png and b/Data/temp.png differ diff --git a/Data/temp1.png b/Data/temp1.png old mode 100644 new mode 100755 index 4511cca..3301163 Binary files a/Data/temp1.png and b/Data/temp1.png differ diff --git a/LICENSE b/LICENSE old mode 100644 new mode 100755 diff --git a/Procfile b/Procfile old mode 100644 new mode 100755 index 2c1c45b..c1eeb53 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -worker: python3 telegram.py +worker: bash worker.sh diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 1e981d5..ea220c1 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ [![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg)](https://www.python.org/) [![Open Source Love svg1](https://badges.frapsoft.com/os/v1/open-source.svg?v=103)](https://github.com/ellerbrock/open-source-badges/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) - - - - - + + + + + diff --git a/apple.py b/apple.py old mode 100644 new mode 100755 index a32de7b..0cc3d2d --- a/apple.py +++ b/apple.py @@ -16,7 +16,7 @@ def get(self, url): url = url + '&l=uk' - + splitted = str(url).split('/') splitted = splitted[:3]+['ua']+splitted[4:] url = '/'.join(splitted) @@ -49,4 +49,5 @@ def getName(self, url): return None if __name__ == "__main__": a = AppleMusic() - a.get('https://itunes.apple.com/uk/album/simplify/1430224633?i=1430225075') + name = a.get('https://itunes.apple.com/us/album/contra-la-pared/1455082839?i=1455082850') + print(name) diff --git a/deezer.py b/deezer.py new file mode 100755 index 0000000..fcdd028 --- /dev/null +++ b/deezer.py @@ -0,0 +1,73 @@ +import requests + +class Deezer(object): + + def __init__(self): + + ''' + Init function + Creating deezer object + :return: None + ''' + + self.__url = 'http://api.deezer.com/' + + + def getSongInfo(self, id): + + try: + + response = requests.get(f'{self.__url}/track/{id}').json() + + return ({ + 'uri' : f"D{response['id']}T", + 'name' : response['title'], + 'artist' : [response['artist']['name']], + 'album' : response['album']['title'], + 'image' : response['album']['cover_xl'], + 'duration_ms' : response['duration'] + }) + + except: return None + + def getAlbum(self, id): + + try: + + response = requests.get(f'{self.__url}/album/{id}').json() + + alb = { + 'name':response['title'], + 'artist':response['artist']['name'], + 'copyright': None, + 'image':response['cover_xl'], + } + + tracks = [] + + for item in response['tracks']['data']: + + tracks.append({ + 'uri' : f"D{item['id']}T", + 'name' : item['title'], + 'artist' : [item['artist']['name']], + 'album' : alb['name'], + 'image' : alb['image'], + 'preview_url' : item['preview'], + 'duration_ms' : item['duration'] + }) + + alb.setdefault( + 'tracks', tracks + ) + + return alb + + except: return None + +if __name__ == '__main__': + + deezer = Deezer() + data = deezer.getSongInfo('636758392') + + print(data) diff --git a/editor.py b/editor.py index 46ab33d..51b211b 100755 --- a/editor.py +++ b/editor.py @@ -1,10 +1,12 @@ #!/usr/bin/python3 import re, os import shutil +import genius #used for mp3 ID3 tagging -from mutagen.id3._frames import TIT2, TALB, TPE1 +from mutagen.id3._frames import TIT2, TALB, TPE1, USLT from mutagen.mp3 import MP3 from mutagen.id3 import ID3, APIC, error + #used for web scraping import urllib.request @@ -20,15 +22,21 @@ class TagEditor(object): @staticmethod def getImageFromSpotify(url, name): - if len(url): - urllib.request.urlretrieve(url, name) - else: + try: + if len(url): + urllib.request.urlretrieve(url, name) + else: + + cachepath = os.getcwd() + '/cache' + datapath = os.getcwd() + '/Data' + os.system(f'cp {datapath}/temp.png {name}') + + except: cachepath = os.getcwd() + '/cache' datapath = os.getcwd() + '/Data' os.system(f'cp {datapath}/temp.png {name}') - @staticmethod def getTags(): pass @@ -63,6 +71,10 @@ def setTags(data): ''' if data: + if data['image'] == 'https://lastfm-img2.akamaized.net/i/u/300x300/2a96cbd8b46e442fc41c2b86b821562f.png': + + data['image'] = None + #download image TagEditor.getImageFromSpotify(data['image'], f"cache/{data['uri']}/{data['uri']}.png") @@ -106,6 +118,14 @@ def setTags(data): text=(data['artist'][0])) ) + #add song artist + audio.tags.add(USLT( + encoding=3, + lang=u'eng', + desc=u'desc', + text=genius.getLyrics(data['artist'][0],data['name'])) + ) + #save result audio.save() ID3(f"cache/{data['uri']}/{data['uri']}.mp3").save(v2_version=3) diff --git a/genius.py b/genius.py new file mode 100755 index 0000000..84acf8e --- /dev/null +++ b/genius.py @@ -0,0 +1,31 @@ +import lyricsgenius +import pickle + + +def getLyrics(artist, song): + + def function(): + + try: + + with open('.genius', 'rb') as f: + data = pickle.load(f) + + return data['token'] + + except: + + return None + + try: + + genius = lyricsgenius.Genius(function()) + genius.verbose, genius.remove_section_headers = False, True + song = genius.search_song(song, artist) + + return song.lyrics + + except:return None + +if __name__ == '__main__': + print(getLyrics('Cage The Elephant', 'Ready To Let Go')) diff --git a/heroku.py b/heroku.py new file mode 100644 index 0000000..7e3310c --- /dev/null +++ b/heroku.py @@ -0,0 +1,37 @@ +import heroku3 +import pickle + + +def restart(): + + def getData(): + + try: + + with open('.heroku_data', 'rb') as f: + data = pickle.load(f) + + return data['token'] + + except: + + sys.exit() + + try: + + heroku_conn = heroku3.from_key(getData()) + app = heroku_conn.apps()['smd-bot'] + dyno = app.dynos()[0] + + print('RESTARTING_DYNO:DONE') + + dyno.restart() + + except: + + print('RESTARTING_DYNO:ERROR') + + +if __name__ == '__main__': + + restart() diff --git a/image.py b/image.py new file mode 100755 index 0000000..5fab581 --- /dev/null +++ b/image.py @@ -0,0 +1,90 @@ +from PIL import Image, ImageFilter, ImageDraw, ImageFont, ImageEnhance + +class Effects(): + + @staticmethod + def rounded(im, rad=5, width=None, height=None, _scale=1): + im = Image.open(im) + + if width is None and height is None: width, height = im.size[0], im.size[1] + im = im.resize((int(width / _scale), int(height / _scale)), Image.ANTIALIAS) + + circle = Image.new('L', (rad * 2, rad * 2), 0) + + draw = ImageDraw.Draw(circle) + draw.ellipse((0, 0, rad * 2, rad * 2), fill=255) + + alpha = Image.new('L', im.size, 255) + w, h = im.size + + alpha.paste(circle.crop((0, 0, rad, rad)), (0, 0)) + alpha.paste(circle.crop((0, rad, rad, rad * 2)), (0, h - rad)) + alpha.paste(circle.crop((rad, 0, rad * 2, rad)), (w - rad, 0)) + alpha.paste(circle.crop((rad, rad, rad * 2, rad * 2)), (w - rad, h - rad)) + + im.putalpha(alpha) + + return im + + @staticmethod + def text(image, name='Name', artist='Artist'): + draw = ImageDraw.Draw(image) + + f1, s1, l1, c1 = 528, 40, 0, 0 + f2, s2, l2, c2 = 535, 40, 0, 0 + + font = ImageFont.truetype("Data/Ultralight.ttf", s1) + font1 = ImageFont.truetype("Data/Heavy.ttf", s2) + font2 = ImageFont.truetype("Data/Heavy.ttf", 20) + + font_size = font.getsize(name) + font_size1 = font1.getsize(artist) + + if font_size[0] > f1: + temp = s1 + while font_size[0] > f1: + font = ImageFont.truetype("Data/Ultralight.ttf", temp) + font_size = font.getsize(name) + temp, l1 = temp - 1, l1 + 1 + + if font_size1[0] > f2: + temp = s2 + while font_size1[0] > f2: + font1 = ImageFont.truetype("Data/Heavy.ttf", temp) + font_size1 = font1.getsize(artist) + temp, l2 = temp - 1, l2 + 1 + + c1, c2 = (f1 - font_size[0]) / 2, (f2 - font_size1[0]) / 2 + + draw.text((520 + c1, 150 + l1 * 2), name, (255,255,255), font=font) + draw.text((520 + c2, 250), artist, (255,255,255), font=font1) + draw.text((665, 365), 'Spotify Music Downloader', (255,255,255), font=font2) + + return image + + @staticmethod + def createPoster(image, name='Name', artist='Artist', file='image.png'): + original = Image.open(image) + logo = Image.open('Data/logo-w.png') + logo = logo.resize((20, 20), Image.ANTIALIAS) + + rounded = Effects.rounded(image, rad=30) + rounded = rounded.resize((440, 440), Image.ANTIALIAS) + + image = Image.open(image) + image = image.resize((1080, 1080), Image.ANTIALIAS) + width, height = image.size + + left, right, top, bottom = 0, width, height/4, 3 * height/4 + cropped = image.crop((left, top, right, bottom)) + blurred = cropped.filter(ImageFilter.GaussianBlur(radius=40)) + enhancer = ImageEnhance.Brightness(blurred) + enhanced_im = enhancer.enhance(.6) + enhanced_im.paste(rounded, (50, 50), rounded) + enhanced_im.paste(logo, (635, 368), logo) + enhanced_im = Effects.text(enhanced_im, name, artist) + enhanced_im.save(file) + + +if __name__ == "__main__": + Effects.createPoster('image.jpg', name='Hard EP', artist='The Neighbourhood', file='Downloads/image.png') diff --git a/lastfm.py b/lastfm.py old mode 100644 new mode 100755 diff --git a/main.py b/main.py index 3c026fa..d989e73 100755 --- a/main.py +++ b/main.py @@ -3,6 +3,7 @@ from youtube import Youtube from editor import TagEditor from lastfm import LastFM +from deezer import Deezer import sys, getopt, shutil import os @@ -17,12 +18,20 @@ class MusicDownloader(object): - def __init__(self): - self.__youtube = Youtube() + def __init__(self, YT_API_KEY_N): + self.__youtube = Youtube(YT_API_KEY_N) self.__spotify = Spotify() self.__editor = TagEditor() self.__last = LastFM() + self.__deezer = Deezer() + def getYTS(self): + + return self.__youtube.getGoogleAPIStatus() + + def FUCK_GOOGLE(self): + + self.__youtube.testYT_D() def __downloadMusicFromYoutube(self, name, uri, dur): @@ -59,6 +68,9 @@ def getData(self, uri): def getLastFMTags(self, name): return self.__last.get(name) + def getDeezerTags(self, id): + return self.__deezer.getSongInfo(id) + def getYoutubeMusicInfo(self, url): return self.__youtube.getNameFromYoutube(url) @@ -302,9 +314,70 @@ def downloadFromYoutubeMusic(self, url, info): else: return False, None + def downloadByDeezerID(self, uri): + #get info + info = self.__deezer.getSongInfo(uri) + + if info: + + fixed_name = f'{info["artist"][0]} - {info["name"]}' + fixed_name = fixed_name.replace('.','') + fixed_name = fixed_name.replace(',','') + fixed_name = fixed_name.replace("'",'') + fixed_name = fixed_name.replace("/","") + + #finding and download from YouTube and tagging + if self.__downloadMusicFromYoutube(fixed_name, info['uri'], info['duration_ms']): + + self.__editor.setTags( + data=info + ) + + cachepath = os.getcwd() + '/cache' + fullpath = os.getcwd() + '/Downloads' + + #logging + logging.info(f'CACHEPATH {cachepath}') + logging.info(f'FULLPATH {fullpath}') + + if not os.path.exists(fullpath): + os.makedirs(fullpath) + + os.rename( + f"{cachepath}/{info['uri']}/{info['uri']}.png", + f"{fullpath}/{info['uri']}.png" + ) + #logging + logging.info(f"MOVE TO Downloads/{info['uri']}.png") + + os.rename( + f"{cachepath}/{info['uri']}/{info['uri']}.mp3", + f"{fullpath}/{info['uri']}.mp3" + ) + #logging + logging.info(f"MOVE TO Downloads/{info['uri']}.mp3") + + #deleting cache + try: + shutil.rmtree(f"cache/{info['uri']}") + #logging + logging.info(f"DELETED cache/{info['uri']}") + except: + #logging + logging.error(f"DELETING cache/{info['uri']}") + + return True + return False + def search(self, query): return self.__spotify.search(query=query) + def getAlbum(self, uri): + return self.__spotify.getAlbum(uri) + + def getAlbumDeezer(self, id): + return self.__deezer.getAlbum(id) + class CLI(object): diff --git a/proxy.py b/proxy.py new file mode 100644 index 0000000..0385b8a --- /dev/null +++ b/proxy.py @@ -0,0 +1,62 @@ +import asyncio +from proxybroker import Broker +import requests + +def getProxy(log=True): + + async def show(proxies): + while True: + proxy = await proxies.get() + if proxy is None: break + print('Found proxy: %s' % proxy) + + return { + 'proxy': f'https://{proxy.host}:{proxy.port}', + 'type':'https', + 'ip':f'https://{proxy.host}', + 'port':proxy.port + } + + + proxies = asyncio.Queue() + broker = Broker(proxies) + tasks = asyncio.gather( + broker.find(types=['HTTPS'], limit=1), + show(proxies)) + + loop = asyncio.get_event_loop() + result = loop.run_until_complete(tasks) + + return result[1] + +def get(): + + proxy = getProxy() + + print(proxy) + + _type = proxy['type'] + _proxy = proxy['proxy'] + + try: + requests.get( + "https://www.youtube.com", + proxies = { + _type:_proxy + } + ) + except IOError: + + print("Connection error!") + return get() + + else: + + print("All was fine") + return proxy + + + +if __name__ == '__main__': + + get() diff --git a/requirements.txt b/requirements.txt old mode 100644 new mode 100755 index bd7110b..6ecb509 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,17 @@ -mutagen==1.41.0 lxml==4.2.3 -Flask==0.12.2 -moviepy==0.2.3.5 -requests==2.18.4 +mutagen==1.41.0 spotipy==2.4.4 -git+git://github.com/nficano/pytube.git -imageio==2.3.0 +youtube_dl==2019.8.13 +Flask==1.0.0 pyperclip==1.6.4 +celery==4.2.1 +requests==2.18.4 +moviepy==0.2.3.5 +pytube==9.5.1 +imageio==2.3.0 +Pillow==6.0.0 beautifulsoup4==4.7.1 +redis==3.2.0 +lyricsgenius==1.4.0 +proxybroker==0.3.2 +heroku3==3.4.0 \ No newline at end of file diff --git a/spotify.py b/spotify.py index 6e66520..93f182b 100755 --- a/spotify.py +++ b/spotify.py @@ -9,6 +9,7 @@ import webbrowser #flask server from flask import Flask, request +import pickle class Spotify(object): @@ -43,27 +44,27 @@ def code(): class User(object): - def __init__( - self, - client_id = '83e4430b4700434baa21228a9c7d11c5', - client_secret = '9bb9e131ff28424cb6a420afdf41d44a' - ): + def __init__(self): - self.__client_id = client_id - self.__client_secret = client_secret self.__grant_type = 'authorization_code' self.__scope = 'user-library-read' + self.__getData() self.__redirect = 'http://localhost:5000/' self.__urlCode = f'https://accounts.spotify.com/authorize?client_id={self.__client_id}&response_type=code&redirect_uri={self.__redirect}&scope={self.__scope}' self.__url = 'https://accounts.spotify.com/api/token' + self.__getRefreshToken() + self.__client = spotipy.Spotify(auth=self.__access_token) + + + def __getAccessToken(self): + #start server #handling the code webbrowser.open_new(self.__urlCode) Spotify.Server.run() self.__code = Spotify.Server.code - self.__body_params = { 'grant_type': self.__grant_type, 'code': self.__code, @@ -71,16 +72,62 @@ def __init__( } #getting access_token by POST request to Spotify API - self.__access_token = requests.post( + response = requests.post( self.__url, data=self.__body_params, auth=( self.__client_id, self.__client_secret ) - ).json()['access_token'] + ).json() - self.__client = spotipy.Spotify(auth=self.__access_token) + self.__access_token = response['access_token'] + self.__refresh_token = response['refresh_token'] + + data = {'refresh_token' : self.__refresh_token} + + with open('.spotify', 'wb') as f: + pickle.dump(data, f) + + + def __getAccessTokenByRefreshToken(self, refresh_token): + response = requests.post('https://accounts.spotify.com/api/token?', + { + 'grant_type': 'refresh_token', + 'refresh_token': str(refresh_token), + 'client_id': self.__client_id, + 'client_secret': self.__client_secret + } + ).json() + self.__access_token = response['access_token'] + + + def __getRefreshToken(self): + try: + + with open('.spotify', 'rb') as f: + data = pickle.load(f) + self.__getAccessTokenByRefreshToken(data['refresh_token']) + + except: + self.__getAccessToken() + + + def __getData(self): + try: + + with open('.spotify', 'rb') as f: + data = pickle.load(f) + + self.__client_id = data['client_id'] + self.__client_secret = data['client_secret'] + + except: + print(''' + A new version is available on GitHub.\n + Download: https://github.com/artyshko/smd + ''') + sys.exit() def getPlaylistTracks(self, playlist_uri): @@ -110,47 +157,64 @@ def getPlaylistTracks(self, playlist_uri): 'name' : data['name'], 'artist' : [ artist['name'] for artist in data['artists']], 'album' : data['album']['name'], - 'image' : data['album']['images'][0]['url'] + 'image' : data['album']['images'][0]['url'], + 'duration_ms':data['duration_ms'] }) return tracks - def __init__( - self, - client_id = '83e4430b4700434baa21228a9c7d11c5', - client_secret = '9bb9e131ff28424cb6a420afdf41d44a' - ): + def __init__(self): ''' Init function Creating spotify object with access_token - :param client_id: spotify client_id parametr - :param client_secret: spotify client_secret parametr :return: None ''' self.__url = 'https://accounts.spotify.com/api/token' - self.__client_id = client_id - self.__client_secret = client_secret self.__grant_type = 'client_credentials' self.__body_params = { 'grant_type': self.__grant_type } + self.__getData() + self.__getAccessToken() + + #initialization of spotify client + self.client = spotipy.Spotify(self.__access_token) + + + def __getData(self): + try: + + with open('.spotify', 'rb') as f: + data = pickle.load(f) + + self.__client_id = data['client_id'] + self.__client_secret = data['client_secret'] + + except: + print(''' + A new version is available on GitHub.\n + Download: https://github.com/artyshko/smd + ''') + sys.exit() + + + def __getAccessToken(self): #getting access_token by POST request to Spotify API - self.__access_token = requests.post( + response = requests.post( self.__url, data=self.__body_params, auth=( self.__client_id, self.__client_secret ) - ).json()['access_token'] + ).json() - #initialization of spotify client - self.client = spotipy.Spotify(self.__access_token) + self.__access_token = response['access_token'] def getSongInfo(self, uri): @@ -185,7 +249,51 @@ def search(self, query): except: return False + def getDuration(self, uri): data = self.client.track(uri) return data['duration_ms'] + + + def getAlbum(self, uri): + try: + + album = self.client.album(uri) + + copyright = None + + try:copyright = album['copyrights'][0]['text'] + except:pass + + alb = { + 'name':album['name'], + 'artist':album['artists'][0]['name'], + 'copyright':copyright, + 'image':album['images'][0]['url'], + } + + tracks = [] + + for data in album['tracks']['items']: + tracks.append({ + 'uri' : str(data['uri'].split(':')[-1]), + 'name' : data['name'], + 'artist' : [ artist['name'] for artist in data['artists']], + 'album' : alb['name'], + 'image' : alb['image'], + 'preview_url' : data['preview_url'], + 'duration_ms' : data['duration_ms'] + }) + + alb.setdefault( + 'tracks', tracks + ) + + return alb + + except: return None + +if __name__ == '__main__': + s = Spotify() + s.getAlbum('0nW0w37lrQ87k7PLZvC4qJ') diff --git a/status_manager.py b/status_manager.py new file mode 100644 index 0000000..1276045 --- /dev/null +++ b/status_manager.py @@ -0,0 +1,27 @@ +import pickle + +class Manager(): + + @staticmethod + def setStatus(status=True): + + with open('.status', 'wb') as f: + + pickle.dump( + { + 'status':status + }, f + ) + @staticmethod + def getStatus(): + + with open('.status', 'rb') as f: + data = pickle.load(f) + + return data['status'] + +if __name__ == '__main__': + + Manager.setStatus(True) + + print(Manager.getStatus()) \ No newline at end of file diff --git a/telegram.py b/telegram.py old mode 100644 new mode 100755 index f315079..5cd89bb --- a/telegram.py +++ b/telegram.py @@ -1,9 +1,17 @@ +from celery import Celery + import requests -import datetime -import io, os +import datetime, time +import io, os, sys import re import main import apple +import random +import urllib.request +import pickle +import image +import status_manager +import heroku import logging @@ -12,13 +20,30 @@ console = logging.StreamHandler() console.setLevel(logging.INFO) +manager = Celery('telegram',broker='redis://smd:1mThquQxrJbyVYVlmLLAmwzLd2t5vDWVO@redis-12274.c52.us-east-1-4.ec2.cloud.redislabs.com:12274') +#manager = Celery('telegram',broker='redis://localhost:6379/0') class BotHandler(object): def __init__(self): - self.token = '752979930:AAFhdyGx0CSOJ-m17wLGN0NhrxvpwCqCPoQ' + self.__getData() self.api_url = "https://api.telegram.org/bot{}/".format(self.token) + def __getData(self): + try: + + with open('.telegram', 'rb') as f: + data = pickle.load(f) + + self.token = data['token'] + + except: + print(''' + A new version is available on GitHub.\n + Download: https://github.com/artyshko/smd + ''') + sys.exit() + def getUpdates(self, offset=None, timeout=30): method = 'getUpdates' @@ -43,6 +68,19 @@ def sendText(self, chat_id, text): return requests.post(self.api_url + method, params) + def sendAlert(self, chat_id, text): + + params = { + 'callback_query_id':1, + 'show_alert':True, + 'chat_id': chat_id, + 'text': text + } + + method = 'answerCallbackQuery' + + return requests.post(self.api_url + method, params) + def sendHTML(self, chat_id, text): params = { @@ -56,7 +94,38 @@ def sendHTML(self, chat_id, text): return requests.post(self.api_url + method, params) - def sendAudio(self, chat_id, name, artist, audio, thumb): + def sendAudioOld(self, chat_id, name, artist, audio, thumb, duration, number=None): + + method = 'sendAudio' + + files = { + 'audio': audio, + 'thumb':thumb + } + + if number: + part = f"{number}. " + + data = { + 'chat_id' : chat_id, + 'title': str(name), + 'performer':str(artist), + 'duration':int(duration * .001), + 'caption':f'{part if number else ""}{str(artist)} {str(name)}\n@SpotifyMusicDownloaderBot', + 'parse_mode':'HTML' + } + + response = requests.post( + self.api_url + method, + files=files, + data=data + ) + #logging + logging.info(f'SEND STATUS {response.status_code} {response.reason}') + + return response.status_code + + def sendAudio(self, chat_id, name, artist, audio, thumb, duration, number=None): method = 'sendAudio' @@ -65,11 +134,15 @@ def sendAudio(self, chat_id, name, artist, audio, thumb): 'thumb':thumb } + if number: + part = f"{number}. " + data = { 'chat_id' : chat_id, 'title': str(name), 'performer':str(artist), - 'caption':f'{str(artist)} {str(name)}', + 'duration':int(duration * .001), + 'caption':f'{part if number else ""}{str(artist)} {str(name)}', 'parse_mode':'HTML' } @@ -141,28 +214,78 @@ def checkLastUpdates(self): class Controller(object): - def __init__(self): + def __init__(self, YT_API_KEY_N): self.bot = BotHandler() self.offset = None - self.downloader = main.MusicDownloader() + + self.downloader = main.MusicDownloader(YT_API_KEY_N) self.apple = apple.AppleMusic() - def classify(self, message): - if str(message).find('open.spotify.com') > 0: - return 'link' + def __restart(self): - elif str(message).find(':track:') > 0: - return 'uri' + #logging + logging.warning(f'RESTARTING "youtube_dl" MODULE') - elif str(message) == '/start': - return 'start' + self.downloader = main.MusicDownloader(7) - else: - return 'text' + return True + + def __send(self, data, user, name, number=None): + + uri = data['uri'] + + os.rename( + f"Downloads/{uri}.mp3", + f"Downloads/{name}.mp3" + ) + + return self.bot.sendAudio( + chat_id=user, + audio=open(f"Downloads/{name}.mp3",'rb'), + thumb=open(f"Downloads/{uri}.png",'rb'), + name=f'{data["name"]}', + artist=f'{data["artist"][0]}', + duration=data['duration_ms'], + number=number + ) - def getTrackFromShazam(self, message): + def __remove(self, data, name): + + uri = data['uri'] + + #deleting song and cover + os.remove(f"Downloads/{name}.mp3") + #logging + logging.info(f"DELETED Downloads/{name}.mp3") + + os.remove(f"Downloads/{uri}.png") + #logging + logging.info(f'DELETED Downloads/{uri}.png') + + def __sendStatus(self, user): + + self.bot.sendText(user, text=f'200 {self.downloader.getYTS()}') + + return True + + def __sendStartMessage(self, user): + + self.bot.sendText( + user, + text='https://telegra.ph/How-to-Spotify-Music-Downloader-Bot-full-instruction-03-09' + ) + + #logging + logging.info('Hello message was sent') + + return True + + def __convertToURI(self, link): + return "spotify:track:" + str(str(link).split('/')[-1]).split('?')[0] + + def __getTrackFromShazam(self, message): slug = str(message).split('/')[-1].split('-') @@ -174,20 +297,22 @@ def getTrackFromShazam(self, message): message = str(message).split(' ') count = 0 + for word in message: - if str(word) == 'by': - count+=1 + count += 1 if str(word) == 'by' else 0 + if count != 1: - try: + try: for word in message: try: + if str(word).lower().find(slug[0]) > -1: title.append(word) slug.pop(0) - else: artist.append(word) + except: artist.append(word) @@ -197,21 +322,293 @@ def getTrackFromShazam(self, message): except:pass return str(song).replace('&','') + else: + new = [] [new.append(word if str(word) != 'by' else '-') for word in message] song = " ".join(new) song = str(song).replace('&','') + return str(song) - def convertToURI(self, link): - return "spotify:track:" + str(str(link).split('/')[-1]).split('?')[0] - def isIncorrect(self, text): - r = re.compile("[а-яА-Я]+") - text = str(text).split(' ') - return True if len([w for w in filter(r.match, text)]) else False + def DL_SPOTIFY_ALBUM(self, message, user): + + try: + link = str(message).split('?')[0] + uri = str(link).split('/')[-1] + data = self.downloader.getAlbum(uri) + path = f"Downloads/{uri}.png" + + + downloadAlbumImage(data['image'], path) + + try: image.Effects.createPoster(path, name=data["name"], artist=data["artist"], file=path) + except: pass + + logging.info(f'Downloaded {path}') + + self.bot.sendPhoto( + chat_id=user, + photo=open(path,'rb'), + text=f'' + ) + + except:pass + + logging.info(f'Sended {path}') + + album = data + count = len(album['tracks']) + + for data, i in zip(album['tracks'], range(count)): + #logging + logging.info(f'S-ALBUM {i+1}/{count} | {data["artist"][0]} - {data["name"]}') + + #fixed incorrect statistic + try: self.downloader.getData(data["uri"]) + except: pass + + if self.downloader.downloadBySpotifyUri(data['uri']): + + self.sendSong(data=data, user=user, number=i+1) + + os.remove(path) + #logging + logging.info(f'DELETED {path}') + + return True + + def DL_QUERY(self, message, user): + + state, data = self.downloader.downloadBySearchQuery(message) + + if not state: + + #in case of downloader didn't find a song + #restarting downloader + #and trying to get data + self.__restart() + state, data = self.downloader.downloadBySearchQuery(message) + + + if state: + + return self.sendSong(data=data, user=user) + + else: + + #logging + logging.error(f'SENDED "Couldn\'t find that" MESSAGE') + self.bot.sendSticker(user, sticker=open(f"Data/s3.webp",'rb'),) + self.bot.sendText(user, text='Couldn\'t find that:(') + + return False + + def DL_YOUTUBE_MUSIC(self, message, user): + + self.__restart() + + link = 'http' + str(message).split('http')[-1] + link = ''.join(str(link).split('music.')).split('&')[0] + + name = self.downloader.getYoutubeMusicInfo(link) + tags = self.downloader.getLastFMTags(name) + + #logging + logging.info(f"LINK {link}") + logging.info(f"NAME {name}") + + try: + + state, data = self.downloader.downloadFromYoutubeMusic(url=link, info=tags) + + if state: + + return self.sendSong(data=data, user=user) + + else: + + #logging + logging.warning(f"This video is unavailable.") + self.bot.sendSticker(user, sticker=open(f"Data/s2.webp",'rb')) + self.bot.sendText(user, text='This video is unavailable for me(') + + return False + + except: + + try: + + self.DL_QUERY(message=name, user=user) + + except: + + #logging + logging.warning(f"This video is unavailable.") + self.bot.sendSticker(user, sticker=open(f"Data/s2.webp",'rb'),) + self.bot.sendText(user, text='This video is unavailable for me(') + + return False + + return True + + def DL_DEEZER_ALBUM(self, message, user): + + uri = message + + data = self.downloader.getAlbumDeezer(uri) + path = f"Downloads/{uri}.png" + + + downloadAlbumImage(data['image'], path) + + try: image.Effects.createPoster(path, name=data["name"], artist=data["artist"], file=path) + except: pass + + logging.info(f'Downloaded {path}') + + self.bot.sendPhoto( + chat_id=user, + photo=open(path,'rb'), + text=f'' + ) + + logging.info(f'Sended {path}') + album = data + count = len(album['tracks']) + + for data, i in zip(album['tracks'], range(count)): + #logging + logging.info(f'D-ALBUM {i+1}/{count} | {data["artist"][0]} - {data["name"]}') + + if self.downloader.downloadByDeezerID(str(data['uri'][1:-1])): + + self.sendSong(data=data, user=user, number=i+1) + + os.remove(path) + #logging + logging.info(f'DELETED {path}') + + return True + + + def sendSong(self, data, user, number=None): + + try: + + name = getCorrect(f'{data["artist"][0]} - {data["name"]}') + code = self.__send(data, name=name, user=user, number=number) + + if int(code) != 200: + + #trying to fix incorrect name + os.rename( + f"Downloads/{name}.mp3", + f"Downloads/{data['uri']}.mp3" + ) + new_name = data['uri'] + code = self.__send(data, user=user, name=new_name, number=number) + + if int(code) != 200: + + #sending sad message + self.bot.sendSticker(user, sticker=open(f"Data/s3.webp",'rb'),) + self.bot.sendText(user, text='Couldn\'t find that:(') + self.__remove(data, name=new_name) + + return False + + self.__remove(data, name=new_name) + + return True + + else: + + self.__remove(data, name) + return True + + except: + + logging.error(f'ERROR IN controller.sendSong()') + self.bot.sendSticker(user, sticker=open(f"Data/s1.webp",'rb'),) + self.bot.sendText(user, text='Something went wrong:(') + + return False + + def worker(self,update): + + if 'message' in list(update.keys()): + #in case of new message + + #get message data + chat_id = update['message']['chat']['id'] + + try: + username = update['message']['chat']['username'] + except: + username = 'unknown' + + + if 'text' in list(update['message'].keys()): + #skipping unsupported messages + #get message + message = update['message']['text'] + + #logging + logging.info(f'USER [{username}]') + logging.info(f'MESSAGE {message}') + + try: + #start controller + self.controller(message, chat_id) + + except: + #logging + logging.error('ERROR IN CONTROLLER') + + try: + + self.downloader = main.MusicDownloader(0) + #restart controller + self.controller(message, chat_id) + + except: + + self.bot.sendSticker(chat_id, sticker=open(f"Data/s1.webp",'rb')) + self.bot.sendText(chat_id, 'Couldn\'t find that :(') + + else: + #logging + logging.warning('UNSUPPORTED MESSAGE') + + self.bot.sendSticker(chat_id, sticker=open(f"Data/s4.webp",'rb')) + self.bot.sendText(chat_id, 'Wooops! Something went wrong.\nERROR CODE 42 - You are so funny!') + + def classify(self, message): + + if str(message).find('open.spotify.com') > 0: + return 'link' + + elif str(message).find(':track:') > 0: + return 'uri' + + elif str(message) == '/start' or str(message) == '/help': + return 'start' + + elif str(message) == '/status': + return 'status' + + elif str(message).find('deezer.com/track/') > 0: + return 'dtrack' + + elif str(message).find('deezer.com/album/') > 0: + return 'dalbum' + + else: + return 'text' def controller(self, message, id): @@ -223,39 +620,29 @@ def controller(self, message, id): #start message if type == 'start': - #logging - logging.info('Sended hello message') - - self.bot.sendPhoto( - chat_id=id, - photo=open(f"Data/header1.png",'rb'), - text='' - ) - self.bot.sendPhoto( - chat_id=id, - photo=open(f"Data/header3.png",'rb'), - text='' - ) - self.bot.sendPhoto( - chat_id=id, - photo=open(f"Data/header5.png",'rb'), - text='' - ) + self.__sendStartMessage(user=id) return True + elif type == 'status': + + #self.bot.sendText(id, text='200 ALIVE') + self.bot.sendText(id, text=f'200 {self.downloader.getYTS()}') + return True elif type == 'text': if str(message).find('I used Shazam to discover') > -1: + #logging logging.info(f"SHAZAM SONG DETECTED") - message = self.getTrackFromShazam(message) + message = self.__getTrackFromShazam(message) logging.info(f"NAME {message}") elif str(message).find('Мое открытие на Shazam:') > -1: + #fix for russian lang new = str(message).split('Мое открытие на Shazam: ')[1] new = str(new).split('. https')[0] @@ -264,6 +651,7 @@ def controller(self, message, id): logging.info(f"NAME {message}") elif str(message).find('youtube.com/watch') > -1: + #logging logging.info(f"YOUTUBE MUSIC DETECTED") @@ -271,158 +659,123 @@ def controller(self, message, id): if str(message).find('music.') > -1: - self.downloader = main.MusicDownloader() - - link = ''.join(str(link).split('music.')).split('&')[0] - - #logging - logging.info(f"LINK {link}") + self.DL_YOUTUBE_MUSIC(message, id) - name = self.downloader.getYoutubeMusicInfo(link) - tags = self.downloader.getLastFMTags(name) + else: #logging - logging.info(f"NAME {name}") - - try: - state, data = self.downloader.downloadFromYoutubeMusic(url=link, info=tags) - - if state: - - code = self.bot.sendAudio( - chat_id=id, - audio=open(f"Downloads/{data['uri']}.mp3",'rb'), - thumb=open(f"Downloads/{data['uri']}.png",'rb'), - name=f'{data["name"]}', - artist=f'{data["artist"][0]}' - ) - - if int(code) != 200: - #logging - logging.warning(f'CODE {code}') - self.bot.sendText(id,text='Something went wrong:(') - - - os.remove(f"Downloads/{data['uri']}.mp3") - #logging - logging.info(f"DELETED Downloads/{data['uri']}.mp3") - - os.remove(f"Downloads/{data['uri']}.png") - #logging - logging.info(f"DELETED Downloads/{data['uri']}.png") - - - else: - #logging - logging.warning(f"This video is unavailable.") - self.bot.sendSticker(id,sticker=open(f"Data/s2.webp",'rb'),) - self.bot.sendText(id,text='This video is unavailable for me(') - - return False - - except: - try: - self.controller(name, id) - except: - #logging - logging.warning(f"This video is unavailable.") - self.bot.sendSticker(id,sticker=open(f"Data/s2.webp",'rb'),) - self.bot.sendText(id,text='This video is unavailable for me(') - - return False - - return True - - else: logging.warning(f"YOUTUBE SONG DETECTED") self.bot.sendSticker(id,sticker=open(f"Data/s5.webp",'rb'),) self.bot.sendText(id,text='You need to use YouTube Music instead of YouTube.') + return False + return True elif str(message).find('youtu.be') > -1: + logging.warning(f"YOUTUBE SONG DETECTED") self.bot.sendSticker(id,sticker=open(f"Data/s5.webp",'rb'),) self.bot.sendText(id,text='You need to use YouTube Music instead of YouTube.') + return True elif str(message).find('itunes.apple.com') > 1: name = self.apple.getName(message) message = name + if not name: + #logging logging.error(f'SENDED "Couldn\'t find that" MESSAGE') self.bot.sendSticker(id,sticker=open(f"Data/s3.webp",'rb'),) self.bot.sendText(id,text='Couldn\'t find that:(') - return False - - state, data = self.downloader.downloadBySearchQuery(message) - - #FIX - if not state: - #logging - self.downloader = main.MusicDownloader() - logging.warning(f'Restarting downloader') - logging.warning(f'Trying do the same') - - state, data = self.downloader.downloadBySearchQuery(message) + return False - if state: + return self.DL_QUERY(message, user=id) - fixed_name = f'{data["artist"][0]} - {data["name"]}' - fixed_name = data['uri'] + elif type == 'dtrack': - code = self.bot.sendAudio( - chat_id=id, - audio=open(f"Downloads/{data['uri']}.mp3",'rb'), - thumb=open(f"Downloads/{data['uri']}.png",'rb'), - name=f'{data["name"]}', - artist=f'{data["artist"][0]}' - ) + #logging + logging.info(f'DEEZER TRACK DETECTED') - if int(code) != 200: - #logging - logging.warning(f'CODE {code}') - self.bot.sendText(id,text='Something went wrong:(') + track = str(str(message).split('/track/')[1]).split('?')[0] + data = self.downloader.getDeezerTags(track) + if data: - os.remove(f"Downloads/{data['uri']}.mp3") #logging - logging.info(f"DELETED Downloads/{data['uri']}.mp3") + logging.info(f'SONG {data["artist"][0]} - {data["name"]}') - os.remove(f"Downloads/{data['uri']}.png") - #logging - logging.info(f"DELETED Downloads/{data['uri']}.png") + if self.downloader.downloadByDeezerID(track): + return self.sendSong(data=data, user=id) else: + #logging - logging.error(f'SENDED "Couldn\'t find that" MESSAGE') + logging.error(f'SENDED "Something went wrong" MESSAGE') self.bot.sendSticker(id,sticker=open(f"Data/s3.webp",'rb'),) self.bot.sendText(id,text='Couldn\'t find that:(') + return False - return True + elif type == 'dalbum': + + #logging + logging.info(f'DEEZER ALBUM DETECTED') + + self.bot.sendSticker( + id, + sticker=open(f"Data/s6.webp",'rb') + ) + + self.bot.sendHTML( + id, + 'Due to a huge load and often fall down, the bot temporary won\'t support albums downloading.\n\nWe Apologize for the Temporary Inconvenience!' + ) + + + #album = str(str(message).split('album/')[1]).split('?')[0] + + #return self.DL_DEEZER_ALBUM(album, id) + return None elif type == 'link': #logging - logging.info(f'Converted open.spotify.com link to spotify URI') + logging.info(f'Converting open.spotify.com link to spotify URI') + + if str(message).find('/album/') > -1: + + logging.info('ALBUM MODE') + + self.bot.sendSticker( + id, + sticker=open(f"Data/s6.webp",'rb') + ) + + self.bot.sendHTML( + id, + 'Due to a huge load and often fall down, the bot temporary won\'t support albums downloading.\n\nWe Apologize for the Temporary Inconvenience!' + ) - message = self.convertToURI(message) + #return self.DL_SPOTIFY_ALBUM(message, user=id) + return None + message = self.__convertToURI(message) - #get data - uri = str(message).split(':')[-1] + #getting data data = self.downloader.getData(message) if not data: - #logging - logging.warning(f'Restarting downloader') - logging.warning(f'Trying do the same') - self.downloader = main.MusicDownloader() + + #in case of downloader didn't find a song + #restarting downloader + #and trying to get data + self.__restart() data = self.downloader.getData(message) if data: @@ -432,106 +785,119 @@ def controller(self, message, id): if self.downloader.downloadBySpotifyUri(message): + return self.sendSong(data=data, user=id) - code = self.bot.sendAudio( - chat_id=id, - audio=open(f"Downloads/{uri}.mp3",'rb'), - thumb=open(f"Downloads/{uri}.png",'rb'), - name=f'{data["name"]}', - artist=f'{data["artist"][0]}' - ) + else: + #logging + logging.error(f'SENDED "Something went wrong" MESSAGE') + self.bot.sendSticker(id,sticker=open(f"Data/s3.webp",'rb'),) + self.bot.sendText(id,text='Couldn\'t find that:(') - if int(code) != 200: - #logging - logging.warning(f'CODE {code}') - self.bot.sendSticker(id,sticker=open(f"Data/s3.webp",'rb'),) - self.bot.sendText(id,text='Something went wrong:(') + return False - os.remove(f"Downloads/{uri}.mp3") - #logging - logging.info(f'DELETED Downloads/{uri}.mp3') +def getCorrect(name): + try: - os.remove(f"Downloads/{uri}.png") - #logging - logging.info(f'DELETED Downloads/{uri}.png') + #checking out incorrect words + r, text = re.compile("[а-яА-Я]+"), str(name).split(' ') + status = True if len([w for w in filter(r.match, name)]) else False - return True + if status: + return f'S{str(random.randint(1000000000,100000000000))}D' - else: + return re.sub(r"[\"#/@;:<>{}`+=~|.!?$%^&*№&]", string=name, repl='') - #logging - logging.error(f'SENDED "Something went wrong" MESSAGE') - self.bot.sendSticker(id,sticker=open(f"Data/s3.webp",'rb'),) - self.bot.sendText(id,text='Couldn\'t find that:(') - return False + except: + return 'music' +def downloadAlbumImage(url, name): + urllib.request.urlretrieve(url, name) - def mainloop(self): - while True: - self.bot.getUpdates(self.offset) - update = self.bot.checkLastUpdates() +@manager.task +def do(update, YT_API_KEY_N=0): - if update: + controller = Controller(YT_API_KEY_N) + controller.worker(update) - update_id = update['update_id'] + if not status_manager.Manager.getStatus(): + bot = BotHandler() - if 'message' in list(update.keys()): - #in case of new message + bot.sendSticker(id,sticker=open(f"Data/s1.webp",'rb'),) + bot.sendText( + chat_id=232027721, + text='SERVER IS DOWN!\nRESTARTING!' + ) + print('HEROKU:RESTART') + heroku.restart() + + else: - #get message data - chat_id = update['message']['chat']['id'] + downloader = main.MusicDownloader(YT_API_KEY_N) + downloader.FUCK_GOOGLE() - try: - username = update['message']['chat']['username'] - except: - username = 'unknown' + del controller + + +def mainloop(): - if 'text' in list(update['message'].keys()): - #skipping unsupported messages - #get message - message = update['message']['text'] + offset = None + wait = 0 + YT_API_KEY_N = 0 - #logging - logging.info(f'USER [{username}]') - logging.info(f'MESSAGE {message}') + bot = BotHandler() + downloader = main.MusicDownloader(YT_API_KEY_N) + downloader.FUCK_GOOGLE() + bot.sendText( + chat_id=232027721, + text='SERVER IS UP' + ) - try: + while True: - #start controller - self.controller(message, chat_id) + print('BLOCKED_STATUS:', not status_manager.Manager.getStatus()) - except: - #logging - logging.error('ERROR IN CONTROLLER') + if status_manager.Manager.getStatus(): - try: + bot.getUpdates(offset) + update = bot.checkLastUpdates() - self.downloader = main.MusicDownloader() - #restart controller - self.controller(message, chat_id) + + try: + + if update: - except: + update_id = update['update_id'] + offset = update_id + 1 - self.bot.sendSticker(chat_id, sticker=open(f"Data/s1.webp",'rb')) - self.bot.sendText(chat_id, 'Couldn\'t find that :(') + #celery task + do.delay(update, YT_API_KEY_N) - else: - #logging - logging.warning('UNSUPPORTED MESSAGE') + if YT_API_KEY_N < 11: + YT_API_KEY_N += 1 + + else: + YT_API_KEY_N = 0 + + except: + offset = None + bot = BotHandler() + else: - self.bot.sendSticker(chat_id, sticker=open(f"Data/s4.webp",'rb')) - self.bot.sendText(chat_id, 'Wooops! Something went wrong.\nERROR CODE 42 - You are so funny!') + time.sleep(10) + wait += 10 - self.offset = update_id + 1 + print(f'BLOCKED:{wait}') + if wait >= 120: + print('HEROKU:RESTART') + heroku.restart() if __name__ == '__main__': logging.info('Starting app') - controller = Controller() - controller.mainloop() + mainloop() diff --git a/token_creator.py b/token_creator.py new file mode 100755 index 0000000..684feb1 --- /dev/null +++ b/token_creator.py @@ -0,0 +1,18 @@ +import pickle + +def create_new_file(file, token): + + with open(file, 'wb') as f: + + pickle.dump( + { + 'token':str(token) + }, f + ) +def create_new_file_from_dict(file, dict): + + with open(file, 'wb') as f: + + pickle.dump( + dict, f + ) diff --git a/worker.sh b/worker.sh new file mode 100755 index 0000000..1c50304 --- /dev/null +++ b/worker.sh @@ -0,0 +1 @@ +python3 telegram.py & celery worker -A telegram --loglevel=info --autoscale=3,2 diff --git a/youtube.py b/youtube.py old mode 100755 new mode 100644 index b036156..8323ffc --- a/youtube.py +++ b/youtube.py @@ -1,33 +1,44 @@ #!/usr/bin/python3 +from __future__ import unicode_literals +import youtube_dl from pytube import YouTube from bs4 import BeautifulSoup import requests import lxml import os - -#IMPORT WITH STDOUT REDIRECTION -#FIX STARTUP PYGAME HELLO MESSAGE -#THANKS @Mad Physicist FROM STACK OVERFLOW +import socket +import proxy +import json +import pickle import contextlib -# with contextlib.redirect_stdout(None): -# from moviepy.editor import * -# import moviepy.editor as mp - import imageio +import shutil +import time +import logging +import proxy +import status_manager +import heroku + + +#fix imageio.plugins.ffmpeg.download() from moviepy.editor import * import moviepy.editor as mp -import logging -logging.basicConfig(level=logging.INFO, - format='%(asctime)s - %(levelname)-2s - %(message)s') +#include loagging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)-2s - %(message)s' +) console = logging.StreamHandler() console.setLevel(logging.INFO) + class Youtube(object): - def __init__(self): + def __init__(self, YT_API_KEY_N): + self.__query = '' self.__host = 'https://www.youtube.com/' self.__url = self.__host + 'results?search_query=' @@ -36,12 +47,16 @@ def __init__(self): 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' } self.__result = [] + self.YT_API_KEY_N = YT_API_KEY_N + + GoogleAPI.loadData() + logging.error(f"YT_API_KEY_{self.YT_API_KEY_N}") + GoogleAPI.setKey(self.YT_API_KEY_N) def getResult(self,i=0): return self.__result[i] - def getFullResult(self): return self.__result @@ -54,8 +69,18 @@ def removeInvallidLinks(self): def get(self, text, dur): - data1 = self.getVideoFromYoutube(text) - data2 = self.getVideoFromYoutube(text + ' Audio') + text = str(text).replace('&','') + + # data1 = self.getVideoFromYoutube(text) + # data2 = self.getVideoFromYoutube(text + ' Audio') + try: + data1 = self.getVideoFromYoutube(text) + data2 = self.getVideoFromYoutube(text + ' Audio') + except: + pass + #data1 = GoogleAPI.search(text) + #data2 = GoogleAPI.search(text + ' Audio') + self.__result = self.classify(data1, data2, dur) @@ -84,6 +109,7 @@ def getVideoFromYoutube(self,text): def download(self, url, path='', filename='video'): + ''' Downloading song from YouTube :param url: video url on YouTube @@ -94,17 +120,14 @@ def download(self, url, path='', filename='video'): #logging logging.info(f"Start downloading") try: + + try:url = str(url).replace('com//watch','com/watch') + except:pass + #logging logging.info(f"Init YouTube") - logging.info(f"URL {url}") - yt = YouTube(url) - #logging - logging.info(f"Get Data") - #downloading - yt = yt.streams.filter( - progressive=True, - file_extension='mp4' - ).order_by('resolution').desc().first() + logging.warning(f"URL {url}") + #logging logging.info(f"Create Directory") @@ -113,9 +136,9 @@ def download(self, url, path='', filename='video'): fullpath = os.getcwd() + '/cache' try: - # if not os.path.exists(fullpath): - # os.makedirs(fullpath) + os.makedirs('cache/'+path) + #logging logging.info(f"Created") except: @@ -125,8 +148,49 @@ def download(self, url, path='', filename='video'): #logging logging.info(f"Start downloading") + if not status_manager.Manager.getStatus(): + + self.__proxy = proxy.getProxy() + + ydl_opts = { + 'outtmpl': f'{fullpath}/{filename}/{filename}', + 'format':'best', + 'proxy':self.__proxy['proxy'] + } + logging.error(f"DOWNLOADING:USING_PROXY") + + else: + ydl_opts = { + 'outtmpl': f'{fullpath}/{filename}/{filename}', + 'format':'best' + } + logging.error(f"DOWNLOADING:WITHOUT_PROXY") - yt.download('cache/'+ path, filename=path) + # #'source_address': f'{socket.gethostbyname(socket.getfqdn())}' + # try:print(f'SERVER_IPv4:{socket.gethostbyname(socket.getfqdn())}') + # except:pass 'proxy':self.__proxy['proxy'] + + try: + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download([url]) + except: + logging.error(f"DOWNLOADING:USING_PROXY[1]") + #trying another one + self.__proxy = proxy.getProxy() + status_manager.Manager.setStatus(False) + + ydl_opts = { + 'outtmpl': f'{fullpath}/{filename}/{filename}', + 'format':'best', + 'proxy':self.__proxy['proxy'] + } + + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download([url]) + + os.system(f'cp {fullpath}/{filename}/{filename} {fullpath}/{filename}/{filename}.mp4') + + #yt.download('cache/'+ path, filename=path) #logging logging.info(f"Downloading successful") @@ -150,7 +214,7 @@ def convertVideoToMusic(self, uri): try: clip = mp.VideoFileClip(f'cache/{uri}/{uri}.mp4').subclip() - clip.audio.write_audiofile(f'cache/{uri}/{uri}.mp3', bitrate='3000k') + clip.audio.write_audiofile(f'cache/{uri}/{uri}.mp3', bitrate='3000k', progress_bar=False) logging.info(f"Converting successful") @@ -169,8 +233,9 @@ def getTrack(self,name): #self.convertVideoToMusic(self.download(self.get(name)[0],filename=name)) return None - def classify(self, data1, data2, duration=229486): + def classify(self, data1, data2, duration=229486): + data1 = data1[:2] if len(data1) >= 2 else data1 data2 = data2[:2] if len(data2) >= 2 else data2 @@ -182,25 +247,55 @@ def classify(self, data1, data2, duration=229486): result = -1 link = None - #logging - logging.info(f"{len(research)} research objects") for item in research: - try: - y = YouTube(item) + try: + try:item = str(item).replace('com//watch','com/watch') + except:pass - item_duration = int(y.length)*1000 + item_duration = GoogleAPI.duration(item) diff = duration - item_duration diff = diff * -1 if diff < 0 else diff - if (result == -1 or diff < result) and not str(y.title).find('8D') > -1: - result, link = diff, item + logging.warning(f'{item} {item_duration}') + except: #logging logging.error(f"Some problems on classify loop") + try: + + try:item = str(item).replace('com//watch','com/watch') + except:pass + + ydl_opts = { + 'outtmpl': f'1', + 'format':'best' + } + + #'source_address': f'{socket.gethostbyname(socket.getfqdn())}' + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + dictMeta = ydl.extract_info(item, download=False) + + item_duration = int(dictMeta['duration'])*1000 + diff = duration - item_duration + diff = diff * -1 if diff < 0 else diff + + logging.warning(f'{item} {item_duration}') + + if (result == -1 or diff < result) and not str(dictMeta['title']).find('8D') > -1: + result, link = diff, item + + except: + + logging.error(f"[1] Some problems on classify loop") + + status_manager.Manager.setStatus(False) + + logging.error(status_manager.Manager.getStatus()) + if link: _result = [link] + data1 + data2 else: @@ -233,8 +328,152 @@ def getNameFromYoutube(self, url): return name + def getGoogleAPIStatus(self): + try: + rep1, rep2 = '', '' + + try: + #GoogleAPI.search('Google sucks') + rep1 = 0 + except:rep1 = 0 + + try: + GoogleAPI.duration(GoogleAPI.YT_V_DEFAULT_URL+'lFGnsdV-sR4') + rep2 = 1 + except:rep2 = 0 + + return f'SERVER IS UP\nDEV[DL:TRUE-W:{self.YT_API_KEY_N}-API:{rep1}{rep2}]' + except: + return 'ALIVE' + + + def testYT_D(self): + #https://www.youtube.com/watch?v=Wch3gJG2GJ4 + res = self.download( + url = 'https://www.youtube.com/watch?v=jhFDyDgMVUI', + path='test', + filename='test' + ) + + fullpath = os.getcwd() + '/cache' + status = os.path.exists(f'{fullpath}/{res}/{res}.mp4') + + try: + shutil.rmtree(f'{fullpath}/{res}') + except: + print('Error while deleting directory') + + if not status or not status_manager.Manager.getStatus(): + + try: + + from telegram import BotHandler + + bot = BotHandler() + bot.sendText( + chat_id=232027721, + text='SERVER IS DOWN\nRESTARTING!' + ) + heroku.restart() + + except:print('NOOOO') + + print('HEROKU:RESTART') + + + # 10 SEC SLEEP + # + # It has to make a pause and won't let to get the last message, + # so it is probably gonna fix "the last message issue" + + time.sleep(10) + + return status + + + + +class GoogleAPI(): + + + YT_API_KEY = None + YT_API_KEY_DATA = {} + + + YT_API_V3_SEARCH = f'https://www.googleapis.com/youtube/v3/search?part=snippet&type=video&q=' + YT_API_V3_VIDEOS = f'https://www.googleapis.com/youtube/v3/videos?&part=contentDetails&id=' + YT_V_DEFAULT_URL = 'https://www.youtube.com/watch?v=' + + + @staticmethod + def loadData(): + + try: + + with open('.youtube', 'rb') as f: + data = pickle.load(f) + + GoogleAPI.YT_API_KEY_DATA = data + + except: + pass + + + @staticmethod + def setKey(key): + + GoogleAPI.YT_API_KEY = GoogleAPI.YT_API_KEY_DATA[f'YT_API_KEY_{key}'] + logging.info(f"LOADED NEW KEY [...{GoogleAPI.YT_API_KEY[:20:]}...]") + + + @staticmethod + def search(query): + + #logging + logging.info("YouTube APIv3 SEARCH") + query = str(query).replace(' ','+') + + + data = json.loads( + requests.get( + f'{GoogleAPI.YT_API_V3_SEARCH}{query}&key={GoogleAPI.YT_API_KEY}' + ).text + )['items'] + + return [GoogleAPI.YT_V_DEFAULT_URL + str(video['id']['videoId']) for video in data] + + + + @staticmethod + def duration(video): + #logging + logging.info("YouTube APIv3 VIDEO_INFO") + + video = str(video).split('watch?v=')[1] + video = str(video).split('&')[0] + + + data = json.loads( + requests.get( + f'{GoogleAPI.YT_API_V3_VIDEOS}{video}&key={GoogleAPI.YT_API_KEY}' + ).text + )['items'][0]['contentDetails']['duration'] + + #google sucks + _google_time_format = data[2:-1] + + try: + + min, sec = str(_google_time_format).split('M') + msec = (int(sec) + (int(min) * 60)) * 1000 + + return msec + + except: return 0 + + if __name__ == "__main__": - y = Youtube() - name = y.getNameFromYoutube('https://youtube.com/watch?v=H4buRu9-Wb4') - print(name) + print(GoogleAPI.YT_API_KEY_DATA) + + [print(i) for i in range(0,12)] diff --git a/youtube.py.bac b/youtube.py.bac new file mode 100755 index 0000000..a126796 --- /dev/null +++ b/youtube.py.bac @@ -0,0 +1,287 @@ +#!/usr/bin/python3 +from __future__ import unicode_literals +import youtube_dl + +from pytube import YouTube +from bs4 import BeautifulSoup +import requests +import lxml +import os +import socket +import proxy + +#IMPORT WITH STDOUT REDIRECTION +#FIX STARTUP PYGAME HELLO MESSAGE +#THANKS @Mad Physicist FROM STACK OVERFLOW +import contextlib +# with contextlib.redirect_stdout(None): +# from moviepy.editor import * +# import moviepy.editor as mp + +import imageio +imageio.plugins.ffmpeg.download() +from moviepy.editor import * +import moviepy.editor as mp + +import logging + +logging.basicConfig(level=logging.INFO, + format='%(asctime)s - %(levelname)-2s - %(message)s') +console = logging.StreamHandler() +console.setLevel(logging.INFO) + +class Youtube(object): + + def __init__(self): + self.__query = '' + self.__host = 'https://www.youtube.com/' + self.__url = self.__host + 'results?search_query=' + self.headers = { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' + } + self.__result = [] + + + + def getResult(self,i=0): + return self.__result[i] + + + def getFullResult(self): + return self.__result + + def removeInvallidLinks(self): + temp = [] + for item in self.getFullResult(): + if 40 < len(item) < 50: + temp.append(item) + self.__result = temp + + def get(self, text, dur): + + text = str(text).replace('&','') + + data1 = self.getVideoFromYoutube(text) + data2 = self.getVideoFromYoutube(text + ' Audio') + + self.__result = self.classify(data1, data2, dur) + + return self.__result + + + def getVideoFromYoutube(self,text): + ''' + Getting song url from YouTube + :param text: name of song + :return: list of results + ''' + + logging.info(f"Finding") + + request = self.__url + str(text).replace(' ','+') + response = requests.get(request, headers=self.headers) + soup = BeautifulSoup(response.text,'lxml') + self.__result = [] + + for link in soup.findAll(attrs={'class': 'yt-uix-tile-link'}): + self.__result.append(self.__host + link['href']) + + self.removeInvallidLinks() + return self.__result + + + def download(self, url, path='', filename='video'): + ''' + Downloading song from YouTube + :param url: video url on YouTube + :param path: local directory + :param filename: name of file + :return: str, filename + ''' + #logging + logging.info(f"Start downloading") + try: + + try:url = str(url).replace('com//watch','com/watch') + except:pass + + #logging + logging.info(f"Init YouTube") + logging.warning(f"URL {url}") + + + #logging + logging.info(f"Create Directory") + + + fullpath = os.getcwd() + '/cache' + + try: + # if not os.path.exists(fullpath): + # os.makedirs(fullpath) + os.makedirs('cache/'+path) + #logging + logging.info(f"Created") + except: + #logging + logging.error(f"Youtube:os.makedirs('cache/'+path)") + + #logging + logging.info(f"Start downloading") + + + print(filename) + ydl_opts = { + 'outtmpl': f'{fullpath}/{filename}/{filename}', + 'format':'best', + 'source_address': f'{socket.gethostbyname(socket.getfqdn())}' + } + + # #'source_address': f'{socket.gethostbyname(socket.getfqdn())}' + # try:print(f'SERVER_IPv4:{socket.gethostbyname(socket.getfqdn())}') + # except:pass 'proxy':self.__proxy['proxy'] + + + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download([url]) + + os.system(f'cp {fullpath}/{filename}/{filename} {fullpath}/{filename}/{filename}.mp4') + + #yt.download('cache/'+ path, filename=path) + + #logging + logging.info(f"Downloading successful") + + return filename + except: return None + + + def convertVideoToMusic(self, uri): + #logging + logging.info(f"Start converting") + + try: + fullpath = os.getcwd() + f'/cache/{uri}/' + if not os.path.exists(fullpath): + os.makedirs(fullpath) + except: + #logging + logging.error(f"Youtube:os.makedirs(fullpath)") + + clip = mp.VideoFileClip(f'cache/{uri}/{uri}.mp4').subclip() + clip.audio.write_audiofile(f'cache/{uri}/{uri}.mp3', bitrate='3000k', progress_bar=False) + + logging.info(f"Converting successful") + + try: + + pass + + except Exception as e: + logging.error(f"Youtube.convertVideoToMusic") + return -1 + + finally: + return 0 + + + def getTrack(self,name): + ''' + quick download and convert to mp3 + ''' + #self.convertVideoToMusic(self.download(self.get(name)[0],filename=name)) + return None + + def classify(self, data1, data2, duration=229486): + + data1 = data1[:2] if len(data1) >= 2 else data1 + data2 = data2[:2] if len(data2) >= 2 else data2 + + research = data2 + data1 + + if duration == 0: + return research + + result = -1 + link = None + + for item in research: + + + try: + + try:item = str(item).replace('com//watch','com/watch') + except:pass + + ydl_opts = { + 'outtmpl': f'1', + 'format':'best', + 'source_address': f'{socket.gethostbyname(socket.getfqdn())}' + } + + #'source_address': f'{socket.gethostbyname(socket.getfqdn())}' + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + dictMeta = ydl.extract_info(item, download=False) + + item_duration = int(dictMeta['duration'])*1000 + diff = duration - item_duration + diff = diff * -1 if diff < 0 else diff + + logging.warning(f'{item} {item_duration}') + + if (result == -1 or diff < result) and not str(dictMeta['title']).find('8D') > -1: + result, link = diff, item + + except: + #logging + logging.error(f"Some problems on classify loop") + + if link: + _result = [link] + data1 + data2 + else: + _result = data1 + data2 + + return _result + + def getNameFromYoutube(self, url): + + response = requests.get(url, headers=self.headers) + soup = BeautifulSoup(response.text,'lxml') + + _title = soup.find('title').text + _title = str(_title).replace(' - YouTube', '') + + _result = [] + __name = None + + if not str(_title).find('-') > -1: + + for link in soup.findAll('meta', attrs={'property': 'og:video:tag'}): + _result.append(link.get('content')) + + if len(_result) > 1: + name = f"{_result[0]} - {_title}" + else: + name = _title + else: + name = _title + + return name + + + +if __name__ == "__main__": + + y = Youtube() + #name = y.get(text="Sean Paul & J Balvin - ‎Contra La Pared", dur=256271) + y.download(url='https://www.youtube.com//watch?v=l91u752OCPo', path='boom',filename='file') + + + + # ydl_opts = { + # 'outtmpl': 'videoo.%(ext)s', + # 'format':'137' + # } + # with youtube_dl.YoutubeDL(ydl_opts) as ydl: + # ydl.download(['https://www.youtube.com/watch?v=dP15zlyra3c'])