diff --git a/dasdf b/dasdf new file mode 100644 index 0000000..bfcb33d --- /dev/null +++ b/dasdf @@ -0,0 +1,275 @@ +commit 5ee1c988b6c77dde854ed7c210326a3c9bb132ad +Author: LittlePeng +AuthorDate: Sat May 11 23:18:14 2013 +0800 +Commit: LittlePeng +CommitDate: Sat May 11 23:18:14 2013 +0800 + + delete` + +commit 47b6c5b717f04110dac5f66f675e3bd65391197a +Author: LittlePeng +AuthorDate: Sat May 11 22:26:14 2013 +0800 +Commit: LittlePeng +CommitDate: Sat May 11 22:26:14 2013 +0800 + + Update README.md + +commit e95c8a34c1b8b16badaa09edb8775872cbae6564 +Author: pengjilu +AuthorDate: Sat May 11 22:16:44 2013 +0800 +Commit: pengjilu +CommitDate: Sat May 11 22:16:44 2013 +0800 + + iasdf + +commit f5a53fb7d42796fda81e87092fe04bec6991f195 +Author: pengjilu +AuthorDate: Sat May 11 21:46:55 2013 +0800 +Commit: pengjilu +CommitDate: Sat May 11 21:46:55 2013 +0800 + + aa + +commit 88b16161c429c0ef87b98f918e0a6f7191ced503 +Author: pengjilu +AuthorDate: Sat May 11 21:45:37 2013 +0800 +Commit: pengjilu +CommitDate: Sat May 11 21:45:37 2013 +0800 + + readme + +commit 4f8a708f775a1fdfd70a408323addcff50c1c4b7 +Author: pengjilu +AuthorDate: Sat May 11 21:34:55 2013 +0800 +Commit: pengjilu +CommitDate: Sat May 11 21:34:55 2013 +0800 + + pic + +commit 3fd9ea1c4241ed1496f965ee9c5b6a69f1177252 +Author: pengjilu +AuthorDate: Sat May 11 21:30:36 2013 +0800 +Commit: pengjilu +CommitDate: Sat May 11 21:30:36 2013 +0800 + + two + +commit 33f93016da193f9f53cbc68f83a14cc8f49970d1 +Author: pengjilu +AuthorDate: Sat May 11 21:29:14 2013 +0800 +Commit: pengjilu +CommitDate: Sat May 11 21:29:14 2013 +0800 + + first + +commit 285fc4b1c8e3a3438e7a9cdbd13a2f826baa8032 +Merge: f328e53 f8a31a3 +Author: Nitin Kumar +AuthorDate: Mon Oct 29 11:34:10 2012 -0700 +Commit: Nitin Kumar +CommitDate: Mon Oct 29 11:34:10 2012 -0700 + + Merge pull request #38 from josegonzalez/requirements + + Added a requirements.txt + +commit f328e539c8345a2972a1921d7ee7751560914743 +Author: Nitin Kumar +AuthorDate: Mon Oct 29 11:08:34 2012 -0700 +Commit: Nitin Kumar +CommitDate: Mon Oct 29 11:08:34 2012 -0700 + + added support for tornado's command line options + +commit f8a31a3c0a5b9be5989638d155523490c44bf88d +Author: Jose Diaz-Gonzalez +AuthorDate: Wed Aug 8 12:02:21 2012 -0400 +Commit: Jose Diaz-Gonzalez +CommitDate: Wed Aug 8 12:02:21 2012 -0400 + + Added a requirements.txt + + Note that the redis version is not specified, so it is possible to + setup a virtualenv with whatever redis version you see fit. + +commit 3f1160f6b517d3606c13554cdb44346a1a3b62fe +Author: kumarnitin +AuthorDate: Sat Aug 4 04:41:08 2012 -0700 +Commit: kumarnitin +CommitDate: Sat Aug 4 04:41:08 2012 -0700 + + Update README.md + +commit 46aa122e5d5527f49a08189ed2150bb31791ca9e +Author: kumarnitin +AuthorDate: Mon Jul 16 19:22:11 2012 -0700 +Commit: kumarnitin +CommitDate: Mon Jul 16 19:22:11 2012 -0700 + + Update master + +commit 5cac95816fb5c436d9024b2b44405e9cbd0c0508 +Merge: f2bfb80 555f261 +Author: kumarnitin +AuthorDate: Mon Jul 16 19:19:22 2012 -0700 +Commit: kumarnitin +CommitDate: Mon Jul 16 19:19:22 2012 -0700 + + Merge pull request #30 from quiver/cmd_option + + add support for passing logging options to redis-live.py + +commit 555f261d26acc473a2e9aa6f85e615f290c4b104 +Author: george +AuthorDate: Mon Jul 16 18:33:34 2012 +0900 +Commit: george +CommitDate: Mon Jul 16 18:33:34 2012 +0900 + + add support for passing logging options to redis-live.py + + * example usage + $ ./redis-live.py --logging=debug + +commit f2bfb80d969ea8ce0a0c5e922407cf52680da4b0 +Author: Nitin Kumar +AuthorDate: Sun Jul 15 18:39:54 2012 -0700 +Commit: Nitin Kumar +CommitDate: Sun Jul 15 18:39:54 2012 -0700 + + fixed a bug + +commit 1ef583150fe30c55de05cb884823d4666cb232e2 +Author: kumarnitin +AuthorDate: Sun Jul 15 17:54:19 2012 -0700 +Commit: kumarnitin +CommitDate: Sun Jul 15 17:54:19 2012 -0700 + + Update master + +commit a4bb6c7228f077a972b648c241c88abc3f5b8d09 +Author: Nitin Kumar +AuthorDate: Sun Jul 15 17:39:50 2012 -0700 +Commit: Nitin Kumar +CommitDate: Sun Jul 15 17:39:50 2012 -0700 + + Fixed a few bugs + +commit 1bbbf16ab8c50ddc60f46be5dc9264e05bc9978c +Author: Lee McFadden +AuthorDate: Sun Jul 15 13:27:16 2012 -0700 +Commit: Lee McFadden +CommitDate: Sun Jul 15 13:27:16 2012 -0700 + + Fixed a few bugs. + +commit f7911212ce05a4d47f4f3e4c34046e86b9099225 +Author: Lee McFadden +AuthorDate: Sun Jul 15 13:09:23 2012 -0700 +Commit: Lee McFadden +CommitDate: Sun Jul 15 13:09:23 2012 -0700 + + Last few tweaks in controllers + +commit 84efb5ae98b09599b0756f6ad99fc1c26387ccae +Author: Lee McFadden +AuthorDate: Sun Jul 15 13:02:54 2012 -0700 +Commit: Lee McFadden +CommitDate: Sun Jul 15 13:02:54 2012 -0700 + + PEP8 for sqliteprovider + +commit 75dd0bcdcb87139bd47d3df932961c5f9b2c36c6 +Author: Lee McFadden +AuthorDate: Sun Jul 15 11:40:01 2012 -0700 +Commit: Lee McFadden +CommitDate: Sun Jul 15 11:40:01 2012 -0700 + + PEP8 method names and a few tweaks + +commit 5bc87ce3a4ef8577b2d7605a74e3bf75e14bdfd7 +Author: Lee McFadden +AuthorDate: Sun Jul 15 11:16:42 2012 -0700 +Commit: Lee McFadden +CommitDate: Sun Jul 15 11:16:42 2012 -0700 + + PEP8 for TopKeysController + +commit 84280468997fa4f3a2e85e1fef55f043b104052b +Author: Lee McFadden +AuthorDate: Sun Jul 15 11:13:12 2012 -0700 +Commit: Lee McFadden +CommitDate: Sun Jul 15 11:13:12 2012 -0700 + + PEP8 for TopCommandsController + +commit 95f3c67bbd7d4ff1283d7eec0462e24c4690f63b +Author: Lee McFadden +AuthorDate: Sun Jul 15 11:09:48 2012 -0700 +Commit: Lee McFadden +CommitDate: Sun Jul 15 11:09:48 2012 -0700 + + Missed a few spots. + +commit d145c7da870a0162fb97d117b30364b99e65b381 +Author: Lee McFadden +AuthorDate: Sun Jul 15 11:08:52 2012 -0700 +Commit: Lee McFadden +CommitDate: Sun Jul 15 11:08:52 2012 -0700 + + PEP8 for MemoryController + +commit 9b991dd158399ea16db5352567c8b001f2be1072 +Author: Lee McFadden +AuthorDate: Fri Jul 13 19:01:19 2012 -0700 +Commit: Lee McFadden +CommitDate: Fri Jul 13 19:01:19 2012 -0700 + + PEP8 for InfoController + +commit 74495c0683c933db3a3d7541811dc52c610dec67 +Author: Lee McFadden +AuthorDate: Fri Jul 13 18:42:51 2012 -0700 +Commit: Lee McFadden +CommitDate: Fri Jul 13 18:42:51 2012 -0700 + + Missed a spot... + +commit 0e3f9142abedee4557960346314b3dde85695225 +Author: Lee McFadden +AuthorDate: Fri Jul 13 18:41:22 2012 -0700 +Commit: Lee McFadden +CommitDate: Fri Jul 13 18:41:22 2012 -0700 + + PEP8'ed BaseController and dependencies + +commit db720a69c8ec3b78b1882bafa2750991f889ce04 +Author: Lee McFadden +AuthorDate: Fri Jul 13 17:53:50 2012 -0700 +Commit: Lee McFadden +CommitDate: Fri Jul 13 17:53:50 2012 -0700 + + PEP8'ed the internal code of this handler. Methods are up next. + +commit 24205a36e7526cad329f1fa97b69478b9619a412 +Author: Lee McFadden +AuthorDate: Fri Jul 13 17:27:50 2012 -0700 +Commit: Lee McFadden +CommitDate: Fri Jul 13 17:27:50 2012 -0700 + + Made RedisLiveSettings and dependencies more pythonic + +commit 205e7d995ae68a9e3ec1a235779738554201773e +Author: Lee McFadden +AuthorDate: Fri Jul 13 15:09:00 2012 -0700 +Commit: Lee McFadden +CommitDate: Fri Jul 13 15:09:00 2012 -0700 + + Fixed whitespace + +commit ed2041c4b103c2b9e9c95d90d3a904208c124918 +Author: Nitin Kumar +AuthorDate: Sun Jul 1 00:43:12 2012 -0700 +Commit: Nitin Kumar +CommitDate: Sun Jul 1 00:43:12 2012 -0700 + + fix for sqlite database lock issue. closed #7 diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..afcea79 --- /dev/null +++ b/src/README.md @@ -0,0 +1,22 @@ +redis-monitor +--------- + +base RedisLive,monitor multiple redis-server in product enviroment: +* 1. monitor multiple redis-instance in one page +* 2. monitor memory,comand per sec,HitRate,keyspace, master-slave change,expire +* 3. sms alert when crash , master-slave stats changed + +### install + in src/script/redis-monitor.sh add redis-monitor as a startup service for redhat + python redis_live.py #start web with port 8888 + python redis_monitor.py # start info collector + #start daemon + python redis_live_daemon.py + python redis_monitor_daemon.py + +### 演示地址 +[Redis-monitor](http://10.10.209.104:8888/index.html) +### overview +![Redis Live](https://raw.github.com/LittlePeng/redis-monitor/master/design/redis-live.png) +![Redis Live](https://raw.github.com/LittlePeng/redis-monitor/master/design/overview.png) + diff --git a/src/api/controller/BaseController.py b/src/api/controller/BaseController.py index e317be2..eb10ca3 100644 --- a/src/api/controller/BaseController.py +++ b/src/api/controller/BaseController.py @@ -10,7 +10,7 @@ class BaseController(tornado.web.RequestHandler): def getStatsPerServer(self, server): try: - connection = redis.Redis(host=server[0], port=(int)(server[1]), db=0) + connection = redis.Redis(host=server[0], port=(int)(server[1]), db=0,socket_timeout=0.1) info = connection.info() # when instances down ,this maybe slowly... info.update({ diff --git a/src/api/controller/CommandsController.py b/src/api/controller/CommandsController.py index 12e10b0..e68e38e 100644 --- a/src/api/controller/CommandsController.py +++ b/src/api/controller/CommandsController.py @@ -19,7 +19,7 @@ def get(self): if from_date==None or to_date==None or len(from_date)==0: end = datetime.datetime.now() - delta = datetime.timedelta(seconds=300) + delta = datetime.timedelta(seconds=900) start = end - delta else: start = dateutil.parser.parse(from_date) diff --git a/src/api/controller/InfoListController.py b/src/api/controller/InfoListController.py index fd8a5a6..b863236 100644 --- a/src/api/controller/InfoListController.py +++ b/src/api/controller/InfoListController.py @@ -5,34 +5,26 @@ class InfoListController(BaseController): def get(self): + group = self.get_argument("group", None) + response = {} response['data']=[] - for server in self.read_server_config(): - info=self.getStatsPerServer(server) + for server in settings.get_redis_servers(): + if(group !=None and group!='all' and server['group'] != group): + continue; + + info=self.getStatsPerServer((server['server'],server['port'])) - info.update({ - "addr" : info.get("server_name")[0].replace(".", "_") + str(info.get("server_name")[1]), + info.update({"addr" : info.get("server_name")[0].replace(".", "_") + str(info.get("server_name")[1]), }) - + info['show_name']=server['group']+'('+server['instance']+')' + info['group']= server['group'] screen_strategy = 'normal' if info.get("status") == 'down': screen_strategy = 'hidden' - info.update({ - "screen_strategy": screen_strategy, - }) + info.update({ "screen_strategy": screen_strategy,}) - #key = info.get("addr") response["data"].append(info) - - - self.write(response) - - def read_server_config(self): - server_list = [] - redis_servers = settings.get_redis_servers() - - for server in redis_servers: - server_list.append([server['server'],server['port']]) - return server_list \ No newline at end of file + self.write(response) \ No newline at end of file diff --git a/src/api/controller/ServerListController.py b/src/api/controller/ServerListController.py index 1d2373d..c2d835c 100644 --- a/src/api/controller/ServerListController.py +++ b/src/api/controller/ServerListController.py @@ -8,15 +8,11 @@ def get(self): self.write(servers) def read_server_config(self): - """Returns a list of servers with the 'id' field added. - """ - # TODO: Move this into the settings module so everything benefits. server_list = [] redis_servers = settings.get_redis_servers() for server in redis_servers: - server_id = "%(server)s:%(port)s" % server - s = dict(server=server['server'], port=server['port'], id=server_id) - server_list.append(s) + server['id']= "%(server)s:%(port)s" % server + server_list.append(server) return server_list diff --git a/src/api/controller/SettingsController.py b/src/api/controller/SettingsController.py index 8cd4fed..5ab150c 100644 --- a/src/api/controller/SettingsController.py +++ b/src/api/controller/SettingsController.py @@ -1,12 +1,13 @@ from BaseController import BaseController from api.util import settings +import os class SettingsController(BaseController): def get(self): server_list="" for server in settings.get_redis_servers(): - server_list+= "%(server)s:%(port)s\r\n" % server + server_list+= "%(server)s:%(port)s %(group)s %(instance)s\r\n" % server sms_repl=0; sms_stats=0; @@ -33,12 +34,20 @@ def post(self): eps=server.split(':') if(len(eps)!=2): raise Exception('server Ip format error.'); + ip=eps[0] - port=eps[1] + eps2 = eps[1].split(' ') + port=(int)(eps2[0]) + group='' + instance='' + + if(len(eps2)>1): + group=eps2[1] + if(len(eps2)>2): + instance=eps2[2] - servers.append({'server':ip,'port':(int)(port)}) + servers.append({'server':ip,'port':port,'group':group,'instance':instance}) settings.save_settings(servers, sms) self.write({"status":200}) except Exception,ex: - self.write({"status":500,"error":ex.message}) - \ No newline at end of file + self.write({"status":500,"error":ex.message}) \ No newline at end of file diff --git a/src/api/controller/SlowlogController.py b/src/api/controller/SlowlogController.py new file mode 100644 index 0000000..8a99ed5 --- /dev/null +++ b/src/api/controller/SlowlogController.py @@ -0,0 +1,17 @@ +from BaseController import BaseController +import datetime +import redis + +class SlowlogController(BaseController): + + def get(self): + data={} + data['data']=[] + server = self.get_argument("server").split(':') + connection = redis.Redis(host=server[0], port=(int)(server[1]), db=0,socket_timeout=1) + logs = connection.execute_command('slowlog','get','128') + for lid,timeticks,run_micro,commands in logs: + timestamp = datetime.datetime.fromtimestamp(int(timeticks)) + cmd=' '.join(commands) + data['data'].append({'id':lid,'time':str(timestamp),'escapeMs':(float)(run_micro)/1000,'cmd':cmd}) + self.write(data) \ No newline at end of file diff --git a/src/api/util/settings.py b/src/api/util/settings.py index d1bd63d..1ad9645 100644 --- a/src/api/util/settings.py +++ b/src/api/util/settings.py @@ -1,21 +1,21 @@ import json import os -curpath='' - def get_settings(): - """Parses the settings from redis-live.conf. - """ - # TODO: Consider YAML. Human writable, machine readable. - global curpath - if(curpath==''): - curpath=os.path.abspath('.') - - return json.load(open(curpath+ "/redis_live.conf")) + return json.load(open(os.path.abspath('.')+ "/redis_live.conf")) def get_redis_servers(): config = get_settings() - return config["RedisServers"] + servers= config["RedisServers"] + data=[] + for server in servers: + server['ep']='%(server)s:%(port)d' % server + if(server.get('group')==None or server.get('group')==''): + server['group']='ungrouped' + if(server.get('instance')==None or server.get('instance')==''): + server['instance']=(str)(server['port']) + data.append(server) + return data def get_redis_alerturi(): config = get_settings() diff --git a/src/daemonized.py b/src/daemonized.py index 4bfd670..5410f82 100644 --- a/src/daemonized.py +++ b/src/daemonized.py @@ -3,8 +3,8 @@ class daemonized(object): def __init__(self): - self.curpath='' - + pass + def daemonize(self): pass @@ -17,7 +17,7 @@ def daemonize(self): % (e.errno,e.strerror)) sys.exit(1) - os.chdir('/') + #os.chdir('/') #detach from terminal os.setsid() #file to be created? @@ -49,9 +49,7 @@ def daemonize(self): def start_daemon(self): self.daemonize() - if(self.curpath!=''): - os.chdir(self.curpath) - + self.run_daemon() def start(self): diff --git a/src/dataprovider/redisprovider.py b/src/dataprovider/redisprovider.py index 4b1aadc..e75ca59 100644 --- a/src/dataprovider/redisprovider.py +++ b/src/dataprovider/redisprovider.py @@ -4,106 +4,81 @@ import json import ast import time +import struct - -def datetime2_unix_str(timestamp): - return (str)((int)(time.mktime(timestamp.timetuple()))) +def datetime2_unix_int(timestamp): + return (int)(time.mktime(timestamp.timetuple())) class RedisStatsProvider(object): - """A Redis based persistance to store and fetch stats""" - def __init__(self): - # redis server to use to store stats stats_server = settings.get_redis_stats_server() self.server = stats_server["server"] self.port = stats_server["port"] self.conn = redis.StrictRedis(host=self.server, port=self.port, db=0) -# def save_memory_info(self, server, timestamp, used, peak): -# -# data = {"timestamp": datetime2_unix_str(timestamp), -# "used": used, -# "peak": peak} -# self.conn.zadd(server + ":memory", datetime2_unix_str(timestamp), data) - - def save_keys_Info(self, server,timestamp, expires, persists,expired,evicted + def save_keys_Info(self, server,rediskey,timestamp, expires, persists,expired,evicted ,hit_rate,commands,used,peak): - data = {"timestamp": datetime2_unix_str(timestamp), - "expires": expires, - "persists": persists, - "expired":expired, - "evicted":evicted, - "hit_rate":hit_rate, - "commands":commands, - "used": used, - "peak": peak} - self.conn.zadd(server + ":keysinfo", datetime2_unix_str(timestamp), data) - + score=datetime2_unix_int(timestamp) + data=struct.pack('iiiiiiiqq', + score, + commands, + expires, + persists, + expired, + evicted, + hit_rate, + peak, + used) + self.conn.zadd(server +':'+ rediskey, score, data) + + def get_keys_info(self, server, from_date, to_date): + data = [] + start = datetime2_unix_int(from_date) + end = datetime2_unix_int(to_date) + key=server + ":info" + if(end-start>=3600*2): + key=key+"_hours" + rows = self.conn.zrangebyscore(key, start, end) + + rate=1 + if(len(rows)> 400): + rate=len(rows)/300 + + index=0 + for row in rows: + index+=1 + if(index%rate==0): + row=struct.unpack('iiiiiiiqq',row) + timestamp = datetime.fromtimestamp(int(row[0])) + item=list(row) + item[0]=tuple(timestamp.timetuple())[:-2] + + data.append(item) + return data + def save_status_info(self, server, timestamp, data): - timestamp=datetime2_unix_str(timestamp) + timestamp=datetime2_unix_int(timestamp) data['timestamp']=timestamp self.conn.zadd(server + ":status", timestamp, json.dumps(data)) - def save_info_command(self, server, timestamp, info): - # info let aof file too large ... do not save - self.conn.set(server + ":Info", json.dumps(info)) + def get_status_info(self, server, from_date, to_date): + data = [] + start = datetime2_unix_int(from_date) + end = datetime2_unix_int(to_date) + rows = self.conn.zrangebyscore(server + ":status", start, end) + for row in rows: + row = ast.literal_eval(row) + timestamp = datetime.fromtimestamp(int(row['timestamp'])) + data.append([tuple(timestamp.timetuple())[:-2], row]) + return data def delete_history(self,server,timestamp): begin=0 - end = int(datetime2_unix_str(timestamp)) - self.conn.zremrangebyscore(server + ":keysinfo", begin, end) + end = datetime2_unix_int(timestamp) + self.conn.zremrangebyscore(server + ":info", begin, end) + self.conn.zremrangebyscore(server + ":info_hours", begin, end-(3600*24*7)) # status for more then 3 month self.conn.zremrangebyscore(server + ":status", begin, end - (3600*24*90)) - - def get_info(self, server): - info = self.conn.get(server + ":Info") - info = json.loads(info) - return info -# def get_memory_info(self, server, from_date, to_date): -# memory_data = [] -# data=self.get_zset_data("memory", server, from_date, to_date) -# for row in data: -# memory_data.append([row[0],row[1]['peak'],row[1]['used']]) -# -# return memory_data def collection_database(self): - self.conn.bgrewriteaof() - - def get_status_info(self, server, from_date, to_date): - return self.get_zset_data("status", server, from_date, to_date) - - def get_keys_info(self, server, from_date, to_date): - rows=self.get_zset_data("keysinfo", server, from_date, to_date) - keys=[] - for row in rows: - keys.append([row[0],row[1]["commands"],row[1]["expires"],row[1]["persists"], - row[1]["expired"],row[1]["evicted"],row[1]["hit_rate"],row[1]['peak'],row[1]['used']]) - return keys - - def get_zset_data(self,keyprefix, server, from_date, to_date): - data = [] - start = int(datetime2_unix_str(from_date)) - end = int(datetime2_unix_str(to_date)) - rows = self.conn.zrangebyscore(server + ":"+keyprefix, start, end) - - count=len(rows) - rate=1 - - if(count> 400): - rate=count/200 - - index=0 - for row in rows: - # TODO: Check to see if there's not a better way to do this. Using - # eval feels like it could be wrong/dangerous... but that's just a - # feeling. - index+=1 - if(index%rate==0): - row = ast.literal_eval(row) - - # convert the timestamp - timestamp = datetime.fromtimestamp(int(row['timestamp'])) - - data.append([tuple(timestamp.timetuple())[:-2], row]) - return data \ No newline at end of file + self.conn.bgrewriteaof() \ No newline at end of file diff --git a/src/redis_live.conf b/src/redis_live.conf index acfdf2e..a1ed1dc 100644 --- a/src/redis_live.conf +++ b/src/redis_live.conf @@ -1,14 +1,18 @@ -{"sms_alert":"192.168.110.207:9999", -"master_slave_sms": "1,1", "RedisStatsServer": {"port": 7009, "server": "10.10.209.104"} -, "DataStoreType": "redis", "RedisServers": [{"port": 7001, "server": "10.10.209.104"} -, {"port": 7002, "server": "10.10.209.104"} -, {"port": 7003, "server": "10.10.209.104"} -, {"port": 7004, "server": "10.10.209.104"} -, {"port": 7001, "server": "10.10.209.150"} -, {"port": 7002, "server": "10.10.209.150"} -, {"port": 7003, "server": "10.10.209.150"} -, {"port": 7004, "server": "10.10.209.150"} -, {"port": 7100, "server": "192.168.245.1"} -, {"port": 7101, "server": "192.168.245.1"} -, {"port": 7100, "server": "192.168.245.2"} -]} +{"master_slave_sms": "1,1", "RedisStatsServer": {"port": 7001, "server": "10.10.40.145"} +, "sms_alert": "192.168.110.207:9999", "DataStoreType": "redis", "RedisServers": [{"instance": "Master1", "group": "PRSOnline", "port": 5707, "server": "10.10.209.104"} +, {"instance": "M2", "group": "PRSOnline", "port": 5708, "server": "10.10.209.104"} +, {"instance": "M3", "group": "PRSOnline", "port": 5709, "server": "10.10.209.104"} +, {"instance": "M4", "group": "PRSOnline", "port": 5710, "server": "10.10.209.104"} +, {"instance": "Slave1", "group": "PRSOnline", "port": 5707, "server": "10.10.209.150"} +, {"instance": "S2", "group": "PRSOnline", "port": 5708, "server": "10.10.209.150"} +, {"instance": "S3", "group": "PRSOnline", "port": 5709, "server": "10.10.209.150"} +, {"instance": "S4", "group": "PRSOnline", "port": 5710, "server": "10.10.209.150"} +, {"instance": "I1", "group": "PRSOffline", "port": 5711, "server": "10.10.209.104"} +, {"instance": "I2", "group": "PRSOffline", "port": 5712, "server": "10.10.209.104"} +, {"instance": "I1", "group": "PRSOffline", "port": 5711, "server": "10.10.209.150"} +, {"instance": "I2", "group": "PRSOffline", "port": 5712, "server": "10.10.209.150"} +, {"instance": "01", "group": "SiteAPRS", "port": 6679, "server": "192.168.251.212"} +, {"instance": "02", "group": "SiteAPRS", "port": 6680, "server": "192.168.251.212"} +, {"instance": "01", "group": "SiteACATS", "port": 6779, "server": "192.168.251.213"} +, {"instance": "02", "group": "SiteACATS", "port": 6780, "server": "192.168.251.213"} +]} diff --git a/src/redis_live.py b/src/redis_live.py index b751d3b..99025c7 100644 --- a/src/redis_live.py +++ b/src/redis_live.py @@ -13,6 +13,7 @@ from api.controller.InfoListController import InfoListController from api.controller.StatusController import StatusController from api.controller.SettingsController import SettingsController +from api.controller.SlowlogController import SlowlogController from daemonized import daemonized class redis_live(daemonized): @@ -21,7 +22,8 @@ def run_daemon(self): define("port", default=8888, help="run on the given port", type=int) define("debug", default=0, help="debug mode", type=int) tornado.options.parse_command_line() - + + print os.path.abspath('.') # Bootup handlers = [ (r"/api/servers", ServerListController), @@ -30,6 +32,7 @@ def run_daemon(self): (r"/api/infolist",InfoListController), (r"/api/commands", CommandsController), (r"/api/settings",SettingsController), + (r"/api/slowlog",SlowlogController), (r"/(.*)", BaseStaticFileHandler, {"path": os.path.abspath('.')+'/www'}) ] diff --git a/src/redis_live_daemon.py b/src/redis_live_daemon.py index a10a12c..4e5347e 100644 --- a/src/redis_live_daemon.py +++ b/src/redis_live_daemon.py @@ -2,11 +2,9 @@ import sys if __name__ == "__main__": - curpath=os.path.split( os.path.realpath( sys.argv[0] ) )[0] + curpath = os.path.split(os.path.realpath(sys.argv[0]))[0] os.chdir(curpath) from redis_live import redis_live - live= redis_live() - live.curpath= curpath - print 'curpath:'+live.curpath - live.start_daemon() \ No newline at end of file + live = redis_live() + live.start_daemon() diff --git a/src/redis_monitor.py b/src/redis_monitor.py index d5aef93..98301f5 100644 --- a/src/redis_monitor.py +++ b/src/redis_monitor.py @@ -14,286 +14,305 @@ import httplib import urllib +class InfoItem(object): + def __init__(self, key): + self.expired_keys = 0 + self.evicted_keys = 0 + self.keyspace_hits = 0 + self.keyspace_misses = 0 + self.total_commands_processed = 0 + self.checkTime = datetime.datetime.now() + self.key = key + + def datetime2_unix_int(self, timestamp): + return (int)(time.mktime(timestamp.timetuple())) + + def gettick_sec(self, current_time): + return self.datetime2_unix_int(current_time) - self.datetime2_unix_int(self.checkTime) + class InfoThread(threading.Thread): - def __init__(self, server, port, password=None): + def __init__(self, server, port, instancename, password=None): threading.Thread.__init__(self) + self.server = server self.port = port self.password = password + self.instancename = instancename self.id = self.server + ":" + str(self.port) + self._stop = threading.Event() - + self.stats_provider = None + + self.monitor_tick = 3 # log times every sec + self.reserved_min = 1440 * 7 # data reserved time,1day=1440 + + self.last = InfoItem('info') # for every tick + self.last2 = InfoItem('info_hours') # for every minutes + + self.last_role_status = "" + self.last_role = {} + self.continueFaildTimes = 0 + def stop(self): - """Stops the thread. - """ self._stop.set() def stopped(self): - """Returns True if the thread is stopped, False otherwise. - """ return self._stop.is_set() - + def run(self): - """Does all the work. - """ - monitor_tick=2 #log times every sec - reserved_min = 1440*7 # data reserved time,1day=1440 - - - stats_provider = RedisLiveDataProvider.get_provider() + self.stats_provider = RedisLiveDataProvider.get_provider() redis_client = redis.StrictRedis(host=self.server, port=self.port, db=0, - password=self.password) + password=self.password, socket_timeout=3) - last_expired_keys=0 - last_evicted_keys=0 - last_keyspace_hits=0 - last_keyspace_misses=0 - last_total_commands_processed=0 - last_role_status="" - last_role={} - - doitems=0 - # process the results from redis + doitems = 0 while not self.stopped(): - time.sleep(monitor_tick) - doitems+=1 + time.sleep(self.monitor_tick) + doitems += 1 try: - redis_info={} + redis_info = {} current_time = datetime.datetime.now() - redis_info = redis_client.info() - - # not save,for it let aof too large in redis -# try: -# redis_info = redis_client.info() -# except Exception, e: -# redis_info['role']='down' -# redis_info['uptime_in_seconds']=0 -# redis_info['total_commands_processed']=0 -# redis_info['used_memory_human']='' -# redis_info['connected_clients']='' -# -# stats_provider.save_info_command(self.id, current_time, -# redis_info) -# print "==============================\n" -# print datetime.datetime.now() -# print traceback.format_exc() -# print "==============================\n" -# continue -# -# info all -# stats_provider.save_info_command(self.id, current_time, -# redis_info) - #try remove history - if(doitems %30==0): - delta = datetime.timedelta(seconds=reserved_min*60) - start = current_time - delta - stats_provider.delete_history(self.id,start) - - - #memory - used_memory = int(redis_info['used_memory']) - - # used_memory_peak not available in older versions of redis try: - peak_memory = int(redis_info['used_memory_peak']) - except: - peak_memory = used_memory - -# stats_provider.save_memory_info(self.id, current_time, -# used_memory, peak_memory) -# - #keys info - databases=[] - for key in sorted(redis_info.keys()): - if key.startswith("db"): - database = redis_info[key] - database['name']=key - databases.append(database) - - expires=0 - persists=0 - for database in databases: - expires+=database.get("expires") - persists+=database.get("keys")-database.get("expires") - - - expired_keys=redis_info["expired_keys"] - evicted_keys=redis_info["evicted_keys"] - keyspace_hits=redis_info["keyspace_hits"] - keyspace_misses=redis_info["keyspace_misses"] - total_commands_processed=redis_info["total_commands_processed"] + redis_info = redis_client.info() + except Exception: + self.servicedown() + print traceback.format_exc() + continue - expired=0 - evicted=0 - if(last_expired_keys>0 or last_evicted_keys>0): - expired=expired_keys-last_expired_keys - if(expired>=0): - expired= (int)(expired/monitor_tick) - last_expired_keys=expired_keys - else: - expired=0 - last_expired_keys=0 - - evicted=evicted_keys-last_evicted_keys - if(evicted>=0): - evicted= (int)(evicted/monitor_tick) - last_evicted_keys= evicted_keys - else: - evicted=0 - last_evicted_keys=0 - else: - last_expired_keys=expired_keys - last_evicted_keys=evicted_keys + # remove history + if(doitems % 30 == 0): + delta = datetime.timedelta(seconds=self.reserved_min * 60) + start = current_time - delta + self.stats_provider.delete_history(self.id, start) - hit_rate=0 - if(last_keyspace_hits>0 or last_keyspace_misses>0): - hits=keyspace_hits-last_keyspace_hits - miss=keyspace_misses - last_keyspace_misses - if(hits>=0 and miss>=0): - total=hits+miss - if(total>0): - hit_rate= (int)((hits*100)/(hits+miss)) - last_keyspace_hits=keyspace_hits - last_keyspace_misses=keyspace_misses - else: - last_keyspace_hits=0 - last_keyspace_misses =0 - else: - last_keyspace_hits=keyspace_hits - last_keyspace_misses=keyspace_misses - - commands=0 - if(last_total_commands_processed>0): - commands=total_commands_processed-last_total_commands_processed - if(commands>=0): - commands=(int)(commands/monitor_tick) - last_total_commands_processed=total_commands_processed - else: - last_total_commands_processed=0 - commands=0 - else: - last_total_commands_processed=total_commands_processed - - stats_provider.save_keys_Info(self.id, current_time, expires, persists, - expired,evicted,hit_rate,commands,used_memory, peak_memory) - #master status - role=redis_info["role"] - role_status={} + self.LogInfo(redis_info, current_time, self.last) - if(role=="master"): - connected_slaves=(int)(redis_info["connected_slaves"]) - slaves="" - for i in range(0,connected_slaves): - slaves+=redis_info["slave"+(str)(i)] - - role_status={"role":role,"slaves":slaves} - else: - master_host=redis_info["master_host"] - master_port=(str)(redis_info["master_port"]) - master_link_status=redis_info["master_link_status"] - master_sync_in_progress=redis_info["master_sync_in_progress"] - role_status={"role":role, - "master_host_port":master_host+":"+master_port, - "master_link_status":master_link_status, - "master_sync_in_progress":master_sync_in_progress } + if(self.last2.gettick_sec(current_time) >= 60): + self.LogInfo(redis_info, current_time, self.last2) - role_cur=json.dumps(role_status) - if(role_cur!=last_role_status): - #monitor first start,not save - if(last_role_status!=""): - stats_provider.save_status_info(self.id, current_time, role_status) - self.sendslavesms(role_status,last_role) - - last_role_status=role_cur - last_role=role_status - - except Exception, e: - last_expired_keys=0 - last_evicted_keys=0 - last_keyspace_hits=0 - last_keyspace_misses=0 - last_total_commands_processed=0 + self.CheckMasterStatus(redis_info, current_time) + except Exception: tb = traceback.format_exc() - print "==============================\n" print datetime.datetime.now() print tb print "==============================\n" - - def sendslavesms(self,current,last): + + def LogInfo(self, redis_info, current_time, last): + ticksec = last.gettick_sec(current_time) + last.checkTime = current_time; + + # memory + used_memory = int(redis_info['used_memory']) try: - self.sendsmsInner(current,last) - except Exception,ex: - print ex + peak_memory = int(redis_info['used_memory_peak']) + except: + peak_memory = used_memory + + # keys info + databases = [] + for key in sorted(redis_info.keys()): + if key.startswith("db"): + database = redis_info[key] + database['name'] = key + databases.append(database) + + expires = 0 + persists = 0 + for database in databases: + expires += database.get("expires") + persists += database.get("keys") - database.get("expires") + + expired_keys = redis_info["expired_keys"] + evicted_keys = redis_info["evicted_keys"] + keyspace_hits = redis_info["keyspace_hits"] + keyspace_misses = redis_info["keyspace_misses"] + total_commands_processed = redis_info["total_commands_processed"] + + expired = 0 + evicted = 0 + if(last.expired_keys > 0 or last.evicted_keys > 0): + expired = expired_keys - last.expired_keys + if(expired >= 0): + expired = (int)(expired / ticksec) + last.expired_keys = expired_keys + else: + expired = 0 + last.expired_keys = 0 + + evicted = evicted_keys - last.evicted_keys + if(evicted >= 0): + evicted = (int)(evicted / ticksec) + last.evicted_keys = evicted_keys + else: + evicted = 0 + last.evicted_keys = 0 + else: + last.expired_keys = expired_keys + last.evicted_keys = evicted_keys + + hit_rate = 0 + if(last.keyspace_hits > 0 or last.keyspace_misses > 0): + hits = keyspace_hits - last.keyspace_hits + miss = keyspace_misses - last.keyspace_misses + if(hits >= 0 and miss >= 0): + total = hits + miss + if(total > 0): + hit_rate = (int)((hits * 100) / (hits + miss)) + last.keyspace_hits = keyspace_hits + last.keyspace_misses = keyspace_misses + else: + last.keyspace_hits = 0 + last.keyspace_misses = 0 + else: + last.keyspace_hits = keyspace_hits + last.keyspace_misses = keyspace_misses + + commands = 0 + if(last.total_commands_processed > 0): + commands = total_commands_processed - last.total_commands_processed + if(commands >= 0): + commands = (int)(commands / ticksec) + last.total_commands_processed = total_commands_processed + else: + last.total_commands_processed = 0 + commands = 0 + else: + last.total_commands_processed = total_commands_processed - def sendsmsInner(self,current,last): - sms_repl=0; - sms_stats=0; + self.stats_provider.save_keys_Info(self.id, last.key , current_time, expires, persists, + expired, evicted, hit_rate, commands, used_memory, peak_memory) + + def CheckMasterStatus(self, redis_info, current_time): + role = redis_info["role"] + role_status = {} + + if(role == "master"): + connected_slaves = (int)(redis_info["connected_slaves"]) + slaves = "" + for i in range(0, connected_slaves): + slaves += redis_info["slave" + (str)(i)] + + role_status = {"role":role, "slaves":slaves} + else: + master_host = redis_info["master_host"] + master_port = (str)(redis_info["master_port"]) + master_link_status = redis_info["master_link_status"] + master_sync_in_progress = redis_info["master_sync_in_progress"] + role_status = {"role":role, + "master_host_port":master_host + ":" + master_port, + "master_link_status":master_link_status, + "master_sync_in_progress":master_sync_in_progress } + + role_cur = json.dumps(role_status) + if(role_cur != self.last_role_status): + # monitor first start,not save + if(self.last_role_status != ""): + self.stats_provider.save_status_info(self.id, current_time, role_status) + self.sendslavesms(role_status, self.last_role) + + self.last_role_status = role_cur + self.last_role = role_status + + def sendslavesms(self, current, last): try: - sms=settings.get_master_slave_sms_type() - sms=sms.split(',') - sms_repl=(int)(sms[0]) - sms_stats=(int)(sms[1]) - except: - pass - if(sms_repl==1 and current['role']!=last['role']): - self.sendsms(self.id+'from:'+last['role']+'changeto:'+current['role']) - elif(sms_stats==1): - self.sendsms(self.id+",status changed:"+json.dumps(last)) - - def ServiceDown(self): - pass - def sendsms(self,content): - url="192.168.110.207:9999" + sms_repl = 0; + sms_stats = 0; + try: + sms = settings.get_master_slave_sms_type() + sms = sms.split(',') + sms_repl = (int)(sms[0]) + sms_stats = (int)(sms[1]) + except: + pass + if( current['role'] != last['role']): + if(sms_repl == 1): + self.sendsms('from: %s changeto: %s' % (last['role'], current['role'])) + elif(sms_stats == 1 and current['role'] =='master'): + stat='slave status OK.' + slv=current['slaves'] + if(slv.find('wait_bgsave')!=-1): + stat='dumping data to prepare send to slave.' + elif(slv.find('send_bulk')!=-1): + stat='sending dump data to slave.' + self.sendsms(stat+'(%s)' % slv) + + except Exception, ex: + print ex + + def servicedown(self): + self.continueFaildTimes += 1 + if(self.continueFaildTimes % 20 == 0): + self.sendsms('ping retry 20 times,all faild!!') + + def sendsms(self, content): + url = "192.168.110.207:9999" try: - url=settings.get_redis_alerturi() + url = settings.get_redis_alerturi() finally: pass - - conn = httplib.HTTPConnection(url) + content = '[redis]%s(%s):' % (self.instancename , self.id) + content + conn = httplib.HTTPConnection(url) print content - conn.request("POST", "/SendSms",body= urllib.urlencode({'text':content})) - r1 = conn.getresponse() + conn.request("POST", "/SendSms", body=urllib.urlencode({'text':content})) + r1 = conn.getresponse() print r1.status, r1.reason class redis_monitor(daemonized): def __init__(self): - self.threads = [] + self.threads = {} self.active = True def run_daemon(self): - redis_servers = settings.get_redis_servers() - - for redis_server in redis_servers: - - info = InfoThread(redis_server["server"], redis_server["port"], - redis_server.get("password", None)) - self.threads.append(info) - info.setDaemon(True) - info.start() - # In this particular case, running a single MONITOR client can reduce - # the throughput by more than 50%. Running more MONITOR clients will - # reduce throughput even more. try: - doitems=0 + doitems = 0 while self.active: - time.sleep(1) - doitems+=1 - stats_provider = RedisLiveDataProvider.get_provider() - #try collection DB like:redis aofrewrite - if(doitems %3600==0): + try: + self.run_info() + except Exception: + print traceback.format_exc() + + time.sleep(10) + doitems += 1 + + # try collection DB like:redis bgrewriteaof + if(doitems % 3600 == 0): + stats_provider = RedisLiveDataProvider.get_provider() stats_provider.collection_database() except (KeyboardInterrupt, SystemExit): self.stop() - + + def run_info(self): + servers = settings.get_redis_servers() + + eids = set(()) + for server in servers: + eid = server['ep'] + eids.add(eid) + instancename = '%(group)s-%(instance)s' % server + info=self.threads.get(eid) + if( info== None): + info = InfoThread(server["server"], server["port"], + instancename,server.get("password", None)) + info.setDaemon(True) + info.start() + self.threads[eid] = info + + print 'add %s' % eid + else: + info.instancename= instancename + + for eid, info2 in self.threads.items(): + if(eid not in eids): + info2.stop() + self.threads.pop(eid) + print 'remove %s' % eid + def stop(self): - """Stops the monitor and all associated threads. - """ print "shutting down..." for t in self.threads: diff --git a/src/redis_monitor_daemon.py b/src/redis_monitor_daemon.py index e73f461..55d2d4a 100644 --- a/src/redis_monitor_daemon.py +++ b/src/redis_monitor_daemon.py @@ -2,11 +2,9 @@ import sys if __name__ == "__main__": - curpath=os.path.split( os.path.realpath( sys.argv[0] ) )[0] + curpath = os.path.split(os.path.realpath(sys.argv[0]))[0] os.chdir(curpath) from redis_monitor import redis_monitor - montor= redis_monitor() - montor.curpath=curpath - print 'curpath:'+montor.curpath - montor.start_daemon() \ No newline at end of file + montor = redis_monitor() + montor.start_daemon() diff --git a/src/www/index.html b/src/www/index.html index 83205f4..0e5d81c 100644 --- a/src/www/index.html +++ b/src/www/index.html @@ -67,6 +67,8 @@ + + + + @@ -28,19 +30,50 @@
Loading...
- + + - \ No newline at end of file diff --git a/src/www/static/css/reset.css b/src/www/static/css/reset.css index c5c6a7d..0f6d0d9 100644 --- a/src/www/static/css/reset.css +++ b/src/www/static/css/reset.css @@ -1,2 +1,2 @@ -html,body,div,span,applet,object,iframe,h1,h2,h3,h4,h5,h6,p,blockquote,pre,a,abbr,acronym,address,big,cite,code,del,dfn,em,font,img,ins,kbd,q,s,samp,small,strike,strong,sub,sup,tt,var,b,u,i,center,dl,dt,dd,ol,ul,li,fieldset,form,label,legend,table,caption,tbody,tfoot,thead,tr,th,td{margin:0;padding:0;border:0;outline:0;font-size:100%;vertical-align:baseline;background:transparent}body{line-height:1}ol,ul{list-style:none}blockquote,q{quotes:none}blockquote:before,blockquote:after,q:before,q:after{content:'';content:none}:focus{outline:0}ins{text-decoration:none}del{text-decoration:line-through}table{border-collapse:collapse;border-spacing:0} +html,body,div,span,applet,object,iframe,h1,h2,h3,h4,h5,h6,p,blockquote,pre,a,abbr,acronym,address,big,cite,code,del,dfn,em,font,img,ins,kbd,q,s,samp,small,strike,strong,sub,sup,tt,var,b,u,i,center,dl,dt,dd,ol,ul,li,fieldset,form,label,legend,table,caption,tbody,tfoot,thead,tr,th,td{margin:0;border:0;outline:0;font-size:100%;vertical-align:baseline;background:transparent}body{line-height:1}ol,ul{list-style:none}blockquote,q{quotes:none}blockquote:before,blockquote:after,q:before,q:after{content:'';content:none}:focus{outline:0}ins{text-decoration:none}del{text-decoration:line-through}table{border-collapse:collapse;border-spacing:0} diff --git a/src/www/static/css/style.css b/src/www/static/css/style.css index 71ce89d..76bf80b 100644 --- a/src/www/static/css/style.css +++ b/src/www/static/css/style.css @@ -24,7 +24,6 @@ a:hover { width:360px; margin-right: auto; margin-left: auto; - margin-top: 40px; float: left; } @@ -113,8 +112,7 @@ h2 { } .line_span { - font-weight:bold; - float: right; + float: right; } .hidden {