From 82d5448eb7dc09be85b9187048285e98ae17ced5 Mon Sep 17 00:00:00 2001 From: Brian Martin Date: Sat, 25 Jan 2020 15:26:35 -0500 Subject: [PATCH 1/7] change randomid to 4 characters --- bumper/plugins/bumper_confserver_portal_iot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bumper/plugins/bumper_confserver_portal_iot.py b/bumper/plugins/bumper_confserver_portal_iot.py index 6455ffdf..486fe371 100644 --- a/bumper/plugins/bumper_confserver_portal_iot.py +++ b/bumper/plugins/bumper_confserver_portal_iot.py @@ -29,7 +29,7 @@ async def handle_devmanager_botcommand(self, request): try: json_body = json.loads(await request.text()) - randomid = "".join(random.sample(string.ascii_letters, 6)) + randomid = "".join(random.sample(string.ascii_letters, 4)) did = "" if "toId" in json_body: # Its a command did = json_body["toId"] From 436830cdd6ac039e6053bb402fc23186246ebc59 Mon Sep 17 00:00:00 2001 From: Kusha Gharahi <3326002+kushagharahi@users.noreply.github.com> Date: Sun, 11 Oct 2020 18:22:24 -0500 Subject: [PATCH 2/7] add LOG_TO_STDOUT env var --- bumper/__init__.py | 102 +++++++++++++++++++++++++++++---------------- docs/Env_Var.md | 3 +- 2 files changed, 69 insertions(+), 36 deletions(-) diff --git a/bumper/__init__.py b/bumper/__init__.py index ff03f003..ccce858e 100644 --- a/bumper/__init__.py +++ b/bumper/__init__.py @@ -26,10 +26,13 @@ def strtobool(strbool): # os.environ['PYTHONASYNCIODEBUG'] = '1' # Uncomment to enable ASYNCIODEBUG bumper_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) +log_to_stdout = os.environ.get("LOG_TO_STDOUT") + # Set defaults from environment variables first # Folders -logs_dir = os.environ.get("BUMPER_LOGS") or os.path.join(bumper_dir, "logs") -os.makedirs(logs_dir, exist_ok=True) # Ensure logs directory exists or create +if not log_to_stdout: + logs_dir = os.environ.get("BUMPER_LOGS") or os.path.join(bumper_dir, "logs") + os.makedirs(logs_dir, exist_ok=True) # Ensure logs directory exists or create data_dir = os.environ.get("BUMPER_DATA") or os.path.join(bumper_dir, "data") os.makedirs(data_dir, exist_ok=True) # Ensure data directory exists or create certs_dir = os.environ.get("BUMPER_CERTS") or os.path.join(bumper_dir, "certs") @@ -80,73 +83,102 @@ def strtobool(strbool): ) bumperlog = logging.getLogger("bumper") -bumper_rotate = RotatingFileHandler("logs/bumper.log", maxBytes=5000000, backupCount=5) -bumper_rotate.setFormatter(logformat) -bumperlog.addHandler(bumper_rotate) +if not log_to_stdout: + bumper_rotate = RotatingFileHandler("logs/bumper.log", maxBytes=5000000, backupCount=5) + bumper_rotate.setFormatter(logformat) + bumperlog.addHandler(bumper_rotate) +else: + bumperlog.addHandler(logging.StreamHandler(sys.stdout)) # Override the logging level # bumperlog.setLevel(logging.INFO) confserverlog = logging.getLogger("confserver") -conf_rotate = RotatingFileHandler( - "logs/confserver.log", maxBytes=5000000, backupCount=5 -) -conf_rotate.setFormatter(logformat) -confserverlog.addHandler(conf_rotate) +if not log_to_stdout: + conf_rotate = RotatingFileHandler( + "logs/confserver.log", maxBytes=5000000, backupCount=5 + ) + conf_rotate.setFormatter(logformat) + confserverlog.addHandler(conf_rotate) +else: + confserverlog.addHandler(logging.StreamHandler(sys.stdout)) # Override the logging level # confserverlog.setLevel(logging.INFO) mqttserverlog = logging.getLogger("mqttserver") -mqtt_rotate = RotatingFileHandler( - "logs/mqttserver.log", maxBytes=5000000, backupCount=5 -) -mqtt_rotate.setFormatter(logformat) -mqttserverlog.addHandler(mqtt_rotate) +if not log_to_stdout: + mqtt_rotate = RotatingFileHandler( + "logs/mqttserver.log", maxBytes=5000000, backupCount=5 + ) + mqtt_rotate.setFormatter(logformat) + mqttserverlog.addHandler(mqtt_rotate) +else: + mqttserverlog.addHandler(logging.StreamHandler(sys.stdout)) # Override the logging level # mqttserverlog.setLevel(logging.INFO) ### Additional MQTT Logs translog = logging.getLogger("transitions") -translog.addHandler(mqtt_rotate) +if not log_to_stdout: + translog.addHandler(mqtt_rotate) +else: + translog.addHandler(logging.StreamHandler(sys.stdout)) translog.setLevel(logging.CRITICAL + 1) # Ignore this logger logging.getLogger("passlib").setLevel(logging.CRITICAL + 1) # Ignore this logger brokerlog = logging.getLogger("hbmqtt.broker") #brokerlog.setLevel( # logging.CRITICAL + 1 #) # Ignore this logger #There are some sublogs that could be set if needed (.plugins) -brokerlog.addHandler(mqtt_rotate) +if not log_to_stdout: + brokerlog.addHandler(mqtt_rotate) +else: + brokerlog.addHandler(logging.StreamHandler(sys.stdout)) protolog = logging.getLogger("hbmqtt.mqtt.protocol") #protolog.setLevel( # logging.CRITICAL + 1 #) # Ignore this logger -protolog.addHandler(mqtt_rotate) +if not log_to_stdout: + protolog.addHandler(mqtt_rotate) +else: + protolog.addHandler(logging.StreamHandler(sys.stdout)) clientlog = logging.getLogger("hbmqtt.client") #clientlog.setLevel(logging.CRITICAL + 1) # Ignore this logger -clientlog.addHandler(mqtt_rotate) - +if not log_to_stdout: + clientlog.addHandler(mqtt_rotate) +else: + clientlog.addHandler(logging.StreamHandler(sys.stdout)) helperbotlog = logging.getLogger("helperbot") -helperbot_rotate = RotatingFileHandler( - "logs/helperbot.log", maxBytes=5000000, backupCount=5 -) -helperbot_rotate.setFormatter(logformat) -helperbotlog.addHandler(helperbot_rotate) +if not log_to_stdout: + helperbot_rotate = RotatingFileHandler( + "logs/helperbot.log", maxBytes=5000000, backupCount=5 + ) + helperbot_rotate.setFormatter(logformat) + helperbotlog.addHandler(helperbot_rotate) +else: + helperbotlog.addHandler(logging.StreamHandler(sys.stdout)) # Override the logging level # helperbotlog.setLevel(logging.INFO) boterrorlog = logging.getLogger("boterror") -boterrorlog_rotate = RotatingFileHandler( - "logs/boterror.log", maxBytes=5000000, backupCount=5 -) -boterrorlog_rotate.setFormatter(logformat) -boterrorlog.addHandler(boterrorlog_rotate) +if not log_to_stdout: + boterrorlog_rotate = RotatingFileHandler( + "logs/boterror.log", maxBytes=5000000, backupCount=5 + ) + boterrorlog_rotate.setFormatter(logformat) + boterrorlog.addHandler(boterrorlog_rotate) +else: + boterrorlog.addHandler(logging.StreamHandler(sys.stdout)) # Override the logging level # boterrorlog.setLevel(logging.INFO) xmppserverlog = logging.getLogger("xmppserver") -xmpp_rotate = RotatingFileHandler( - "logs/xmppserver.log", maxBytes=5000000, backupCount=5 -) -xmpp_rotate.setFormatter(logformat) -xmppserverlog.addHandler(xmpp_rotate) +if not log_to_stdout: + xmpp_rotate = RotatingFileHandler( + "logs/xmppserver.log", maxBytes=5000000, backupCount=5 + ) + xmpp_rotate.setFormatter(logformat) + xmppserverlog.addHandler(xmpp_rotate) +else: + xmppserverlog.addHandler(logging.StreamHandler(sys.stdout)) # Override the logging level # xmppserverlog.setLevel(logging.INFO) diff --git a/docs/Env_Var.md b/docs/Env_Var.md index f5b06978..4d3378be 100644 --- a/docs/Env_Var.md +++ b/docs/Env_Var.md @@ -11,4 +11,5 @@ Bumper has a number of environment variables to help with custom deployments and | BUMPER_KEY | {full path to bumper.key location} | The private server key (bumper.key) to be used by the Bumper server | | BUMPER_LOGS | {full path to logs directory} | The directory where logs should be stored | | BUMPER_DATA | {full path to data directory} | The directory where persistent data should be stored (bumper.db) | -| BUMPER_DEBUG | true | Run Bumper with debug mode/logging | \ No newline at end of file +| BUMPER_DEBUG | true | Run Bumper with debug mode/logging | +| LOG_TO_STDOUT | true | Instead of logging to logs/, logs to to STDOUT | \ No newline at end of file From 7cbac37c4d20ecbf613d8cbdb2b2c7d8ac4edd5a Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Fri, 21 May 2021 13:46:05 +0200 Subject: [PATCH 3/7] proxy forward also other command than p2p --- bumper/mqttserver.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/bumper/mqttserver.py b/bumper/mqttserver.py index f6f61ed2..1845c0ad 100644 --- a/bumper/mqttserver.py +++ b/bumper/mqttserver.py @@ -4,8 +4,9 @@ import asyncio import os import hbmqtt +import websockets from hbmqtt.broker import Broker -from hbmqtt.client import MQTTClient +from hbmqtt.client import MQTTClient, ConnectException from hbmqtt.mqtt.constants import QOS_0, QOS_1, QOS_2 import pkg_resources import time @@ -278,7 +279,7 @@ async def _connect_coro(self): #Override default to ignore ssl verification reader = StreamReaderAdapter(conn_reader) writer = StreamWriterAdapter(conn_writer) elif scheme in ('ws', 'wss'): - websocket = await websockets.connect( + websocket = await websockets.connect( self.session.broker_uri, subprotocols=['mqtt'], loop=self._loop, @@ -322,14 +323,18 @@ async def get_msg(self): msgdata = str(message.data.decode("utf-8")) proxymodelog.info(f"MQTT Proxy Client - Message Received From Ecovacs - Topic: {message.topic} - Message: {msgdata}") - ttopic = message.topic.split("/") - self.ecohelpername = ttopic[3] - ttopic[3] = "proxyhelper" - ttopic_comb = "/".join(ttopic) - proxymodelog.info(f"MQTT Proxy Client - Converted Topic From {message.topic} TO {ttopic_comb}") - proxymodelog.info(f"MQTT Proxy Client - Proxy Forward Message to Helperbot - Topic: {ttopic_comb} - Message: {msgdata.encode()}") + topic = message.topic + ttopic = topic.split("/") + if ttopic[1] == "p2p": + self.ecohelpername = ttopic[3] + ttopic[3] = "proxyhelper" + topic = "/".join(ttopic) + proxymodelog.info(f"MQTT Proxy Client - Converted Topic From {message.topic} TO {topic}") + + proxymodelog.info( + f"MQTT Proxy Client - Proxy Forward Message to Robot - Topic: {topic} - Message: {msgdata.encode()}") await bumper.mqtt_helperbot.Client.publish( - ttopic_comb, msgdata.encode(), QOS_0 + topic, msgdata.encode(), QOS_0 ) except Exception as e: From e7d1819857370d6466ce09ebc769ad744a853924 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 2 Jun 2021 12:07:27 +0200 Subject: [PATCH 4/7] mqtt connect on port 443 --- bumper/mqttserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bumper/mqttserver.py b/bumper/mqttserver.py index 1845c0ad..f8e72ce3 100644 --- a/bumper/mqttserver.py +++ b/bumper/mqttserver.py @@ -400,7 +400,7 @@ async def authenticate(self, *args, **kwargs): try: await self.proxyclients[client_id].connect( - f"mqtts://{username}:{password}@{mqtt_server}:8883", + f"mqtts://{username}:{password}@{mqtt_server}:443", ) except Exception as e: mqttserverlog.error(f"MQTT Proxy Mode - Exception connecting with proxy to ecovacs - {e}") From 5583b4a99be87977c94138564df6c9dcab4b3548 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 2 Jun 2021 12:07:39 +0200 Subject: [PATCH 5/7] update Dockerfile --- Dockerfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index dde7ee98..912db451 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,11 +22,15 @@ RUN apk add build-base FROM base -COPY . /bumper - -WORKDIR /bumper +COPY requirements.txt /requirements.txt # install required python packages RUN pip3 install -r requirements.txt +WORKDIR /bumper + +# Copy only required folders instead of all +COPY create_certs/ create_certs/ +COPY bumper/ bumper/ + ENTRYPOINT ["python3", "-m", "bumper"] From 2f61eef73eb2cad863ca2d92c583eada56796707 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 2 Jun 2021 14:00:25 +0200 Subject: [PATCH 6/7] forward to correct recipient --- bumper/mqttserver.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bumper/mqttserver.py b/bumper/mqttserver.py index f8e72ce3..7f6cdbf1 100644 --- a/bumper/mqttserver.py +++ b/bumper/mqttserver.py @@ -3,6 +3,8 @@ import logging import asyncio import os +from typing import Dict + import hbmqtt import websockets from hbmqtt.broker import Broker @@ -227,7 +229,9 @@ def __init__(self, address, **kwargs): mqttserverlog.exception("{}".format(e)) class BumperProxyModeMQTTClient(MQTTClient): - ecohelpername = "" + + eco_helper_names: Dict[str, str] = {} + async def _connect_coro(self): #Override default to ignore ssl verification kwargs = dict() @@ -326,7 +330,7 @@ async def get_msg(self): topic = message.topic ttopic = topic.split("/") if ttopic[1] == "p2p": - self.ecohelpername = ttopic[3] + self.eco_helper_names[ttopic[10]] = ttopic[3] ttopic[3] = "proxyhelper" topic = "/".join(ttopic) proxymodelog.info(f"MQTT Proxy Client - Converted Topic From {message.topic} TO {topic}") @@ -341,7 +345,7 @@ async def get_msg(self): proxymodelog.error(f"MQTT Proxy Client - get_msg Exception - {e}") class BumperMQTTServer_Plugin: - proxyclients = {} + proxyclients: Dict[str, BumperProxyModeMQTTClient] = {} def __init__(self, context): self.context = context try: @@ -519,7 +523,7 @@ async def handle_helperbot_msg(self, client_id, message): if not str(message.topic).split("/")[3] == "proxyhelper": # if from proxyhelper, don't send back to ecovacs...yet if str(message.topic).split("/")[6] == "proxyhelper": ttopic = message.topic.split("/") - ttopic[6] = self.proxyclients[client_id].ecohelpername + ttopic[6] = self.proxyclients[client_id].eco_helper_names.pop(ttopic[10], "") ttopic_join = "/".join(ttopic) proxymodelog.info(f"MQTT Proxy Client - Bot Message Converted Topic From {message.topic} TO {ttopic_join} with message: {msgdata}") else: From a3e9463821ec231372b03c971a789f83ec281336 Mon Sep 17 00:00:00 2001 From: Robert Resch Date: Wed, 2 Jun 2021 14:02:36 +0200 Subject: [PATCH 7/7] fix ip --- bumper/__init__.py | 2 +- bumper/db.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bumper/__init__.py b/bumper/__init__.py index 88f7e5a1..e513ad21 100644 --- a/bumper/__init__.py +++ b/bumper/__init__.py @@ -195,7 +195,7 @@ def strtobool(strbool): logging.getLogger("asyncio").setLevel(logging.CRITICAL + 1) # Ignore this logger - +# iptables -A PREROUTING -t nat -i wlp0s20f3 -p tcp --dport 443 -j REDIRECT --to-port 8883 mqtt_listen_port = 8883 conf1_listen_port = 443 conf2_listen_port = 8007 diff --git a/bumper/db.py b/bumper/db.py index dfa001e7..05940291 100644 --- a/bumper/db.py +++ b/bumper/db.py @@ -50,7 +50,7 @@ def config_proxyMode_defaults(): {"type":"app","host":"eco-us-api.ecovacs.com","ip":"47.89.135.130","match":"eco-"}, {"type":"app","host":"ecovacs.com","ip":"47.90.210.46"}, {"type":"app","host":"ecouser.net","ip":"116.62.93.217"}, - {"type":"mqtt_server","host":"mq-ww.ecouser.net","ip":"47.254.52.46"}, + {"type":"mqtt_server","host":"mq-ww.ecouser.net","ip":"47.254.143.26"}, ] opendb = db_get() with opendb: