Merge pull request #435 from ton-blockchain/dev

Dev

Author: igroman787

modules/__init__.py

@@ -59,8 +59,6 @@ class Setting:
     'defaultCustomOverlaysUrl': Setting(None, 'https://ton-blockchain.github.io/fallback_custom_overlays.json', 'Default custom overlays config url'),
     'debug': Setting(None, False, 'Debug mtc console mode. Prints Traceback on errors'),
     'subscribe_tg_channel': Setting('validator', False, 'Disables warning about subscribing to the `TON STATUS` channel'),
-    'BotToken': Setting('alert-bot', None, 'Alerting Telegram bot token'),
-    'ChatId': Setting('alert-bot', None, 'Alerting Telegram chat id'),
     'auto_backup': Setting('validator', None, 'Make validator backup every election'),
     'auto_backup_path': Setting('validator', '/tmp/mytoncore/auto_backups/', 'Path to store auto-backups'),
     'prometheus_url': Setting('prometheus', None, 'Prometheus pushgateway url'),

modules/alert_bot.py

@@ -11,6 +11,7 @@
 @dataclasses.dataclass
 class Alert:
     severity: str
+    description: str
     text: str
     timeout: int
 
@@ -29,75 +30,89 @@ def init_alerts():
     ALERTS = {
         "low_wallet_balance": Alert(
             "low",
-            "Validator wallet {wallet} balance is low: {balance} TON.",
+            "Validator's wallet balance is less than 10 TON",
+            "Validator's wallet <code>{wallet}</code> balance is less than 10 TON: {balance} TON.",
             18 * HOUR
         ),
         "db_usage_80": Alert(
             "high",
+            "Node's db usage is more than 80%",
             """TON DB usage > 80%. Clean the TON database: 
             https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming 
             or (and) set node\'s archive ttl to lower value.""",
             24 * HOUR
         ),
         "db_usage_95": Alert(
             "critical",
+            "Node's db usage is more than 95%",
             """TON DB usage > 95%. Disk is almost full, clean the TON database immediately: 
             https://docs.ton.org/participate/nodes/node-maintenance-and-security#database-grooming 
             or (and) set node\'s archive ttl to lower value.""",
             6 * HOUR
         ),
         "low_efficiency": Alert(
             "high",
-            """Validator efficiency is low: {efficiency}%.""",
+            "Validator had efficiency less than 90% in the validation round",
+            """Validator efficiency is less than 90%: <b>{efficiency}%</b>.""",
             VALIDATION_PERIOD // 3
         ),
         "out_of_sync": Alert(
             "critical",
-            "Node is out of sync on {sync} sec.",
+            "Node is out of sync on more than 20 sec",
+            "Node is out of sync on more than 20 sec: <b>{sync} sec</b>.",
             300
         ),
         "service_down": Alert(
             "critical",
+            "Node is not running (service is down)",
             "validator.service is down.",
             300
         ),
         "adnl_connection_failed": Alert(
             "high",
+            "Node is not answering to ADNL connection",
             "ADNL connection to node failed",
             3 * HOUR
         ),
         "zero_block_created": Alert(
             "critical",
+            f"Validator has not created any blocks in the {int(VALIDATION_PERIOD // 3 // 3600)} hours",
             "Validator has not created any blocks in the last {hours} hours.",
             VALIDATION_PERIOD // 3
         ),
         "validator_slashed": Alert(
             "high",
+            "Validator has been slashed in the previous validation round",
             "Validator has been slashed in previous round for {amount} TON",
             FREEZE_PERIOD
         ),
         "stake_not_accepted": Alert(
             "high",
             "Validator's stake has not been accepted",
+            "Validator's stake has not been accepted",
             ELECTIONS_START_BEFORE
         ),
         "stake_accepted": Alert(
             "info",
+            "Validator's stake has been accepted (info alert with no sound)",
             "Validator's stake {stake} TON has been accepted",
             ELECTIONS_START_BEFORE
         ),
         "stake_returned": Alert(
             "info",
-            "Validator's stake {stake} TON has been returned on address {address}. The reward amount is {reward} TON.",
+            "Validator's stake has been returned (info alert with no sound)",
+            "Validator's stake {stake} TON has been returned on address <code>{address}</code>. The reward amount is {reward} TON.",
             60
         ),
         "stake_not_returned": Alert(
             "high",
-            "Validator's stake has not been returned on address {address}.",
+            "Validator's stake has not been returned",
+            "Validator's stake has not been returned on address <code>{address}.</code>",
             60
         ),
         "voting": Alert(
             "high",
+            "There is an active network proposal that has many votes (more than 50% of required) but is not voted by the validator",
             "Found proposals with hashes `{hashes}` that have significant amount of votes, but current validator didn't vote for them. Please check @tonstatus for more details.",
             VALIDATION_PERIOD
         ),
@@ -115,18 +130,19 @@ def __init__(self, ton, local, *args, **kwargs):
         self.inited = False
         self.hostname = None
         self.ip = None
+        self.adnl = None
         self.token = None
         self.chat_id = None
         self.last_db_check = 0
 
-    def send_message(self, text: str, silent: bool = False):
+    def send_message(self, text: str, silent: bool = False, disable_web_page_preview: bool = False):
         if self.token is None:
             raise Exception("send_message error: token is not initialized")
         if self.chat_id is None:
             raise Exception("send_message error: chat_id is not initialized")
         request_url = f"https://api.telegram.org/bot{self.token}/sendMessage"
-        data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML', 'disable_notification': silent}
-        response = requests.post(request_url, data=data, timeout=3)
+        data = {'chat_id': self.chat_id, 'text': text, 'parse_mode': 'HTML', 'disable_notification': silent, 'link_preview_options': {'is_disabled': disable_web_page_preview}}
+        response = requests.post(request_url, json=data, timeout=3)
         if response.status_code != 200:
             raise Exception(f"send_message error: {response.text}")
         response = response.json()
@@ -141,16 +157,23 @@ def send_alert(self, alert_name: str, *args, **kwargs):
         alert = ALERTS.get(alert_name)
         if alert is None:
             raise Exception(f"Alert {alert_name} not found")
-        text = f'''
-❗️ <b>MyTonCtrl Alert {alert_name}</b> ❗️
+        alert_name_readable = alert_name.replace('_', ' ').title()
+        text = '🆘' if alert.severity != 'info' else ''
+        text += f''' <b>Node {self.hostname}: {alert_name_readable} </b> 
+
+{alert.text.format(*args, **kwargs)}
 
 Hostname: <code>{self.hostname}</code>
 Node IP: <code>{self.ip}</code>
+ADNL: <code>{self.adnl}</code>'''
+
+        if self.ton.using_validator():
+            text += f"\nWallet: <code>{self.wallet}</code>"
+
+        text += f'''
 Time: <code>{time_}</code> (<code>{int(time.time())}</code>)
+Alert name: <code>{alert_name}</code>
 Severity: <code>{alert.severity}</code>
-
-Alert text:
-<blockquote> {alert.text.format(*args, **kwargs)} </blockquote>
 '''
         if time.time() - last_sent > alert.timeout:
             self.send_message(text, alert.severity == "info")  # send info alerts without sound
@@ -174,6 +197,9 @@ def init(self):
         from modules.validator import ValidatorModule
         self.validator_module = ValidatorModule(self.ton, self.local)
         self.hostname = get_hostname()
+        adnl = self.ton.GetAdnlAddr()
+        self.adnl = adnl
+        self.wallet = self.ton.GetValidatorWallet().addrB64
         self.ip = self.ton.get_node_ip()
         self.set_global_vars()
         init_alerts()
@@ -230,6 +256,41 @@ def test_alert(self, args):
             self.init()
         self.send_message('Test alert')
 
+    def setup_alert_bot(self, args):
+        if len(args) != 2:
+            raise Exception("Usage: setup_alert_bot <bot_token> <chat_id>")
+        self.token = args[0]
+        self.chat_id = args[1]
+        init_alerts()
+        try:
+            self.send_welcome_message()
+            self.ton.local.db['BotToken'] = args[0]
+            self.ton.local.db['ChatId'] = args[1]
+            color_print("setup_alert_bot - {green}OK{endc}")
+        except Exception as e:
+            self.local.add_log(f"Error while sending welcome message: {e}", "error")
+            self.local.add_log(f"If you want the bot to write to a multi-person chat group, make sure the bot is added to that chat group. If it is not - do it and run the command `setup_alert_bot <bot_token> <chat_id>` again.", "info")
+            color_print("setup_alert_bot - {red}Error{endc}")
+
+    def send_welcome_message(self):
+        message = f"""
+This is alert bot. You have connected validator with ADNL <code>{self.ton.GetAdnlAddr()}</code>.
+
+I don't process any commands, I only send notifications. 
+
+Current notifications enabled:
+
+"""
+        for alert in ALERTS.values():
+            message += f"- {alert.description}\n"
+
+        message += """
+If you want, you can disable some notifications in mytonctrl by the <a href="https://docs.ton.org/v3/guidelines/nodes/maintenance-guidelines/mytonctrl-private-alerting#endisbling-alerts"> instruction</a>.
+
+Full bot documentation <a href="https://docs.ton.org/v3/guidelines/nodes/maintenance-guidelines/mytonctrl-private-alerting">here</a>.
+"""
+        self.send_message(text=message, disable_web_page_preview=True)
+
     def check_db_usage(self):
         if time.time() - self.last_db_check < 600:
             return
@@ -303,6 +364,7 @@ def check_adnl_connection_failed(self):
         utils_module = UtilitiesModule(self.ton, self.local)
         ok, error = utils_module.check_adnl_connection()
         if not ok:
+            self.local.add_log(error, "warning")
             self.send_alert("adnl_connection_failed")
 
     def get_myself_from_election(self, config: dict):
@@ -371,7 +433,8 @@ def check_voting(self):
     def check_status(self):
         if not self.ton.using_alert_bot():
             return
-        if not self.inited:
+
+        if not self.inited or self.token != self.ton.local.db.get("BotToken") or self.chat_id != self.ton.local.db.get("ChatId"):
             self.init()
 
         self.local.try_function(self.check_db_usage)
@@ -391,3 +454,4 @@ def add_console_commands(self, console):
         console.AddItem("disable_alert", self.disable_alert, self.local.translate("disable_alert_cmd"))
         console.AddItem("list_alerts", self.print_alerts, self.local.translate("list_alerts_cmd"))
         console.AddItem("test_alert", self.test_alert, self.local.translate("test_alert_cmd"))
+        console.AddItem("setup_alert_bot", self.setup_alert_bot, self.local.translate("setup_alert_bot_cmd"))

modules/backups.py

@@ -56,8 +56,8 @@ def run_restore_backup(args):
         return run_as_root(["bash", restore_script_path] + args)
 
     def restore_backup(self, args):
-        if len(args) == 0 or len(args) > 2:
-            color_print("{red}Bad args. Usage:{endc} restore_backup <filename> [-y]")
+        if len(args) == 0 or len(args) > 3:
+            color_print("{red}Bad args. Usage:{endc} restore_backup <filename> [-y] [--skip-create-backup]")
             return
         if '-y' not in args:
             res = input(
@@ -67,11 +67,14 @@ def restore_backup(self, args):
                 return
         else:
             args.pop(args.index('-y'))
-        print('Before proceeding, mtc will create a backup of current configuration.')
-        try:
-            self.create_backup([])
-        except:
-            color_print("{red}Could not create backup{endc}")
+        if '--skip-create-backup' in args:
+            args.pop(args.index('--skip-create-backup'))
+        else:
+            print('Before proceeding, mtc will create a backup of current configuration.')
+            try:
+                self.create_backup([])
+            except:
+                color_print("{red}Could not create backup{endc}")
 
         ip = str(ip2int(get_own_ip()))
         command_args = ["-m", self.ton.local.buffer.my_work_dir, "-n", args[0], "-i", ip]

modules/utilities.py

@@ -355,15 +355,15 @@ def check_adnl_connection(self):
                 response = requests.post(url, json=data, timeout=5).json()
             except Exception as e:
                 ok = False
-                error = f'{{red}}Failed to check ADNL connection to local node: {type(e)}: {e}{{endc}}'
+                error = f'Failed to check ADNL connection to local node: {type(e)}: {e}'
                 continue
             result = response.get("ok")
             if result:
                 ok = True
                 break
             if not result:
                 ok = False
-                error = f'{{red}}Failed to check ADNL connection to local node: {response.get("message")}{{endc}}'
+                error = f'Failed to check ADNL connection to local node: {response.get("message")}'
         return ok, error
 
     def get_pool_data(self, args):

mytoncore/mytoncore.py

@@ -117,12 +117,24 @@ def CheckConfigFile(self, fift, liteClient):
 				subprocess.run(args)
 				self.dbFile = mconfig_path
 				self.Refresh()
-		elif os.path.isfile(backup_path) == False:
-			self.local.add_log("Create backup config file", "info")
-			args = ["cp", mconfig_path, backup_path]
-			subprocess.run(args)
+		elif not os.path.isfile(backup_path) or time.time() - os.path.getmtime(backup_path) > 3600:
+			self.local.try_function(self.create_self_db_backup)
 	#end define
 
+	def create_self_db_backup(self):
+		self.local.add_log("Create backup config file", "info")
+		mconfig_path = self.local.buffer.db_path
+		backup_path = mconfig_path + ".backup"
+		backup_tmp_path = backup_path + '.tmp'
+		subprocess.run(["cp", mconfig_path, backup_tmp_path])
+		try:
+			with open(backup_tmp_path, "r") as file:
+				json.load(file)
+			os.rename(backup_tmp_path, backup_path)  # atomic opetation
+		except:
+			self.local.add_log("Could not update backup, backup_tmp file is broken", "warning")
+			os.remove(backup_tmp_path)
+
 	def GetVarFromWorkerOutput(self, text, search):
 		if ':' not in search:
 			search += ':'
@@ -3037,6 +3049,7 @@ def SetSettings(self, name, data):
 		except: pass
 		self.local.db[name] = data
 		self.local.save()
+		self.create_self_db_backup()
 	#end define
 
 	def migrate_to_modes(self):

mytonctrl/mytonctrl.py

@@ -488,6 +488,7 @@ def check_adnl(local, ton):
 	utils_module = UtilitiesModule(ton, local)
 	ok, error = utils_module.check_adnl_connection()
 	if not ok:
+		error = "{red}" + error + "{endc}"
 		print_warning(local, error)
 #end define
 
@@ -728,7 +729,9 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency,
 	validatorStatus_color = GetColorStatus(validatorStatus_bool)
 	mytoncoreStatus_text = local.translate("local_status_mytoncore_status").format(mytoncoreStatus_color, mytoncoreUptime_text)
 	validatorStatus_text = local.translate("local_status_validator_status").format(validatorStatus_color, validatorUptime_text)
-	validator_out_of_sync_text = local.translate("local_status_validator_out_of_sync").format(GetColorInt(validator_status.out_of_sync, 20, logic="less", ending=" s"))
+	validator_out_of_sync_text = local.translate("local_status_validator_out_of_sync").format(GetColorInt(validator_status.out_of_sync, 20, logic="less"))
+	master_out_of_sync_text = local.translate("local_status_master_out_of_sync").format(GetColorInt(validator_status.masterchain_out_of_sync, 20, logic="less", ending=" sec"))
+	shard_out_of_sync_text = local.translate("local_status_shard_out_of_sync").format(GetColorInt(validator_status.shardchain_out_of_sync, 5, logic="less", ending=" blocks"))
 
 	validator_out_of_ser_text = local.translate("local_status_validator_out_of_ser").format(f'{validator_status.out_of_ser} blocks ago')
 
@@ -776,6 +779,8 @@ def PrintLocalStatus(local, ton, adnlAddr, validatorIndex, validatorEfficiency,
 	if not is_node_remote:
 		print(validatorStatus_text)
 	print(validator_out_of_sync_text)
+	print(master_out_of_sync_text)
+	print(shard_out_of_sync_text)
 	print(validator_out_of_ser_text)
 	print(dbStatus_text)
 	print(mtcVersion_text)
@@ -879,7 +884,7 @@ def GetSettings(ton, args):
 	print(json.dumps(result, indent=2))
 #end define
 
-def SetSettings(ton, args):
+def SetSettings(local, ton, args):
 	try:
 		name = args[0]
 		value = args[1]

mytonctrl/resources/translate.json

@@ -324,6 +324,16 @@
 		"ru": "Рассинхронизация локального валидатора: {0}",
 		"zh_TW": "本地驗證者不同步: {0}"
 	},
+	"local_status_master_out_of_sync": {
+		"en": "Masterchain out of sync: {0}",
+		"ru": "Рассинхронизация Мастерчейна локального валидатора: {0}",
+		"zh_TW": "主鏈不同步: {0}"
+	},
+	"local_status_shard_out_of_sync": {
+		"en": "Shardchain out of sync: {0}",
+		"ru": "Рассинхронизация Шардчейна локального валидатора: {0}",
+		"zh_TW": "分片鏈不同步: {0}"
+	},
 	"local_status_validator_out_of_ser": {
 		"en": "Local validator last state serialization: {0}",
 		"ru": "Серализация стейта локального валидатора была: {0}",
@@ -489,6 +499,11 @@
 		"ru": "Отправить тестовое оповещение через Telegram Bot",
 		"zh_TW": "通過 Telegram Bot 發送測試警報"
 	},
+	"setup_alert_bot_cmd": {
+		"en": "Setup Telegram Bot for alerts",
+		"ru": "Настроить Telegram Bot для оповещений",
+		"zh_TW": "設置 Telegram Bot 以接收警報"
+	},
 	"benchmark_cmd": {
 		"en": "Run benchmark",
 		"ru": "Запустить бенчмарк",