From a504d38418a5c1596c4278e6615cc7f5c34b8730 Mon Sep 17 00:00:00 2001 From: mihalin Date: Sun, 23 Jan 2022 00:25:49 +0300 Subject: [PATCH 1/9] handle deactivated error --- server/custom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/custom.py b/server/custom.py index 7a5315a..540014b 100644 --- a/server/custom.py +++ b/server/custom.py @@ -86,7 +86,7 @@ async def message_handler(message: types.Message, *args, **kwargs): try: await message.copy_to(chat_id) - except (exceptions.MessageError, exceptions.BotBlocked): + except (exceptions.MessageError, exceptions.Unauthorized): await message.reply("Невозможно переслать сообщение (автор заблокировал бота?)", parse_mode="HTML") return From 02df39c9fd06059d3d5fe22422a81eaf8df46673 Mon Sep 17 00:00:00 2001 From: mihalin Date: Wed, 16 Feb 2022 18:45:17 +0300 Subject: [PATCH 2/9] redis timeout --- olgram/settings.py | 12 ++++++++++-- server/custom.py | 6 ++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/olgram/settings.py b/olgram/settings.py index 53761f4..1b30b34 100644 --- a/olgram/settings.py +++ b/olgram/settings.py @@ -2,8 +2,10 @@ from dotenv import load_dotenv from abc import ABC import os import logging -from olgram.utils.crypto import Cryptor from functools import lru_cache +from datetime import timedelta +import typing as ty +from olgram.utils.crypto import Cryptor load_dotenv() @@ -84,7 +86,13 @@ class ServerSettings(AbstractSettings): def append_text(cls) -> str: return "\n\nЭтот бот создан с помощью @OlgramBot" - logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) + @classmethod + @lru_cache + def redis_timeout_ms(cls) -> ty.Optional[int]: + return int(timedelta(days=14).total_seconds() * 1000.0) + + +logging.basicConfig(level=os.environ.get("LOGLEVEL", "WARNING")) class BotSettings(AbstractSettings): diff --git a/server/custom.py b/server/custom.py index 540014b..b3dc473 100644 --- a/server/custom.py +++ b/server/custom.py @@ -52,7 +52,8 @@ async def message_handler(message: types.Message, *args, **kwargs): # сообщение нужно переслать в супер-чат new_message = await message.forward(super_chat_id) - await _redis.set(_message_unique_id(bot.pk, new_message.message_id), message.chat.id) + await _redis.set(_message_unique_id(bot.pk, new_message.message_id), message.chat.id, + pexpire=ServerSettings.redis_timeout_ms()) # И отправить пользователю специальный текст, если он указан if bot.second_text: @@ -67,7 +68,8 @@ async def message_handler(message: types.Message, *args, **kwargs): chat_id = message.reply_to_message.forward_from_chat if not chat_id: return SendMessage(chat_id=message.chat.id, - text="Невозможно переслать сообщение: автор не найден", + text="Невозможно переслать сообщение: автор не найден " + "(сообщение слишком старое?)", parse_mode="HTML") chat_id = int(chat_id) From d4582d9a9de3541d25e15bb62ee9f949241dc1ef Mon Sep 17 00:00:00 2001 From: mihalin Date: Wed, 16 Feb 2022 19:56:03 +0300 Subject: [PATCH 3/9] threads first iteration --- olgram/settings.py | 5 +++++ server/custom.py | 21 +++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/olgram/settings.py b/olgram/settings.py index 1b30b34..e713633 100644 --- a/olgram/settings.py +++ b/olgram/settings.py @@ -91,6 +91,11 @@ class ServerSettings(AbstractSettings): def redis_timeout_ms(cls) -> ty.Optional[int]: return int(timedelta(days=14).total_seconds() * 1000.0) + @classmethod + @lru_cache + def thread_timeout_ms(cls) -> int: + return int(timedelta(days=1).total_seconds() * 1000.0) + logging.basicConfig(level=os.environ.get("LOGLEVEL", "WARNING")) diff --git a/server/custom.py b/server/custom.py index b3dc473..bc66721 100644 --- a/server/custom.py +++ b/server/custom.py @@ -30,6 +30,10 @@ def _message_unique_id(bot_id: int, message_id: int) -> str: return f"{bot_id}_{message_id}" +def _thread_uniqie_id(bot_id: int, chat_id: int) -> str: + return f"thread_{bot_id}_{chat_id}" + + async def message_handler(message: types.Message, *args, **kwargs): _logger.info("message handler") bot = db_bot_instance.get() @@ -40,6 +44,7 @@ async def message_handler(message: types.Message, *args, **kwargs): text=bot.start_text + ServerSettings.append_text()) super_chat_id = await bot.super_chat_id() + is_super_group = super_chat_id < 0 if message.chat.id != super_chat_id: # Это обычный чат @@ -50,8 +55,20 @@ async def message_handler(message: types.Message, *args, **kwargs): return SendMessage(chat_id=message.chat.id, text="Вы заблокированы в этом боте") - # сообщение нужно переслать в супер-чат - new_message = await message.forward(super_chat_id) + if is_super_group: + thread_first_message = await _redis.get(_thread_uniqie_id(bot.pk, message.chat.id)) + if thread_first_message: + # переслать в супер-чат, отвечая на предыдущее сообщение + new_message = await message.copy_to(super_chat_id, reply_to_message_id=thread_first_message) + else: + # переслать супер-чат + new_message = await message.forward(super_chat_id) + await _redis.set(_thread_uniqie_id(bot.pk, message.chat.id), new_message.message_id, + pexpire=ServerSettings.thread_timeout_ms()) + else: + # сообщение нужно переслать в супер-чат + new_message = await message.forward(super_chat_id) + await _redis.set(_message_unique_id(bot.pk, new_message.message_id), message.chat.id, pexpire=ServerSettings.redis_timeout_ms()) From 878abc6a0f20bb3576c329ed606f133fec03db4e Mon Sep 17 00:00:00 2001 From: mihalin Date: Wed, 16 Feb 2022 20:52:08 +0300 Subject: [PATCH 4/9] threads first iteration --- server/custom.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/custom.py b/server/custom.py index bc66721..ef4808a 100644 --- a/server/custom.py +++ b/server/custom.py @@ -55,18 +55,18 @@ async def message_handler(message: types.Message, *args, **kwargs): return SendMessage(chat_id=message.chat.id, text="Вы заблокированы в этом боте") + # Пересылаем сообщение в супер-чат if is_super_group: thread_first_message = await _redis.get(_thread_uniqie_id(bot.pk, message.chat.id)) if thread_first_message: # переслать в супер-чат, отвечая на предыдущее сообщение - new_message = await message.copy_to(super_chat_id, reply_to_message_id=thread_first_message) + new_message = await message.copy_to(super_chat_id, reply_to_message_id=int(thread_first_message)) else: # переслать супер-чат new_message = await message.forward(super_chat_id) await _redis.set(_thread_uniqie_id(bot.pk, message.chat.id), new_message.message_id, pexpire=ServerSettings.thread_timeout_ms()) - else: - # сообщение нужно переслать в супер-чат + else: # личные сообщения не поддерживают потоки сообщений: простой forward new_message = await message.forward(super_chat_id) await _redis.set(_message_unique_id(bot.pk, new_message.message_id), message.chat.id, From 36a0bc0f95e79f0a9980de6af5a4ae724b259bac Mon Sep 17 00:00:00 2001 From: mihalin Date: Thu, 17 Feb 2022 02:56:11 +0300 Subject: [PATCH 5/9] =?UTF-8?q?=D0=B5=D1=89=D1=91=20=D0=BD=D0=B5=D0=BC?= =?UTF-8?q?=D0=BD=D0=BE=D0=B3=D0=BE=20=D1=81=D1=82=D0=B0=D1=82=D0=B8=D1=81?= =?UTF-8?q?=D1=82=D0=B8=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- olgram/commands/info.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/olgram/commands/info.py b/olgram/commands/info.py index b73492b..eb1ef34 100644 --- a/olgram/commands/info.py +++ b/olgram/commands/info.py @@ -22,6 +22,8 @@ async def info(message: types.Message, state: FSMContext): bots_count = len(await models.Bot.all()) user_count = len(await models.User.all()) + templates_count = len(await models.DefaultAnswer.all()) await message.answer(f"Количество ботов: {bots_count}\n" - f"Количество пользователей: {user_count}\n") + f"Количество пользователей: {user_count}\n" + f"Шаблонов ответов: {templates_count}\n") From 5cff8da9cd1d49d8987228c1c87f3cc67c80edc9 Mon Sep 17 00:00:00 2001 From: mihalin Date: Fri, 18 Feb 2022 07:51:51 +0300 Subject: [PATCH 6/9] webhook less connections --- server/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/server.py b/server/server.py index 727aa0f..569c435 100644 --- a/server/server.py +++ b/server/server.py @@ -33,7 +33,8 @@ async def register_token(bot: Bot) -> bool: if ServerSettings.use_custom_cert(): certificate = open(ServerSettings.public_path(), 'rb') - res = await a_bot.set_webhook(url_for_bot(bot), certificate=certificate, drop_pending_updates=True) + res = await a_bot.set_webhook(url_for_bot(bot), certificate=certificate, drop_pending_updates=True, + max_connections=10) await a_bot.session.close() del a_bot return res From 6f602f417f0df9192d6680f61d1d344b30252aae Mon Sep 17 00:00:00 2001 From: mihalin Date: Fri, 18 Feb 2022 21:47:40 +0300 Subject: [PATCH 7/9] =?UTF-8?q?=D0=9D=D0=B5=D0=BC=D0=BD=D0=BE=D0=B3=D0=BE?= =?UTF-8?q?=20=D1=81=D1=82=D0=B0=D1=82=D0=B8=D1=81=D1=82=D0=B8=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- olgram/commands/info.py | 10 ++++-- olgram/commands/menu.py | 31 +++++++++++++++++++ .../models/9_20220218211744_update.sql | 6 ++++ olgram/models/models.py | 3 ++ server/custom.py | 7 +++++ 5 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 olgram/migrations/models/9_20220218211744_update.sql diff --git a/olgram/commands/info.py b/olgram/commands/info.py index eb1ef34..11e2600 100644 --- a/olgram/commands/info.py +++ b/olgram/commands/info.py @@ -20,10 +20,16 @@ async def info(message: types.Message, state: FSMContext): await message.answer("Недостаточно прав") return - bots_count = len(await models.Bot.all()) + bots = await models.Bot.all() + bots_count = len(bots) user_count = len(await models.User.all()) templates_count = len(await models.DefaultAnswer.all()) + income_messages = sum([bot.incoming_messages_count for bot in bots]) + outgoing_messages = sum([bot.outgoing_messages_count for bot in bots]) + await message.answer(f"Количество ботов: {bots_count}\n" f"Количество пользователей: {user_count}\n" - f"Шаблонов ответов: {templates_count}\n") + f"Шаблонов ответов: {templates_count}\n" + f"Входящих сообщений у всех ботов: {income_messages}\n" + f"Исходящих сообщений у всех ботов: {outgoing_messages}\n") diff --git a/olgram/commands/menu.py b/olgram/commands/menu.py index 79029cb..fa59a6e 100644 --- a/olgram/commands/menu.py +++ b/olgram/commands/menu.py @@ -107,6 +107,11 @@ async def send_bot_menu(bot: Bot, call: types.CallbackQuery): callback_data=menu_callback.new(level=2, bot_id=bot.id, operation="delete", chat=empty)) ) + keyboard.insert( + types.InlineKeyboardButton(text="Статистика", + callback_data=menu_callback.new(level=2, bot_id=bot.id, operation="stat", + chat=empty)) + ) keyboard.insert( types.InlineKeyboardButton(text="<< Назад", callback_data=menu_callback.new(level=0, bot_id=empty, operation=empty, chat=empty)) @@ -174,6 +179,30 @@ async def send_bot_text_menu(bot: Bot, call: ty.Optional[types.CallbackQuery] = await AioBot.get_current().send_message(chat_id, text, reply_markup=keyboard, parse_mode="HTML") +async def send_bot_statistic_menu(bot: Bot, call: ty.Optional[types.CallbackQuery] = None, + chat_id: ty.Optional[int] = None): + if call: + await call.answer() + keyboard = types.InlineKeyboardMarkup(row_width=2) + keyboard.insert( + types.InlineKeyboardButton(text="<< Назад", + callback_data=menu_callback.new(level=1, bot_id=bot.id, operation=empty, chat=empty)) + ) + + text = dedent(f""" + Статистика по боту @{bot.name} + + Входящих сообщений: {bot.incoming_messages_count} + Ответов: {bot.outgoing_messages_count} + Шаблоны ответов: {len(await bot.answers)} + Забанено пользователей: {len(await bot.banned_users)} + """) + if call: + await edit_or_create(call, text, keyboard, parse_mode="HTML") + else: + await AioBot.get_current().send_message(chat_id, text, reply_markup=keyboard, parse_mode="HTML") + + async def send_bot_second_text_menu(bot: Bot, call: ty.Optional[types.CallbackQuery] = None, chat_id: ty.Optional[int] = None): if call: @@ -329,6 +358,8 @@ async def callback(call: types.CallbackQuery, callback_data: dict, state: FSMCon return await send_chats_menu(bot, call) if operation == "delete": return await send_bot_delete_menu(bot, call) + if operation == "stat": + return await send_bot_statistic_menu(bot, call) if operation == "text": await state.set_state("wait_start_text") async with state.proxy() as proxy: diff --git a/olgram/migrations/models/9_20220218211744_update.sql b/olgram/migrations/models/9_20220218211744_update.sql new file mode 100644 index 0000000..71ca41c --- /dev/null +++ b/olgram/migrations/models/9_20220218211744_update.sql @@ -0,0 +1,6 @@ +-- upgrade -- +ALTER TABLE "bot" ADD "outgoing_messages_count" BIGINT NOT NULL DEFAULT 0; +ALTER TABLE "bot" ADD "incoming_messages_count" BIGINT NOT NULL DEFAULT 0; +-- downgrade -- +ALTER TABLE "bot" DROP COLUMN "outgoing_messages_count"; +ALTER TABLE "bot" DROP COLUMN "incoming_messages_count"; diff --git a/olgram/models/models.py b/olgram/models/models.py index 20f2e6d..18cd4e4 100644 --- a/olgram/models/models.py +++ b/olgram/models/models.py @@ -38,6 +38,9 @@ class Bot(Model): on_delete=fields.relational.CASCADE, null=True) + incoming_messages_count = fields.BigIntField(default=0) + outgoing_messages_count = fields.BigIntField(default=0) + def decrypted_token(self): cryptor = DatabaseSettings.cryptor() return cryptor.decrypt(self.token) diff --git a/server/custom.py b/server/custom.py index ef4808a..f72f5da 100644 --- a/server/custom.py +++ b/server/custom.py @@ -7,6 +7,7 @@ from contextvars import ContextVar from aiohttp.web_exceptions import HTTPNotFound from aioredis.commands import create_redis_pool from aioredis import Redis +from tortoise.expressions import F import logging import typing as ty from olgram.settings import ServerSettings @@ -55,6 +56,9 @@ async def message_handler(message: types.Message, *args, **kwargs): return SendMessage(chat_id=message.chat.id, text="Вы заблокированы в этом боте") + bot.incoming_messages_count = F("incoming_messages_count") + 1 + await bot.save(update_fields=["incoming_messages_count"]) + # Пересылаем сообщение в супер-чат if is_super_group: thread_first_message = await _redis.get(_thread_uniqie_id(bot.pk, message.chat.id)) @@ -109,6 +113,9 @@ async def message_handler(message: types.Message, *args, **kwargs): await message.reply("Невозможно переслать сообщение (автор заблокировал бота?)", parse_mode="HTML") return + + bot.outgoing_messages_count = F("outgoing_messages_count") + 1 + await bot.save(update_fields=["outgoing_messages_count"]) elif super_chat_id > 0: # в супер-чате кто-то пишет сообщение сам себе, только для личных сообщений await message.forward(super_chat_id) From 1768d9e7ea096b421e6b7d9c34a27275eb68ce97 Mon Sep 17 00:00:00 2001 From: mihalin Date: Fri, 18 Feb 2022 21:50:23 +0300 Subject: [PATCH 8/9] =?UTF-8?q?=D0=9D=D0=B5=D0=BC=D0=BD=D0=BE=D0=B3=D0=BE?= =?UTF-8?q?=20=D1=81=D1=82=D0=B0=D1=82=D0=B8=D1=81=D1=82=D0=B8=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- olgram/commands/menu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/olgram/commands/menu.py b/olgram/commands/menu.py index fa59a6e..cedf2be 100644 --- a/olgram/commands/menu.py +++ b/olgram/commands/menu.py @@ -191,7 +191,7 @@ async def send_bot_statistic_menu(bot: Bot, call: ty.Optional[types.CallbackQuer text = dedent(f""" Статистика по боту @{bot.name} - + Входящих сообщений: {bot.incoming_messages_count} Ответов: {bot.outgoing_messages_count} Шаблоны ответов: {len(await bot.answers)} From 4be45985a04171dc14834c1320a6626bf9820f7d Mon Sep 17 00:00:00 2001 From: mihalin Date: Fri, 18 Feb 2022 22:09:37 +0300 Subject: [PATCH 9/9] minor edition --- olgram/commands/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/olgram/commands/info.py b/olgram/commands/info.py index 11e2600..330451c 100644 --- a/olgram/commands/info.py +++ b/olgram/commands/info.py @@ -29,7 +29,7 @@ async def info(message: types.Message, state: FSMContext): outgoing_messages = sum([bot.outgoing_messages_count for bot in bots]) await message.answer(f"Количество ботов: {bots_count}\n" - f"Количество пользователей: {user_count}\n" + f"Количество пользователей (у конструктора): {user_count}\n" f"Шаблонов ответов: {templates_count}\n" f"Входящих сообщений у всех ботов: {income_messages}\n" f"Исходящих сообщений у всех ботов: {outgoing_messages}\n")