diff --git a/admin/.~lock.Menu.xlsx# b/admin/.~lock.Menu.xlsx# new file mode 100644 index 0000000..4cab078 --- /dev/null +++ b/admin/.~lock.Menu.xlsx# @@ -0,0 +1 @@ +,pi3c,pi3code,12.02.2024 22:20,file:///home/pi3c/.config/libreoffice/4; diff --git a/admin/Menu.xlsx b/admin/Menu.xlsx index ac02620..216bc3d 100644 Binary files a/admin/Menu.xlsx and b/admin/Menu.xlsx differ diff --git a/bg_tasks/bg_task.py b/bg_tasks/bg_task.py index 3703ea6..2b86b54 100644 --- a/bg_tasks/bg_task.py +++ b/bg_tasks/bg_task.py @@ -19,7 +19,7 @@ celery_app = Celery( celery_app.conf.beat_schedule = { 'run-task-every-15-seconds': { 'task': 'bg_tasks.bg_task.periodic_task', - 'schedule': 15.0, + 'schedule': 30.0, }, } diff --git a/bg_tasks/parser.py b/bg_tasks/parser.py new file mode 100644 index 0000000..29d616f --- /dev/null +++ b/bg_tasks/parser.py @@ -0,0 +1,94 @@ +import os + +import gspread +import openpyxl + +file = os.path.join(os.path.curdir, 'admin', 'Menu.xlsx') + + +async def gsheets_to_rows() -> list[list[str | int | float]]: + """Получение всех строк из Google Sheets""" + + def to_int(val: str) -> int | str: + try: + res = int(val) + except ValueError: + return val + return res + + def to_float(val: str) -> float | str: + val = val.replace(',', '.') + try: + res = float(val) + except ValueError: + return val + return res + + gc = gspread.service_account(filename='creds.json') + sh = gc.open('Menu') + data = sh.sheet1.get_all_values() + for row in data: + row[:3] = list(map(to_int, row[:3])) + row[-2:] = list(map(to_float, row[-2:])) + + return data + + +async def local_xlsx_to_rows() -> list[list[str | int | float]]: + """Получение всех строк из локального файла Menu""" + data = [] + wb = openpyxl.load_workbook(file).worksheets[0] + for row in wb.iter_rows(values_only=True): + data.append(list(row)) + return data + + +async def rows_to_dict(rows: list[list]) -> tuple: + """Парсит строки полученные и источников в словарь""" + + menus = {} + submenus = {} + dishes = {} + + menu_num = None + submenu_num = None + + for row in rows: + if all(row[:3]): + menu = { + row[0]: { + 'data': {'title': row[1], 'description': row[2]}, + 'id': None, + } + } + menu_num = row[0] + menus.update(menu) + + elif all(row[1:4]): + submenu = { + (menu_num, row[1]): { + 'data': {'title': row[2], 'description': row[3]}, + 'parent_num': menu_num, + 'id': None, + 'parent_menu': None, + } + } + submenu_num = row[1] + submenus.update(submenu) + + elif all(row[3:6]): + dish = { + (menu_num, submenu_num, row[2]): { + 'data': { + 'title': row[3], + 'description': row[4], + 'price': row[5], + }, + 'parent_num': (menu_num, submenu_num), + 'id': None, + 'parent_submenu': None, + 'discont': row[6], + }, + } + dishes.update(dish) + return menus, submenus, dishes diff --git a/bg_tasks/updater.py b/bg_tasks/updater.py index 68c2f76..01af762 100644 --- a/bg_tasks/updater.py +++ b/bg_tasks/updater.py @@ -1,16 +1,14 @@ import os import pickle -from uuid import UUID -import openpyxl import redis.asyncio as redis # type: ignore -from sqlalchemy import delete +from sqlalchemy import delete, update from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from fastfood.config import settings from fastfood.models import Dish, Menu, SubMenu -file = os.path.join(os.path.curdir, 'admin', 'Menu.xlsx') +from .parser import file, gsheets_to_rows, local_xlsx_to_rows, rows_to_dict redis = redis.Redis.from_url(url=settings.REDIS_URL) @@ -22,16 +20,10 @@ async_session_maker = async_sessionmaker( ) -async def refresh_cache(disconts: dict) -> None: - """Очищает кэш при обновлении БД и ставит отметку времени обновления - и сохраняет данные скидок на товар - """ - await redis.flushall() - - for key in disconts.keys(): - await redis.set(f'DISCONT:{str(key)}', pickle.dumps(disconts[key])) - - await redis.set('XLSX_MOD_TIME', pickle.dumps(os.path.getmtime(file))) +async def clear_cache(pattern: str) -> None: + keys = [key async for key in redis.scan_iter(pattern)] + if keys: + await redis.delete(*keys) async def is_changed_xls() -> bool: @@ -51,96 +43,253 @@ async def is_changed_xls() -> bool: return True -async def xlsx_to_dict() -> dict: - """Парсит Menu.xlsx в словарь""" - wb = openpyxl.load_workbook(file).worksheets[0] +async def on_menu_change( + new_menu: dict, old_menu: dict, session: AsyncSession +) -> dict | None: + if new_menu and not old_menu: + # Создаем меню + menu = Menu( + title=new_menu['data']['title'], + description=new_menu['data']['description'], + ) + session.add(menu) + await session.flush() + new_menu['id'] = str(menu.id) + elif new_menu and old_menu: + # Обновляем меню + await session.execute( + update(Menu).where(Menu.id == old_menu['id']).values(**(new_menu['data'])) + ) + new_menu['id'] = old_menu['id'] - data = {} + else: + # Удаляем меню + await session.execute(delete(Menu).where(Menu.id == old_menu['id'])) - menu = None - submenu = None - dish = None + await session.commit() + # Чистим кэш + await clear_cache('MENUS*') + await clear_cache('summary') - for row in wb.iter_rows(values_only=True): - if row[0] is not None: - menu = row[0] - data[menu] = { - 'id': None, - 'title': row[1], - 'description': row[2], - 'submenus': dict(), - } - elif row[1] is not None: - submenu = row[1] - data[menu]['submenus'][submenu] = { - 'id': None, - 'title': row[2], - 'description': row[3], - 'dishes': dict(), - } - elif row[2] is not None: - dish = row[2] - data[menu]['submenus'][submenu]['dishes'][dish] = { - 'id': None, - 'title': row[3], - 'description': row[4], - 'price': row[5], - 'discont': row[6], - } - return data + return new_menu -async def refresh_all_data(data: dict) -> dict[UUID, int | float]: - """Удаляет старые данные и сохраняет новые. - Создает и возвращает список со скидками с привязкой по UUID товара +async def menus_updater(menus: dict, session: AsyncSession) -> None: + """Проверяет пункты меню на изменения + При необходимости запускае обновление БД + через фенкцию on_menu_change """ + cached_menus = await redis.get('ALL_MENUS') - disconts = {} + if cached_menus is not None: + cached_menus = pickle.loads(cached_menus) + else: + cached_menus = {} - async with async_session_maker() as session: - await session.execute(delete(Menu)) - await session.commit() + for key in menus.keys(): + if key not in cached_menus.keys(): + # Создание меню + menu = await on_menu_change(menus[key], {}, session) + menus[key] = menu + elif key in cached_menus.keys(): + # Обновление меню + if menus[key].get('data') != cached_menus[key].get('data'): + menu = await on_menu_change(menus[key], cached_menus[key], session) + menus[key] = menu + else: + menus[key]['id'] = cached_menus[key]['id'] - for menu_key in data.keys(): - menu = Menu( - title=data[menu_key].get('title'), - description=data[menu_key].get('description'), - ) - session.add(menu) - await session.flush() + for key in {k: cached_menus[k] for k in set(cached_menus) - set(menus)}: + # Проверяем на удаленные меню + await on_menu_change({}, cached_menus.pop(key), session) - submenus = data[menu_key]['submenus'] - for sub_key in submenus.keys(): - submenu = SubMenu( - title=submenus[sub_key]['title'], - description=submenus[sub_key]['description'], - parent_menu=menu.id, + await redis.set('ALL_MENUS', pickle.dumps(menus)) + + +async def on_submenu_change( + new_sub: dict, old_sub: dict, session: AsyncSession +) -> dict: + if new_sub and not old_sub: + # Создаем меню + submenu = SubMenu( + title=new_sub['data']['title'], + description=new_sub['data']['description'], + ) + submenu.parent_menu = new_sub['parent_menu'] + + session.add(submenu) + await session.flush() + new_sub['id'] = str(submenu.id) + new_sub['parent_menu'] = str(submenu.parent_menu) + elif new_sub and old_sub: + # Обновляем меню + await session.execute( + update(SubMenu) + .where(SubMenu.id == old_sub['id']) + .values(**(new_sub['data'])) + ) + new_sub['id'] = old_sub['id'] + new_sub['parent_menu'] = old_sub['parent_menu'] + + else: + # Удаляем меню + await session.execute(delete(SubMenu).where(SubMenu.id == old_sub['id'])) + + await clear_cache('MENUS*') + await clear_cache('summary') + + await session.commit() + return new_sub + + +async def submenus_updater(submenus: dict, session: AsyncSession) -> None: + """Проверяет пункты подменю на изменения + При необходимости запускае обновление БД + """ + # Получаем Меню из кэша для получения их ID по померу в таблице + cached_menus = await redis.get('ALL_MENUS') + if cached_menus is not None: + cached_menus = pickle.loads(cached_menus) + else: + cached_menus = {} + + # Получаем подмен из кэша + cached_sub = await redis.get('ALL_SUBMENUS') + + if cached_sub is not None: + cached_sub = pickle.loads(cached_sub) + else: + cached_sub = {} + + for key in submenus.keys(): + parent = cached_menus[submenus[key]['parent_num']]['id'] + submenus[key]['parent_menu'] = parent + + if key not in cached_sub.keys(): + # Получаем и ставим UUID parent_menu + submenus[key]['parent_menu'] = parent + + submenu = await on_submenu_change(submenus[key], {}, session) + submenus[key] = submenu + elif key in cached_sub.keys(): + # Обновление меню + if submenus[key].get('data') != cached_sub[key].get('data'): + submenu = await on_submenu_change( + submenus[key], cached_sub[key], session ) - session.add(submenu) - await session.flush() + submenus[key] = submenu + else: + submenus[key]['id'] = cached_sub[key]['id'] + submenus[key]['parent_menu'] = cached_sub[key]['parent_menu'] - dishes = data[menu_key]['submenus'][sub_key]['dishes'] - print(dishes) - for dish_key in dishes.keys(): - dish = Dish( - title=dishes[dish_key]['title'], - description=dishes[dish_key]['description'], - price=dishes[dish_key]['price'], - parent_submenu=submenu.id, - ) - session.add(dish) - await session.flush() - if dishes[dish_key]['discont'] is not None: - disconts[dish.id] = dishes[dish_key]['discont'] + for key in {k: cached_sub[k] for k in set(cached_sub) - set(submenus)}: + # Проверяем на удаленные меню + await on_submenu_change({}, cached_sub.pop(key), session) - await session.commit() - return disconts + await redis.set('ALL_SUBMENUS', pickle.dumps(submenus)) + + +async def on_dish_change(new_dish: dict, old_dish, session: AsyncSession) -> dict: + if new_dish and not old_dish: + dish = Dish( + title=new_dish['data']['title'], + description=new_dish['data']['description'], + price=new_dish['data']['price'], + ) + dish.parent_submenu = new_dish['parent_submenu'] + + session.add(dish) + await session.flush() + new_dish['id'] = str(dish.id) + new_dish['parent_submenu'] = str(dish.parent_submenu) + new_dish['data']['price'] = str(dish.price) + elif new_dish and old_dish: + # Обновляем меню + await session.execute( + update(Dish).where(Dish.id == old_dish['id']).values(**(new_dish['data'])) + ) + new_dish['id'] = old_dish['id'] + new_dish['parent_submenu'] = old_dish['parent_submenu'] + new_dish['data']['price'] = old_dish['data']['price'] + + else: + # Удаляем меню + await session.execute(delete(Dish).where(Dish.id == old_dish['id'])) + + await clear_cache('MENUS*') + await clear_cache('summary') + + await session.commit() + return new_dish + + +async def dishes_updater(dishes: dict, session: AsyncSession) -> None: + """Проверяет блюда на изменения + При необходимости запускае обновление БД + """ + cached_submenus = await redis.get('ALL_SUBMENUS') + if cached_submenus is not None: + cached_submenus = pickle.loads(cached_submenus) + else: + cached_submenus = {} + + # Получаем подмен из кэша + cached_dishes = await redis.get('ALL_DISHES') + + if cached_dishes is not None: + cached_dishes = pickle.loads(cached_dishes) + else: + cached_dishes = {} + + await clear_cache('DISCONT*') + + for key in {k: cached_dishes[k] for k in set(cached_dishes) - set(dishes)}: + # Проверяем на удаленные меню + await on_submenu_change({}, cached_dishes.pop(key), session) + + for key in dishes.keys(): + parent = cached_submenus[dishes[key]['parent_num']]['id'] + dishes[key]['parent_submenu'] = parent + + if key not in cached_dishes.keys(): + # Получаем и ставим UUID parent_menu + dishes[key]['parent_submenu'] = parent + + dish = await on_dish_change(dishes[key], {}, session) + dishes[key] = dish + elif key in cached_dishes.keys(): + # Обновление меню + if dishes[key].get('data') != cached_dishes[key].get('data'): + dish = await on_dish_change(dishes[key], cached_dishes[key], session) + dishes[key] = dish + else: + dishes[key]['id'] = cached_dishes[key]['id'] + dishes[key]['parent_submenu'] = cached_dishes[key]['parent_submenu'] + + if dishes[key]['discont'] is not None: + await redis.set( + f"DISCONT:{dishes[key]['id']}", pickle.dumps(dishes[key]['discont']) + ) + + await redis.set('ALL_DISHES', pickle.dumps(dishes)) + + +async def updater(rows): + menus, submenus, dishes = await rows_to_dict(rows) + async with async_session_maker() as session: + await menus_updater(menus, session) + await submenus_updater(submenus, session) + await dishes_updater(dishes, session) async def main() -> None: """Главная функция фоновой задачи""" changed = await is_changed_xls() if changed: - menu_data = await xlsx_to_dict() - discont_data = await refresh_all_data(menu_data) - await refresh_cache(discont_data) + rows = await local_xlsx_to_rows() + await updater(rows) + + +async def main_gsheets() -> None: + rows = await gsheets_to_rows() + await updater(rows) diff --git a/fastfood/repository/redis.py b/fastfood/repository/redis.py index 80f0e35..81ff28d 100644 --- a/fastfood/repository/redis.py +++ b/fastfood/repository/redis.py @@ -12,15 +12,15 @@ def get_key(level: str, **kwargs) -> str: case 'menus': return 'MENUS' case 'menu': - return f"{kwargs.get('menu_id')}" + return f"MENUS:{kwargs.get('menu_id')}" case 'submenus': - return f"{kwargs.get('menu_id')}:SUBMENUS" + return f"MENUS:{kwargs.get('menu_id')}:SUBMENUS" case 'submenu': - return f"{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}" + return f"MENUS:{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}" case 'dishes': - return f"{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:DISHES" + return f"MENUS:{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:DISHES" case 'dish': - return f"{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:{kwargs.get('dish_id')}" + return f"MENUS:{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:{kwargs.get('dish_id')}" return 'summary'