upd фоновая задача теперь не дропает базу

admin/.~lock.Menu.xlsx#

@@ -0,0 +1 @@
+,pi3c,pi3code,12.02.2024 22:20,file:///home/pi3c/.config/libreoffice/4;

bg_tasks/bg_task.py

@@ -19,7 +19,7 @@ celery_app = Celery(
 celery_app.conf.beat_schedule = {
     'run-task-every-15-seconds': {
         'task': 'bg_tasks.bg_task.periodic_task',
-        'schedule': 15.0,
+        'schedule': 30.0,
     },
 }
 

bg_tasks/parser.py

@@ -0,0 +1,94 @@
+import os
+
+import gspread
+import openpyxl
+
+file = os.path.join(os.path.curdir, 'admin', 'Menu.xlsx')
+
+
+async def gsheets_to_rows() -> list[list[str | int | float]]:
+    """Получение всех строк из Google Sheets"""
+
+    def to_int(val: str) -> int | str:
+        try:
+            res = int(val)
+        except ValueError:
+            return val
+        return res
+
+    def to_float(val: str) -> float | str:
+        val = val.replace(',', '.')
+        try:
+            res = float(val)
+        except ValueError:
+            return val
+        return res
+
+    gc = gspread.service_account(filename='creds.json')
+    sh = gc.open('Menu')
+    data = sh.sheet1.get_all_values()
+    for row in data:
+        row[:3] = list(map(to_int, row[:3]))
+        row[-2:] = list(map(to_float, row[-2:]))
+
+    return data
+
+
+async def local_xlsx_to_rows() -> list[list[str | int | float]]:
+    """Получение всех строк из локального файла Menu"""
+    data = []
+    wb = openpyxl.load_workbook(file).worksheets[0]
+    for row in wb.iter_rows(values_only=True):
+        data.append(list(row))
+    return data
+
+
+async def rows_to_dict(rows: list[list]) -> tuple:
+    """Парсит строки полученные и источников в словарь"""
+
+    menus = {}
+    submenus = {}
+    dishes = {}
+
+    menu_num = None
+    submenu_num = None
+
+    for row in rows:
+        if all(row[:3]):
+            menu = {
+                row[0]: {
+                    'data': {'title': row[1], 'description': row[2]},
+                    'id': None,
+                }
+            }
+            menu_num = row[0]
+            menus.update(menu)
+
+        elif all(row[1:4]):
+            submenu = {
+                (menu_num, row[1]): {
+                    'data': {'title': row[2], 'description': row[3]},
+                    'parent_num': menu_num,
+                    'id': None,
+                    'parent_menu': None,
+                }
+            }
+            submenu_num = row[1]
+            submenus.update(submenu)
+
+        elif all(row[3:6]):
+            dish = {
+                (menu_num, submenu_num, row[2]): {
+                    'data': {
+                        'title': row[3],
+                        'description': row[4],
+                        'price': row[5],
+                    },
+                    'parent_num': (menu_num, submenu_num),
+                    'id': None,
+                    'parent_submenu': None,
+                    'discont': row[6],
+                },
+            }
+            dishes.update(dish)
+    return menus, submenus, dishes

bg_tasks/updater.py

@@ -1,16 +1,14 @@
 import os
 import pickle
-from uuid import UUID
 
-import openpyxl
 import redis.asyncio as redis  # type: ignore
-from sqlalchemy import delete
+from sqlalchemy import delete, update
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 
 from fastfood.config import settings
 from fastfood.models import Dish, Menu, SubMenu
 
-file = os.path.join(os.path.curdir, 'admin', 'Menu.xlsx')
+from .parser import file, gsheets_to_rows, local_xlsx_to_rows, rows_to_dict
 
 redis = redis.Redis.from_url(url=settings.REDIS_URL)
 
@@ -22,16 +20,10 @@ async_session_maker = async_sessionmaker(
 )
 
 
-async def refresh_cache(disconts: dict) -> None:
+async def clear_cache(pattern: str) -> None:
-    """Очищает кэш при обновлении БД и ставит отметку времени обновления
+    keys = [key async for key in redis.scan_iter(pattern)]
-    и сохраняет данные скидок на товар
+    if keys:
-    """
+        await redis.delete(*keys)
-    await redis.flushall()
-
-    for key in disconts.keys():
-        await redis.set(f'DISCONT:{str(key)}', pickle.dumps(disconts[key]))
-
-    await redis.set('XLSX_MOD_TIME', pickle.dumps(os.path.getmtime(file)))
 
 
 async def is_changed_xls() -> bool:
@@ -51,96 +43,253 @@ async def is_changed_xls() -> bool:
     return True
 
 
-async def xlsx_to_dict() -> dict:
+async def on_menu_change(
-    """Парсит Menu.xlsx в словарь"""
+    new_menu: dict, old_menu: dict, session: AsyncSession
-    wb = openpyxl.load_workbook(file).worksheets[0]
+) -> dict | None:
-
+    if new_menu and not old_menu:
-    data = {}
+        # Создаем меню
-
-    menu = None
-    submenu = None
-    dish = None
-
-    for row in wb.iter_rows(values_only=True):
-        if row[0] is not None:
-            menu = row[0]
-            data[menu] = {
-                'id': None,
-                'title': row[1],
-                'description': row[2],
-                'submenus': dict(),
-            }
-        elif row[1] is not None:
-            submenu = row[1]
-            data[menu]['submenus'][submenu] = {
-                'id': None,
-                'title': row[2],
-                'description': row[3],
-                'dishes': dict(),
-            }
-        elif row[2] is not None:
-            dish = row[2]
-            data[menu]['submenus'][submenu]['dishes'][dish] = {
-                'id': None,
-                'title': row[3],
-                'description': row[4],
-                'price': row[5],
-                'discont': row[6],
-            }
-    return data
-
-
-async def refresh_all_data(data: dict) -> dict[UUID, int | float]:
-    """Удаляет старые данные и сохраняет новые.
-    Создает и возвращает список со скидками с привязкой по UUID товара
-    """
-
-    disconts = {}
-
-    async with async_session_maker() as session:
-        await session.execute(delete(Menu))
-        await session.commit()
-
-        for menu_key in data.keys():
         menu = Menu(
-                title=data[menu_key].get('title'),
+            title=new_menu['data']['title'],
-                description=data[menu_key].get('description'),
+            description=new_menu['data']['description'],
         )
         session.add(menu)
         await session.flush()
-
+        new_menu['id'] = str(menu.id)
-            submenus = data[menu_key]['submenus']
+    elif new_menu and old_menu:
-            for sub_key in submenus.keys():
+        # Обновляем меню
-                submenu = SubMenu(
+        await session.execute(
-                    title=submenus[sub_key]['title'],
+            update(Menu).where(Menu.id == old_menu['id']).values(**(new_menu['data']))
-                    description=submenus[sub_key]['description'],
-                    parent_menu=menu.id,
         )
-                session.add(submenu)
+        new_menu['id'] = old_menu['id']
-                await session.flush()
 
-                dishes = data[menu_key]['submenus'][sub_key]['dishes']
+    else:
-                print(dishes)
+        # Удаляем меню
-                for dish_key in dishes.keys():
+        await session.execute(delete(Menu).where(Menu.id == old_menu['id']))
-                    dish = Dish(
-                        title=dishes[dish_key]['title'],
-                        description=dishes[dish_key]['description'],
-                        price=dishes[dish_key]['price'],
-                        parent_submenu=submenu.id,
-                    )
-                    session.add(dish)
-                    await session.flush()
-                    if dishes[dish_key]['discont'] is not None:
-                        disconts[dish.id] = dishes[dish_key]['discont']
 
     await session.commit()
-        return disconts
+    # Чистим кэш
+    await clear_cache('MENUS*')
+    await clear_cache('summary')
+
+    return new_menu
+
+
+async def menus_updater(menus: dict, session: AsyncSession) -> None:
+    """Проверяет пункты меню на изменения
+    При необходимости запускае обновление БД
+    через фенкцию on_menu_change
+    """
+    cached_menus = await redis.get('ALL_MENUS')
+
+    if cached_menus is not None:
+        cached_menus = pickle.loads(cached_menus)
+    else:
+        cached_menus = {}
+
+    for key in menus.keys():
+        if key not in cached_menus.keys():
+            # Создание меню
+            menu = await on_menu_change(menus[key], {}, session)
+            menus[key] = menu
+        elif key in cached_menus.keys():
+            # Обновление меню
+            if menus[key].get('data') != cached_menus[key].get('data'):
+                menu = await on_menu_change(menus[key], cached_menus[key], session)
+                menus[key] = menu
+            else:
+                menus[key]['id'] = cached_menus[key]['id']
+
+    for key in {k: cached_menus[k] for k in set(cached_menus) - set(menus)}:
+        # Проверяем на удаленные меню
+        await on_menu_change({}, cached_menus.pop(key), session)
+
+    await redis.set('ALL_MENUS', pickle.dumps(menus))
+
+
+async def on_submenu_change(
+    new_sub: dict, old_sub: dict, session: AsyncSession
+) -> dict:
+    if new_sub and not old_sub:
+        # Создаем меню
+        submenu = SubMenu(
+            title=new_sub['data']['title'],
+            description=new_sub['data']['description'],
+        )
+        submenu.parent_menu = new_sub['parent_menu']
+
+        session.add(submenu)
+        await session.flush()
+        new_sub['id'] = str(submenu.id)
+        new_sub['parent_menu'] = str(submenu.parent_menu)
+    elif new_sub and old_sub:
+        # Обновляем меню
+        await session.execute(
+            update(SubMenu)
+            .where(SubMenu.id == old_sub['id'])
+            .values(**(new_sub['data']))
+        )
+        new_sub['id'] = old_sub['id']
+        new_sub['parent_menu'] = old_sub['parent_menu']
+
+    else:
+        # Удаляем меню
+        await session.execute(delete(SubMenu).where(SubMenu.id == old_sub['id']))
+
+    await clear_cache('MENUS*')
+    await clear_cache('summary')
+
+    await session.commit()
+    return new_sub
+
+
+async def submenus_updater(submenus: dict, session: AsyncSession) -> None:
+    """Проверяет пункты подменю на изменения
+    При необходимости запускае обновление БД
+    """
+    # Получаем Меню из кэша для получения их ID по померу в таблице
+    cached_menus = await redis.get('ALL_MENUS')
+    if cached_menus is not None:
+        cached_menus = pickle.loads(cached_menus)
+    else:
+        cached_menus = {}
+
+    # Получаем подмен из кэша
+    cached_sub = await redis.get('ALL_SUBMENUS')
+
+    if cached_sub is not None:
+        cached_sub = pickle.loads(cached_sub)
+    else:
+        cached_sub = {}
+
+    for key in submenus.keys():
+        parent = cached_menus[submenus[key]['parent_num']]['id']
+        submenus[key]['parent_menu'] = parent
+
+        if key not in cached_sub.keys():
+            # Получаем и ставим UUID parent_menu
+            submenus[key]['parent_menu'] = parent
+
+            submenu = await on_submenu_change(submenus[key], {}, session)
+            submenus[key] = submenu
+        elif key in cached_sub.keys():
+            # Обновление меню
+            if submenus[key].get('data') != cached_sub[key].get('data'):
+                submenu = await on_submenu_change(
+                    submenus[key], cached_sub[key], session
+                )
+                submenus[key] = submenu
+            else:
+                submenus[key]['id'] = cached_sub[key]['id']
+                submenus[key]['parent_menu'] = cached_sub[key]['parent_menu']
+
+    for key in {k: cached_sub[k] for k in set(cached_sub) - set(submenus)}:
+        # Проверяем на удаленные меню
+        await on_submenu_change({}, cached_sub.pop(key), session)
+
+    await redis.set('ALL_SUBMENUS', pickle.dumps(submenus))
+
+
+async def on_dish_change(new_dish: dict, old_dish, session: AsyncSession) -> dict:
+    if new_dish and not old_dish:
+        dish = Dish(
+            title=new_dish['data']['title'],
+            description=new_dish['data']['description'],
+            price=new_dish['data']['price'],
+        )
+        dish.parent_submenu = new_dish['parent_submenu']
+
+        session.add(dish)
+        await session.flush()
+        new_dish['id'] = str(dish.id)
+        new_dish['parent_submenu'] = str(dish.parent_submenu)
+        new_dish['data']['price'] = str(dish.price)
+    elif new_dish and old_dish:
+        # Обновляем меню
+        await session.execute(
+            update(Dish).where(Dish.id == old_dish['id']).values(**(new_dish['data']))
+        )
+        new_dish['id'] = old_dish['id']
+        new_dish['parent_submenu'] = old_dish['parent_submenu']
+        new_dish['data']['price'] = old_dish['data']['price']
+
+    else:
+        # Удаляем меню
+        await session.execute(delete(Dish).where(Dish.id == old_dish['id']))
+
+    await clear_cache('MENUS*')
+    await clear_cache('summary')
+
+    await session.commit()
+    return new_dish
+
+
+async def dishes_updater(dishes: dict, session: AsyncSession) -> None:
+    """Проверяет блюда на изменения
+    При необходимости запускае обновление БД
+    """
+    cached_submenus = await redis.get('ALL_SUBMENUS')
+    if cached_submenus is not None:
+        cached_submenus = pickle.loads(cached_submenus)
+    else:
+        cached_submenus = {}
+
+    # Получаем подмен из кэша
+    cached_dishes = await redis.get('ALL_DISHES')
+
+    if cached_dishes is not None:
+        cached_dishes = pickle.loads(cached_dishes)
+    else:
+        cached_dishes = {}
+
+    await clear_cache('DISCONT*')
+
+    for key in {k: cached_dishes[k] for k in set(cached_dishes) - set(dishes)}:
+        # Проверяем на удаленные меню
+        await on_submenu_change({}, cached_dishes.pop(key), session)
+
+    for key in dishes.keys():
+        parent = cached_submenus[dishes[key]['parent_num']]['id']
+        dishes[key]['parent_submenu'] = parent
+
+        if key not in cached_dishes.keys():
+            # Получаем и ставим UUID parent_menu
+            dishes[key]['parent_submenu'] = parent
+
+            dish = await on_dish_change(dishes[key], {}, session)
+            dishes[key] = dish
+        elif key in cached_dishes.keys():
+            # Обновление меню
+            if dishes[key].get('data') != cached_dishes[key].get('data'):
+                dish = await on_dish_change(dishes[key], cached_dishes[key], session)
+                dishes[key] = dish
+            else:
+                dishes[key]['id'] = cached_dishes[key]['id']
+                dishes[key]['parent_submenu'] = cached_dishes[key]['parent_submenu']
+
+        if dishes[key]['discont'] is not None:
+            await redis.set(
+                f"DISCONT:{dishes[key]['id']}", pickle.dumps(dishes[key]['discont'])
+            )
+
+        await redis.set('ALL_DISHES', pickle.dumps(dishes))
+
+
+async def updater(rows):
+    menus, submenus, dishes = await rows_to_dict(rows)
+    async with async_session_maker() as session:
+        await menus_updater(menus, session)
+        await submenus_updater(submenus, session)
+        await dishes_updater(dishes, session)
 
 
 async def main() -> None:
     """Главная функция фоновой задачи"""
     changed = await is_changed_xls()
     if changed:
-        menu_data = await xlsx_to_dict()
+        rows = await local_xlsx_to_rows()
-        discont_data = await refresh_all_data(menu_data)
+        await updater(rows)
-        await refresh_cache(discont_data)
+
+
+async def main_gsheets() -> None:
+    rows = await gsheets_to_rows()
+    await updater(rows)

fastfood/repository/redis.py

@@ -12,15 +12,15 @@ def get_key(level: str, **kwargs) -> str:
         case 'menus':
             return 'MENUS'
         case 'menu':
-            return f"{kwargs.get('menu_id')}"
+            return f"MENUS:{kwargs.get('menu_id')}"
         case 'submenus':
-            return f"{kwargs.get('menu_id')}:SUBMENUS"
+            return f"MENUS:{kwargs.get('menu_id')}:SUBMENUS"
         case 'submenu':
-            return f"{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}"
+            return f"MENUS:{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}"
         case 'dishes':
-            return f"{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:DISHES"
+            return f"MENUS:{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:DISHES"
         case 'dish':
-            return f"{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:{kwargs.get('dish_id')}"
+            return f"MENUS:{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:{kwargs.get('dish_id')}"
 
     return 'summary'