upd фоновая задача теперь не дропает базу
parent
afdf1c5e2b
commit
9ba42aae9f
|
|
@ -0,0 +1 @@
|
|||
,pi3c,pi3code,12.02.2024 22:20,file:///home/pi3c/.config/libreoffice/4;
|
||||
BIN
admin/Menu.xlsx
BIN
admin/Menu.xlsx
Binary file not shown.
|
|
@ -19,7 +19,7 @@ celery_app = Celery(
|
|||
celery_app.conf.beat_schedule = {
|
||||
'run-task-every-15-seconds': {
|
||||
'task': 'bg_tasks.bg_task.periodic_task',
|
||||
'schedule': 15.0,
|
||||
'schedule': 30.0,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
import os
|
||||
|
||||
import gspread
|
||||
import openpyxl
|
||||
|
||||
file = os.path.join(os.path.curdir, 'admin', 'Menu.xlsx')
|
||||
|
||||
|
||||
async def gsheets_to_rows() -> list[list[str | int | float]]:
|
||||
"""Получение всех строк из Google Sheets"""
|
||||
|
||||
def to_int(val: str) -> int | str:
|
||||
try:
|
||||
res = int(val)
|
||||
except ValueError:
|
||||
return val
|
||||
return res
|
||||
|
||||
def to_float(val: str) -> float | str:
|
||||
val = val.replace(',', '.')
|
||||
try:
|
||||
res = float(val)
|
||||
except ValueError:
|
||||
return val
|
||||
return res
|
||||
|
||||
gc = gspread.service_account(filename='creds.json')
|
||||
sh = gc.open('Menu')
|
||||
data = sh.sheet1.get_all_values()
|
||||
for row in data:
|
||||
row[:3] = list(map(to_int, row[:3]))
|
||||
row[-2:] = list(map(to_float, row[-2:]))
|
||||
|
||||
return data
|
||||
|
||||
|
||||
async def local_xlsx_to_rows() -> list[list[str | int | float]]:
|
||||
"""Получение всех строк из локального файла Menu"""
|
||||
data = []
|
||||
wb = openpyxl.load_workbook(file).worksheets[0]
|
||||
for row in wb.iter_rows(values_only=True):
|
||||
data.append(list(row))
|
||||
return data
|
||||
|
||||
|
||||
async def rows_to_dict(rows: list[list]) -> tuple:
|
||||
"""Парсит строки полученные и источников в словарь"""
|
||||
|
||||
menus = {}
|
||||
submenus = {}
|
||||
dishes = {}
|
||||
|
||||
menu_num = None
|
||||
submenu_num = None
|
||||
|
||||
for row in rows:
|
||||
if all(row[:3]):
|
||||
menu = {
|
||||
row[0]: {
|
||||
'data': {'title': row[1], 'description': row[2]},
|
||||
'id': None,
|
||||
}
|
||||
}
|
||||
menu_num = row[0]
|
||||
menus.update(menu)
|
||||
|
||||
elif all(row[1:4]):
|
||||
submenu = {
|
||||
(menu_num, row[1]): {
|
||||
'data': {'title': row[2], 'description': row[3]},
|
||||
'parent_num': menu_num,
|
||||
'id': None,
|
||||
'parent_menu': None,
|
||||
}
|
||||
}
|
||||
submenu_num = row[1]
|
||||
submenus.update(submenu)
|
||||
|
||||
elif all(row[3:6]):
|
||||
dish = {
|
||||
(menu_num, submenu_num, row[2]): {
|
||||
'data': {
|
||||
'title': row[3],
|
||||
'description': row[4],
|
||||
'price': row[5],
|
||||
},
|
||||
'parent_num': (menu_num, submenu_num),
|
||||
'id': None,
|
||||
'parent_submenu': None,
|
||||
'discont': row[6],
|
||||
},
|
||||
}
|
||||
dishes.update(dish)
|
||||
return menus, submenus, dishes
|
||||
|
|
@ -1,16 +1,14 @@
|
|||
import os
|
||||
import pickle
|
||||
from uuid import UUID
|
||||
|
||||
import openpyxl
|
||||
import redis.asyncio as redis # type: ignore
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy import delete, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
from fastfood.config import settings
|
||||
from fastfood.models import Dish, Menu, SubMenu
|
||||
|
||||
file = os.path.join(os.path.curdir, 'admin', 'Menu.xlsx')
|
||||
from .parser import file, gsheets_to_rows, local_xlsx_to_rows, rows_to_dict
|
||||
|
||||
redis = redis.Redis.from_url(url=settings.REDIS_URL)
|
||||
|
||||
|
|
@ -22,16 +20,10 @@ async_session_maker = async_sessionmaker(
|
|||
)
|
||||
|
||||
|
||||
async def refresh_cache(disconts: dict) -> None:
|
||||
"""Очищает кэш при обновлении БД и ставит отметку времени обновления
|
||||
и сохраняет данные скидок на товар
|
||||
"""
|
||||
await redis.flushall()
|
||||
|
||||
for key in disconts.keys():
|
||||
await redis.set(f'DISCONT:{str(key)}', pickle.dumps(disconts[key]))
|
||||
|
||||
await redis.set('XLSX_MOD_TIME', pickle.dumps(os.path.getmtime(file)))
|
||||
async def clear_cache(pattern: str) -> None:
|
||||
keys = [key async for key in redis.scan_iter(pattern)]
|
||||
if keys:
|
||||
await redis.delete(*keys)
|
||||
|
||||
|
||||
async def is_changed_xls() -> bool:
|
||||
|
|
@ -51,96 +43,253 @@ async def is_changed_xls() -> bool:
|
|||
return True
|
||||
|
||||
|
||||
async def xlsx_to_dict() -> dict:
|
||||
"""Парсит Menu.xlsx в словарь"""
|
||||
wb = openpyxl.load_workbook(file).worksheets[0]
|
||||
|
||||
data = {}
|
||||
|
||||
menu = None
|
||||
submenu = None
|
||||
dish = None
|
||||
|
||||
for row in wb.iter_rows(values_only=True):
|
||||
if row[0] is not None:
|
||||
menu = row[0]
|
||||
data[menu] = {
|
||||
'id': None,
|
||||
'title': row[1],
|
||||
'description': row[2],
|
||||
'submenus': dict(),
|
||||
}
|
||||
elif row[1] is not None:
|
||||
submenu = row[1]
|
||||
data[menu]['submenus'][submenu] = {
|
||||
'id': None,
|
||||
'title': row[2],
|
||||
'description': row[3],
|
||||
'dishes': dict(),
|
||||
}
|
||||
elif row[2] is not None:
|
||||
dish = row[2]
|
||||
data[menu]['submenus'][submenu]['dishes'][dish] = {
|
||||
'id': None,
|
||||
'title': row[3],
|
||||
'description': row[4],
|
||||
'price': row[5],
|
||||
'discont': row[6],
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
async def refresh_all_data(data: dict) -> dict[UUID, int | float]:
|
||||
"""Удаляет старые данные и сохраняет новые.
|
||||
Создает и возвращает список со скидками с привязкой по UUID товара
|
||||
"""
|
||||
|
||||
disconts = {}
|
||||
|
||||
async with async_session_maker() as session:
|
||||
await session.execute(delete(Menu))
|
||||
await session.commit()
|
||||
|
||||
for menu_key in data.keys():
|
||||
async def on_menu_change(
|
||||
new_menu: dict, old_menu: dict, session: AsyncSession
|
||||
) -> dict | None:
|
||||
if new_menu and not old_menu:
|
||||
# Создаем меню
|
||||
menu = Menu(
|
||||
title=data[menu_key].get('title'),
|
||||
description=data[menu_key].get('description'),
|
||||
title=new_menu['data']['title'],
|
||||
description=new_menu['data']['description'],
|
||||
)
|
||||
session.add(menu)
|
||||
await session.flush()
|
||||
|
||||
submenus = data[menu_key]['submenus']
|
||||
for sub_key in submenus.keys():
|
||||
submenu = SubMenu(
|
||||
title=submenus[sub_key]['title'],
|
||||
description=submenus[sub_key]['description'],
|
||||
parent_menu=menu.id,
|
||||
new_menu['id'] = str(menu.id)
|
||||
elif new_menu and old_menu:
|
||||
# Обновляем меню
|
||||
await session.execute(
|
||||
update(Menu).where(Menu.id == old_menu['id']).values(**(new_menu['data']))
|
||||
)
|
||||
session.add(submenu)
|
||||
await session.flush()
|
||||
new_menu['id'] = old_menu['id']
|
||||
|
||||
dishes = data[menu_key]['submenus'][sub_key]['dishes']
|
||||
print(dishes)
|
||||
for dish_key in dishes.keys():
|
||||
dish = Dish(
|
||||
title=dishes[dish_key]['title'],
|
||||
description=dishes[dish_key]['description'],
|
||||
price=dishes[dish_key]['price'],
|
||||
parent_submenu=submenu.id,
|
||||
)
|
||||
session.add(dish)
|
||||
await session.flush()
|
||||
if dishes[dish_key]['discont'] is not None:
|
||||
disconts[dish.id] = dishes[dish_key]['discont']
|
||||
else:
|
||||
# Удаляем меню
|
||||
await session.execute(delete(Menu).where(Menu.id == old_menu['id']))
|
||||
|
||||
await session.commit()
|
||||
return disconts
|
||||
# Чистим кэш
|
||||
await clear_cache('MENUS*')
|
||||
await clear_cache('summary')
|
||||
|
||||
return new_menu
|
||||
|
||||
|
||||
async def menus_updater(menus: dict, session: AsyncSession) -> None:
|
||||
"""Проверяет пункты меню на изменения
|
||||
При необходимости запускае обновление БД
|
||||
через фенкцию on_menu_change
|
||||
"""
|
||||
cached_menus = await redis.get('ALL_MENUS')
|
||||
|
||||
if cached_menus is not None:
|
||||
cached_menus = pickle.loads(cached_menus)
|
||||
else:
|
||||
cached_menus = {}
|
||||
|
||||
for key in menus.keys():
|
||||
if key not in cached_menus.keys():
|
||||
# Создание меню
|
||||
menu = await on_menu_change(menus[key], {}, session)
|
||||
menus[key] = menu
|
||||
elif key in cached_menus.keys():
|
||||
# Обновление меню
|
||||
if menus[key].get('data') != cached_menus[key].get('data'):
|
||||
menu = await on_menu_change(menus[key], cached_menus[key], session)
|
||||
menus[key] = menu
|
||||
else:
|
||||
menus[key]['id'] = cached_menus[key]['id']
|
||||
|
||||
for key in {k: cached_menus[k] for k in set(cached_menus) - set(menus)}:
|
||||
# Проверяем на удаленные меню
|
||||
await on_menu_change({}, cached_menus.pop(key), session)
|
||||
|
||||
await redis.set('ALL_MENUS', pickle.dumps(menus))
|
||||
|
||||
|
||||
async def on_submenu_change(
|
||||
new_sub: dict, old_sub: dict, session: AsyncSession
|
||||
) -> dict:
|
||||
if new_sub and not old_sub:
|
||||
# Создаем меню
|
||||
submenu = SubMenu(
|
||||
title=new_sub['data']['title'],
|
||||
description=new_sub['data']['description'],
|
||||
)
|
||||
submenu.parent_menu = new_sub['parent_menu']
|
||||
|
||||
session.add(submenu)
|
||||
await session.flush()
|
||||
new_sub['id'] = str(submenu.id)
|
||||
new_sub['parent_menu'] = str(submenu.parent_menu)
|
||||
elif new_sub and old_sub:
|
||||
# Обновляем меню
|
||||
await session.execute(
|
||||
update(SubMenu)
|
||||
.where(SubMenu.id == old_sub['id'])
|
||||
.values(**(new_sub['data']))
|
||||
)
|
||||
new_sub['id'] = old_sub['id']
|
||||
new_sub['parent_menu'] = old_sub['parent_menu']
|
||||
|
||||
else:
|
||||
# Удаляем меню
|
||||
await session.execute(delete(SubMenu).where(SubMenu.id == old_sub['id']))
|
||||
|
||||
await clear_cache('MENUS*')
|
||||
await clear_cache('summary')
|
||||
|
||||
await session.commit()
|
||||
return new_sub
|
||||
|
||||
|
||||
async def submenus_updater(submenus: dict, session: AsyncSession) -> None:
|
||||
"""Проверяет пункты подменю на изменения
|
||||
При необходимости запускае обновление БД
|
||||
"""
|
||||
# Получаем Меню из кэша для получения их ID по померу в таблице
|
||||
cached_menus = await redis.get('ALL_MENUS')
|
||||
if cached_menus is not None:
|
||||
cached_menus = pickle.loads(cached_menus)
|
||||
else:
|
||||
cached_menus = {}
|
||||
|
||||
# Получаем подмен из кэша
|
||||
cached_sub = await redis.get('ALL_SUBMENUS')
|
||||
|
||||
if cached_sub is not None:
|
||||
cached_sub = pickle.loads(cached_sub)
|
||||
else:
|
||||
cached_sub = {}
|
||||
|
||||
for key in submenus.keys():
|
||||
parent = cached_menus[submenus[key]['parent_num']]['id']
|
||||
submenus[key]['parent_menu'] = parent
|
||||
|
||||
if key not in cached_sub.keys():
|
||||
# Получаем и ставим UUID parent_menu
|
||||
submenus[key]['parent_menu'] = parent
|
||||
|
||||
submenu = await on_submenu_change(submenus[key], {}, session)
|
||||
submenus[key] = submenu
|
||||
elif key in cached_sub.keys():
|
||||
# Обновление меню
|
||||
if submenus[key].get('data') != cached_sub[key].get('data'):
|
||||
submenu = await on_submenu_change(
|
||||
submenus[key], cached_sub[key], session
|
||||
)
|
||||
submenus[key] = submenu
|
||||
else:
|
||||
submenus[key]['id'] = cached_sub[key]['id']
|
||||
submenus[key]['parent_menu'] = cached_sub[key]['parent_menu']
|
||||
|
||||
for key in {k: cached_sub[k] for k in set(cached_sub) - set(submenus)}:
|
||||
# Проверяем на удаленные меню
|
||||
await on_submenu_change({}, cached_sub.pop(key), session)
|
||||
|
||||
await redis.set('ALL_SUBMENUS', pickle.dumps(submenus))
|
||||
|
||||
|
||||
async def on_dish_change(new_dish: dict, old_dish, session: AsyncSession) -> dict:
|
||||
if new_dish and not old_dish:
|
||||
dish = Dish(
|
||||
title=new_dish['data']['title'],
|
||||
description=new_dish['data']['description'],
|
||||
price=new_dish['data']['price'],
|
||||
)
|
||||
dish.parent_submenu = new_dish['parent_submenu']
|
||||
|
||||
session.add(dish)
|
||||
await session.flush()
|
||||
new_dish['id'] = str(dish.id)
|
||||
new_dish['parent_submenu'] = str(dish.parent_submenu)
|
||||
new_dish['data']['price'] = str(dish.price)
|
||||
elif new_dish and old_dish:
|
||||
# Обновляем меню
|
||||
await session.execute(
|
||||
update(Dish).where(Dish.id == old_dish['id']).values(**(new_dish['data']))
|
||||
)
|
||||
new_dish['id'] = old_dish['id']
|
||||
new_dish['parent_submenu'] = old_dish['parent_submenu']
|
||||
new_dish['data']['price'] = old_dish['data']['price']
|
||||
|
||||
else:
|
||||
# Удаляем меню
|
||||
await session.execute(delete(Dish).where(Dish.id == old_dish['id']))
|
||||
|
||||
await clear_cache('MENUS*')
|
||||
await clear_cache('summary')
|
||||
|
||||
await session.commit()
|
||||
return new_dish
|
||||
|
||||
|
||||
async def dishes_updater(dishes: dict, session: AsyncSession) -> None:
|
||||
"""Проверяет блюда на изменения
|
||||
При необходимости запускае обновление БД
|
||||
"""
|
||||
cached_submenus = await redis.get('ALL_SUBMENUS')
|
||||
if cached_submenus is not None:
|
||||
cached_submenus = pickle.loads(cached_submenus)
|
||||
else:
|
||||
cached_submenus = {}
|
||||
|
||||
# Получаем подмен из кэша
|
||||
cached_dishes = await redis.get('ALL_DISHES')
|
||||
|
||||
if cached_dishes is not None:
|
||||
cached_dishes = pickle.loads(cached_dishes)
|
||||
else:
|
||||
cached_dishes = {}
|
||||
|
||||
await clear_cache('DISCONT*')
|
||||
|
||||
for key in {k: cached_dishes[k] for k in set(cached_dishes) - set(dishes)}:
|
||||
# Проверяем на удаленные меню
|
||||
await on_submenu_change({}, cached_dishes.pop(key), session)
|
||||
|
||||
for key in dishes.keys():
|
||||
parent = cached_submenus[dishes[key]['parent_num']]['id']
|
||||
dishes[key]['parent_submenu'] = parent
|
||||
|
||||
if key not in cached_dishes.keys():
|
||||
# Получаем и ставим UUID parent_menu
|
||||
dishes[key]['parent_submenu'] = parent
|
||||
|
||||
dish = await on_dish_change(dishes[key], {}, session)
|
||||
dishes[key] = dish
|
||||
elif key in cached_dishes.keys():
|
||||
# Обновление меню
|
||||
if dishes[key].get('data') != cached_dishes[key].get('data'):
|
||||
dish = await on_dish_change(dishes[key], cached_dishes[key], session)
|
||||
dishes[key] = dish
|
||||
else:
|
||||
dishes[key]['id'] = cached_dishes[key]['id']
|
||||
dishes[key]['parent_submenu'] = cached_dishes[key]['parent_submenu']
|
||||
|
||||
if dishes[key]['discont'] is not None:
|
||||
await redis.set(
|
||||
f"DISCONT:{dishes[key]['id']}", pickle.dumps(dishes[key]['discont'])
|
||||
)
|
||||
|
||||
await redis.set('ALL_DISHES', pickle.dumps(dishes))
|
||||
|
||||
|
||||
async def updater(rows):
|
||||
menus, submenus, dishes = await rows_to_dict(rows)
|
||||
async with async_session_maker() as session:
|
||||
await menus_updater(menus, session)
|
||||
await submenus_updater(submenus, session)
|
||||
await dishes_updater(dishes, session)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
"""Главная функция фоновой задачи"""
|
||||
changed = await is_changed_xls()
|
||||
if changed:
|
||||
menu_data = await xlsx_to_dict()
|
||||
discont_data = await refresh_all_data(menu_data)
|
||||
await refresh_cache(discont_data)
|
||||
rows = await local_xlsx_to_rows()
|
||||
await updater(rows)
|
||||
|
||||
|
||||
async def main_gsheets() -> None:
|
||||
rows = await gsheets_to_rows()
|
||||
await updater(rows)
|
||||
|
|
|
|||
|
|
@ -12,15 +12,15 @@ def get_key(level: str, **kwargs) -> str:
|
|||
case 'menus':
|
||||
return 'MENUS'
|
||||
case 'menu':
|
||||
return f"{kwargs.get('menu_id')}"
|
||||
return f"MENUS:{kwargs.get('menu_id')}"
|
||||
case 'submenus':
|
||||
return f"{kwargs.get('menu_id')}:SUBMENUS"
|
||||
return f"MENUS:{kwargs.get('menu_id')}:SUBMENUS"
|
||||
case 'submenu':
|
||||
return f"{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}"
|
||||
return f"MENUS:{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}"
|
||||
case 'dishes':
|
||||
return f"{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:DISHES"
|
||||
return f"MENUS:{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:DISHES"
|
||||
case 'dish':
|
||||
return f"{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:{kwargs.get('dish_id')}"
|
||||
return f"MENUS:{kwargs.get('menu_id')}:{kwargs.get('submenu_id')}:{kwargs.get('dish_id')}"
|
||||
|
||||
return 'summary'
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue