From 2abc198a39f980ad02e2d328701feb0e9c6e4f3f Mon Sep 17 00:00:00 2001 From: AlexSserb Date: Wed, 25 Dec 2024 21:14:00 +0400 Subject: [PATCH] feat: filling deals from excel file --- marketplaces/factory.py | 1 - parsers/__init__.py | 1 + parsers/deal_parser.py | 143 ++++++++++++++++++++++++++++++++++++++++ routers/deal.py | 28 +++++++- schemas/deal.py | 43 +++++++++++- services/deal.py | 98 ++++++++++++++++++++++++--- 6 files changed, 302 insertions(+), 12 deletions(-) create mode 100644 parsers/__init__.py create mode 100644 parsers/deal_parser.py diff --git a/marketplaces/factory.py b/marketplaces/factory.py index 270dc28..a978298 100644 --- a/marketplaces/factory.py +++ b/marketplaces/factory.py @@ -1,4 +1,3 @@ -from audioop import ratecv from typing import Union from sqlalchemy.ext.asyncio import AsyncSession diff --git a/parsers/__init__.py b/parsers/__init__.py new file mode 100644 index 0000000..5619d72 --- /dev/null +++ b/parsers/__init__.py @@ -0,0 +1 @@ +from .deal_parser import DealParser diff --git a/parsers/deal_parser.py b/parsers/deal_parser.py new file mode 100644 index 0000000..5bfec43 --- /dev/null +++ b/parsers/deal_parser.py @@ -0,0 +1,143 @@ +from io import BytesIO +from typing import Optional + +from openpyxl.reader.excel import load_workbook +from openpyxl.utils.cell import get_column_letter +from sqlalchemy import select, exists +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from models import ProductBarcode, Product, ShippingWarehouse, BaseMarketplace +from schemas.deal import ParsedProductRowSchema, ParseDealsExcelResponse, ParsedCityBreakdownSchema, \ + OptionalShippingWarehouseSchema +from schemas.marketplace import MarketplaceSchema, BaseMarketplaceSchema +from schemas.product import ProductSchema + + +class DealParser: + def __init__(self, session: AsyncSession): + self._session = session + self._warehouses: dict[str, OptionalShippingWarehouseSchema] = {} + self._marketplaces: dict[str, BaseMarketplaceSchema] = {} + self._errors: list[str] = [] + self._warehouses_and_marketplaces: dict[ + int, dict[str, Optional[MarketplaceSchema] | OptionalShippingWarehouseSchema] + ] = {} + + async def _set_warehouses(self): + stmt = select(ShippingWarehouse) + warehouses = (await self._session.execute(stmt)).scalars().all() + for warehouse in warehouses: + self._warehouses[warehouse.name.lower()] = OptionalShippingWarehouseSchema.model_validate(warehouse) + + async def _set_marketplaces(self): + stmt = select(BaseMarketplace) + marketplaces = (await self._session.execute(stmt)).scalars().all() + for marketplace in marketplaces: + self._marketplaces[marketplace.key] = BaseMarketplaceSchema.model_validate(marketplace) + + async def _parse_product_barcode(self, barcode_value: str) -> list[ProductSchema]: + products_stmt = ( + select(Product) + .join(ProductBarcode) + .where(ProductBarcode.barcode == barcode_value) + .options( + selectinload(Product.barcodes) + .noload(ProductBarcode.product), + selectinload(Product.barcode_image), + ) + ) + products = (await self._session.execute(products_stmt)).scalars().all() + + return [ProductSchema.model_validate(product) for product in products] + + async def _parse_warehouse_and_marketplace_labels(self, value: str) -> dict[ + str, Optional[MarketplaceSchema] | OptionalShippingWarehouseSchema + ]: + stripped = value.split() + base_mp_value, warehouse_value = stripped[0], " ".join(stripped[1:]) + base_mp_lower, warehouse_lower = base_mp_value.lower(), warehouse_value.lower() + + base_mp = self._marketplaces.get(base_mp_lower) + warehouse = self._warehouses.get(warehouse_lower) + if not warehouse: + warehouse = OptionalShippingWarehouseSchema(name=warehouse_value) + + return { + "marketplace": base_mp, + "warehouse": warehouse, + } + + async def _barcode_exists(self, barcode: str) -> bool: + stmt = select(exists(ProductBarcode).where(ProductBarcode.barcode == barcode)) + barcode_exists = (await self._session.execute(stmt)).scalar() + return barcode_exists + + async def _parse_warehouses_and_marketplaces(self, ws): + mp_wh_row, mp_wh_start_col = 3, 9 + + await self._set_warehouses() + await self._set_marketplaces() + + self._warehouses_and_marketplaces = {} + while True: + value = ws.cell(row=mp_wh_row, column=mp_wh_start_col).value + if not value: + break + mp_and_wh = await self._parse_warehouse_and_marketplace_labels(value) + if mp_and_wh["marketplace"]: + self._warehouses_and_marketplaces[mp_wh_start_col] = mp_and_wh + else: + self._errors.append(f"В ячейке {get_column_letter(mp_wh_start_col)}3 не распознан маркетплейс") + mp_wh_start_col += 1 + + async def _parse_barcodes(self, ws) -> list[ParsedProductRowSchema]: + rows: list[ParsedProductRowSchema] = [] + barcode_row, barcode_col, count_empty_rows = 4, 3, 0 + barcode_col_name = get_column_letter(barcode_col) + max_empty_rows_between_barcodes = 6 + + while True: + barcode: Optional[int] = ws.cell(row=barcode_row, column=barcode_col).value + if barcode and type(barcode) == int: + count_empty_rows = 0 + str_barcode = str(barcode) + barcode_exists = await self._barcode_exists(str_barcode) + if barcode_exists: + products: list[ProductSchema] = await self._parse_product_barcode(str_barcode) + + breakdowns: list[ParsedCityBreakdownSchema] = [] + + for col, data in self._warehouses_and_marketplaces.items(): + quantity: Optional[str] = ws.cell(row=barcode_row, column=col).value + if quantity: + warehouse_excel_info = ParsedCityBreakdownSchema( + base_marketplace=data["marketplace"], + shippingWarehouse=data["warehouse"], + quantity=int(quantity), + ) + breakdowns.append(warehouse_excel_info) + if len(products) > 0 and len(breakdowns) > 0: + rows.append( + ParsedProductRowSchema(products=products, breakdowns=breakdowns, barcode=str(barcode)) + ) + else: + self._errors.append(f"Штрихкод из ячейки {barcode_col_name}{barcode_row} не найден") + else: + count_empty_rows += 1 + if count_empty_rows > max_empty_rows_between_barcodes: + break + + barcode_row += 1 + + return rows + + async def parse(self, file_bytes: bytes) -> ParseDealsExcelResponse: + wb = load_workbook(filename=BytesIO(file_bytes)) + ws = wb.worksheets[0] + + await self._parse_warehouses_and_marketplaces(ws) + + rows = await self._parse_barcodes(ws) + + return ParseDealsExcelResponse(rows=rows, errors=self._errors) diff --git a/routers/deal.py b/routers/deal.py index 4744d25..6314080 100644 --- a/routers/deal.py +++ b/routers/deal.py @@ -2,13 +2,14 @@ import base64 from io import BytesIO from typing import Annotated -from fastapi import APIRouter, Depends, Response +from fastapi import APIRouter, Depends, Response, UploadFile from sqlalchemy.ext.asyncio import AsyncSession from backend.dependecies import SessionDependency, CurrentUserDependency from backend.session import get_session from generators.deal_pdf_generator.generator import DealTechSpecPdfGenerator from models import User +from parsers import DealParser from schemas.barcode import GetDealProductsBarcodesPdfRequest, GetDealProductsBarcodesPdfResponse from schemas.deal import * from services.auth import get_current_user, authorized_user, guest_user @@ -258,6 +259,31 @@ async def get_available_employees_to_assign( return await DealService(session).get_available_employees_to_assign(deal_id) +@deal_router.post( + '/prefill/excel/parse', + response_model=ParseDealsExcelResponse, + operation_id='parse_deals_excel', +) +async def parse_deals_excel( + session: Annotated[AsyncSession, Depends(get_session)], + upload_file: UploadFile, +): + file_bytes = upload_file.file.read() + return await DealParser(session).parse(file_bytes) + + +@deal_router.post( + '/prefill/excel/create', + response_model=CreateDealsFromExcelResponse, + operation_id='create_deals_excel', +) +async def create_deals_from_excel( + session: Annotated[AsyncSession, Depends(get_session)], + request: CreateDealsFromExcelRequest, + user: CurrentUserDependency, +): + return await DealService(session).create_deals_from_excel(request, user) + # endregion # region Deal services diff --git a/schemas/deal.py b/schemas/deal.py index 91459a4..ef7a90b 100644 --- a/schemas/deal.py +++ b/schemas/deal.py @@ -12,7 +12,7 @@ from schemas.marketplace import BaseMarketplaceSchema from schemas.product import ProductSchema from schemas.service import ServiceSchema, ServicePriceCategorySchema from schemas.shipping import PalletSchema, BoxSchema -from schemas.shipping_warehouse import ShippingWarehouseSchema +from schemas.shipping_warehouse import ShippingWarehouseSchema, BaseShippingWarehouseSchema from schemas.user import UserSchema @@ -127,6 +127,33 @@ class DealGeneralInfoSchema(BaseSchema): manager: Optional[UserSchema] = None +class OptionalShippingWarehouseSchema(BaseShippingWarehouseSchema): + id: Optional[int] = None + + +class ParsedCityBreakdownSchema(BaseSchema): + base_marketplace: BaseMarketplaceSchema + shipping_warehouse: OptionalShippingWarehouseSchema + quantity: int + + +class ParsedProductRowSchema(BaseSchema): + barcode: str + products: list[ProductSchema] + breakdowns: list[ParsedCityBreakdownSchema] + + +class CityBreakdownFromExcelSchema(BaseSchema): + base_marketplace: BaseMarketplaceSchema + shipping_warehouse: OptionalShippingWarehouseSchema + quantity: int + + +class ProductFromExcelSchema(BaseSchema): + product_id: int + cities_breakdown: list[CityBreakdownFromExcelSchema] + + # endregion Entities # region Requests @@ -269,6 +296,11 @@ class ManageEmployeeRequest(BaseSchema): is_assign: bool +class CreateDealsFromExcelRequest(BaseSchema): + client_id: int + products: list[ProductFromExcelSchema] + + class DealAddToGroupRequest(BaseSchema): deal_id: int group_id: int @@ -409,6 +441,15 @@ class GetAvailableEmployeesToAssignResponse(BaseSchema): employees: list[UserSchema] +class ParseDealsExcelResponse(BaseSchema): + rows: list[ParsedProductRowSchema] + errors: list[str] + + +class CreateDealsFromExcelResponse(OkMessageSchema): + pass + + class DealAddToGroupResponse(OkMessageSchema): pass diff --git a/services/deal.py b/services/deal.py index 13a11fa..65b819c 100644 --- a/services/deal.py +++ b/services/deal.py @@ -1218,6 +1218,92 @@ class DealService(BaseService): free_employees = (await self.session.execute(stmt_free_employees)).scalars().all() return GetAvailableEmployeesToAssignResponse(employees=free_employees) + async def _create_deal_from_excel( + self, + client: Client, + breakdown: CityBreakdownFromExcelSchema, + user: User, + ) -> Deal: + rank = await self._get_rank_for_deal(DealStatus.CREATED) + deal = Deal( + name=f"{client.name} - {breakdown.base_marketplace.key.upper()} - {breakdown.shipping_warehouse.name}", + created_at=datetime.datetime.now(), + current_status=DealStatus.CREATED, + lexorank=rank, + client_id=client.id, + base_marketplace_key=breakdown.base_marketplace.key, + shipping_warehouse_id=breakdown.shipping_warehouse.id, + ) + + self.session.add(deal) + await self.session.flush() + await self.change_status(deal, DealStatus.AWAITING_ACCEPTANCE, user) + + return deal + + async def _create_group(self) -> DealGroup: + group = models.DealGroup( + name='', + lexorank=lexorank.middle(lexorank.Bucket.BUCEKT_0).__str__(), + ) + self.session.add(group) + await self.session.flush() + return group + + async def _get_or_create_warehouse( + self, + shipping_warehouse: OptionalShippingWarehouseSchema, + ) -> OptionalShippingWarehouseSchema: + if not shipping_warehouse.id: + stmt = select(ShippingWarehouse).where(ShippingWarehouse.name == shipping_warehouse.name) + row = (await self.session.execute(stmt)).first() + warehouse_model: Optional[ShippingWarehouse] = row[0] if row else None + if warehouse_model: + shipping_warehouse.id = warehouse_model.id + else: + warehouse = await ShippingWarehouseService(self.session).create_by_name(shipping_warehouse.name) + shipping_warehouse.id = warehouse.id + + return shipping_warehouse + + async def create_deals_from_excel( + self, + request: CreateDealsFromExcelRequest, + user: User, + ) -> CreateDealsFromExcelResponse: + client: Optional[Client] = await self.session.get(Client, request.client_id) + if not client: + return CreateDealsFromExcelResponse(ok=False, message=f"Клиент с ID {request.client_id} не найден") + + deals_dict: dict[str, Deal] = {} + group = await self._create_group() + + for product_data in request.products: + for breakdown in product_data.cities_breakdown: + breakdown.shipping_warehouse = await self._get_or_create_warehouse(breakdown.shipping_warehouse) + + key = f"{breakdown.shipping_warehouse.id} - {breakdown.base_marketplace.key}" + deal = deals_dict.get(key) + if not deal: + deal = await self._create_deal_from_excel(client, breakdown, user) + deals_dict[key] = deal + + insert_stmt = insert(deal_relations).values({ + 'deal_id': deal.id, + 'group_id': group.id + }) + await self.session.execute(insert_stmt) + + deal_product = DealProduct( + deal_id=deal.id, + product_id=product_data.product_id, + quantity=breakdown.quantity, + ) + self.session.add(deal_product) + + await self.session.commit() + return CreateDealsFromExcelResponse(ok=True, message="Сделки успешно созданы") + async def add_to_group(self, user: User, request: DealAddToGroupRequest) -> DealAddToGroupResponse: try: group_bill_request = await self.session.get(GroupBillRequest, request.group_id) @@ -1254,14 +1340,8 @@ class DealService(BaseService): try: # getting lexorank for grop - group = models.DealGroup( - name='', - lexorank=lexorank.middle(lexorank.Bucket.BUCEKT_0).__str__(), + group = await self._create_group() - ) - - self.session.add(group) - await self.session.flush() for deal_id in [request.dragging_deal_id, request.hovered_deal_id]: insert_stmt = insert(deal_relations).values({ 'deal_id': deal_id, @@ -1339,7 +1419,7 @@ class DealService(BaseService): await self.session.rollback() return DealGroupChangeStatusResponse(ok=False, message=str(e)) - async def get_deals_grouped(self, deal:models.Deal)->List[models.Deal]: + async def get_deals_grouped(self, deal: models.Deal) -> List[models.Deal]: if not deal.group: return [deal] deals = await self.session.scalars( @@ -1354,4 +1434,4 @@ class DealService(BaseService): .where(Deal.group == deal.group) ) deals = list(deals.all()) - return deals \ No newline at end of file + return deals