rewritten crap

sender/base.py

@@ -0,0 +1,114 @@
+import logging
+from abc import ABC, abstractmethod
+from enum import IntEnum
+from typing import TYPE_CHECKING, TypedDict
+
+from utils import chunk_list
+
+if TYPE_CHECKING:
+    from marketplaces.base import BaseMarketplaceApi
+    from updaters.base import BaseMarketplaceUpdater
+
+
+class SendStockStatus(IntEnum):
+    SUCCESS = 0
+    SHOULD_RETRY = 1
+    ERROR = 2
+
+
+class StockRequest(TypedDict):
+    request_data: dict
+    product_id: int
+    full_stock: int
+
+
+class BaseStocksSender(ABC):
+    updater: "BaseMarketplaceUpdater"
+    api: "BaseMarketplaceApi"
+
+    def __init__(self, updater: "BaseMarketplaceUpdater"):
+        self.updater = updater
+        self.api = updater.marketplace_api
+
+    @property
+    @abstractmethod
+    def max_retries(self) -> int:
+        raise NotImplementedError()
+
+    @property
+    @abstractmethod
+    def chunk_size(self) -> int:
+        raise NotImplementedError()
+
+    def get_lock(self):
+        lock = self.updater.redis_client.lock(self.updater.get_lock_key(), timeout=60 * 10, blocking=True)
+        return lock
+
+    @abstractmethod
+    async def _process_chunk(self, chunk: list[dict]) -> SendStockStatus:
+        raise NotImplementedError()
+
+    async def __process_chunk_with_retries(self, chunk: list[dict]) -> SendStockStatus:
+        for attempt in range(self.max_retries + 1):
+            status = await self._process_chunk(chunk)
+            if status != SendStockStatus.SHOULD_RETRY:
+                return status
+        return SendStockStatus.SHOULD_RETRY
+
+    async def process_chunk(self, chunk: list[dict]) -> SendStockStatus:
+        try:
+            return await self.__process_chunk_with_retries(chunk)
+        except Exception as e:
+            logging.error(f'[{self.updater.marketplace.id}] {e}')
+            return SendStockStatus.ERROR
+
+    @abstractmethod
+    async def after_chunk_processed(self):
+        return
+
+    @abstractmethod
+    async def before_chunk_processed(self):
+        return
+
+    async def __send(self, chunks: list[list[StockRequest]]) -> list[int]:
+        invalid_product_ids = []
+        is_global_error = False
+        try:
+            for chunk in chunks:
+                request = []
+                product_ids = []
+                for stock in chunk:
+                    request.append(stock['request_data'])
+                    product_ids.append(stock['product_id'])
+
+                await self.before_chunk_processed()
+                try:
+                    status = await self.process_chunk(request)
+                    if status == SendStockStatus.ERROR:
+                        is_global_error = True
+                        break
+                    if status == SendStockStatus.SHOULD_RETRY:
+                        invalid_product_ids.extend(product_ids)
+                except Exception as e:
+                    invalid_product_ids.extend(product_ids)
+                    logging.error(f'[{self.updater.marketplace.id}]: {e}')
+                    continue
+                await self.after_chunk_processed()
+        except Exception as e:
+            logging.error(f'{self.updater.marketplace.id}: {e}')
+
+        if is_global_error:
+            invalid_product_ids = []
+            for chunk in chunks:
+                for stock in chunk:
+                    invalid_product_ids.append(stock['product_id'])
+        return invalid_product_ids
+
+    async def send(self, stocks: list[StockRequest]) -> list[int]:
+        if not stocks:
+            return []
+        chunks = chunk_list(stocks, self.chunk_size)
+        lock = self.get_lock()
+        async with lock:
+            result = await self.__send(chunks)
+        return result

sender/factory.py

@@ -0,0 +1,26 @@
+from typing import TYPE_CHECKING, Union
+
+from database.sipro.enums.general import BaseMarketplace
+from sender.ozon import OzonStocksSender
+from sender.wildberries import WildberriesStocksSender
+from sender.yandexmarket import YandexmarketStocksSender
+
+if TYPE_CHECKING:
+    from updaters.base import BaseMarketplaceUpdater
+
+
+class SenderFactory:
+    @staticmethod
+    def get_sender(updater: "BaseMarketplaceUpdater") -> Union[
+        WildberriesStocksSender,
+        OzonStocksSender,
+        YandexmarketStocksSender
+    ]:
+        match updater.marketplace.base_marketplace:
+            case BaseMarketplace.WILDBERRIES:
+                return WildberriesStocksSender(updater)
+            case BaseMarketplace.OZON:
+                return OzonStocksSender(updater)
+            case BaseMarketplace.YANDEX_MARKET:
+                return YandexmarketStocksSender(updater)
+        raise ValueError()

sender/ozon.py

@@ -0,0 +1,43 @@
+import asyncio
+import logging
+
+from starlette.responses import JSONResponse
+
+from sender.base import BaseStocksSender, SendStockStatus
+
+
+class OzonStocksSender(BaseStocksSender):
+    @property
+    def max_retries(self) -> int:
+        return 5
+
+    @property
+    def chunk_size(self) -> int:
+        return 100
+
+    async def _process_chunk(self, chunk: list[dict]) -> SendStockStatus:
+        response = await self.api.update_stocks(data=chunk)
+        status_code = response.status
+        if status_code == 200:
+            return SendStockStatus.SUCCESS
+        if response.content_type != JSONResponse.media_type:
+            return SendStockStatus.ERROR
+        json_data = await response.json()
+        error_code = json_data.get('code')
+        error_message = json_data.get('message')
+        if error_code == 8:
+            await asyncio.sleep(1)
+            return SendStockStatus.SHOULD_RETRY
+        logging.error(f'[{self.updater.marketplace.id}]: {error_message}')
+        if status_code in [
+            404,
+            500,
+        ]:
+            return SendStockStatus.SHOULD_RETRY
+        return SendStockStatus.ERROR
+
+    async def after_chunk_processed(self):
+        return await asyncio.sleep(80 / 100)
+
+    async def before_chunk_processed(self):
+        pass

sender/wildberries.py

@@ -0,0 +1,54 @@
+import asyncio
+from typing import TYPE_CHECKING
+
+from marketplaces import WildberriesMarketplaceApi
+from sender.base import SendStockStatus, BaseStocksSender
+
+if TYPE_CHECKING:
+    from updaters.base import BaseMarketplaceUpdater
+
+
+class WildberriesStocksSender(BaseStocksSender):
+    def __init__(self, updater: "BaseMarketplaceUpdater"):
+        super().__init__(updater)
+        self.remaining = 1
+        self.sleep_time = 60 / 300
+
+    @property
+    def max_retries(self) -> int:
+        return 5
+
+    async def _process_chunk(self, chunk: list[dict]) -> SendStockStatus:
+        response = await self.api.update_stocks(chunk)
+        headers = response.headers
+        status_code = response.status
+
+        if status_code in [
+            401,  # Invalid token
+            403,  # Access denied
+            404,  # Not found
+            400,  # Other
+        ]:
+            return SendStockStatus.ERROR
+
+        # If there is rate limit
+        if status_code == 429:
+            delay_time = float(headers.get('X-Ratelimit-Reset', self.sleep_time))
+            await asyncio.sleep(delay_time)
+            self.remaining = int(headers.get('X-Ratelimit-Limit', 1))
+            return SendStockStatus.SHOULD_RETRY
+        self.remaining = int(headers.get('X-Ratelimit-Remaining', 0))
+        return SendStockStatus.SUCCESS
+
+    async def after_chunk_processed(self):
+        if self.remaining <= 0:
+            await asyncio.sleep(self.sleep_time)
+            self.remaining = 1
+
+    async def before_chunk_processed(self):
+        if self.remaining <= 0:
+            await asyncio.sleep(self.sleep_time)
+            self.remaining = 1
+    @property
+    def chunk_size(self) -> int:
+        return 1000

sender/yandexmarket.py

@@ -0,0 +1,51 @@
+import asyncio
+import logging
+import time
+from typing import TYPE_CHECKING
+
+from sender.base import BaseStocksSender, SendStockStatus
+
+if TYPE_CHECKING:
+    from updaters.base import BaseMarketplaceUpdater
+
+
+class YandexmarketStocksSender(BaseStocksSender):
+    def __init__(self, updater: "BaseMarketplaceUpdater"):
+        super().__init__(updater)
+
+        self.start_time = time.time()
+
+        self.total_stocks_sent = 0
+
+    @property
+    def max_retries(self) -> int:
+        return 5
+
+    @property
+    def chunk_size(self) -> int:
+        return 2000
+
+    async def _process_chunk(self, chunk: list[dict]) -> SendStockStatus:
+        response = await self.api.update_stocks(chunk)
+        status_code = response.status
+        if status_code == 200:
+            self.total_stocks_sent += len(chunk)
+            return SendStockStatus.SUCCESS
+        if status_code == 420:
+            time_to_sleep = 60 - (time.time() - self.start_time)
+            await asyncio.sleep(time_to_sleep)
+            return SendStockStatus.SHOULD_RETRY
+        response_text = await response.text()
+        logging.error(f'[{self.updater.marketplace.id}]: {response_text}')
+        return SendStockStatus.ERROR
+
+    async def after_chunk_processed(self):
+        time_delta = time.time() - self.start_time
+        if self.total_stocks_sent >= 100_000 and time_delta < 60:
+            time_to_sleep = 60 - time_delta
+            await asyncio.sleep(time_to_sleep)
+            self.start_time = time.time()
+            self.total_stocks_sent = 0
+
+    async def before_chunk_processed(self):
+        pass