feat: expenses in statistics

services/statistics.py

@@ -1,12 +1,11 @@
-from collections import defaultdict
 from datetime import datetime, timedelta
 
 from fastapi import HTTPException
-from sqlalchemy import select, and_, union_all, func, Subquery
+from sqlalchemy import select, and_, union_all, func, Subquery, literal, text
 
 from enums.profit_table_group_by import ProfitTableGroupBy
 from models import DealService, Deal, DealStatusHistory, DealProductService, DealProduct, Service, Client, \
-    ShippingWarehouse, BaseMarketplace, User
+    ShippingWarehouse, BaseMarketplace, User, PaymentRecord
 from schemas.statistics import GetProfitChartDataResponse, GetProfitChartDataRequest, ProfitChartDataItem, \
     GetProfitTableDataResponse, GetProfitTableDataRequest, ProfitTableDataItem, CommonProfitFilters
 from services.base import BaseService
@@ -18,7 +17,7 @@ class StatisticsService(BaseService):
         return (
             select(
                 Deal.id.label('deal_id'),
-                Deal.created_at.label('deal_date'),
+                Deal.created_at.label('date'),
                 Deal.current_status,
             )
             .where(Deal.created_at.between(date_from, date_to))
@@ -38,7 +37,7 @@ class StatisticsService(BaseService):
         return (
             select(
                 Deal.id.label('deal_id'),
-                last_statuses.c.changed_at.label('deal_date'),
+                last_statuses.c.changed_at.label('date'),
                 Deal.current_status,
             )
             .join(last_statuses, last_statuses.c.deal_id == Deal.id)
@@ -50,46 +49,39 @@ class StatisticsService(BaseService):
         sub_status_history = StatisticsService._get_sub_status_history()
         return (
             select(sub_status_history)
-            .where(sub_status_history.c.deal_date.between(date_from, date_to))
+            .where(sub_status_history.c.date.between(date_from, date_to))
             .subquery()
         )
 
-    @staticmethod
-    def _get_deals_dates(deal_status_id: int, date_from: datetime.date, date_to: datetime.date):
+    def _get_deals_dates(self, deal_status_id: int):
         if deal_status_id == -1:
-            return StatisticsService._get_sub_deals_created_at(date_from, date_to)
-        return StatisticsService._get_filtered_sub_status_history(date_from, date_to)
+            return StatisticsService._get_sub_deals_created_at(self.date_from, self.date_to)
+        return StatisticsService._get_filtered_sub_status_history(self.date_from, self.date_to)
 
     @staticmethod
-    def _fill_dates_gaps(rows, date_from: datetime.date, date_to: datetime.date, is_chart: bool = True) -> (
+    def _to_schema(rows, is_chart: bool = True) -> (
             list[ProfitChartDataItem] | list[ProfitTableDataItem]
     ):
-        dates = defaultdict(lambda: {"deals_count": 0, "profit": 0, "revenue": 0})
-        for row in rows:
-            dates[row.deal_date.date()] = {
-                "deals_count": row.deals_count,
-                "profit": row.profit,
-                "revenue": row.revenue
-            }
-
         data = []
-        while date_from < date_to:
+        for row in rows:
             if is_chart:
                 data_item = ProfitChartDataItem(
-                    date=date_from,
-                    deals_count=dates[date_from]["deals_count"],
-                    profit=dates[date_from]["profit"],
-                    revenue=dates[date_from]["revenue"],
+                    date=row.date.date(),
+                    deals_count=row.deals_count,
+                    profit=row.profit,
+                    revenue=row.revenue,
+                    expenses=row.expenses,
                 )
             else:
                 data_item = ProfitTableDataItem(
-                    grouped_value=date_from,
-                    deals_count=dates[date_from]["deals_count"],
-                    profit=dates[date_from]["profit"],
-                    revenue=dates[date_from]["revenue"],
+                    grouped_value=row.date.date(),
+                    deals_count=row.deals_count,
+                    profit=row.profit,
+                    revenue=row.revenue,
+                    expenses=row.expenses,
                 )
             data.append(data_item)
-            date_from += timedelta(days=1)
+
         return data
 
     @staticmethod
@@ -99,20 +91,20 @@ class StatisticsService(BaseService):
                 Deal.id.label("deal_id"),
                 func.date_trunc(
                     "day",
-                    sub_filtered_status_history.c.deal_date,
-                ).label("deal_date"),
+                    sub_filtered_status_history.c.date,
+                ).label("date"),
                 func.sum(DealService.price * DealService.quantity).label("revenue"),
-                func.sum((DealService.price - Service.cost) * DealService.quantity).label("profit"),
+                func.sum(DealService.price * DealService.quantity).label("profit"),
             )
             .join(DealService, Deal.id == DealService.deal_id)
             .join(Service, DealService.service_id == Service.id)
             .join(sub_filtered_status_history, Deal.id == sub_filtered_status_history.c.deal_id)
             .where(Deal.is_deleted == False)
-            .group_by(Deal.id, "deal_date")
+            .group_by(Deal.id, "date")
         )
 
     @staticmethod
-    def _apply_chart_filters(request: CommonProfitFilters, stmt_deal_services, stmt_deal_product_services):
+    def _apply_filters(request: CommonProfitFilters, stmt_deal_services, stmt_deal_product_services):
         if request.client_id != -1:
             stmt_deal_services = stmt_deal_services.where(Deal.client_id == request.client_id)
             stmt_deal_product_services = stmt_deal_product_services.where(Deal.client_id == request.client_id)
@@ -138,7 +130,7 @@ class StatisticsService(BaseService):
             select(
                 Deal.id.label("deal_id"),
                 func.sum(DealProductService.price * DealProduct.quantity).label("revenue"),
-                func.sum((DealProductService.price - Service.cost) * DealProduct.quantity).label("profit"),
+                func.sum(DealProductService.price * DealProduct.quantity).label("profit"),
             )
             .join(DealProduct, Deal.id == DealProduct.deal_id)
             .join(
@@ -160,37 +152,49 @@ class StatisticsService(BaseService):
                 sub_deal_product_services.c.deal_id,
                 func.date_trunc(
                     "day",
-                    sub_deals_dates.c.deal_date
-                ).label("deal_date"),
+                    sub_deals_dates.c.date
+                ).label("date"),
                 sub_deal_product_services.c.revenue.label("revenue"),
                 sub_deal_product_services.c.profit.label("profit"),
             )
             .join(sub_deals_dates, sub_deal_product_services.c.deal_id == sub_deals_dates.c.deal_id)
         )
 
+    def _group_by_date(self, stmt):
+        all_dates = self._generate_date_range(["deals_count", "revenue", "profit", "expenses"])
+        deals = (
+            select(
+                stmt.c.date,
+                func.count(stmt.c.deal_id).label("deals_count"),
+                func.sum(stmt.c.revenue).label("revenue"),
+                func.sum(stmt.c.profit).label("profit"),
+                literal(0).label("expenses"),
+            )
+            .group_by(stmt.c.date)
+            .order_by(stmt.c.date.asc())
+        )
+        missing_dates = (
+            select(all_dates)
+            .where(all_dates.c.date.not_in(
+                select(deals.c.date)
+            ))
+        )
+        deals_with_filled_gaps = union_all(deals, missing_dates).subquery()
+        return (
+            select(deals_with_filled_gaps)
+            .order_by(deals_with_filled_gaps.c.date.asc())
+        )
+
     @staticmethod
     def _group_by_deals(stmt_union: Subquery):
         return (
             select(
                 stmt_union.c.deal_id,
-                stmt_union.c.deal_date,
+                stmt_union.c.date,
                 func.sum(stmt_union.c.profit).label("profit"),
                 func.sum(stmt_union.c.revenue).label("revenue"),
             )
-            .group_by(stmt_union.c.deal_id, stmt_union.c.deal_date)
-        )
-
-    @staticmethod
-    def _group_by_date(stmt):
-        return (
-            select(
-                stmt.c.deal_date,
-                func.count(stmt.c.deal_id).label("deals_count"),
-                func.sum(stmt.c.revenue).label("revenue"),
-                func.sum(stmt.c.profit).label("profit"),
-            )
-            .group_by(stmt.c.deal_date)
-            .order_by(stmt.c.deal_date.asc())
+            .group_by(stmt_union.c.deal_id, stmt_union.c.date)
         )
 
     @staticmethod
@@ -272,6 +276,60 @@ class StatisticsService(BaseService):
             .group_by(managers.c.id, "grouped_value")
         )
 
+    def _generate_date_range(self, additional_columns: list[str]):
+        cols = [literal(0).label(col_label) for col_label in additional_columns]
+        date_from = self.date_from + timedelta(days=1)
+        return select(
+            func.date_trunc(
+                "day",
+                func.generate_series(
+                    date_from,
+                    self.date_to,
+                    text("'1 day'")
+                ),
+            ).label("date"),
+            *cols,
+        ).cte()
+
+    def _get_expenses_sub(self) -> Subquery:
+        all_dates = self._generate_date_range(["expenses"])
+        payment_records = (
+            select(
+                func.sum(PaymentRecord.amount).label("expenses"),
+                func.date_trunc(
+                    "day",
+                    PaymentRecord.start_date,
+                ).label("date"),
+            )
+            .where(PaymentRecord.start_date.between(self.date_from, self.date_to))
+            .group_by("date")
+            .subquery()
+        )
+        payments_with_filled_gaps = (
+            select(
+                all_dates.c.date,
+                (all_dates.c.expenses + func.coalesce(payment_records.c.expenses, 0)).label("expenses"),
+            )
+            .join(payment_records, all_dates.c.date == payment_records.c.date, isouter=True)
+            .order_by(all_dates.c.date)
+            .subquery()
+        )
+        return payments_with_filled_gaps
+
+    def _apply_expenses(self, deals_by_dates: Subquery):
+        expenses_by_dates = self._get_expenses_sub()
+
+        return (
+            select(
+                deals_by_dates.c.date,
+                deals_by_dates.c.deals_count,
+                deals_by_dates.c.revenue,
+                (func.coalesce(deals_by_dates.c.profit, 0) - func.coalesce(expenses_by_dates.c.expenses, 0)).label("profit"),
+                expenses_by_dates.c.expenses,
+            )
+            .join(expenses_by_dates, expenses_by_dates.c.date == deals_by_dates.c.date)
+        )
+
     async def _get_data_rows_grouped_by_date(
             self,
             stmt_deal_services,
@@ -284,21 +342,23 @@ class StatisticsService(BaseService):
         sub_union = union_all(stmt_deal_services, stmt_join_deals_statuses).subquery()
 
         sub_grouped_by_deals = self._group_by_deals(sub_union)
-        stmt_grouped_by_date = self._group_by_date(sub_grouped_by_deals)
+        sub_deals_grouped_by_date = self._group_by_date(sub_grouped_by_deals).subquery()
 
-        result = await self.session.execute(stmt_grouped_by_date)
+        stmt_deals_applied_expenses = self._apply_expenses(sub_deals_grouped_by_date)
+
+        result = await self.session.execute(stmt_deals_applied_expenses)
         rows = result.all()
         return rows
 
-    async def get_data_grouped_by_date(self, request: CommonProfitFilters, is_chart: bool = True):
-        date_from, date_to = request.date_range
-        date_to += timedelta(days=1)
+    async def _get_data_grouped_by_date(self, request: CommonProfitFilters, is_chart: bool = True):
+        self.date_from, self.date_to = request.date_range
+        self.date_to += timedelta(days=1)
 
-        sub_deals_dates = self._get_deals_dates(request.deal_status_id, date_from, date_to)
+        sub_deals_dates = self._get_deals_dates(request.deal_status_id)
 
         stmt_deal_services = self._get_stmt_deal_services(sub_deals_dates)
         stmt_deal_product_services = self._get_stmt_product_services()
-        stmt_deal_services, stmt_deal_product_services = self._apply_chart_filters(
+        stmt_deal_services, stmt_deal_product_services = self._apply_filters(
             request,
             stmt_deal_services,
             stmt_deal_product_services
@@ -307,14 +367,14 @@ class StatisticsService(BaseService):
         rows = await self._get_data_rows_grouped_by_date(
             stmt_deal_services, stmt_deal_product_services, sub_deals_dates
         )
-        return self._fill_dates_gaps(rows, date_from, date_to, is_chart)
+        return self._to_schema(rows, is_chart)
 
     async def get_profit_chart_data(self, request: GetProfitChartDataRequest) -> GetProfitChartDataResponse:
-        data = await self.get_data_grouped_by_date(request)
+        data = await self._get_data_grouped_by_date(request)
         return GetProfitChartDataResponse(data=data)
 
     async def _get_table_grouped_by_dates(self, request: GetProfitTableDataRequest) -> GetProfitTableDataResponse:
-        data = await self.get_data_grouped_by_date(request, False)
+        data = await self._get_data_grouped_by_date(request, False)
         return GetProfitTableDataResponse(data=data)
 
     async def _table_data_from_stmt(self, stmt) -> GetProfitTableDataResponse:
@@ -332,10 +392,10 @@ class StatisticsService(BaseService):
         return GetProfitTableDataResponse(data=data)
 
     def _get_common_table_grouped(self, request: GetProfitTableDataRequest):
-        date_from, date_to = request.date_range
-        date_to += timedelta(days=1)
+        self.date_from, self.date_to = request.date_range
+        self.date_to += timedelta(days=1)
 
-        sub_deals_dates = self._get_deals_dates(request.deal_status_id, date_from, date_to)
+        sub_deals_dates = self._get_deals_dates(request.deal_status_id)
 
         stmt_deal_services = self._get_stmt_deal_services(sub_deals_dates)
 
@@ -347,14 +407,14 @@ class StatisticsService(BaseService):
                 func.date_trunc(
                     "day",
                     Deal.created_at
-                ).label("deal_date"),
+                ).label("date"),
                 stmt_deal_product_services.c.revenue.label("revenue"),
                 stmt_deal_product_services.c.profit.label("profit"),
             )
             .join(Deal, Deal.id == stmt_deal_product_services.c.deal_id)
         )
 
-        stmt_deal_services, stmt_deal_product_services = self._apply_chart_filters(
+        stmt_deal_services, stmt_deal_product_services = self._apply_filters(
             request,
             stmt_deal_services,
             stmt_deal_product_services
@@ -380,7 +440,7 @@ class StatisticsService(BaseService):
         stmt_deal_services = self._get_stmt_deal_services(sub_deals_dates)
         stmt_deal_product_services = self._get_stmt_product_services()
 
-        stmt_deal_services, stmt_deal_product_services = self._apply_chart_filters(
+        stmt_deal_services, stmt_deal_product_services = self._apply_filters(
             request,
             stmt_deal_services,
             stmt_deal_product_services