feat: add protobuf message compression and decompression for Redis; refactor product fetching logic

This commit is contained in:
2025-08-17 06:16:15 +03:00
parent 38acc4a443
commit abbcc0a81a
1396 changed files with 609 additions and 451436 deletions

View File

@@ -2,16 +2,17 @@ package ozon
import (
"context"
"encoding/json"
"fmt"
"github.com/hibiken/asynq"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/samber/lo"
pb "sipro-mps/api/generated/v1/ozon/products"
"sipro-mps/internal/marketplace"
"sipro-mps/internal/ozon/products/mapping/generated"
"sipro-mps/internal/redis"
"sipro-mps/internal/tasks/types"
"github.com/hibiken/asynq"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/samber/lo"
"sipro-mps/internal/ozon/products"
)
@@ -52,15 +53,12 @@ func (p *FetchProductsProcessor) ProcessTask(ctx context.Context, task *asynq.Ta
productsProto := lo.Map(productsRaw, func(item products.OzonProduct, _ int) *products.PbProduct {
return converter.ToProto(&item)
})
productsJson, err := json.Marshal(productsProto)
redisKey := fmt.Sprintf("ozon:products:%s", identifier)
err = redis.WriteProtoMessage(ctx, redisKey, &pb.GetListOfProductsResponse{Products: productsProto})
if err != nil {
return nil
}
redisClient := *redis.Client
productsKey := fmt.Sprintf("ozon:products:%s", identifier)
err = redisClient.Do(ctx, redisClient.B().Set().Key(productsKey).Value(string(productsJson)).Build()).Error()
if err != nil {
return err
fmt.Printf("Failed to write products to Redis for marketplace %s: %v\n", identifier, err)
return asynq.RevokeTask
}
return nil
}