fakz9/Sipro-Marketplaces
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add protobuf message compression and decompression for Redis; refactor product fetching logic

Browse Source
This commit is contained in:
admin
2025-08-17 06:16:15 +03:00
parent 38acc4a443
commit abbcc0a81a
1396 changed files with 609 additions and 451436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
internal/ozon/products/repository_api.go
View File

@@ -2,10 +2,7 @@ package products
import (
"context"
"encoding/json"
"fmt"
api "git.denco.store/fakz9/ozon-api-client/ozon"
"github.com/samber/lo"
pb "sipro-mps/api/generated/v1/ozon/products"
"sipro-mps/internal/marketplace"
"sipro-mps/internal/ozon"
@@ -13,8 +10,12 @@ import (
"sipro-mps/internal/redis"
"sipro-mps/internal/tasks/client"
"sipro-mps/internal/tasks/types"
"sipro-mps/pkg/utils"
"strconv"
"sync"
api "git.denco.store/fakz9/ozon-api-client/ozon"
"github.com/samber/lo"
)
type apiRepository struct {
@@ -135,28 +136,13 @@ func (a *apiRepository) StreamAllProductsCache(ctx context.Context, marketplaceI
errChan <- fmt.Errorf("getting marketplace identifier: %w", err)
return
}
r := *redis.Client
key := fmt.Sprintf("ozon:products:%s", identifier)
var cachedMessage pb.GetListOfProductsResponse
err = redis.ReadProtoMessage(ctx, key, &cachedMessage)
if err == nil && len(cachedMessage.Products) > 0 {
resultChan <- utils.DerefSlice(cachedMessage.Products)
_ = client.EnqueueFetchProductsTask(types.TypeOzonFetchProducts, marketplaceId)
jsonString, err := r.Do(ctx, r.B().Get().Key(key).Build()).ToString()
if err == nil && jsonString != "null" {
var result []PbProduct
err = json.Unmarshal([]byte(jsonString), &result)
if err != nil {
errChan <- fmt.Errorf("unmarshalling products from cache: %w", err)
return
}
task, err := types.NewFetchProductsTask(types.TypeOzonFetchProducts, marketplaceId)
if err != nil {
errChan <- fmt.Errorf("creating fetch products task: %w", err)
return
}
_, err = client.Client.Enqueue(task)
if err != nil {
errChan <- fmt.Errorf("enqueueing fetch products task: %w", err)
return
}
resultChan <- result
return
}
@@ -167,17 +153,11 @@ func (a *apiRepository) StreamAllProductsCache(ctx context.Context, marketplaceI
converter := generated.ConverterImpl{}
var allProducts []PbProduct
defer func() {
value, err := json.Marshal(allProducts)
if err != nil {
errChan <- fmt.Errorf("marshalling products to JSON: %w", err)
return
}
err = r.Do(ctx, r.B().Set().Key(key).Value(string(value)).Build()).Error()
if err != nil {
errChan <- fmt.Errorf("setting products in cache: %w", err)
if len(allProducts) == 0 {
return
}
message := pb.GetListOfProductsResponse{Products: utils.ToPtrs(allProducts)}
_ = redis.WriteProtoMessage(ctx, key, &message)
}()
for {
select {
@@ -219,7 +199,28 @@ func (a *apiRepository) StreamProductAttributesCache(ctx context.Context, market
errChan <- err
return
}
identifider, err := mp.GetIdentifier()
if err != nil {
errChan <- fmt.Errorf("getting marketplace identifier: %w", err)
return
}
key := fmt.Sprintf("ozon:products:attributes:%s", identifider)
var cachedMessage pb.GetProductAttributesResponse
err = redis.ReadProtoMessage(ctx, key, &cachedMessage)
if err == nil && len(cachedMessage.Items) > 0 {
resultChan <- utils.DerefSlice(cachedMessage.Items)
return
}
converter := generated.ConverterImpl{}
var allAttributes []PbProductAttributes
defer func() {
if len(allAttributes) == 0 {
return
}
message := pb.GetProductAttributesResponse{Items: utils.ToPtrs(allAttributes)}
_ = redis.WriteProtoMessage(ctx, key, &message)
}()
for _, chunk := range lo.Chunk(productIds, 1000) {
chunkStrings := lo.Map(chunk, func(item int, index int) string {
return strconv.Itoa(item)
@@ -236,9 +237,11 @@ func (a *apiRepository) StreamProductAttributesCache(ctx context.Context, market
errChan <- err
return
}
resultChan <- lo.Map(response.Result, func(item api.GetDescriptionOfProductsResult, index int) pb.ProductAttributes {
attrs := lo.Map(response.Result, func(item api.GetDescriptionOfProductsResult, index int) pb.ProductAttributes {
return *converter.AttributesToProto(&item)
})
resultChan <- attrs
allAttributes = append(allAttributes, attrs...)
}
//ozonClient.Products().GetDescriptionOfProducts()

97
internal/redis/client.go
View File

@@ -1,9 +1,17 @@
package redis
import (
"bytes"
"compress/flate"
"compress/zlib"
"context"
"github.com/redis/rueidis"
"fmt"
"io"
"os"
"time"
"github.com/golang/protobuf/proto"
"github.com/redis/rueidis"
)
var Client *rueidis.Client
@@ -35,3 +43,90 @@ func CloseClient() {
(*Client).Close()
}
}
// WriteProtoMessage compresses and writes a protobuf message to Redis
func WriteProtoMessage(ctx context.Context, key string, message proto.Message, ttl ...time.Duration) error {
if Client == nil {
return fmt.Errorf("redis client not initialized")
}
if message == nil {
return fmt.Errorf("message is nil")
}
// Marshal protobuf message
bytesMessage, err := proto.Marshal(message)
if err != nil {
return fmt.Errorf("failed to marshal proto message: %w", err)
}
// Compress with zlib
var buf bytes.Buffer
w, err := zlib.NewWriterLevel(&buf, flate.BestCompression)
if err != nil {
return fmt.Errorf("failed to create zlib writer: %w", err)
}
defer w.Close() // гарантированное закрытие
if _, err := w.Write(bytesMessage); err != nil {
return fmt.Errorf("failed to write to zlib writer: %w", err)
}
if err := w.Close(); err != nil { // финализируем сжатие
return fmt.Errorf("failed to close zlib writer: %w", err)
}
var ttlDuration time.Duration
if len(ttl) > 0 {
ttlDuration = ttl[0]
} else {
ttlDuration = 3 * time.Hour // Default TTL of 24 hours
}
// Write to Redis
if err := (*Client).Do(ctx, (*Client).B().
Set().
Key(key).
Value(rueidis.BinaryString(buf.Bytes())).
Ex(ttlDuration).
Build()).
Error(); err != nil {
return fmt.Errorf("failed to write compressed data to Redis: %w", err)
}
return nil
}
// ReadProtoMessage reads and decompresses a protobuf message from Redis
func ReadProtoMessage(ctx context.Context, key string, message proto.Message) error {
if Client == nil {
return fmt.Errorf("redis client not initialized")
}
if message == nil {
return fmt.Errorf("message is nil")
}
// Get bytes from Redis
resp, err := (*Client).Do(ctx, (*Client).B().Get().Key(key).Build()).AsBytes()
if err != nil {
return fmt.Errorf("failed to read data from Redis: %w", err)
}
if resp == nil {
return fmt.Errorf("no data found for key: %s", key)
}
// Decompress
reader, err := zlib.NewReader(bytes.NewReader(resp))
if err != nil {
return fmt.Errorf("failed to create zlib reader: %w", err)
}
defer reader.Close()
decompressed, err := io.ReadAll(reader)
if err != nil {
return fmt.Errorf("failed to decompress data: %w", err)
}
// Unmarshal protobuf
if err := proto.Unmarshal(decompressed, message); err != nil {
return fmt.Errorf("failed to unmarshal proto message: %w", err)
}
return nil
}

3
internal/redis/lock.go
View File

@@ -1,9 +1,10 @@
package redis
import (
"os"
"github.com/redis/rueidis"
"github.com/redis/rueidis/rueidislock"
"os"
)
var Locker *rueidislock.Locker

16
internal/tasks/client/client.go
View File

@@ -1,8 +1,10 @@
package client
import (
"github.com/hibiken/asynq"
"sipro-mps/internal/config"
"sipro-mps/internal/tasks/types"
"github.com/hibiken/asynq"
)
var Client *asynq.Client
@@ -23,3 +25,15 @@ func CloseClient() {
}
}
}
func EnqueueFetchProductsTask(taskType string, marketplaceId int) error {
task, err := types.NewFetchProductsTask(taskType, marketplaceId)
if err != nil {
return err
}
_, err = Client.Enqueue(task)
if err != nil {
return err
}
return nil
}

22
internal/tasks/ozon/ozon.go
View File

@@ -2,16 +2,17 @@ package ozon
import (
"context"
"encoding/json"
"fmt"
"github.com/hibiken/asynq"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/samber/lo"
pb "sipro-mps/api/generated/v1/ozon/products"
"sipro-mps/internal/marketplace"
"sipro-mps/internal/ozon/products/mapping/generated"
"sipro-mps/internal/redis"
"sipro-mps/internal/tasks/types"
"github.com/hibiken/asynq"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/samber/lo"
"sipro-mps/internal/ozon/products"
)
@@ -52,15 +53,12 @@ func (p *FetchProductsProcessor) ProcessTask(ctx context.Context, task *asynq.Ta
productsProto := lo.Map(productsRaw, func(item products.OzonProduct, _ int) *products.PbProduct {
return converter.ToProto(&item)
})
productsJson, err := json.Marshal(productsProto)
redisKey := fmt.Sprintf("ozon:products:%s", identifier)
err = redis.WriteProtoMessage(ctx, redisKey, &pb.GetListOfProductsResponse{Products: productsProto})
if err != nil {
return nil
}
redisClient := *redis.Client
productsKey := fmt.Sprintf("ozon:products:%s", identifier)
err = redisClient.Do(ctx, redisClient.B().Set().Key(productsKey).Value(string(productsJson)).Build()).Error()
if err != nil {
return err
fmt.Printf("Failed to write products to Redis for marketplace %s: %v\n", identifier, err)
return asynq.RevokeTask
}
return nil
}

23
internal/tasks/wb/wb.go
View File

@@ -4,15 +4,16 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/hibiken/asynq"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/samber/lo"
pb "sipro-mps/api/generated/v1/wb/products"
"sipro-mps/internal/marketplace"
"sipro-mps/internal/redis"
"sipro-mps/internal/tasks/types"
"sipro-mps/internal/wb/products"
"sipro-mps/internal/wb/products/mapping/generated"
"github.com/hibiken/asynq"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/samber/lo"
)
type FetchProductsProcessor struct {
@@ -22,7 +23,7 @@ type FetchProductsProcessor struct {
func (p *FetchProductsProcessor) ProcessTask(ctx context.Context, task *asynq.Task) error {
var payload types.FetchProductsTask
if err := json.Unmarshal(task.Payload(), &payload); err != nil {
return asynq.SkipRetry
return asynq.RevokeTask
}
marketplaceRepo := marketplace.NewDBRepository(p.Dbpool)
repo := products.NewAPIRepository(marketplaceRepo)
@@ -38,7 +39,7 @@ func (p *FetchProductsProcessor) ProcessTask(ctx context.Context, task *asynq.Ta
_, cancel, err := locker.TryWithContext(ctx, fmt.Sprintf("wb:products:marketplace:%s:lock", sellerId))
if err != nil {
fmt.Printf("Failed to acquire lock for marketplace %s: %v\n", sellerId, err)
return asynq.SkipRetry
return asynq.RevokeTask
}
fmt.Println("Working on marketplace", payload.MarketplaceId, "with seller ID", sellerId)
@@ -54,15 +55,11 @@ func (p *FetchProductsProcessor) ProcessTask(ctx context.Context, task *asynq.Ta
productsProto := lo.Map(productsRaw, func(item products.WbProduct, _ int) *pb.Product {
return converter.ToProto(&item)
})
redisClient := *redis.Client
productsJson, err := json.Marshal(productsProto)
err = redis.WriteProtoMessage(ctx, redisKey, &pb.GetProductsResponse{Products: productsProto})
if err != nil {
return fmt.Errorf("failed to marshal products: %w", err)
fmt.Printf(
"Failed to write products to Redis for marketplace %s: %v\n")
return asynq.RevokeTask
}
err = redisClient.Do(ctx, redisClient.B().Set().Key(redisKey).Value(string(productsJson)).Build()).Error()
if err != nil {
return err
}
return nil
}

9
internal/wb/products/adapter_grpc.go
View File

@@ -2,10 +2,11 @@ package products
import (
"fmt"
"github.com/samber/lo"
"google.golang.org/grpc"
pb "sipro-mps/api/generated/v1/wb/products"
"sipro-mps/internal/marketplace"
"sipro-mps/pkg/utils"
"google.golang.org/grpc"
)
type AdapterGRPC struct {
@@ -41,9 +42,7 @@ func (a *AdapterGRPC) GetProducts(req *pb.GetProductsRequest, stream pb.Products
return nil
}
resp := &pb.GetProductsResponse{
Products: lo.Map(products, func(p pb.Product, _ int) *pb.Product {
return &p
}),
Products: utils.ToPtrs(products),
}
if err := stream.Send(resp); err != nil {
fmt.Println("error sending response", err)

124
internal/wb/products/repository_api.go
View File

@@ -5,12 +5,14 @@ import (
"fmt"
pb "sipro-mps/api/generated/v1/wb/products"
"sipro-mps/internal/marketplace"
"sipro-mps/internal/redis"
"sipro-mps/internal/tasks/client"
"sipro-mps/internal/tasks/types"
"sipro-mps/internal/wb"
"sipro-mps/internal/wb/products/mapping/generated"
wbapi "sipro-mps/pkg/api/wb/client"
"sipro-mps/pkg/utils"
"github.com/deliveryhero/pipeline/v2"
"github.com/samber/lo"
)
@@ -104,87 +106,51 @@ func (a apiRepository) StreamAllProductsCache(ctx context.Context, marketplaceId
errChan <- fmt.Errorf("getting marketplace identifier: %w", err)
return
}
client, err := wb.GetClientFromMarketplace(mp)
if err != nil {
converter := generated.ConverterImpl{}
key := fmt.Sprintf("wb:products:%s", identifier)
var cachedMessage pb.GetProductsResponse
err = redis.ReadProtoMessage(ctx, key, &cachedMessage)
if err == nil && len(cachedMessage.Products) > 0 {
resultChan <- utils.DerefSlice(cachedMessage.Products)
_ = client.EnqueueFetchProductsTask(types.TypeWbFetchProducts, marketplaceId)
return
}
converter := generated.ConverterImpl{}
transform := pipeline.NewProcessor(func(_ context.Context, products []WbProduct) ([]pb.Product, error) {
return lo.Map(products, func(item WbProduct, _ int) pb.Product {
return *converter.ToProto(&item)
}), nil
}, nil)
inputChan := make(chan []WbProduct)
fetchProducts(ctx, client, identifier, inputChan, nil)
for out := range pipeline.Process(ctx, transform, inputChan) {
resultChan <- out
}
//c := *redis.Client
//key := fmt.Sprintf("wb:products:%s", sellerId)
//jsonString, err := c.Do(ctx, c.B().Get().Key(key).Build()).ToString()
//if err == nil && jsonString != "null" {
// var result []pb.Product
// err = json.Unmarshal([]byte(jsonString), &result)
// if err != nil {
// errChan <- fmt.Errorf("unmarshalling products from cache: %w", err)
// return
// }
// task, err := types.NewFetchProductsTask(types.TypeWbFetchProducts, marketplaceId)
// if err != nil {
// errChan <- fmt.Errorf("creating fetch products task: %w", err)
// return
// }
// _, err = client.Client.Enqueue(task)
// if err != nil {
// errChan <- fmt.Errorf("enqueueing fetch products task: %w", err)
// return
// }
//
// resultChan <- result
// return
//}
//if !errors.As(err, &rueidis.Nil) && err != nil {
// errChan <- fmt.Errorf("fetching products from cache: %w", err)
// return
//}
//converter := generated.ConverterImpl{}
//
//innerResultChan := make(chan []WbProduct)
//innerErrChan := make(chan error)
//go a.StreamAllProducts(ctx, marketplaceId, innerResultChan, innerErrChan)
//var allProducts []pb.Product
//defer func() {
// jsonData, err := json.Marshal(allProducts)
// if err != nil {
// errChan <- fmt.Errorf("marshalling products to cache: %w", err)
// return
// }
// err = c.Do(ctx, c.B().Set().Key(key).Value(string(jsonData)).Build()).Error()
// if err != nil {
// errChan <- fmt.Errorf("setting products to cache: %w", err)
// return
// }
//}()
//for {
// select {
// case err, ok := <-innerErrChan:
// if !ok {
// return
// }
// errChan <- fmt.Errorf("streaming products: %w", err)
// return
// case products, ok := <-innerResultChan:
// if !ok {
// return
// }
// pbProducts := lo.Map(products, func(p WbProduct, _ int) pb.Product {
// return *converter.ToProto(&p)
// })
// allProducts = append(allProducts, pbProducts...)
// resultChan <- pbProducts
// }
//}
innerResultChan := make(chan []WbProduct)
innerErrChan := make(chan error)
go a.StreamAllProducts(ctx, marketplaceId, innerResultChan, innerErrChan)
var allProducts []pb.Product
defer func() {
if len(allProducts) == 0 {
return
}
message := pb.GetProductsResponse{
Products: utils.ToPtrs(allProducts),
}
_ = redis.WriteProtoMessage(ctx, key, &message)
}()
for {
select {
case err, ok := <-innerErrChan:
if !ok {
return
}
errChan <- fmt.Errorf("streaming products: %w", err)
return
case products, ok := <-innerResultChan:
if !ok {
return
}
pbProducts := lo.Map(products, func(p WbProduct, _ int) pb.Product {
return *converter.ToProto(&p)
})
allProducts = append(allProducts, pbProducts...)
resultChan <- pbProducts
}
}
}
func (a apiRepository) GetAllProducts(ctx context.Context, marketplaceId int) ([]WbProduct, error) {

2
internal/ym/common.go
View File

@@ -2,9 +2,9 @@ package ym
import (
"errors"
"git.denco.store/fakz9/yandex-go-client"
"net/http"
"sipro-mps/internal/marketplace"
"sipro-mps/pkg/api/yandex/ymclient"
"strings"
"github.com/tidwall/gjson"

3
internal/ym/products/mapping/converter.go
View File

@@ -1,8 +1,8 @@
package mapping
import (
"git.denco.store/fakz9/yandex-go-client"
proto "sipro-mps/api/generated/v1/yandexmarket/products"
"sipro-mps/pkg/api/yandex/ymclient"
)
//go:generate go run github.com/jmattheis/goverter/cmd/goverter gen .
@@ -12,6 +12,7 @@ import (
// goverter:output:package generated
// goverter:ignoreUnexported yes
// goverter:matchIgnoreCase yes
// goverter:ignoreMissing yes
// goverter:useZeroValueOnPointerInconsistency yes
// goverter:extend Int64ToFloat32 Int64ToInt32 Float32ToInt64 Int32ToInt64 PointerInt32ToInt64
type Converter interface {

36
internal/ym/products/mapping/generated/generated.go
View File

@@ -4,17 +4,17 @@
package generated
import (
yandexgoclient "git.denco.store/fakz9/yandex-go-client"
products "sipro-mps/api/generated/v1/yandexmarket/products"
mapping "sipro-mps/internal/ym/products/mapping"
ymclient "sipro-mps/pkg/api/yandex/ymclient"
)
type ConverterImpl struct{}
func (c *ConverterImpl) ProtoOfferToYmOffer(source *products.CalculateProductTariffsRequest_Offers) *ymclient.CalculateTariffsOfferDTO {
var pYmclientCalculateTariffsOfferDTO *ymclient.CalculateTariffsOfferDTO
func (c *ConverterImpl) ProtoOfferToYmOffer(source *products.CalculateProductTariffsRequest_Offers) *yandexgoclient.CalculateTariffsOfferDTO {
var pYmclientCalculateTariffsOfferDTO *yandexgoclient.CalculateTariffsOfferDTO
if source != nil {
var ymclientCalculateTariffsOfferDTO ymclient.CalculateTariffsOfferDTO
var ymclientCalculateTariffsOfferDTO yandexgoclient.CalculateTariffsOfferDTO
ymclientCalculateTariffsOfferDTO.CategoryId = (*source).CategoryId
ymclientCalculateTariffsOfferDTO.Price = mapping.Int64ToFloat32((*source).Price)
ymclientCalculateTariffsOfferDTO.Length = mapping.Int64ToFloat32((*source).Length)
@@ -27,23 +27,23 @@ func (c *ConverterImpl) ProtoOfferToYmOffer(source *products.CalculateProductTar
}
return pYmclientCalculateTariffsOfferDTO
}
func (c *ConverterImpl) ProtoParametersToYmParameters(source *products.CalculateProductTariffsRequest_Parameters) *ymclient.CalculateTariffsParametersDTO {
var pYmclientCalculateTariffsParametersDTO *ymclient.CalculateTariffsParametersDTO
func (c *ConverterImpl) ProtoParametersToYmParameters(source *products.CalculateProductTariffsRequest_Parameters) *yandexgoclient.CalculateTariffsParametersDTO {
var pYmclientCalculateTariffsParametersDTO *yandexgoclient.CalculateTariffsParametersDTO
if source != nil {
var ymclientCalculateTariffsParametersDTO ymclient.CalculateTariffsParametersDTO
var ymclientCalculateTariffsParametersDTO yandexgoclient.CalculateTariffsParametersDTO
pInt64 := (*source).CampaignId
ymclientCalculateTariffsParametersDTO.CampaignId = &pInt64
pYmclientSellingProgramType := ymclient.SellingProgramType((*source).SellingProgram)
pYmclientSellingProgramType := yandexgoclient.SellingProgramType((*source).SellingProgram)
ymclientCalculateTariffsParametersDTO.SellingProgram = &pYmclientSellingProgramType
pYmclientPaymentFrequencyType := ymclient.PaymentFrequencyType((*source).Frequency)
pYmclientPaymentFrequencyType := yandexgoclient.PaymentFrequencyType((*source).Frequency)
ymclientCalculateTariffsParametersDTO.Frequency = &pYmclientPaymentFrequencyType
pYmclientCurrencyType := ymclient.CurrencyType((*source).Currency)
pYmclientCurrencyType := yandexgoclient.CurrencyType((*source).Currency)
ymclientCalculateTariffsParametersDTO.Currency = &pYmclientCurrencyType
pYmclientCalculateTariffsParametersDTO = &ymclientCalculateTariffsParametersDTO
}
return pYmclientCalculateTariffsParametersDTO
}
func (c *ConverterImpl) YmOfferToProtoOffer(source *ymclient.GetOfferDTO) *products.GetProductsResponse_Offer {
func (c *ConverterImpl) YmOfferToProtoOffer(source *yandexgoclient.GetOfferDTO) *products.GetProductsResponse_Offer {
var pProductsGetProductsResponse_Offer *products.GetProductsResponse_Offer
if source != nil {
var productsGetProductsResponse_Offer products.GetProductsResponse_Offer
@@ -57,7 +57,7 @@ func (c *ConverterImpl) YmOfferToProtoOffer(source *ymclient.GetOfferDTO) *produ
}
return pProductsGetProductsResponse_Offer
}
func (c *ConverterImpl) YmOfferToProtoResponseOffer(source *ymclient.CalculateTariffsOfferInfoDTO) *products.CalculateProductTariffsResponse_Offers {
func (c *ConverterImpl) YmOfferToProtoResponseOffer(source *yandexgoclient.CalculateTariffsOfferInfoDTO) *products.CalculateProductTariffsResponse_Offers {
var pProductsCalculateProductTariffsResponse_Offers *products.CalculateProductTariffsResponse_Offers
if source != nil {
var productsCalculateProductTariffsResponse_Offers products.CalculateProductTariffsResponse_Offers
@@ -72,7 +72,7 @@ func (c *ConverterImpl) YmOfferToProtoResponseOffer(source *ymclient.CalculateTa
}
return pProductsCalculateProductTariffsResponse_Offers
}
func (c *ConverterImpl) YmTariffToProtoTariff(source *ymclient.CalculatedTariffDTO) *products.CalculateProductTariffsResponse_Tariff {
func (c *ConverterImpl) YmTariffToProtoTariff(source *yandexgoclient.CalculatedTariffDTO) *products.CalculateProductTariffsResponse_Tariff {
var pProductsCalculateProductTariffsResponse_Tariff *products.CalculateProductTariffsResponse_Tariff
if source != nil {
var productsCalculateProductTariffsResponse_Tariff products.CalculateProductTariffsResponse_Tariff
@@ -93,7 +93,7 @@ func (c *ConverterImpl) YmTariffToProtoTariff(source *ymclient.CalculatedTariffD
}
return pProductsCalculateProductTariffsResponse_Tariff
}
func (c *ConverterImpl) pYmclientGetPriceWithDiscountDTOToPProductsGetProductsResponse_Offer_BasicPrice(source *ymclient.GetPriceWithDiscountDTO) *products.GetProductsResponse_Offer_BasicPrice {
func (c *ConverterImpl) pYmclientGetPriceWithDiscountDTOToPProductsGetProductsResponse_Offer_BasicPrice(source *yandexgoclient.GetPriceWithDiscountDTO) *products.GetProductsResponse_Offer_BasicPrice {
var pProductsGetProductsResponse_Offer_BasicPrice *products.GetProductsResponse_Offer_BasicPrice
if source != nil {
var productsGetProductsResponse_Offer_BasicPrice products.GetProductsResponse_Offer_BasicPrice
@@ -102,7 +102,7 @@ func (c *ConverterImpl) pYmclientGetPriceWithDiscountDTOToPProductsGetProductsRe
}
return pProductsGetProductsResponse_Offer_BasicPrice
}
func (c *ConverterImpl) pYmclientOfferWeightDimensionsDTOToPProductsGetProductsResponse_Offer_WeightDimensions(source *ymclient.OfferWeightDimensionsDTO) *products.GetProductsResponse_Offer_WeightDimensions {
func (c *ConverterImpl) pYmclientOfferWeightDimensionsDTOToPProductsGetProductsResponse_Offer_WeightDimensions(source *yandexgoclient.OfferWeightDimensionsDTO) *products.GetProductsResponse_Offer_WeightDimensions {
var pProductsGetProductsResponse_Offer_WeightDimensions *products.GetProductsResponse_Offer_WeightDimensions
if source != nil {
var productsGetProductsResponse_Offer_WeightDimensions products.GetProductsResponse_Offer_WeightDimensions
@@ -114,7 +114,7 @@ func (c *ConverterImpl) pYmclientOfferWeightDimensionsDTOToPProductsGetProductsR
}
return pProductsGetProductsResponse_Offer_WeightDimensions
}
func (c *ConverterImpl) ymclientCalculateTariffsOfferDTOToPProductsCalculateProductTariffsResponse_Offer(source ymclient.CalculateTariffsOfferDTO) *products.CalculateProductTariffsResponse_Offer {
func (c *ConverterImpl) ymclientCalculateTariffsOfferDTOToPProductsCalculateProductTariffsResponse_Offer(source yandexgoclient.CalculateTariffsOfferDTO) *products.CalculateProductTariffsResponse_Offer {
var productsCalculateProductTariffsResponse_Offer products.CalculateProductTariffsResponse_Offer
productsCalculateProductTariffsResponse_Offer.CategoryId = source.CategoryId
productsCalculateProductTariffsResponse_Offer.Price = mapping.Float32ToInt64(source.Price)
@@ -127,7 +127,7 @@ func (c *ConverterImpl) ymclientCalculateTariffsOfferDTOToPProductsCalculateProd
}
return &productsCalculateProductTariffsResponse_Offer
}
func (c *ConverterImpl) ymclientCalculatedTariffDTOToPProductsCalculateProductTariffsResponse_Tariff(source ymclient.CalculatedTariffDTO) *products.CalculateProductTariffsResponse_Tariff {
func (c *ConverterImpl) ymclientCalculatedTariffDTOToPProductsCalculateProductTariffsResponse_Tariff(source yandexgoclient.CalculatedTariffDTO) *products.CalculateProductTariffsResponse_Tariff {
var productsCalculateProductTariffsResponse_Tariff products.CalculateProductTariffsResponse_Tariff
productsCalculateProductTariffsResponse_Tariff.Type = string(source.Type)
if source.Amount != nil {
@@ -144,7 +144,7 @@ func (c *ConverterImpl) ymclientCalculatedTariffDTOToPProductsCalculateProductTa
}
return &productsCalculateProductTariffsResponse_Tariff
}
func (c *ConverterImpl) ymclientTariffParameterDTOToPProductsCalculateProductTariffsResponse_Parameter(source ymclient.TariffParameterDTO) *products.CalculateProductTariffsResponse_Parameter {
func (c *ConverterImpl) ymclientTariffParameterDTOToPProductsCalculateProductTariffsResponse_Parameter(source yandexgoclient.TariffParameterDTO) *products.CalculateProductTariffsResponse_Parameter {
var productsCalculateProductTariffsResponse_Parameter products.CalculateProductTariffsResponse_Parameter
productsCalculateProductTariffsResponse_Parameter.Name = source.Name
productsCalculateProductTariffsResponse_Parameter.Value = source.Value

24
internal/ym/products/repository_api.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"math"
"sipro-mps/internal/redis"
"strconv"
"time"
@@ -11,8 +12,8 @@ import (
"sipro-mps/internal/marketplace"
"sipro-mps/internal/ym"
"sipro-mps/internal/ym/products/mapping/generated"
"sipro-mps/pkg/api/yandex/ymclient"
"git.denco.store/fakz9/yandex-go-client"
"github.com/samber/lo"
)
@@ -100,9 +101,25 @@ func (r *apiRepository) GetProducts(ctx context.Context, marketplaceID int, req
errChan <- err
return
}
key := fmt.Sprintf("ym:products:%d", businessID)
var cachedMessage pb.GetProductsResponse
err = redis.ReadProtoMessage(ctx, key, &cachedMessage)
if err == nil && len(cachedMessage.Offers) > 0 {
resultChan <- cachedMessage.Offers
return
}
r.setOfferMappingsRateLimit(businessID)
// all offers
var allOffers []*pb.GetProductsResponse_Offer
defer func() {
if len(allOffers) == 0 {
return
}
message := pb.GetProductsResponse{
Offers: allOffers,
}
_ = redis.WriteProtoMessage(ctx, key, &message)
}()
for _, chunk := range lo.Chunk(req.OfferIds, defaultChunkSize) {
offers, err := r.fetchOfferMappings(ctx, client, businessID, chunk)
if err != nil {
@@ -110,6 +127,7 @@ func (r *apiRepository) GetProducts(ctx context.Context, marketplaceID int, req
return
}
resultChan <- offers
allOffers = append(allOffers, offers...)
}
}