fakz9/Sipro-Marketplaces
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add protobuf message compression and decompression for Redis; refactor product fetching logic

Browse Source
This commit is contained in:
admin
2025-08-17 06:16:15 +03:00
parent 38acc4a443
commit abbcc0a81a
1396 changed files with 609 additions and 451436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
internal/ozon/products/repository_api.go
View File

@@ -2,10 +2,7 @@ package products
import (
"context"
"encoding/json"
"fmt"
api "git.denco.store/fakz9/ozon-api-client/ozon"
"github.com/samber/lo"
pb "sipro-mps/api/generated/v1/ozon/products"
"sipro-mps/internal/marketplace"
"sipro-mps/internal/ozon"
@@ -13,8 +10,12 @@ import (
"sipro-mps/internal/redis"
"sipro-mps/internal/tasks/client"
"sipro-mps/internal/tasks/types"
"sipro-mps/pkg/utils"
"strconv"
"sync"
api "git.denco.store/fakz9/ozon-api-client/ozon"
"github.com/samber/lo"
)
type apiRepository struct {
@@ -135,28 +136,13 @@ func (a *apiRepository) StreamAllProductsCache(ctx context.Context, marketplaceI
errChan <- fmt.Errorf("getting marketplace identifier: %w", err)
return
}
r := *redis.Client
key := fmt.Sprintf("ozon:products:%s", identifier)
var cachedMessage pb.GetListOfProductsResponse
err = redis.ReadProtoMessage(ctx, key, &cachedMessage)
if err == nil && len(cachedMessage.Products) > 0 {
resultChan <- utils.DerefSlice(cachedMessage.Products)
_ = client.EnqueueFetchProductsTask(types.TypeOzonFetchProducts, marketplaceId)
jsonString, err := r.Do(ctx, r.B().Get().Key(key).Build()).ToString()
if err == nil && jsonString != "null" {
var result []PbProduct
err = json.Unmarshal([]byte(jsonString), &result)
if err != nil {
errChan <- fmt.Errorf("unmarshalling products from cache: %w", err)
return
}
task, err := types.NewFetchProductsTask(types.TypeOzonFetchProducts, marketplaceId)
if err != nil {
errChan <- fmt.Errorf("creating fetch products task: %w", err)
return
}
_, err = client.Client.Enqueue(task)
if err != nil {
errChan <- fmt.Errorf("enqueueing fetch products task: %w", err)
return
}
resultChan <- result
return
}
@@ -167,17 +153,11 @@ func (a *apiRepository) StreamAllProductsCache(ctx context.Context, marketplaceI
converter := generated.ConverterImpl{}
var allProducts []PbProduct
defer func() {
value, err := json.Marshal(allProducts)
if err != nil {
errChan <- fmt.Errorf("marshalling products to JSON: %w", err)
return
}
err = r.Do(ctx, r.B().Set().Key(key).Value(string(value)).Build()).Error()
if err != nil {
errChan <- fmt.Errorf("setting products in cache: %w", err)
if len(allProducts) == 0 {
return
}
message := pb.GetListOfProductsResponse{Products: utils.ToPtrs(allProducts)}
_ = redis.WriteProtoMessage(ctx, key, &message)
}()
for {
select {
@@ -219,7 +199,28 @@ func (a *apiRepository) StreamProductAttributesCache(ctx context.Context, market
errChan <- err
return
}
identifider, err := mp.GetIdentifier()
if err != nil {
errChan <- fmt.Errorf("getting marketplace identifier: %w", err)
return
}
key := fmt.Sprintf("ozon:products:attributes:%s", identifider)
var cachedMessage pb.GetProductAttributesResponse
err = redis.ReadProtoMessage(ctx, key, &cachedMessage)
if err == nil && len(cachedMessage.Items) > 0 {
resultChan <- utils.DerefSlice(cachedMessage.Items)
return
}
converter := generated.ConverterImpl{}
var allAttributes []PbProductAttributes
defer func() {
if len(allAttributes) == 0 {
return
}
message := pb.GetProductAttributesResponse{Items: utils.ToPtrs(allAttributes)}
_ = redis.WriteProtoMessage(ctx, key, &message)
}()
for _, chunk := range lo.Chunk(productIds, 1000) {
chunkStrings := lo.Map(chunk, func(item int, index int) string {
return strconv.Itoa(item)
@@ -236,9 +237,11 @@ func (a *apiRepository) StreamProductAttributesCache(ctx context.Context, market
errChan <- err
return
}
resultChan <- lo.Map(response.Result, func(item api.GetDescriptionOfProductsResult, index int) pb.ProductAttributes {
attrs := lo.Map(response.Result, func(item api.GetDescriptionOfProductsResult, index int) pb.ProductAttributes {
return *converter.AttributesToProto(&item)
})
resultChan <- attrs
allAttributes = append(allAttributes, attrs...)
}
//ozonClient.Products().GetDescriptionOfProducts()