fakz9/Sipro-Marketplaces
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor marketplace product fetching and caching logic; update environment configuration for Redis and PostgreSQL

Browse Source
This commit is contained in:
admin
2025-07-07 19:20:05 +03:00
parent c7be7e2cea
commit 3976c7d0cf
22 changed files with 319 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
internal/ozon/products/adapter_grpc.go
View File

@@ -6,7 +6,6 @@ import (
"google.golang.org/grpc"
pb "sipro-mps/api/generated/v1/ozon/products"
"sipro-mps/internal/marketplace"
"sipro-mps/internal/ozon/products/mapping/generated"
)
type AdapterGRPC struct {
@@ -30,12 +29,10 @@ func RegisterAdapterGRPC(server *grpc.Server, marketplaceRepo marketplace.Reposi
func (g *AdapterGRPC) GetListOfProducts(req *pb.GetListOfProductsRequest, stream pb.ProductsService_GetListOfProductsServer) error {
ctx := stream.Context()
fmt.Printf("GetListOfProducts called with req: %+v\n", req.MarketplaceId)
converter := generated.ConverterImpl{}
resultChan := make(chan []OzonProduct)
resultChan := make(chan []pb.Product)
errChan := make(chan error)
go g.repo.StreamAllProducts(ctx, int(req.MarketplaceId), resultChan, errChan) // Запускаем в горутине
go g.repo.StreamAllProductsCache(ctx, int(req.MarketplaceId), resultChan, errChan) // Запускаем в горутине
for {
select {
case <-ctx.Done():
@@ -45,11 +42,10 @@ func (g *AdapterGRPC) GetListOfProducts(req *pb.GetListOfProductsRequest, stream
if !ok {
return nil // Завершаем, только если результат закрыт
}
protoProducts := lo.Map(products, func(product OzonProduct, _ int) *pb.Product {
return converter.ToProto(&product)
})
resp := &pb.GetListOfProductsResponse{
Products: protoProducts,
Products: lo.Map(products, func(p pb.Product, _ int) *pb.Product {
return &p
}),
}
if err := stream.Send(resp); err != nil {
fmt.Println("GetListOfProducts: error sending response:", err)

6
internal/ozon/products/entities.go
View File

@@ -1,5 +1,9 @@
package products
import "git.denco.store/fakz9/ozon-api-client/ozon"
import (
"git.denco.store/fakz9/ozon-api-client/ozon"
pb "sipro-mps/api/generated/v1/ozon/products"
)
type OzonProduct = ozon.ProductDetails
type PbProduct = pb.Product

1
internal/ozon/products/repository.go
View File

@@ -5,4 +5,5 @@ import "context"
type Repository interface {
GetAllProducts(ctx context.Context, marketplaceId int) ([]OzonProduct, error)
StreamAllProducts(ctx context.Context, marketplaceId int, resultChan chan<- []OzonProduct, errChan chan<- error)
StreamAllProductsCache(ctx context.Context, marketplaceId int, resultChan chan<- []PbProduct, errChan chan<- error)
}

102
internal/ozon/products/repository_api.go
View File

@@ -2,11 +2,16 @@ package products
import (
"context"
"encoding/json"
"fmt"
api "git.denco.store/fakz9/ozon-api-client/ozon"
"github.com/samber/lo"
"sipro-mps/internal/marketplace"
"sipro-mps/internal/ozon"
"sipro-mps/internal/ozon/products/mapping/generated"
"sipro-mps/internal/redis"
"sipro-mps/internal/tasks/client"
"sipro-mps/internal/tasks/types"
"sync"
)
@@ -81,7 +86,7 @@ func (a *apiRepository) GetAllProducts(ctx context.Context, marketplaceId int) (
if err != nil {
return nil, err
}
client, err := ozon.GetClientFromMarketplace(mp)
clientFromMarketplace, err := ozon.GetClientFromMarketplace(mp)
if err != nil {
return nil, err
}
@@ -89,15 +94,13 @@ func (a *apiRepository) GetAllProducts(ctx context.Context, marketplaceId int) (
productIdsChan := make(chan []int64)
producsChan := make(chan []OzonProduct)
errChan := make(chan error)
go fetchProductIds(ctx, client, productIdsChan, errChan)
go fetchProducts(ctx, client, productIdsChan, producsChan, errChan)
go fetchProductIds(ctx, clientFromMarketplace, productIdsChan, errChan)
go fetchProducts(ctx, clientFromMarketplace, productIdsChan, producsChan, errChan)
for products := range producsChan {
for _, product := range products {
fmt.Println(product.Name)
items = append(items, product)
}
}
fmt.Println(len(items))
return items, nil
}
@@ -107,12 +110,95 @@ func (a *apiRepository) StreamAllProducts(ctx context.Context, marketplaceId int
errChan <- err
return
}
client, err := ozon.GetClientFromMarketplace(mp)
clientFromMarketplace, err := ozon.GetClientFromMarketplace(mp)
if err != nil {
errChan <- err
return
}
productIdsChan := make(chan []int64)
go fetchProductIds(ctx, client, productIdsChan, errChan)
go fetchProducts(ctx, client, productIdsChan, resultChan, errChan)
go fetchProductIds(ctx, clientFromMarketplace, productIdsChan, errChan)
go fetchProducts(ctx, clientFromMarketplace, productIdsChan, resultChan, errChan)
}
func (a *apiRepository) StreamAllProductsCache(ctx context.Context, marketplaceId int, resultChan chan<- []PbProduct, errChan chan<- error) {
defer close(resultChan)
defer close(errChan)
mp, err := a.marketplaceRepository.GetMarketplaceByID(ctx, marketplaceId)
if err != nil {
errChan <- err
return
}
identifier, err := mp.GetIdentifier()
if err != nil {
errChan <- fmt.Errorf("getting marketplace identifier: %w", err)
return
}
r := *redis.Client
key := fmt.Sprintf("ozon:products:%s", identifier)
jsonString, err := r.Do(ctx, r.B().Get().Key(key).Build()).ToString()
if err == nil && jsonString != "null" {
var result []PbProduct
err = json.Unmarshal([]byte(jsonString), &result)
if err != nil {
errChan <- fmt.Errorf("unmarshalling products from cache: %w", err)
return
}
task, err := types.NewFetchProductsTask(types.TypeOzonFetchProducts, marketplaceId)
if err != nil {
errChan <- fmt.Errorf("creating fetch products task: %w", err)
return
}
_, err = client.Client.Enqueue(task)
if err != nil {
errChan <- fmt.Errorf("enqueueing fetch products task: %w", err)
return
}
resultChan <- result
return
}
innerResultChan := make(chan []OzonProduct)
innerErrChan := make(chan error)
go a.StreamAllProducts(ctx, marketplaceId, innerResultChan, innerErrChan)
converter := generated.ConverterImpl{}
var allProducts []PbProduct
defer func() {
value, err := json.Marshal(allProducts)
if err != nil {
errChan <- fmt.Errorf("marshalling products to JSON: %w", err)
return
}
err = r.Do(ctx, r.B().Set().Key(key).Value(string(value)).Build()).Error()
if err != nil {
errChan <- fmt.Errorf("setting products in cache: %w", err)
return
}
}()
for {
select {
case err, ok := <-innerErrChan:
if !ok {
return
}
if err != nil {
errChan <- fmt.Errorf("streaming products: %w", err)
return
}
case products, ok := <-innerResultChan:
if !ok && len(products) == 0 {
return
}
pbProducts := lo.Map(products, func(p OzonProduct, _ int) PbProduct {
return *converter.ToProto(&p)
})
allProducts = append(allProducts, pbProducts...)
resultChan <- pbProducts
if !ok {
return
}
}
}
}