Add Wildberries product fetching and rate limiting functionality

internal/wb/products/adapter_grpc.go

@@ -0,0 +1,63 @@
+package products
+
+import (
+	"fmt"
+	"github.com/samber/lo"
+	"google.golang.org/grpc"
+	pb "sipro-mps/api/generated/v1/wb/products"
+	"sipro-mps/internal/marketplace"
+)
+
+type AdapterGRPC struct {
+	pb.UnimplementedProductsServiceServer
+	repo Repository
+}
+
+func NewAdapterGRPC(repo Repository) *AdapterGRPC {
+	return &AdapterGRPC{
+		repo: repo,
+	}
+}
+
+func RegisterAdapterGRPC(server *grpc.Server, marketplacesRepository marketplace.Repository) (*Repository, error) {
+	repo := NewAPIRepository(marketplacesRepository)
+	adapter := NewAdapterGRPC(repo)
+	pb.RegisterProductsServiceServer(server, adapter)
+	return &repo, nil
+}
+func (a *AdapterGRPC) GetProducts(req *pb.GetProductsRequest, stream pb.ProductsService_GetProductsServer) error {
+	ctx := stream.Context()
+	resultChan := make(chan []pb.Product)
+	errChan := make(chan error)
+	go a.repo.StreamAllProductsCache(ctx, int(req.MarketplaceId), resultChan, errChan)
+	for {
+		select {
+		case <-ctx.Done():
+			fmt.Println("context done")
+			return ctx.Err()
+		case products, ok := <-resultChan:
+			if !ok {
+				fmt.Println("result channel closed")
+				return nil
+			}
+			resp := &pb.GetProductsResponse{
+				Products: lo.Map(products, func(p pb.Product, _ int) *pb.Product {
+					return &p
+				}),
+			}
+			if err := stream.Send(resp); err != nil {
+				fmt.Println("error sending response", err)
+				return err
+			}
+		case err, ok := <-errChan:
+			if !ok {
+				return nil
+			}
+			if ok && err != nil {
+				fmt.Println("error in channel", err)
+				return err
+			}
+		}
+
+	}
+}

internal/wb/products/entities.go

@@ -0,0 +1,9 @@
+package products
+
+import (
+	pb "sipro-mps/api/generated/v1/wb/products"
+	"sipro-mps/pkg/api/wb/client"
+)
+
+type WbProduct = api.ContentV2GetCardsListPostOKCardsItem
+type PbProduct = pb.Product

internal/wb/products/mapping/converter.go

@@ -0,0 +1,28 @@
+package mapping
+
+import wbclient "sipro-mps/pkg/api/wb/client"
+
+// import (
+//
+//	proto "sipro-mps/api/generated/v1/wb/products"
+//	internal "sipro-mps/internal/wb/products"
+//	wbclient "sipro-mps/pkg/api/wb/client"
+//
+// )
+//
+// //go:generate go run github.com/jmattheis/goverter/cmd/goverter gen  -global "ignoreUnexported yes" .
+//
+// // goverter:converter
+// // goverter:extend  OptIntToInt64 OptStringToString
+//
+//	type Converter interface {
+//		// goverter:ignore state sizeCache unknownFields
+//
+//		ToProto(details *internal.WbProduct) *proto.Product
+//	}
+func OptIntToInt64(i wbclient.OptInt) int64 {
+	return int64(i.Value)
+}
+func OptStringToString(s wbclient.OptString) string {
+	return s.Value
+}

internal/wb/products/mapping/generated/generated.go

@@ -0,0 +1,40 @@
+// Code generated by github.com/jmattheis/goverter, DO NOT EDIT.
+//go:build !goverter
+
+package generated
+
+import (
+	products "sipro-mps/api/generated/v1/wb/products"
+	mapping "sipro-mps/internal/wb/products/mapping"
+	client "sipro-mps/pkg/api/wb/client"
+)
+
+type ConverterImpl struct{}
+
+func (c *ConverterImpl) ToProto(source *client.ContentV2GetCardsListPostOKCardsItem) *products.Product {
+	var pProductsProduct *products.Product
+	if source != nil {
+		var productsProduct products.Product
+		productsProduct.NmID = mapping.OptIntToInt64((*source).NmID)
+		productsProduct.SubjectID = mapping.OptIntToInt64((*source).SubjectID)
+		productsProduct.VendorCode = mapping.OptStringToString((*source).VendorCode)
+		if (*source).Sizes != nil {
+			productsProduct.Sizes = make([]*products.Product_Size, len((*source).Sizes))
+			for i := 0; i < len((*source).Sizes); i++ {
+				productsProduct.Sizes[i] = c.apiContentV2GetCardsListPostOKCardsItemSizesItemToPProductsProduct_Size((*source).Sizes[i])
+			}
+		}
+		pProductsProduct = &productsProduct
+	}
+	return pProductsProduct
+}
+func (c *ConverterImpl) apiContentV2GetCardsListPostOKCardsItemSizesItemToPProductsProduct_Size(source client.ContentV2GetCardsListPostOKCardsItemSizesItem) *products.Product_Size {
+	var productsProduct_Size products.Product_Size
+	if source.Skus != nil {
+		productsProduct_Size.Skus = make([]string, len(source.Skus))
+		for i := 0; i < len(source.Skus); i++ {
+			productsProduct_Size.Skus[i] = source.Skus[i]
+		}
+	}
+	return &productsProduct_Size
+}

internal/wb/products/repository.go

@@ -0,0 +1,13 @@
+package products
+
+import (
+	"context"
+	"sipro-mps/internal/marketplace"
+)
+
+type Repository interface {
+	GetAllProducts(ctx context.Context, marketplaceId int) ([]WbProduct, error)
+	StreamAllProducts(ctx context.Context, marketplaceId int, resultChan chan<- []WbProduct, errChan chan<- error)
+	StreamAllProductsCache(ctx context.Context, marketplaceId int, resultChan chan<- []PbProduct, errChan chan<- error)
+	ParseMarketplace(ctx context.Context, marketplaceId int) (*marketplace.Marketplace, string, error)
+}

internal/wb/products/repository_api.go

@@ -0,0 +1,224 @@
+package products
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/go-faster/errors"
+	"github.com/redis/rueidis"
+	"github.com/samber/lo"
+	pb "sipro-mps/api/generated/v1/wb/products"
+	"sipro-mps/internal/marketplace"
+	"sipro-mps/internal/redis"
+	"sipro-mps/internal/tasks/client"
+	"sipro-mps/internal/tasks/types"
+	"sipro-mps/internal/wb"
+	"sipro-mps/internal/wb/products/mapping/generated"
+	wbapi "sipro-mps/pkg/api/wb/client"
+)
+
+const (
+	maxRetries            = 5
+	maxProductsPerRequest = 100
+)
+
+type apiRepository struct {
+	marketplaceRepository marketplace.Repository
+}
+
+func (a apiRepository) ParseMarketplace(ctx context.Context, marketplaceId int) (*marketplace.Marketplace, string, error) {
+	marketplaceByID, err := a.marketplaceRepository.GetMarketplaceByID(ctx, marketplaceId)
+	if err != nil {
+		return nil, "", err
+	}
+	_, claims, err := wb.DecodeWildberriesJwt(marketplaceByID.AuthDataJson)
+	if err != nil {
+		return nil, "", err
+	}
+	sellerId := claims["sid"].(string)
+	return marketplaceByID, sellerId, nil
+}
+
+func fetchProducts(
+	ctx context.Context,
+	client *wbapi.Client,
+	sellerId string,
+	resultChan chan<- []WbProduct,
+	errChan chan<- error,
+) {
+	defer close(resultChan)
+	defer close(errChan)
+	request := wbapi.ContentV2GetCardsListPostReq{}
+	request.Settings.SetTo(wbapi.ContentV2GetCardsListPostReqSettings{})
+
+	request.Settings.Value.Cursor.SetTo(wbapi.ContentV2GetCardsListPostReqSettingsCursor{})
+	request.Settings.Value.Cursor.Value.Limit.SetTo(maxProductsPerRequest)
+
+	request.Settings.Value.Filter.SetTo(wbapi.ContentV2GetCardsListPostReqSettingsFilter{})
+	request.Settings.Value.Filter.Value.WithPhoto.SetTo(-1)
+	currentRetry := 0
+	for {
+		response, err := client.ContentV2GetCardsListPost(ctx, &request, wbapi.ContentV2GetCardsListPostParams{Locale: wbapi.NewOptString("ru")})
+		if err != nil {
+			currentRetry++
+			if currentRetry >= maxRetries {
+				errChan <- fmt.Errorf("fetching product IDs: %w", err)
+				return
+			}
+			continue
+		}
+		currentRetry = 0
+
+		switch r := response.(type) {
+		case *wbapi.ContentV2GetCardsListPostOKHeaders:
+			err = wb.SyncRateLimitRemaining(ctx, sellerId, r.XRatelimitRemaining.Value)
+			if err != nil {
+				errChan <- fmt.Errorf("syncing rate limit: %w", err)
+				return
+			}
+			resultChan <- r.Response.Cards
+			if r.Response.Cursor.Value.Total.Value < maxProductsPerRequest {
+				return
+			}
+			request.Settings.Value.Cursor.Value.UpdatedAt.SetTo(r.Response.Cursor.Value.UpdatedAt.Value)
+			request.Settings.Value.Cursor.Value.NmID.SetTo(r.Response.Cursor.Value.NmID.Value)
+		case *wbapi.R429Headers:
+			err = wb.SetRateLimitRetry(ctx, sellerId, r.XRatelimitRetry.Value, r.XRatelimitLimit.Value, r.XRatelimitReset.Value)
+			if err != nil {
+				errChan <- fmt.Errorf("setting rate limit retry: %w", err)
+				return
+			}
+		default:
+			errChan <- fmt.Errorf("unexpected response type: %T", r)
+			return
+		}
+	}
+}
+
+func (a apiRepository) StreamAllProductsCache(ctx context.Context, marketplaceId int, resultChan chan<- []pb.Product, errChan chan<- error) {
+	defer close(resultChan)
+	defer close(errChan)
+	_, sellerId, err := a.ParseMarketplace(ctx, marketplaceId)
+	if err != nil {
+		errChan <- err
+		return
+	}
+	c := *redis.Client
+	key := fmt.Sprintf("wb:products:%s", sellerId)
+	jsonString, err := c.Do(ctx, c.B().Get().Key(key).Build()).ToString()
+	if err == nil && jsonString != "null" {
+		var result []pb.Product
+		err = json.Unmarshal([]byte(jsonString), &result)
+		if err != nil {
+			errChan <- fmt.Errorf("unmarshalling products from cache: %w", err)
+			return
+		}
+		task, err := types.NewFetchProductsTask(marketplaceId)
+		if err != nil {
+			errChan <- fmt.Errorf("creating fetch products task: %w", err)
+			return
+		}
+		_, err = client.Client.Enqueue(task)
+		if err != nil {
+			errChan <- fmt.Errorf("enqueueing fetch products task: %w", err)
+			return
+		}
+
+		resultChan <- result
+		return
+	}
+	if !errors.As(err, &rueidis.Nil) && err != nil {
+		errChan <- fmt.Errorf("fetching products from cache: %w", err)
+		return
+	}
+	converter := generated.ConverterImpl{}
+
+	innerResultChan := make(chan []WbProduct)
+	innerErrChan := make(chan error)
+	go a.StreamAllProducts(ctx, marketplaceId, innerResultChan, innerErrChan)
+	var allProducts []pb.Product
+	defer func() {
+		jsonData, err := json.Marshal(allProducts)
+		if err != nil {
+			errChan <- fmt.Errorf("marshalling products to cache: %w", err)
+			return
+		}
+		err = c.Do(ctx, c.B().Set().Key(key).Value(string(jsonData)).Build()).Error()
+		if err != nil {
+			errChan <- fmt.Errorf("setting products to cache: %w", err)
+			return
+		}
+	}()
+	for {
+		select {
+		case err, ok := <-innerErrChan:
+			if !ok {
+				return
+			}
+			errChan <- fmt.Errorf("streaming products: %w", err)
+			return
+		case products, ok := <-innerResultChan:
+			if !ok {
+				return
+			}
+			pbProducts := lo.Map(products, func(p WbProduct, _ int) pb.Product {
+				return *converter.ToProto(&p)
+			})
+			allProducts = append(allProducts, pbProducts...)
+			resultChan <- pbProducts
+		}
+	}
+
+}
+func (a apiRepository) GetAllProducts(ctx context.Context, marketplaceId int) ([]WbProduct, error) {
+	marketplaceByID, sellerId, err := a.ParseMarketplace(ctx, marketplaceId)
+	fromMarketplace, err := wb.GetClientFromMarketplace(marketplaceByID)
+	if err != nil {
+		return nil, err
+	}
+	resultChan := make(chan []WbProduct)
+	errChan := make(chan error)
+	go fetchProducts(ctx, fromMarketplace, sellerId, resultChan, errChan)
+
+	var products []WbProduct
+	isWaiting := true
+	for {
+		if !isWaiting {
+			break
+		}
+		select {
+		case err, ok := <-errChan:
+			if !ok {
+				isWaiting = false
+				continue
+			}
+			return nil, err
+		case newProducts, ok := <-resultChan:
+			if !ok {
+				isWaiting = false
+			}
+			products = append(products, newProducts...)
+		}
+	}
+	return products, nil
+}
+
+func (a apiRepository) StreamAllProducts(ctx context.Context, marketplaceId int, resultChan chan<- []WbProduct, errChan chan<- error) {
+	marketplaceByID, sellerId, err := a.ParseMarketplace(ctx, marketplaceId)
+	if err != nil {
+		errChan <- err
+		return
+	}
+	fromMarketplace, err := wb.GetClientFromMarketplace(marketplaceByID)
+	if err != nil {
+		errChan <- err
+		return
+	}
+	go fetchProducts(ctx, fromMarketplace, sellerId, resultChan, errChan)
+}
+
+func NewAPIRepository(marketplaceRepository marketplace.Repository) Repository {
+	return &apiRepository{
+		marketplaceRepository: marketplaceRepository,
+	}
+}