Add Wildberries product fetching and rate limiting functionality

internal/tasks/client/client.go

@@ -0,0 +1,25 @@
+package client
+
+import (
+	"github.com/hibiken/asynq"
+	"sipro-mps/internal/config"
+)
+
+var Client *asynq.Client
+
+// InitClient initializes the Asynq client with the provided Redis configuration.
+func InitClient(redisConfig config.RedisConfig) {
+	client := asynq.NewClient(asynq.RedisClientOpt{
+		Addr:     redisConfig.Addr,
+		Password: redisConfig.Password,
+	})
+	Client = client
+}
+
+func CloseClient() {
+	if Client != nil {
+		if err := Client.Close(); err != nil {
+			panic(err)
+		}
+	}
+}

internal/tasks/server/server.go

@@ -0,0 +1,39 @@
+package server
+
+import (
+	"github.com/hibiken/asynq"
+	"github.com/jackc/pgx/v5/pgxpool"
+	"sipro-mps/internal/config"
+	"sipro-mps/internal/tasks/types"
+	"sipro-mps/internal/tasks/wb"
+)
+
+type AsynqServer struct {
+	redisConfig *config.RedisConfig
+	dbpool      *pgxpool.Pool
+}
+
+func NewAsynqServer(redisConfig *config.RedisConfig, dbpool *pgxpool.Pool) *AsynqServer {
+	return &AsynqServer{
+		redisConfig: redisConfig,
+		dbpool:      dbpool,
+	}
+}
+func (s *AsynqServer) createMux() *asynq.ServeMux {
+	mux := asynq.NewServeMux()
+
+	// Register task handlers here
+	mux.Handle(types.TypeWbFetchProducts, &wb.FetchProductsProcessor{Dbpool: s.dbpool})
+	return mux
+}
+
+func (s *AsynqServer) Run() {
+	srv := asynq.NewServer(
+		asynq.RedisClientOpt{Addr: s.redisConfig.Addr, Password: s.redisConfig.Password},
+		asynq.Config{Concurrency: 10},
+	)
+	mux := s.createMux()
+	if err := srv.Run(mux); err != nil {
+		panic(err)
+	}
+}

internal/tasks/types/common.go

@@ -0,0 +1,5 @@
+package types
+
+const (
+	TypeWbFetchProducts = "wb:fetch_products"
+)

internal/tasks/types/wb.go

@@ -0,0 +1,19 @@
+package types
+
+import (
+	"encoding/json"
+	"github.com/hibiken/asynq"
+	"time"
+)
+
+type FetchProductsTask struct {
+	MarketplaceId int
+}
+
+func NewFetchProductsTask(marketplaceId int) (*asynq.Task, error) {
+	payload, err := json.Marshal(&FetchProductsTask{MarketplaceId: marketplaceId})
+	if err != nil {
+		return nil, err
+	}
+	return asynq.NewTask(TypeWbFetchProducts, payload, asynq.MaxRetry(2), asynq.Timeout(20*time.Minute)), nil
+}

internal/tasks/wb/wb.go

@@ -0,0 +1,65 @@
+package wb
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/hibiken/asynq"
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/samber/lo"
+	pb "sipro-mps/api/generated/v1/wb/products"
+	mp_repo "sipro-mps/internal/marketplace"
+	"sipro-mps/internal/redis"
+	"sipro-mps/internal/tasks/types"
+	wb_products_repo "sipro-mps/internal/wb/products"
+	conv "sipro-mps/internal/wb/products/mapping/generated"
+)
+
+type FetchProductsProcessor struct {
+	Dbpool *pgxpool.Pool
+	wbRepo wb_products_repo.Repository
+}
+
+func (p *FetchProductsProcessor) ProcessTask(ctx context.Context, task *asynq.Task) error {
+	var payload types.FetchProductsTask
+	if err := json.Unmarshal(task.Payload(), &payload); err != nil {
+		return asynq.SkipRetry
+	}
+	marketplaceRepo := mp_repo.NewDBRepository(p.Dbpool)
+	repo := wb_products_repo.NewAPIRepository(marketplaceRepo)
+	_, sellerId, err := repo.ParseMarketplace(ctx, payload.MarketplaceId)
+	if err != nil {
+		return fmt.Errorf("failed to parse marketplace %d: %w", payload.MarketplaceId, err)
+	}
+	locker := *redis.Locker
+	_, cancel, err := locker.TryWithContext(ctx, fmt.Sprintf("wb:products:marketplace:%s:lock", sellerId))
+	if err != nil {
+		fmt.Printf("Failed to acquire lock for marketplace %s: %v\n", sellerId, err)
+		return asynq.SkipRetry
+
+	}
+	fmt.Println("Working on marketplace", payload.MarketplaceId, "with seller ID", sellerId)
+
+	defer cancel()
+
+	redisKey := fmt.Sprintf("wb:products:%s", sellerId)
+	productsRaw, err := repo.GetAllProducts(ctx, payload.MarketplaceId)
+	if err != nil {
+		return fmt.Errorf("failed to fetch products for marketplace %d: %w", payload.MarketplaceId, err)
+	}
+	converter := conv.ConverterImpl{}
+	products := lo.Map(productsRaw, func(item wb_products_repo.WbProduct, _ int) *pb.Product {
+		return converter.ToProto(&item)
+	})
+	redisClient := *redis.Client
+	productsJson, err := json.Marshal(products)
+	if err != nil {
+		return fmt.Errorf("failed to marshal products: %w", err)
+	}
+	err = redisClient.Do(ctx, redisClient.B().Set().Key(redisKey).Value(string(productsJson)).Build()).Error()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}