feat : SCRAPER IS HERE , version 0.5.9.17 , change product response

add digikala scraper , change product response model and add pager
release
Amir Hossein Khademi 2024-02-15 10:39:00 +03:30
parent a47b7d2010
commit c41afe4b16
24 changed files with 393 additions and 35 deletions

View File

@ -1 +1 @@
0.5.8.16
0.5.9.17

View File

@ -22,24 +22,9 @@ public class FileController : ICarterModule
public async Task<IResult> GetFilesAsync([FromQuery]StorageFileType? fileType,[FromServices] IStorageService storageService, CancellationToken cancellationToken)
=> TypedResults.Ok(await storageService.GetStorageFiles(fileType: fileType ?? StorageFileType.Image));
public async Task<IResult> UploadFileAsync([FromBody] FileUploadRequest uploadRequest, [FromServices] IStorageService storageService, CancellationToken cancellationToken)
{
var bytes = Convert.FromBase64String(uploadRequest.StringBaseFile);
using var originalMedFileStream = new MemoryStream(bytes);
using var originalThumbFileStream = new MemoryStream(bytes);
using var thumbnailFileStream = new MemoryStream();
using var mediumFileStream = new MemoryStream();
await uploadRequest.ImageResize(originalMedFileStream, mediumFileStream, 1280);
await uploadRequest.ImageResize(originalThumbFileStream, thumbnailFileStream, 200);
var medFileName = await storageService.UploadObjectFromFileAsync(uploadRequest.FileName, $"{uploadRequest.FileUploadType.ToDisplay()}/Med", uploadRequest.ContentType, mediumFileStream);
await storageService.UploadObjectFromFileAsync(medFileName, $"{uploadRequest.FileUploadType.ToDisplay()}/Thumb", uploadRequest.ContentType, thumbnailFileStream, false);
var response = new FileUploadResponse
{
FileName = medFileName,
FileLocation = $"{uploadRequest.FileUploadType.ToDisplay()}/Med/{medFileName}",
FileUrl = $"https://storage.vesmook.com/{uploadRequest.FileUploadType.ToDisplay()}/Med/{medFileName}"
};
return TypedResults.Ok(response);
public async Task<IResult> UploadFileAsync([FromBody] FileUploadRequest uploadRequest, [FromServices] IUploadFileService uploadFileService, CancellationToken cancellationToken)
{
return TypedResults.Ok(await uploadFileService.UploadImageAsync(uploadRequest));
}
}

View File

@ -0,0 +1,25 @@
namespace NetinaShop.Api.Controller;
public class ScraperController : ICarterModule
{
public void AddRoutes(IEndpointRouteBuilder app)
{
var group = app.NewVersionedApi("Scraper")
.MapGroup("api/scraper")
.RequireAuthorization(builder => builder.AddAuthenticationSchemes("Bearer").RequireAuthenticatedUser());
group.MapGet("digi", GetDigiProductsAsync)
.WithDisplayName("GetDigiProducts")
.HasApiVersion(1.0);
group.MapPost("digi/{productId}", AddProductToShopAsync)
.WithDisplayName("AddProductToShop")
.HasApiVersion(1.0);
}
public async Task<IResult> GetDigiProductsAsync([FromQuery] string productName, [FromServices] IDigikalaScraper digikalaScraper, CancellationToken cancellationToken)
=> TypedResults.Ok(await digikalaScraper.GetProductsByNameAsync(productName));
public async Task<IResult> AddProductToShopAsync(string productId, [FromQuery] string productName, [FromServices] IDigikalaScraper digikalaScraper, CancellationToken cancellationToken)
=> TypedResults.Ok(await digikalaScraper.AddProductToShopAsync(productId, productName,cancellationToken));
}

View File

@ -6,8 +6,8 @@
<ImplicitUsings>enable</ImplicitUsings>
<InvariantGlobalization>true</InvariantGlobalization>
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
<AssemblyVersion>0.5.7.15</AssemblyVersion>
<FileVersion>0.5.7.15</FileVersion>
<AssemblyVersion>0.5.9.17</AssemblyVersion>
<FileVersion>0.5.9.17</FileVersion>
</PropertyGroup>
<ItemGroup>

View File

@ -0,0 +1,9 @@
using NetinaShop.Domain.Dtos.ScraperDtos.Response;
namespace NetinaShop.Core.Abstracts;
public interface IDigikalaScraper : IScopedDependency
{
public Task<List<ScraperProductDto>> GetProductsByNameAsync(string productName);
public Task<bool> AddProductToShopAsync(string productId, string productName , CancellationToken cancellationToken = default);
}

View File

@ -0,0 +1,6 @@
namespace NetinaShop.Core.Abstracts;
public interface IUploadFileService : IScopedDependency
{
Task<FileUploadResponse> UploadImageAsync(FileUploadRequest uploadRequest);
}

View File

@ -27,6 +27,7 @@
<Folder Include="EntityServices\ProductHandlers\" />
<Folder Include="EntityServices\ReviewHandlers\" />
<Folder Include="Models\Api\" />
<Folder Include="Models\Scraper\" />
<Folder Include="Utilities\" />
</ItemGroup>

View File

@ -1,7 +1,7 @@
namespace NetinaShop.Domain.CommandQueries.Queries;
public sealed record GetProductQuery(Guid Id) : IRequest<ProductLDto>;
public sealed record GetProductQuery(Guid Id) : IRequest<GetProductResponseDto>;
public sealed record GetProductsQuery(
Guid[]? BrandIds,
bool? IsActive,
@ -11,4 +11,4 @@ public sealed record GetProductsQuery(
QuerySortBy SortBy = QuerySortBy.None ,
Guid CategoryId = default ,
double MinPrice = -1 ,
double MaxPrice = 0) : IRequest<List<ProductSDto>>;
double MaxPrice = 0) : IRequest<GetProductsResponseDto>;

View File

@ -0,0 +1,6 @@
namespace NetinaShop.Domain.Dtos.ResponseDtos;
public class GetProductResponseDto
{
public ProductLDto Product { get; set; } = new ProductLDto();
}

View File

@ -0,0 +1,8 @@
namespace NetinaShop.Domain.Dtos.ResponseDtos;
public class GetProductsResponseDto
{
public List<ProductSDto> Products { get; set; } = new List<ProductSDto>();
public PagerResponseDto Pager { get; set; } = new PagerResponseDto();
}

View File

@ -0,0 +1,8 @@
namespace NetinaShop.Domain.Dtos.ResponseDtos;
public class PagerResponseDto
{
public int CurrentPage { get; set; }
public int TotalItems { get; set; }
public int TotalPage { get; set; }
}

View File

@ -0,0 +1,14 @@
namespace NetinaShop.Domain.Dtos.ScraperDtos.Response;
public class ScraperProductDto
{
public string PersianName { get; set; } = string.Empty;
public string EnglishName { get; set; } = string.Empty;
public string Summery { get; set; } = string.Empty;
public double Cost { get; set; }
public string MainImage { get; set; } = string.Empty;
public string BrandName { get; set; } = string.Empty;
public string CategoryName { get; set; } = string.Empty;
public string ScraperId { get; set; } = string.Empty;
public string ScraperUrl { get; set; } = string.Empty;
}

View File

@ -4,4 +4,5 @@ public static class RestAddress
{
public static string BaseKaveNegar => "https://api.kavenegar.com/v1/";
public static string BaseZarinpal => "https://api.zarinpal.com/pg";
public static string DigikalaApi => "https://api.digikala.com";
}

View File

@ -0,0 +1,65 @@
namespace NetinaShop.Infrastructure.Models.Scrapers.Digikala;
public class GetDigikalProductResponseDto
{
public int status { get; set; }
public Data data { get; set; }
public class Attribute
{
public string title { get; set; }
public List<string> values { get; set; }
}
public class Data
{
public Product product { get; set; }
public Seo seo { get; set; }
}
public class Seo
{
public string title { get; set; }
public string description { get; set; }
public Header header { get; set; }
}
public class Header
{
public string title { get; set; }
public string description { get; set; }
public string canonical_url { get; set; }
}
public class Images
{
public Main main { get; set; }
}
public class Main
{
public List<object> storage_ids { get; set; }
public List<string> url { get; set; }
public object thumbnail_url { get; set; }
public object temporary_id { get; set; }
public List<string> webp_url { get; set; }
}
public class Product
{
public int id { get; set; }
public string title_fa { get; set; }
public string title_en { get; set; }
public string status { get; set; }
public Images images { get; set; }
public List<Specification> specifications { get; set; }
}
public class Specification
{
public string title { get; set; }
public List<Attribute> attributes { get; set; }
}
}

View File

@ -0,0 +1,53 @@
namespace NetinaShop.Infrastructure.Models.Scrapers.Digikala;
public class GetDigikalProductsResponseDto
{
public int status { get; set; }
public GetDigikalProductsResponseDtoData data { get; set; }
public class GetDigikalProductsResponseDtoData
{
public List<GetDigikalProductsResponseDtoProduct> products { get; set; }
}
public class GetDigikalProductsResponseDtoImages
{
public GetDigikalProductsResponseDtoMain main { get; set; }
}
public class GetDigikalProductsResponseDtoMain
{
public List<object> storage_ids { get; set; }
public List<string> url { get; set; }
public object thumbnail_url { get; set; }
public object temporary_id { get; set; }
public List<string> webp_url { get; set; }
}
public class GetDigikalProductsResponseDtoProduct
{
public int id { get; set; }
public string title_fa { get; set; }
public string title_en { get; set; }
public GetDigikalProductsResponseDtoUrl url { get; set; }
public GetDigikalProductsResponseDtoImages images { get; set; }
//public GetDigikalProductsResponseDtoVarient default_variant { get; set; }
}
public class GetDigikalProductsResponseDtoVarient
{
//public GetDigikalProductsResponseDtoPrice price { get; set; }
}
public class GetDigikalProductsResponseDtoPrice
{
//public string rrp_price { get; set; }
}
public class GetDigikalProductsResponseDtoUrl
{
public string uri { get; set; }
}
}

View File

@ -35,4 +35,9 @@
<Using Include="Refit" />
</ItemGroup>
<ItemGroup>
<Folder Include="Services\Scrapers\" />
</ItemGroup>
</Project>

View File

@ -0,0 +1,12 @@
using NetinaShop.Infrastructure.Models.Scrapers.Digikala;
namespace NetinaShop.Infrastructure.RestServices;
public interface IDigikalaRestApi
{
[Get("/v1/search/")]
Task<GetDigikalProductsResponseDto> SearchProductAsync([Query] string q);
[Get("/v1/product/{productId}/")]
Task<GetDigikalProductResponseDto> GetProductAsync(string productId);
}

View File

@ -6,10 +6,12 @@ public interface IRestApiWrapper : IScopedDependency
{
IKaveNegarRestApi KaveNegarRestApi { get; }
IZarinpalRestApi ZarinpalRestApi { get; }
IDigikalaRestApi DigikalaRestApi { get; }
}
public class RestApiWrapper : IRestApiWrapper
{
public IKaveNegarRestApi KaveNegarRestApi => RestService.For<IKaveNegarRestApi>(RestAddress.BaseKaveNegar);
public IZarinpalRestApi ZarinpalRestApi => RestService.For<IZarinpalRestApi>(RestAddress.BaseZarinpal);
public IDigikalaRestApi DigikalaRestApi => RestService.For<IDigikalaRestApi>(RestAddress.DigikalaApi);
}

View File

@ -0,0 +1,113 @@
using NetinaShop.Domain.CommandQueries.Commands;
using NetinaShop.Domain.Dtos.ScraperDtos.Response;
using NetinaShop.Domain.Dtos.SmallDtos;
using NetinaShop.Domain.Entities.Products;
using NetinaShop.Repository.Repositories.Base.Contracts;
using System.Linq;
using MediatR;
using Microsoft.EntityFrameworkCore;
using NetinaShop.Domain.Entities.Brands;
using NetinaShop.Domain.Entities.ProductCategories;
namespace NetinaShop.Infrastructure.Services.Scrapers;
public class DigikalaScraper : IDigikalaScraper
{
private readonly IRestApiWrapper _apiWrapper;
private readonly IRepositoryWrapper _repositoryWrapper;
private readonly IMediator _mediator;
private readonly IUploadFileService _uploadFileService;
public DigikalaScraper(IRestApiWrapper apiWrapper, IRepositoryWrapper repositoryWrapper,IMediator mediator, IUploadFileService uploadFileService)
{
_apiWrapper = apiWrapper;
_repositoryWrapper = repositoryWrapper;
_mediator = mediator;
_uploadFileService = uploadFileService;
}
public async Task<List<ScraperProductDto>> GetProductsByNameAsync(string productName)
{
var products = await _apiWrapper.DigikalaRestApi.SearchProductAsync(productName);
return products.data.products.Select(s => new ScraperProductDto
{
PersianName = s.title_fa,
EnglishName = s.title_en,
MainImage = s.images.main.url.First(),
//Cost = long.TryParse(s.default_variant.price.rrp_price,out long result) ? result : 0,
ScraperId = s.id.ToString(),
ScraperUrl = $"https://digikala.com/{s.url.uri}"
}).ToList();
}
public async Task<bool> AddProductToShopAsync(string productId, string productName, CancellationToken cancellationToken = default)
{
var response = await _apiWrapper.DigikalaRestApi.GetProductAsync(productId);
var digiProduct = response.data;
var dbProduct = await _repositoryWrapper.SetRepository<Product>()
.TableNoTracking
.FirstOrDefaultAsync(p => p.PersianName.ToLower().Trim().Contains(productName.ToLower().Trim()), cancellationToken);
var specifications = new List<SpecificationSDto>();
foreach (var specification in digiProduct.product.specifications)
{
foreach (var attribute in specification.attributes)
{
specifications.Add(new SpecificationSDto { Value = string.Join(",", attribute.values), Title = attribute.title });
}
}
using var httClient = new HttpClient();
var imageBytes = await httClient.GetByteArrayAsync(digiProduct.product.images.main.url.FirstOrDefault(), cancellationToken);
var imageBase64 = Convert.ToBase64String(imageBytes);
var uploadFile = new FileUploadRequest
{
StringBaseFile = imageBase64,
FileName = digiProduct.product.title_fa.Replace(" ", "_") + ".jpg",
FileUploadType = FileUploadType.Image,
ContentType = "image/jpeg"
};
var uploadResponse = await _uploadFileService.UploadImageAsync(uploadFile);
var files = new List<StorageFileSDto>
{
new StorageFileSDto
{
FileLocation = uploadResponse.FileLocation,
FileName = uploadResponse.FileName,
IsPrimary = true,
}
};
if (dbProduct != null)
{
var request = new UpdateProductCommand(dbProduct.Id, productName, digiProduct.product.title_en,
digiProduct.seo.description,
dbProduct.ExpertCheck, dbProduct.Tags, dbProduct.Warranty, dbProduct.BeDisplayed, dbProduct.Cost,
dbProduct.PackingCost, dbProduct.Stock, dbProduct.HasExpressDelivery
, dbProduct.MaxOrderCount, false, dbProduct.BrandId, dbProduct.CategoryId, new DiscountSDto(), specifications, files);
await _mediator.Send(request, cancellationToken);
}
else
{
var nonBrand = await _repositoryWrapper.SetRepository<Brand>()
.TableNoTracking
.FirstOrDefaultAsync(b => b.Name == "بدون برند", cancellationToken);
if (nonBrand == null)
throw new AppException("NoneBrand is not exist");
var nonCat = await _repositoryWrapper.SetRepository<ProductCategory>()
.TableNoTracking
.FirstOrDefaultAsync(b => b.Name == "دسته بندی نشده", cancellationToken);
if (nonCat == null)
throw new AppException("NoneCategory is not exist");
var request = new CreateProductCommand(productName, digiProduct.product.title_en,
digiProduct.seo.description,
string.Empty, string.Empty, string.Empty,true, 0,
0, 0, false
, 5, false, nonBrand.Id, nonCat.Id, new DiscountSDto(), specifications, files);
await _mediator.Send(request, cancellationToken);
}
return true;
}
}

View File

@ -0,0 +1,33 @@
using NetinaShop.Core.Utilities;
namespace NetinaShop.Infrastructure.Services;
public class UploadFileService : IUploadFileService
{
private readonly IStorageService _storageService;
public UploadFileService(IStorageService storageService)
{
_storageService = storageService;
}
public async Task<FileUploadResponse> UploadImageAsync(FileUploadRequest uploadRequest)
{
var bytes = Convert.FromBase64String(uploadRequest.StringBaseFile);
using var originalMedFileStream = new MemoryStream(bytes);
using var originalThumbFileStream = new MemoryStream(bytes);
using var thumbnailFileStream = new MemoryStream();
using var mediumFileStream = new MemoryStream();
await uploadRequest.ImageResize(originalMedFileStream, mediumFileStream, 1280);
await uploadRequest.ImageResize(originalThumbFileStream, thumbnailFileStream, 200);
var medFileName = await _storageService.UploadObjectFromFileAsync(uploadRequest.FileName, $"{uploadRequest.FileUploadType.ToDisplay()}/Med", uploadRequest.ContentType, mediumFileStream);
await _storageService.UploadObjectFromFileAsync(medFileName, $"{uploadRequest.FileUploadType.ToDisplay()}/Thumb", uploadRequest.ContentType, thumbnailFileStream, false);
var response = new FileUploadResponse
{
FileName = medFileName,
FileLocation = $"{uploadRequest.FileUploadType.ToDisplay()}/Med/{medFileName}",
FileUrl = $"https://storage.vesmook.com/{uploadRequest.FileUploadType.ToDisplay()}/Med/{medFileName}"
};
return response;
}
}

View File

@ -19,9 +19,12 @@ public class GetBrandsQueryHandler : IRequestHandler<GetBrandsQuery, List<BrandS
List<BrandSDto> brands = new List<BrandSDto>();
if (request.CategoryId != default)
{
var products = await _mediator.Send(new GetProductsQuery(BrandIds: null,SpecialOffer: null, Page:0, SortBy: QuerySortBy.None,CategoryId: request.CategoryId, IsActive : null),
cancellationToken);
var brandGrouped = products.GroupBy(p => p.BrandId);
//var products = await _mediator.Send(new GetProductsQuery(BrandIds: null,SpecialOffer: null, Page:0, SortBy: QuerySortBy.None,CategoryId: request.CategoryId, IsActive : null),
// cancellationToken);
var brandGrouped = _repositoryWrapper.SetRepository<Product>()
.TableNoTracking
.Where(p => p.CategoryId == request.CategoryId)
.GroupBy(p=>p.BrandId);
foreach (var grouping in brandGrouped)
{
if (grouping.Key != default)

View File

@ -13,6 +13,7 @@ public class CreateProductCommandHandler : IRequestHandler<CreateProductCommand,
public async Task<ProductLDto> Handle(CreateProductCommand request, CancellationToken cancellationToken)
{
var ent = Product.Create(request.PersianName, request.EnglishName, request.Summery, request.ExpertCheck,
request.Tags, request.Warranty,request.BeDisplayed,request.Cost,request.PackingCost,
request.HasExpressDelivery,
@ -31,6 +32,8 @@ public class CreateProductCommandHandler : IRequestHandler<CreateProductCommand,
ent.AddFile(file.Name, file.FileLocation, file.FileName, file.IsHeader, file.IsPrimary, file.FileType);
}
_repositoryWrapper.SetRepository<Product>().Add(ent);
await _repositoryWrapper.SaveChangesAsync(cancellationToken);

View File

@ -2,7 +2,7 @@
namespace NetinaShop.Repository.Handlers.Products;
public class GetProductQueryHandler : IRequestHandler<GetProductQuery, ProductLDto>
public class GetProductQueryHandler : IRequestHandler<GetProductQuery, GetProductResponseDto>
{
private readonly IRepositoryWrapper _repositoryWrapper;
@ -10,7 +10,7 @@ public class GetProductQueryHandler : IRequestHandler<GetProductQuery, ProductLD
{
_repositoryWrapper = repositoryWrapper;
}
public async Task<ProductLDto> Handle(GetProductQuery request, CancellationToken cancellationToken)
public async Task<GetProductResponseDto> Handle(GetProductQuery request, CancellationToken cancellationToken)
{
var ent = await _repositoryWrapper.SetRepository<Product>().TableNoTracking
.Where(b => b.Id == request.Id)
@ -33,6 +33,10 @@ public class GetProductQueryHandler : IRequestHandler<GetProductQuery, ProductLD
else
ent.IsSpecialOffer = true;
return ent;
var response = new GetProductResponseDto
{
Product = ent
};
return response;
}
}

View File

@ -1,11 +1,8 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Internal;
using NetinaShop.Domain.Dtos.LargDtos;
using NetinaShop.Domain.Dtos.SmallDtos;
namespace NetinaShop.Repository.Handlers.Products;
public class GetProductsQueryHandler : IRequestHandler<GetProductsQuery, List<ProductSDto>>
public class GetProductsQueryHandler : IRequestHandler<GetProductsQuery, GetProductsResponseDto>
{
private readonly IRepositoryWrapper _repositoryWrapper;
private readonly IMediator _mediator;
@ -15,8 +12,9 @@ public class GetProductsQueryHandler : IRequestHandler<GetProductsQuery, List<Pr
_repositoryWrapper = repositoryWrapper;
_mediator = mediator;
}
public async Task<List<ProductSDto>> Handle(GetProductsQuery request, CancellationToken cancellationToken)
public async Task<GetProductsResponseDto> Handle(GetProductsQuery request, CancellationToken cancellationToken)
{
var response = new GetProductsResponseDto();
var products = _repositoryWrapper.SetRepository<Product>().TableNoTracking;
if (request.IsActive != null)
products = products.Where(p => p.IsEnable == request.IsActive);
@ -64,6 +62,9 @@ public class GetProductsQueryHandler : IRequestHandler<GetProductsQuery, List<Pr
}
}
response.Pager.CurrentPage = request.Page;
response.Pager.TotalItems = await products.CountAsync(cancellationToken);
response.Pager.TotalPage = response.Pager.TotalItems % 20 == 0 ? response.Pager.TotalItems / 20 : (response.Pager.TotalItems / 20) + 1;
List<ProductSDto> productSDtos = await products
.Skip(request.Page * 20)
@ -76,6 +77,7 @@ public class GetProductsQueryHandler : IRequestHandler<GetProductsQuery, List<Pr
await _mediator.Send(new CalculateProductDiscountCommand(productSDto), cancellationToken);
}
return productSDtos;
response.Products = productSDtos;
return response;
}
}