diff --git a/Controllers/ScraperController.cs b/Controllers/ScraperController.cs new file mode 100644 index 0000000..fb2907a --- /dev/null +++ b/Controllers/ScraperController.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using ViraScraper.Repositories; +using ViraScraper.Repositories.Contracts; +using ViraScraper.Services.Contracts; + +namespace ViraScraper.Controllers +{ + [Route("api/[controller]")] + [ApiController] + public class ScraperController : ControllerBase + { + private readonly IServiceWrapper _serviceWrapper; + + public ScraperController(IServiceWrapper serviceWrapper) + { + _serviceWrapper = serviceWrapper; + } + [HttpGet()] + public async Task GetScrape() + { + try + { + return Ok(await _serviceWrapper.HashtagService.GetHshtags("باران")); + return Ok(); + } + catch (Exception e) + { + return BadRequest(e.Message); + } + } + /*[HttpGet] + public async Task GetScrape(string hashtag) + { + try + { + return Ok(await _serviceWrapper.HashtagService.GetHshtags(hashtag)); + } + catch (Exception e) + { + return BadRequest(e.Message); + } + }*/ + } +} \ No newline at end of file diff --git a/Migrations/20191212061434_Init.Designer.cs b/Migrations/20191212061434_Init.Designer.cs new file mode 100644 index 0000000..cf424fe --- /dev/null +++ b/Migrations/20191212061434_Init.Designer.cs @@ -0,0 +1,84 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ViraScraper.Models; + +namespace ViraScraper.Migrations +{ + [DbContext(typeof(ScrapDbContext))] + [Migration("20191212061434_Init")] + partial class Init + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "3.1.0") + .HasAnnotation("Relational:MaxIdentifierLength", 128) + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + modelBuilder.Entity("ViraScraper.Models.Hashtag", b => + { + b.Property("HashtagId") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + b.Property("CreationDate") + .HasColumnType("datetime2"); + + b.Property("HashtagType") + .HasColumnType("int"); + + b.Property("Tag") + .HasColumnType("nvarchar(max)"); + + b.HasKey("HashtagId"); + + b.ToTable("Hashtags"); + }); + + modelBuilder.Entity("ViraScraper.Models.Synonyms", b => + { + b.Property("SynonymsId") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + b.Property("HashtagAId") + .HasColumnType("int"); + + b.Property("HashtagBId") + .HasColumnType("int"); + + b.HasKey("SynonymsId"); + + b.HasIndex("HashtagAId"); + + b.HasIndex("HashtagBId"); + + b.ToTable("Synonymses"); + }); + + modelBuilder.Entity("ViraScraper.Models.Synonyms", b => + { + b.HasOne("ViraScraper.Models.Hashtag", null) + .WithMany() + .HasForeignKey("HashtagAId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("ViraScraper.Models.Hashtag", null) + .WithMany() + .HasForeignKey("HashtagBId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Migrations/20191212061434_Init.cs b/Migrations/20191212061434_Init.cs new file mode 100644 index 0000000..6b82c3d --- /dev/null +++ b/Migrations/20191212061434_Init.cs @@ -0,0 +1,71 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace ViraScraper.Migrations +{ + public partial class Init : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "Hashtags", + columns: table => new + { + HashtagId = table.Column(nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + Tag = table.Column(nullable: true), + CreationDate = table.Column(nullable: false), + HashtagType = table.Column(nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Hashtags", x => x.HashtagId); + }); + + migrationBuilder.CreateTable( + name: "Synonymses", + columns: table => new + { + SynonymsId = table.Column(nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + HashtagAId = table.Column(nullable: false), + HashtagBId = table.Column(nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Synonymses", x => x.SynonymsId); + table.ForeignKey( + name: "FK_Synonymses_Hashtags_HashtagAId", + column: x => x.HashtagAId, + principalTable: "Hashtags", + principalColumn: "HashtagId", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_Synonymses_Hashtags_HashtagBId", + column: x => x.HashtagBId, + principalTable: "Hashtags", + principalColumn: "HashtagId", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateIndex( + name: "IX_Synonymses_HashtagAId", + table: "Synonymses", + column: "HashtagAId"); + + migrationBuilder.CreateIndex( + name: "IX_Synonymses_HashtagBId", + table: "Synonymses", + column: "HashtagBId"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "Synonymses"); + + migrationBuilder.DropTable( + name: "Hashtags"); + } + } +} diff --git a/Migrations/ScrapDbContextModelSnapshot.cs b/Migrations/ScrapDbContextModelSnapshot.cs new file mode 100644 index 0000000..57c88f8 --- /dev/null +++ b/Migrations/ScrapDbContextModelSnapshot.cs @@ -0,0 +1,82 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ViraScraper.Models; + +namespace ViraScraper.Migrations +{ + [DbContext(typeof(ScrapDbContext))] + partial class ScrapDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "3.1.0") + .HasAnnotation("Relational:MaxIdentifierLength", 128) + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + modelBuilder.Entity("ViraScraper.Models.Hashtag", b => + { + b.Property("HashtagId") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + b.Property("CreationDate") + .HasColumnType("datetime2"); + + b.Property("HashtagType") + .HasColumnType("int"); + + b.Property("Tag") + .HasColumnType("nvarchar(max)"); + + b.HasKey("HashtagId"); + + b.ToTable("Hashtags"); + }); + + modelBuilder.Entity("ViraScraper.Models.Synonyms", b => + { + b.Property("SynonymsId") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + b.Property("HashtagAId") + .HasColumnType("int"); + + b.Property("HashtagBId") + .HasColumnType("int"); + + b.HasKey("SynonymsId"); + + b.HasIndex("HashtagAId"); + + b.HasIndex("HashtagBId"); + + b.ToTable("Synonymses"); + }); + + modelBuilder.Entity("ViraScraper.Models.Synonyms", b => + { + b.HasOne("ViraScraper.Models.Hashtag", null) + .WithMany() + .HasForeignKey("HashtagAId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("ViraScraper.Models.Hashtag", null) + .WithMany() + .HasForeignKey("HashtagBId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Models/Hashtag.cs b/Models/Hashtag.cs new file mode 100644 index 0000000..0dd6ade --- /dev/null +++ b/Models/Hashtag.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; +using System.Linq; +using System.Threading.Tasks; + +namespace ViraScraper.Models +{ + public enum HashtagType + { + Instagram, + Twitter, + Facebook + + } + public class Hashtag + { + public int HashtagId { get; set; } + public string Tag { get; set; } + public DateTime CreationDate { get; set; } + public HashtagType HashtagType { get; set; } + + [NotMapped] + public virtual IEnumerable Synonymses { get; set; } + + public override bool Equals(object obj) + { + var hsb = obj as Hashtag; + if (hsb.Tag == Tag) + return true; + else + return false; + } + } +} diff --git a/Models/JsonModels/InstaHashtagRootObject.cs b/Models/JsonModels/InstaHashtagRootObject.cs new file mode 100644 index 0000000..64e928d --- /dev/null +++ b/Models/JsonModels/InstaHashtagRootObject.cs @@ -0,0 +1,1134 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ViraScraper.Models.JsonModels +{ + + public class InstaExplorerRoot + { + public Config config { get; set; } + public string country_code { get; set; } + public string language_code { get; set; } + public string locale { get; set; } + public EntryData entry_data { get; set; } + public string hostname { get; set; } + public string deployment_stage { get; set; } + public string platform { get; set; } + public string nonce { get; set; } + public double mid_pct { get; set; } + public ZeroData zero_data { get; set; } + public int cache_schema_version { get; set; } + public ServerChecks server_checks { get; set; } + public Knobx knobx { get; set; } + public ToCache to_cache { get; set; } + public string device_id { get; set; } + public Encryption encryption { get; set; } + public string rollout_hash { get; set; } + public string bundle_variant { get; set; } + public bool is_canary { get; set; } + } + public class Config + { + public string csrf_token { get; set; } + public object viewer { get; set; } + public object viewerId { get; set; } + } + + public class PageInfo + { + public bool has_next_page { get; set; } + public string end_cursor { get; set; } + } + + public class Node2 + { + public string text { get; set; } + } + + public class Edge2 + { + public Node2 node { get; set; } + } + + public class EdgeMediaToCaption + { + public List edges { get; set; } + } + + public class EdgeMediaToComment + { + public int count { get; set; } + } + + public class Dimensions + { + public int height { get; set; } + public int width { get; set; } + } + + public class EdgeLikedBy + { + public int count { get; set; } + } + + public class EdgeMediaPreviewLike + { + public int count { get; set; } + } + + public class Owner + { + public string id { get; set; } + } + + public class ThumbnailResource + { + public string src { get; set; } + public int config_width { get; set; } + public int config_height { get; set; } + } + + public class Node + { + public bool comments_disabled { get; set; } + public string __typename { get; set; } + public string id { get; set; } + public EdgeMediaToCaption edge_media_to_caption { get; set; } + public string shortcode { get; set; } + public EdgeMediaToComment edge_media_to_comment { get; set; } + public int taken_at_timestamp { get; set; } + public Dimensions dimensions { get; set; } + public string display_url { get; set; } + public EdgeLikedBy edge_liked_by { get; set; } + public EdgeMediaPreviewLike edge_media_preview_like { get; set; } + public Owner owner { get; set; } + public string thumbnail_src { get; set; } + public List thumbnail_resources { get; set; } + public bool is_video { get; set; } + public string accessibility_caption { get; set; } + public int? video_view_count { get; set; } + } + + public class Edge + { + public Node node { get; set; } + } + + public class EdgeHashtagToMedia + { + public int count { get; set; } + public PageInfo page_info { get; set; } + public List edges { get; set; } + } + + public class Node4 + { + public string text { get; set; } + } + + public class Edge4 + { + public Node4 node { get; set; } + } + + public class EdgeMediaToCaption2 + { + public List edges { get; set; } + } + + public class EdgeMediaToComment2 + { + public int count { get; set; } + } + + public class Dimensions2 + { + public int height { get; set; } + public int width { get; set; } + } + + public class EdgeLikedBy2 + { + public int count { get; set; } + } + + public class EdgeMediaPreviewLike2 + { + public int count { get; set; } + } + + public class Owner2 + { + public string id { get; set; } + } + + public class ThumbnailResource2 + { + public string src { get; set; } + public int config_width { get; set; } + public int config_height { get; set; } + } + + public class Node3 + { + public string __typename { get; set; } + public string id { get; set; } + public EdgeMediaToCaption2 edge_media_to_caption { get; set; } + public string shortcode { get; set; } + public EdgeMediaToComment2 edge_media_to_comment { get; set; } + public int taken_at_timestamp { get; set; } + public Dimensions2 dimensions { get; set; } + public string display_url { get; set; } + public EdgeLikedBy2 edge_liked_by { get; set; } + public EdgeMediaPreviewLike2 edge_media_preview_like { get; set; } + public Owner2 owner { get; set; } + public string thumbnail_src { get; set; } + public List thumbnail_resources { get; set; } + public bool is_video { get; set; } + public string accessibility_caption { get; set; } + } + + public class Edge3 + { + public Node3 node { get; set; } + } + + public class EdgeHashtagToTopPosts + { + public List edges { get; set; } + } + + public class EdgeHashtagToContentAdvisory + { + public int count { get; set; } + public List edges { get; set; } + } + + public class Node5 + { + public string name { get; set; } + } + + public class Edge5 + { + public Node5 node { get; set; } + } + + public class EdgeHashtagToRelatedTags + { + public List edges { get; set; } + } + + public class EdgeHashtagToNullState + { + public List edges { get; set; } + } + + public class Hashtag + { + public string id { get; set; } + public string name { get; set; } + public bool allow_following { get; set; } + public bool is_following { get; set; } + public bool is_top_media_only { get; set; } + public string profile_pic_url { get; set; } + public EdgeHashtagToMedia edge_hashtag_to_media { get; set; } + public EdgeHashtagToTopPosts edge_hashtag_to_top_posts { get; set; } + public EdgeHashtagToContentAdvisory edge_hashtag_to_content_advisory { get; set; } + public EdgeHashtagToRelatedTags edge_hashtag_to_related_tags { get; set; } + public EdgeHashtagToNullState edge_hashtag_to_null_state { get; set; } + } + + public class Graphql + { + public Hashtag hashtag { get; set; } + } + + public class TagPage + { + public Graphql graphql { get; set; } + } + + public class EntryData + { + public List TagPage { get; set; } + } + + public class ZeroData + { + } + + public class ServerChecks + { + } + + public class Knobx + { + public bool __invalid_name__4 { get; set; } + public bool __invalid_name__17 { get; set; } + public bool __invalid_name__20 { get; set; } + public int __invalid_name__21 { get; set; } + public bool __invalid_name__22 { get; set; } + } + + public class Gatekeepers + { + public bool __invalid_name__4 { get; set; } + public bool __invalid_name__5 { get; set; } + public bool __invalid_name__6 { get; set; } + public bool __invalid_name__7 { get; set; } + public bool __invalid_name__8 { get; set; } + public bool __invalid_name__9 { get; set; } + public bool __invalid_name__10 { get; set; } + public bool __invalid_name__11 { get; set; } + public bool __invalid_name__12 { get; set; } + public bool __invalid_name__13 { get; set; } + public bool __invalid_name__14 { get; set; } + public bool __invalid_name__15 { get; set; } + public bool __invalid_name__16 { get; set; } + public bool __invalid_name__18 { get; set; } + public bool __invalid_name__19 { get; set; } + public bool __invalid_name__23 { get; set; } + public bool __invalid_name__24 { get; set; } + public bool __invalid_name__26 { get; set; } + public bool __invalid_name__27 { get; set; } + public bool __invalid_name__28 { get; set; } + public bool __invalid_name__29 { get; set; } + public bool __invalid_name__31 { get; set; } + public bool __invalid_name__32 { get; set; } + public bool __invalid_name__34 { get; set; } + public bool __invalid_name__35 { get; set; } + public bool __invalid_name__38 { get; set; } + public bool __invalid_name__40 { get; set; } + public bool __invalid_name__41 { get; set; } + public bool __invalid_name__43 { get; set; } + public bool __invalid_name__59 { get; set; } + public bool __invalid_name__61 { get; set; } + public bool __invalid_name__62 { get; set; } + public bool __invalid_name__63 { get; set; } + public bool __invalid_name__64 { get; set; } + public bool __invalid_name__65 { get; set; } + public bool __invalid_name__67 { get; set; } + public bool __invalid_name__68 { get; set; } + public bool __invalid_name__69 { get; set; } + public bool __invalid_name__71 { get; set; } + public bool __invalid_name__72 { get; set; } + public bool __invalid_name__73 { get; set; } + public bool __invalid_name__74 { get; set; } + public bool __invalid_name__75 { get; set; } + public bool __invalid_name__76 { get; set; } + public bool __invalid_name__77 { get; set; } + public bool __invalid_name__78 { get; set; } + public bool __invalid_name__79 { get; set; } + public bool __invalid_name__80 { get; set; } + public bool __invalid_name__81 { get; set; } + public bool __invalid_name__82 { get; set; } + } + + public class P + { + } + + public class AppUpsell + { + public string g { get; set; } + public P p { get; set; } + } + + public class P2 + { + } + + public class IglAppUpsell + { + public string g { get; set; } + public P2 p { get; set; } + } + + public class P3 + { + } + + public class Notif + { + public string g { get; set; } + public P3 p { get; set; } + } + + public class P4 + { + public string during_reg { get; set; } + public string default_value { get; set; } + public string storage_version { get; set; } + } + + public class Onetaplogin + { + public string g { get; set; } + public P4 p { get; set; } + } + + public class P5 + { + public string has_back_removed { get; set; } + } + + public class MultiregIter + { + public string g { get; set; } + public P5 p { get; set; } + } + + public class P6 + { + } + + public class FelixClearFbCookie + { + public string g { get; set; } + public P6 p { get; set; } + } + + public class P7 + { + } + + public class FelixCreationDurationLimits + { + public string g { get; set; } + public P7 p { get; set; } + } + + public class P8 + { + } + + public class FelixCreationFbCrossposting + { + public string g { get; set; } + public P8 p { get; set; } + } + + public class P9 + { + } + + public class FelixCreationFbCrosspostingV2 + { + public string g { get; set; } + public P9 p { get; set; } + } + + public class P10 + { + } + + public class FelixCreationValidation + { + public string g { get; set; } + public P10 p { get; set; } + } + + public class P11 + { + } + + public class MwebTopicalExplore + { + public string g { get; set; } + public P11 p { get; set; } + } + + public class P12 + { + } + + public class PostOptions + { + public string g { get; set; } + public P12 p { get; set; } + } + + public class P13 + { + } + + public class Iglscioi + { + public string g { get; set; } + public P13 p { get; set; } + } + + public class P14 + { + } + + public class StickerTray + { + public string g { get; set; } + public P14 p { get; set; } + } + + public class P15 + { + } + + public class WebSentry + { + public string g { get; set; } + public P15 p { get; set; } + } + + public class P16 + { + public bool __invalid_name__4 { get; set; } + public bool __invalid_name__7 { get; set; } + public bool __invalid_name__8 { get; set; } + public bool __invalid_name__9 { get; set; } + } + + public class __invalid_type__0 + { + public P16 p { get; set; } + public bool qex { get; set; } + } + + public class P17 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__2 + { + public P17 p { get; set; } + public bool qex { get; set; } + } + + public class P18 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__4 + { + public P18 p { get; set; } + public bool qex { get; set; } + } + + public class P19 + { + public bool __invalid_name__1 { get; set; } + } + + public class __invalid_type__5 + { + public P19 p { get; set; } + public bool qex { get; set; } + } + + public class P20 + { + public bool __invalid_name__1 { get; set; } + public bool __invalid_name__5 { get; set; } + public bool __invalid_name__6 { get; set; } + public bool __invalid_name__7 { get; set; } + public bool __invalid_name__9 { get; set; } + public bool __invalid_name__10 { get; set; } + } + + public class __invalid_type__6 + { + public P20 p { get; set; } + public bool qex { get; set; } + } + + public class P21 + { + public bool __invalid_name__2 { get; set; } + } + + public class __invalid_type__10 + { + public P21 p { get; set; } + public bool qex { get; set; } + } + + public class P22 + { + public int __invalid_name__0 { get; set; } + } + + public class __invalid_type__12 + { + public P22 p { get; set; } + public bool qex { get; set; } + } + + public class P23 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__13 + { + public P23 p { get; set; } + public bool qex { get; set; } + } + + public class P24 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__16 + { + public P24 p { get; set; } + public bool qex { get; set; } + } + + public class P25 + { + public bool __invalid_name__1 { get; set; } + } + + public class __invalid_type__17 + { + public P25 p { get; set; } + public bool qex { get; set; } + } + + public class P26 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__19 + { + public P26 p { get; set; } + public bool qex { get; set; } + } + + public class P27 + { + public bool __invalid_name__2 { get; set; } + } + + public class __invalid_type__21 + { + public P27 p { get; set; } + public bool qex { get; set; } + } + + public class P28 + { + public bool __invalid_name__1 { get; set; } + public double __invalid_name__2 { get; set; } + public double __invalid_name__3 { get; set; } + public double __invalid_name__4 { get; set; } + public double __invalid_name__10 { get; set; } + public int __invalid_name__11 { get; set; } + public int __invalid_name__12 { get; set; } + public bool __invalid_name__13 { get; set; } + } + + public class __invalid_type__22 + { + public P28 p { get; set; } + public bool qex { get; set; } + } + + public class P29 + { + public bool __invalid_name__0 { get; set; } + public bool __invalid_name__1 { get; set; } + } + + public class __invalid_type__23 + { + public P29 p { get; set; } + public bool qex { get; set; } + } + + public class P30 + { + } + + public class __invalid_type__25 + { + public P30 p { get; set; } + public bool qex { get; set; } + } + + public class P31 + { + public string __invalid_name__0 { get; set; } + } + + public class __invalid_type__26 + { + public P31 p { get; set; } + public bool qex { get; set; } + } + + public class P32 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__28 + { + public P32 p { get; set; } + public bool qex { get; set; } + } + + public class P33 + { + } + + public class __invalid_type__29 + { + public P33 p { get; set; } + public bool qex { get; set; } + } + + public class P34 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__30 + { + public P34 p { get; set; } + public bool qex { get; set; } + } + + public class P35 + { + } + + public class __invalid_type__31 + { + public P35 p { get; set; } + public bool qex { get; set; } + } + + public class P36 + { + } + + public class __invalid_type__33 + { + public P36 p { get; set; } + public bool qex { get; set; } + } + + public class P37 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__34 + { + public P37 p { get; set; } + public bool qex { get; set; } + } + + public class P38 + { + public bool __invalid_name__0 { get; set; } + public bool __invalid_name__1 { get; set; } + public bool __invalid_name__2 { get; set; } + public bool __invalid_name__3 { get; set; } + public bool __invalid_name__4 { get; set; } + } + + public class __invalid_type__36 + { + public P38 p { get; set; } + public bool qex { get; set; } + } + + public class P39 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__37 + { + public P39 p { get; set; } + public bool qex { get; set; } + } + + public class P40 + { + public bool __invalid_name__0 { get; set; } + public bool __invalid_name__6 { get; set; } + public bool __invalid_name__7 { get; set; } + public bool __invalid_name__8 { get; set; } + public bool __invalid_name__9 { get; set; } + } + + public class __invalid_type__39 + { + public P40 p { get; set; } + public bool qex { get; set; } + } + + public class P41 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__40 + { + public P41 p { get; set; } + public bool qex { get; set; } + } + + public class P42 + { + public bool __invalid_name__3 { get; set; } + } + + public class __invalid_type__41 + { + public P42 p { get; set; } + public bool qex { get; set; } + } + + public class P43 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__42 + { + public P43 p { get; set; } + public bool qex { get; set; } + } + + public class P44 + { + public bool __invalid_name__0 { get; set; } + public bool __invalid_name__1 { get; set; } + public bool __invalid_name__2 { get; set; } + } + + public class __invalid_type__43 + { + public P44 p { get; set; } + public bool qex { get; set; } + } + + public class P45 + { + public string __invalid_name__1 { get; set; } + public double __invalid_name__2 { get; set; } + } + + public class __invalid_type__44 + { + public P45 p { get; set; } + public bool qex { get; set; } + } + + public class P46 + { + public bool __invalid_name__2 { get; set; } + public int __invalid_name__4 { get; set; } + public bool __invalid_name__7 { get; set; } + public int __invalid_name__8 { get; set; } + public bool __invalid_name__12 { get; set; } + public bool __invalid_name__13 { get; set; } + public bool __invalid_name__14 { get; set; } + public bool __invalid_name__15 { get; set; } + public int __invalid_name__16 { get; set; } + public int __invalid_name__17 { get; set; } + public bool __invalid_name__18 { get; set; } + public int __invalid_name__19 { get; set; } + public bool __invalid_name__22 { get; set; } + public string __invalid_name__23 { get; set; } + public bool __invalid_name__24 { get; set; } + } + + public class __invalid_type__45 + { + public P46 p { get; set; } + public bool qex { get; set; } + } + + public class P47 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__46 + { + public P47 p { get; set; } + public bool qex { get; set; } + } + + public class P48 + { + public bool __invalid_name__0 { get; set; } + public bool __invalid_name__1 { get; set; } + public bool __invalid_name__2 { get; set; } + public bool __invalid_name__3 { get; set; } + public bool __invalid_name__4 { get; set; } + public bool __invalid_name__5 { get; set; } + public bool __invalid_name__6 { get; set; } + public bool __invalid_name__7 { get; set; } + public bool __invalid_name__8 { get; set; } + public bool __invalid_name__9 { get; set; } + public bool __invalid_name__10 { get; set; } + public bool __invalid_name__11 { get; set; } + } + + public class __invalid_type__47 + { + public P48 p { get; set; } + public bool qex { get; set; } + } + + public class P49 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__48 + { + public P49 p { get; set; } + public bool qex { get; set; } + } + + public class P50 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__49 + { + public P50 p { get; set; } + public bool qex { get; set; } + } + + public class P51 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__50 + { + public P51 p { get; set; } + public bool qex { get; set; } + } + + public class P52 + { + public int __invalid_name__0 { get; set; } + } + + public class __invalid_type__53 + { + public P52 p { get; set; } + public bool qex { get; set; } + } + + public class P53 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__54 + { + public P53 p { get; set; } + public bool qex { get; set; } + } + + public class P54 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__55 + { + public P54 p { get; set; } + public bool qex { get; set; } + } + + public class P55 + { + public double __invalid_name__0 { get; set; } + public bool __invalid_name__1 { get; set; } + } + + public class __invalid_type__58 + { + public P55 p { get; set; } + public bool qex { get; set; } + } + + public class P56 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__59 + { + public P56 p { get; set; } + public bool qex { get; set; } + } + + public class P57 + { + public bool __invalid_name__0 { get; set; } + public bool __invalid_name__1 { get; set; } + public bool __invalid_name__2 { get; set; } + public bool __invalid_name__3 { get; set; } + public bool __invalid_name__4 { get; set; } + } + + public class __invalid_type__60 + { + public P57 p { get; set; } + public bool qex { get; set; } + } + + public class P58 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__62 + { + public P58 p { get; set; } + public bool qex { get; set; } + } + + public class P59 + { + public int __invalid_name__0 { get; set; } + } + + public class __invalid_type__63 + { + public P59 p { get; set; } + public bool qex { get; set; } + } + + public class P60 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__64 + { + public P60 p { get; set; } + public bool qex { get; set; } + } + + public class P61 + { + } + + public class __invalid_type__65 + { + public P61 p { get; set; } + public bool qex { get; set; } + } + + public class P62 + { + public bool __invalid_name__0 { get; set; } + } + + public class __invalid_type__66 + { + public P62 p { get; set; } + public bool qex { get; set; } + } + + public class Qe + { + public AppUpsell app_upsell { get; set; } + public IglAppUpsell igl_app_upsell { get; set; } + public Notif notif { get; set; } + public Onetaplogin onetaplogin { get; set; } + public MultiregIter multireg_iter { get; set; } + public FelixClearFbCookie felix_clear_fb_cookie { get; set; } + public FelixCreationDurationLimits felix_creation_duration_limits { get; set; } + public FelixCreationFbCrossposting felix_creation_fb_crossposting { get; set; } + public FelixCreationFbCrosspostingV2 felix_creation_fb_crossposting_v2 { get; set; } + public FelixCreationValidation felix_creation_validation { get; set; } + public MwebTopicalExplore mweb_topical_explore { get; set; } + public PostOptions post_options { get; set; } + public Iglscioi iglscioi { get; set; } + public StickerTray sticker_tray { get; set; } + public WebSentry web_sentry { get; set; } + public __invalid_type__0 __invalid_name__0 { get; set; } + public __invalid_type__2 __invalid_name__2 { get; set; } + public __invalid_type__4 __invalid_name__4 { get; set; } + public __invalid_type__5 __invalid_name__5 { get; set; } + public __invalid_type__6 __invalid_name__6 { get; set; } + public __invalid_type__10 __invalid_name__10 { get; set; } + public __invalid_type__12 __invalid_name__12 { get; set; } + public __invalid_type__13 __invalid_name__13 { get; set; } + public __invalid_type__16 __invalid_name__16 { get; set; } + public __invalid_type__17 __invalid_name__17 { get; set; } + public __invalid_type__19 __invalid_name__19 { get; set; } + public __invalid_type__21 __invalid_name__21 { get; set; } + public __invalid_type__22 __invalid_name__22 { get; set; } + public __invalid_type__23 __invalid_name__23 { get; set; } + public __invalid_type__25 __invalid_name__25 { get; set; } + public __invalid_type__26 __invalid_name__26 { get; set; } + public __invalid_type__28 __invalid_name__28 { get; set; } + public __invalid_type__29 __invalid_name__29 { get; set; } + public __invalid_type__30 __invalid_name__30 { get; set; } + public __invalid_type__31 __invalid_name__31 { get; set; } + public __invalid_type__33 __invalid_name__33 { get; set; } + public __invalid_type__34 __invalid_name__34 { get; set; } + public __invalid_type__36 __invalid_name__36 { get; set; } + public __invalid_type__37 __invalid_name__37 { get; set; } + public __invalid_type__39 __invalid_name__39 { get; set; } + public __invalid_type__40 __invalid_name__40 { get; set; } + public __invalid_type__41 __invalid_name__41 { get; set; } + public __invalid_type__42 __invalid_name__42 { get; set; } + public __invalid_type__43 __invalid_name__43 { get; set; } + public __invalid_type__44 __invalid_name__44 { get; set; } + public __invalid_type__45 __invalid_name__45 { get; set; } + public __invalid_type__46 __invalid_name__46 { get; set; } + public __invalid_type__47 __invalid_name__47 { get; set; } + public __invalid_type__48 __invalid_name__48 { get; set; } + public __invalid_type__49 __invalid_name__49 { get; set; } + public __invalid_type__50 __invalid_name__50 { get; set; } + public __invalid_type__53 __invalid_name__53 { get; set; } + public __invalid_type__54 __invalid_name__54 { get; set; } + public __invalid_type__55 __invalid_name__55 { get; set; } + public __invalid_type__58 __invalid_name__58 { get; set; } + public __invalid_type__59 __invalid_name__59 { get; set; } + public __invalid_type__60 __invalid_name__60 { get; set; } + public __invalid_type__62 __invalid_name__62 { get; set; } + public __invalid_type__63 __invalid_name__63 { get; set; } + public __invalid_type__64 __invalid_name__64 { get; set; } + public __invalid_type__65 __invalid_name__65 { get; set; } + public __invalid_type__66 __invalid_name__66 { get; set; } + } + + public class ToCache + { + public Gatekeepers gatekeepers { get; set; } + public Qe qe { get; set; } + public bool probably_has_app { get; set; } + public bool cb { get; set; } + } + + public class Encryption + { + public string key_id { get; set; } + public string public_key { get; set; } + } + +} diff --git a/Models/ScrapDbContext.cs b/Models/ScrapDbContext.cs new file mode 100644 index 0000000..c8b39c3 --- /dev/null +++ b/Models/ScrapDbContext.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +namespace ViraScraper.Models +{ + public class ScrapDbContext : DbContext + { + public DbSet Hashtags { get; set; } + public DbSet Synonymses { get; set; } + + public ScrapDbContext(DbContextOptions options) : base(options) { } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity() + .HasOne() + .WithMany() + .HasForeignKey(s=>s.HashtagAId) + .OnDelete(DeleteBehavior.Restrict); + + modelBuilder.Entity() + .HasOne() + .WithMany() + .HasForeignKey(s => s.HashtagBId) + .OnDelete(DeleteBehavior.Restrict); + + base.OnModelCreating(modelBuilder); + } + + public override void Dispose() + { + + } + } +} diff --git a/Models/Synonyms.cs b/Models/Synonyms.cs new file mode 100644 index 0000000..ea259d7 --- /dev/null +++ b/Models/Synonyms.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; +using System.Linq; +using System.Threading.Tasks; + +namespace ViraScraper.Models +{ + public class Synonyms + { + [Key] + public int SynonymsId { get; set; } + public int HashtagAId { get; set; } + public int HashtagBId { get; set; } + + [NotMapped] + public Hashtag HashtagA { get; set; } + [NotMapped] + public Hashtag HashtagB { get; set; } + } +} diff --git a/Program.cs b/Program.cs new file mode 100644 index 0000000..57eefda --- /dev/null +++ b/Program.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ViraScraper +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} diff --git a/Properties/launchSettings.json b/Properties/launchSettings.json new file mode 100644 index 0000000..94d46d1 --- /dev/null +++ b/Properties/launchSettings.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:50934", + "sslPort": 0 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "weatherforecast", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "ViraScraper": { + "commandName": "Project", + "launchBrowser": true, + "launchUrl": "weatherforecast", + "applicationUrl": "http://localhost:5000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/Repositories/Contracts/IHashtagRepository.cs b/Repositories/Contracts/IHashtagRepository.cs new file mode 100644 index 0000000..23ca57f --- /dev/null +++ b/Repositories/Contracts/IHashtagRepository.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using ViraScraper.Models; + +namespace ViraScraper.Repositories.Contracts +{ + public interface IHashtagRepository : IRepositoryBase + { + } +} diff --git a/Repositories/Contracts/IRepositoryBase.cs b/Repositories/Contracts/IRepositoryBase.cs new file mode 100644 index 0000000..867c577 --- /dev/null +++ b/Repositories/Contracts/IRepositoryBase.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; + +namespace ViraScraper.Repositories +{ + public interface IRepositoryBase + { + IQueryable GetAll(); + IQueryable GetAll(Expression> expression); + void Insert(T entity); + void InsertS(T entity); + void InsertRang(IEnumerable entitys); + void InsertSRang(IEnumerable entitys); + void Update(T entity); + void Remove(T entity); + } +} diff --git a/Repositories/Contracts/IRepositoryWrapper.cs b/Repositories/Contracts/IRepositoryWrapper.cs new file mode 100644 index 0000000..06ffed5 --- /dev/null +++ b/Repositories/Contracts/IRepositoryWrapper.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace ViraScraper.Repositories.Contracts +{ + public interface IRepositoryWrapper + { + IHashtagRepository Hashtag { get; } + ISynonymsRepository Synonyms { get; } + } +} diff --git a/Repositories/Contracts/ISynonymsRepository.cs b/Repositories/Contracts/ISynonymsRepository.cs new file mode 100644 index 0000000..d5502c2 --- /dev/null +++ b/Repositories/Contracts/ISynonymsRepository.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using ViraScraper.Models; + +namespace ViraScraper.Repositories.Contracts +{ + public interface ISynonymsRepository : IRepositoryBase + { + } +} diff --git a/Repositories/HashtagRepository.cs b/Repositories/HashtagRepository.cs new file mode 100644 index 0000000..65983c3 --- /dev/null +++ b/Repositories/HashtagRepository.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using ViraScraper.Models; +using ViraScraper.Repositories.Contracts; + +namespace ViraScraper.Repositories +{ + public class HashtagRepository : RepositoryBase,IHashtagRepository + { + public HashtagRepository(ScrapDbContext context) : base(context) { } + } +} diff --git a/Repositories/RepositoryBase.cs b/Repositories/RepositoryBase.cs new file mode 100644 index 0000000..f3ed22f --- /dev/null +++ b/Repositories/RepositoryBase.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using ViraScraper.Models; + +namespace ViraScraper.Repositories +{ + public class RepositoryBase : IRepositoryBase where T : class + { + protected readonly ScrapDbContext _context; + + public RepositoryBase(ScrapDbContext context) + { + _context = context; + } + public IQueryable GetAll() + { + return _context.Set().AsNoTracking(); + } + + public IQueryable GetAll(Expression> expression) + { + return _context.Set().Where(expression).AsNoTracking(); + } + + public async void Insert(T entity) + { + await _context.AddAsync(entity); + await _context.SaveChangesAsync(); + } + + public async void InsertRang(IEnumerable entitys) + { + await _context.AddRangeAsync(entitys); + await _context.SaveChangesAsync(); + } + + public async void InsertS(T entity) + { + var ent = _context.Entry(entity); + if (ent == null) + { + await _context.AddAsync(entity); + await _context.SaveChangesAsync(); + } + + } + + public async void InsertSRang(IEnumerable entitys) + { + foreach (var entity in entitys) + { + var ent = _context.Set().AsNoTracking().FirstOrDefault(e => e.Equals(entity)); + if (ent == null) + { + await _context.AddAsync(entity); + } + } + await _context.SaveChangesAsync(); + } + + public async void Remove(T entity) + { + var ent = _context.Entry(entity); + ent.State = EntityState.Detached; + ent.State = EntityState.Deleted; + await _context.SaveChangesAsync(); + } + + public async void Update(T entity) + { + var ent = _context.Entry(entity); + ent.State = EntityState.Detached; + ent.State = EntityState.Modified; + await _context.SaveChangesAsync(); + } + } +} diff --git a/Repositories/RepositoryWrapper.cs b/Repositories/RepositoryWrapper.cs new file mode 100644 index 0000000..dc08ce9 --- /dev/null +++ b/Repositories/RepositoryWrapper.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using ViraScraper.Models; +using ViraScraper.Repositories.Contracts; + +namespace ViraScraper.Repositories +{ + public class RepositoryWrapper : IRepositoryWrapper + { + protected readonly ScrapDbContext _context; + private IHashtagRepository hashtag; + private ISynonymsRepository synonyms; + public RepositoryWrapper(ScrapDbContext context) + { + _context = context; + } + public IHashtagRepository Hashtag + { + get + { + if(hashtag==null) + hashtag = new HashtagRepository(_context); + return hashtag; + } + } + + public ISynonymsRepository Synonyms + { + get + { + if (synonyms == null) + synonyms = new SynonymsRepository(_context); + return synonyms; + } + } + } +} diff --git a/Repositories/SynonymsRepository.cs b/Repositories/SynonymsRepository.cs new file mode 100644 index 0000000..a8cd3a0 --- /dev/null +++ b/Repositories/SynonymsRepository.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using ViraScraper.Models; +using ViraScraper.Repositories.Contracts; + +namespace ViraScraper.Repositories +{ + public class SynonymsRepository : RepositoryBase , ISynonymsRepository + { + public SynonymsRepository(ScrapDbContext context) : base(context) { } + } +} diff --git a/Services/Contracts/IHashtagService.cs b/Services/Contracts/IHashtagService.cs new file mode 100644 index 0000000..b2de63c --- /dev/null +++ b/Services/Contracts/IHashtagService.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using ViraScraper.Models; + +namespace ViraScraper.Services.Contracts +{ + public interface IHashtagService + { + Task> GetHshtags(string hashtag); + IEnumerable GetHshtags(string hashtag,HashtagType type); + } +} diff --git a/Services/Contracts/IServiceWrapper.cs b/Services/Contracts/IServiceWrapper.cs new file mode 100644 index 0000000..8b9e3a1 --- /dev/null +++ b/Services/Contracts/IServiceWrapper.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace ViraScraper.Services.Contracts +{ + public interface IServiceWrapper + { + IHashtagService HashtagService { get; } + } +} diff --git a/Services/HashtagService.cs b/Services/HashtagService.cs new file mode 100644 index 0000000..a6ff9e5 --- /dev/null +++ b/Services/HashtagService.cs @@ -0,0 +1,148 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using HtmlAgilityPack; +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json; +using ViraScraper.Models; +using ViraScraper.Models.JsonModels; +using ViraScraper.Repositories; +using ViraScraper.Repositories.Contracts; +using ViraScraper.Services.Contracts; +using Hashtag = ViraScraper.Models.Hashtag; + +namespace ViraScraper.Services +{ + public class HashtagService : IHashtagService + { + private readonly IRepositoryWrapper _repository; + + public HashtagService(IRepositoryWrapper repository) + { + _repository = repository; + } + public async Task> GetHshtags(string hashtag) + { + try + { + List returnedList = new List(); + var hashtagList = _repository.Hashtag.GetAll(h => h.Tag.Contains(hashtag)).ToList(); + return hashtagList; + } + finally + { + Task.Run(() => UpdateInstagramHashtag(hashtag)); + } + } + + public IEnumerable GetHshtags(string hashtag, HashtagType type) + { + return null; + } + + private void UpdateInstagramHashtag(string hash) + { + string uri = $"https://www.instagram.com/explore/tags/{hash}/"; + HttpClient client = new HttpClient(); + var html = client.GetStringAsync(uri).Result; + var htmlDocument = new HtmlDocument(); + htmlDocument.LoadHtml(html); + string postList = htmlDocument.DocumentNode.Descendants("script").Where(n => n.InnerHtml.Contains("window._sharedData")).ToList().FirstOrDefault()?.InnerHtml; + postList = postList.Substring(20, postList.Length - 21); + List hashtagList = new List(); + Dictionary synonymses = new Dictionary(); + var root = JsonConvert.DeserializeObject(postList); + foreach (var tagPage in root.entry_data.TagPage) + { + foreach (var edge in tagPage.graphql.hashtag.edge_hashtag_to_top_posts.edges) + { + foreach (var edgeC in edge.node.edge_media_to_caption.edges) + { + var hashtags = edgeC.node.text.Split('#'); + hashtags[0] = " "; + hashtags[hashtags.Length - 1] = " "; + foreach (var hashtag in hashtags) + { + foreach (var str in hashtag.Split(' ')) + { + hashtagList.AddRange(hashtag.Split('\n').Where(s => s.Contains(hash))); + if (synonymses.ContainsKey(str)) + { + int v = synonymses.GetValueOrDefault(str) + 1; + synonymses.Remove(str); + synonymses.Add(str, v); + } + else + synonymses.Add(str, 0); + } + } + } + } + + foreach (var hashtagToMedium in tagPage.graphql.hashtag.edge_hashtag_to_media.edges) + { + foreach (var edgeC in hashtagToMedium.node.edge_media_to_caption.edges) + { + var hashtags = edgeC.node.text.Split('#'); + hashtags[0] = " "; + hashtags[hashtags.Length - 1] = " "; + foreach (var hashtag in hashtags) + { + foreach (var str in hashtag.Split(' ')) + { + hashtagList.AddRange(hashtag.Split('\n').Where(s => s.Contains(hash))); + if (synonymses.ContainsKey(str)) + { + int v = synonymses.GetValueOrDefault(str) + 1; + synonymses.Remove(str); + synonymses.Add(str, v); + } + else + synonymses.Add(str, 0); + } + } + } + } + } + hashtagList = hashtagList.Distinct().ToList(); + hashtagList.Sort((s1, s2) => { return s2.Length.CompareTo(s1.Length); }); + hashtagList.RemoveRange(0, hashtagList.Where(s => s.Length > 30).Count()); + hashtagList.ForEach(s => + { + s = Regex.Replace(s, @"\t|\n|\r", ""); + s = Regex.Replace(s, @"\s+", ""); + }); + hashtagList = hashtagList.Distinct().ToList(); + List hashList = new List(); + hashtagList.ForEach(h => + { + hashList.Add(new Hashtag + { + Tag = h, + HashtagType = HashtagType.Instagram, + CreationDate = DateTime.Now + }); + }); + _repository.Hashtag.InsertSRang(hashList); + + UpdateInstagramSynonyms(hash,synonymses); + + } + + private void UpdateInstagramSynonyms(string hash, Dictionary synonymses) + { + + + var synonymsesList = synonymses.ToList(); + synonymsesList.Sort((paira,pairb)=>pairb.Value.CompareTo(paira.Value)); + synonymsesList.RemoveAll(s => s.Key == ""); + synonymsesList.RemoveAll(s => s.Key.Contains(hash)); + synonymsesList.RemoveAll(s => s.Key == " "); + synonymsesList.RemoveAll(s => s.Key.Length<4); + } + } +} diff --git a/Services/ServiceWrapper.cs b/Services/ServiceWrapper.cs new file mode 100644 index 0000000..772bf1b --- /dev/null +++ b/Services/ServiceWrapper.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using ViraScraper.Repositories.Contracts; +using ViraScraper.Services.Contracts; + +namespace ViraScraper.Services +{ + public class ServiceWrapper : IServiceWrapper + { + private readonly IRepositoryWrapper _repositoryWrapper; + + public ServiceWrapper(IRepositoryWrapper repository) + { + _repositoryWrapper = repository; + } + + private IHashtagService _hashtagService; + public IHashtagService HashtagService + { + get + { + if (_hashtagService == null) + _hashtagService = new HashtagService(_repositoryWrapper); + return _hashtagService; + } + } + } +} diff --git a/Startup.cs b/Startup.cs new file mode 100644 index 0000000..30a531e --- /dev/null +++ b/Startup.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.OpenApi.Models; +using ViraScraper.Models; +using ViraScraper.Repositories; +using ViraScraper.Repositories.Contracts; +using ViraScraper.Services; +using ViraScraper.Services.Contracts; + +namespace ViraScraper +{ + public class Startup + { + public Startup(IConfiguration configuration) + { + Configuration = configuration; + } + + public IConfiguration Configuration { get; } + + // This method gets called by the runtime. Use this method to add services to the container. + public void ConfigureServices(IServiceCollection services) + { + services.AddDbContext(options => + { + options.UseSqlServer(Configuration.GetConnectionString("LocalConnection")); + }, ServiceLifetime.Singleton); + services.AddControllers(); + services.AddTransient(); + services.AddTransient(); + + + + #region Swagger + + services.AddSwaggerGen(swagger => + { + swagger.SwaggerDoc("v1", new OpenApiInfo { Title = "EShop Api", Version = "v1" }); + }); + + #endregion + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseRouting(); + + app.UseAuthorization(); + + app.UseEndpoints(endpoints => + { + endpoints.MapControllers(); + }); + + app.UseSwagger(); + app.UseSwaggerUI(c => { c.SwaggerEndpoint("/swagger/v1/swagger.json", "EShop Api Swagger"); }); + } + } +} diff --git a/ViraScraper.csproj b/ViraScraper.csproj new file mode 100644 index 0000000..4480a54 --- /dev/null +++ b/ViraScraper.csproj @@ -0,0 +1,22 @@ + + + + netcoreapp3.1 + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/ViraScraper.sln b/ViraScraper.sln new file mode 100644 index 0000000..f37b11c --- /dev/null +++ b/ViraScraper.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29519.181 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ViraScraper", "ViraScraper.csproj", "{04D98376-DD9D-4A2A-805A-78C048D4B178}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {04D98376-DD9D-4A2A-805A-78C048D4B178}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {04D98376-DD9D-4A2A-805A-78C048D4B178}.Debug|Any CPU.Build.0 = Debug|Any CPU + {04D98376-DD9D-4A2A-805A-78C048D4B178}.Release|Any CPU.ActiveCfg = Release|Any CPU + {04D98376-DD9D-4A2A-805A-78C048D4B178}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {4F73DBD3-7BF6-4627-A6E3-7B6B9B753740} + EndGlobalSection +EndGlobal diff --git a/WeatherForecast.cs b/WeatherForecast.cs new file mode 100644 index 0000000..22a1306 --- /dev/null +++ b/WeatherForecast.cs @@ -0,0 +1,15 @@ +using System; + +namespace ViraScraper +{ + public class WeatherForecast + { + public DateTime Date { get; set; } + + public int TemperatureC { get; set; } + + public int TemperatureF => 32 + (int)(TemperatureC / 0.5556); + + public string Summary { get; set; } + } +} diff --git a/appsettings.Development.json b/appsettings.Development.json new file mode 100644 index 0000000..8983e0f --- /dev/null +++ b/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/appsettings.json b/appsettings.json new file mode 100644 index 0000000..bd07577 --- /dev/null +++ b/appsettings.json @@ -0,0 +1,14 @@ +{ + "ConnectionStrings": { + "LocalConnection": "Server=.;Database=ViraScraper;Trusted_Connection=true;MultipleActiveResultSets=true", + "ServerConnection": "Data Source=localhost;Initial Catalog=hashshar_test01;Integrated Security=False;User ID=sa;Password=0099;Connect Timeout=15;Encrypt=False;Packet Size=4096" + }, + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*" +}