Commit ac011d89 authored by velayat's avatar velayat
Browse files

merge files

parent 1e92e877
import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose';
import { Document, Types } from 'mongoose';
export type LottoryResultDocument = LottoryResult & Document;
@Schema({ timestamps: true })
export class LottoryResult {
@Prop()
username: string;
@Prop()
index: number;
}
export const LottoryResultSchema = SchemaFactory.createForClass(LottoryResult);
import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose' import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose';
import { Document, Types } from 'mongoose' import { Document, Types } from 'mongoose';
export type AccountFollowersDocument = AccountFollowers & Document export type AccountFollowersDocument = AccountFollowers & Document;
@Schema({timestamps:true}) @Schema({ timestamps: true })
export class AccountFollowers { export class AccountFollowers {
@Prop() @Prop()
_id: Types.ObjectId _id: Types.ObjectId;
@Prop() @Prop()
username: string username: string;
@Prop() @Prop()
user_id: string user_id: string;
@Prop() @Prop()
full_name: string full_name: string;
@Prop() @Prop()
bussines_username: string bussines_username: string;
@Prop({type:Object}) @Prop({ type: Object })
follower_obejct: Object follower_obejct: Object;
@Prop() @Prop()
follow_date: number follow_date: number;
} }
export const AccountFollowersSchema = SchemaFactory.createForClass(AccountFollowers) export const AccountFollowersSchema =
SchemaFactory.createForClass(AccountFollowers);
...@@ -6,20 +6,43 @@ import { GetUserScore } from './dto/get-user-score'; ...@@ -6,20 +6,43 @@ import { GetUserScore } from './dto/get-user-score';
export class AppController { export class AppController {
constructor(private readonly appService: AppService) {} constructor(private readonly appService: AppService) {}
@Get('get-comments') @Get('get-followers')
async getCommentsFromIG() { async getFollowers() {
return await this.appService.getComments(); return await this.appService.getFollowers();
} }
@Post()
@Get('get-followers') async getUserScore(@Body() getUserScoreDto: GetUserScore) {
async getFollowers() { return await this.appService.calculateUserScore(getUserScoreDto.username);
return await this.appService.getFollowers();
} }
@Get('calculate-result')
async calculateResult() {
return await this.appService.getResults();
}
@Post() @Get('get-results')
async getUserScore(@Body() getUserScoreDto: GetUserScore) { async getResults() {
return await this.appService.calculateUserScore(getUserScoreDto.username); return await this.appService.getFinalResults();
} }
@Post('search')
async getUserResults(@Body('username') username: string) {
return await this.appService.getUserResult(username);
}
@Get('shuffle')
async shuffle() {
return await this.appService.getShuffleData();
}
@Get('add-lottory-result')
async addResultDb() {
return await this.appService.addResultsToDB();
}
@Get('get-lottory-result')
async getResultDb() {
return await this.appService.getResultDb();
}
} }
import { Module } from '@nestjs/common'; import { Module, OnApplicationBootstrap } from '@nestjs/common';
import { MongooseModule } from '@nestjs/mongoose'; import { MongooseModule } from '@nestjs/mongoose';
import { AccountFollowersSchema } from './account.followers'; import { AccountFollowersSchema } from './account.followers';
import { AppController } from './app.controller'; import { AppController } from './app.controller';
import { AppService } from './app.service'; import { AppService } from './app.service';
import { CommentSchema } from './comment.schema'; import { CommentSchema } from './comment.schema';
import { LottoryResultSchema } from './LottoryResult.schema';
import { RequestSchema } from './request.schema'; import { RequestSchema } from './request.schema';
import { ResultSchema } from './result.schema';
@Module({ @Module({
imports: [ imports: [
MongooseModule.forRoot('mongodb://localhost/instagram-lottry'), MongooseModule.forRoot(
MongooseModule.forFeature([{ name: 'Request', schema: RequestSchema }]), 'mongodb://netware:Netware%40408009@185.231.180.248:27017/instagram-lottry?serverSelectionTimeoutMS=5000&connectTimeoutMS=10000&authSource=admin&authMechanism=SCRAM-SHA-256',
MongooseModule.forFeature([{ name: 'Comment', schema: CommentSchema }]), ),
MongooseModule.forFeature([{ name: 'AccountFollower', schema: AccountFollowersSchema }]), MongooseModule.forFeature([{ name: 'Request', schema: RequestSchema }]),
MongooseModule.forFeature([{ name: 'Comment', schema: CommentSchema }]),
MongooseModule.forFeature([{ name: 'Result', schema: ResultSchema }]),
MongooseModule.forFeature([
{ name: 'LottoryResult', schema: LottoryResultSchema },
]),
MongooseModule.forFeature([
{ name: 'AccountFollower', schema: AccountFollowersSchema },
]),
], ],
controllers: [AppController], controllers: [AppController],
providers: [AppService], providers: [AppService],
......
import { HttpException, Injectable } from '@nestjs/common'; import {
HttpException,
Injectable,
NotFoundException,
OnApplicationBootstrap,
} from '@nestjs/common';
import { InjectModel } from '@nestjs/mongoose'; import { InjectModel } from '@nestjs/mongoose';
import { Model, Types } from 'mongoose'; import { Model, Types } from 'mongoose';
import { CommentDocument } from './comment.schema'; import { CommentDocument } from './comment.schema';
...@@ -11,14 +16,15 @@ const { username, password } = process.env; ...@@ -11,14 +16,15 @@ const { username, password } = process.env;
import * as _ from 'lodash'; import * as _ from 'lodash';
import FollowerPrivateData from './followers_data'; import FollowerPrivateData from './followers_data';
import { CleanedComments, MentionDocument } from './interface/IcleandComment'; import { CleanedComments, MentionDocument } from './interface/IcleandComment';
import { import { AccountFollowersDocument } from './account.followers';
AccountFollowers,
AccountFollowersDocument,
} from './account.followers';
import { CommentStatus, UserAllMention } from './interface/UserAllMentions'; import { CommentStatus, UserAllMention } from './interface/UserAllMentions';
import { ResultDocument } from './result.schema';
import { LottoryResultDocument } from './LottoryResult.schema';
@Injectable() @Injectable()
export class AppService { export class AppService implements OnApplicationBootstrap {
client: any;
constructor( constructor(
@InjectModel('Request') @InjectModel('Request')
private requestModel: Model<RequestDocument>, private requestModel: Model<RequestDocument>,
...@@ -26,7 +32,30 @@ export class AppService { ...@@ -26,7 +32,30 @@ export class AppService {
private commentModel: Model<CommentDocument>, private commentModel: Model<CommentDocument>,
@InjectModel('AccountFollower') @InjectModel('AccountFollower')
private followerModel: Model<AccountFollowersDocument>, private followerModel: Model<AccountFollowersDocument>,
) { @InjectModel('Result')
private resultModel: Model<ResultDocument>,
@InjectModel('LottoryResult')
private lotoryResultModel: Model<LottoryResultDocument>,
) {}
async onApplicationBootstrap() {
// this.client = await this.login('shahriarvelayat', 'shve8864@@');
this.client = null;
}
private async login(username, password) {
try {
const client = new Instagram({ username, password });
await client.login();
console.log(`user logged in. details :\n
username: ${username},
password: ${password},
`);
return client;
} catch (err) {
console.log(err);
}
} }
async getFollowers(postShortCode = 'CRWNkkchs2x') { async getFollowers(postShortCode = 'CRWNkkchs2x') {
...@@ -109,96 +138,6 @@ export class AppService { ...@@ -109,96 +138,6 @@ export class AppService {
} }
} }
async getComments(postShortCode = 'CRWNkkchs2x') {
try {
const username = 'jangomangoss';
const password = 'kaka7701';
const client = new Instagram({ username, password });
await client.login();
console.log('user logged in...');
let hasNextPage = true;
let requestCount = 0;
let commentCount = 0;
let cursor = '';
const reqList = await this.requestModel
.find({ type: 'comment' })
.sort({ createdAt: -1 });
console.log('Request History:', reqList.length);
if (reqList.length != 0) {
const nextCursor = await this.getCommentsNextCursor(
client,
postShortCode,
reqList[0].cursor,
);
cursor = nextCursor;
}
while (hasNextPage) {
console.log('seted cursor', cursor);
console.log('sending request....');
const collectedComments = await this.sendCommentRequest(
client,
postShortCode,
cursor,
);
console.log('request sended. request count:', requestCount);
requestCount++;
await this.requestModel.create({
_id: new Types.ObjectId(),
cursor: cursor,
type: 'comment',
});
cursor = collectedComments.cursor;
hasNextPage = collectedComments.hasNextPage;
for await (const comment of collectedComments.comments) {
const check = await this.commentModel.findOne({
comment_id: comment.comment_id,
});
if (!check) {
await this.commentModel.create({
_id: new Types.ObjectId(),
comment_id: comment.comment_id,
owner_username: comment.owner_id,
owner_id: comment.owner_username,
text: comment.text,
comment_object: comment.commnet_object,
date: comment.date,
});
commentCount += 1;
}
}
console.log('================nex request info==================');
console.log('nextCursor:', cursor);
console.log('has a next page', hasNextPage);
console.log(commentCount, 'comment imported from pervios requests');
console.log(
'================ End of this Request Proccess ==================',
);
}
return {
status: 'successfull',
totalAdded: commentCount,
};
} catch (err) {
console.log(err);
throw new HttpException(err.message, 500);
}
}
async getCommentsNextCursor(client, cursor: string, postShortCode: string) {
const incomingComments = await client.getMediaComments({
shortcode: postShortCode,
first: '49',
after: cursor,
});
return incomingComments.page_info.end_cursor;
}
async getFollowersNextCursor(client, cursor: string) { async getFollowersNextCursor(client, cursor: string) {
const azadiGoldUser = await client.getUserByUsername({ const azadiGoldUser = await client.getUserByUsername({
username: 'azadi.gold', username: 'azadi.gold',
...@@ -210,42 +149,6 @@ export class AppService { ...@@ -210,42 +149,6 @@ export class AppService {
return followers.page_info.end_cursor; return followers.page_info.end_cursor;
} }
async sendCommentRequest(client, postShortCode, cursor) {
try {
const comments: IncomingComment[] = new Array<IncomingComment>();
const incomingComments = await client.getMediaComments({
shortcode: postShortCode,
first: '49',
after: cursor,
});
await this.delay(_.random(2000, 10000));
console.log(
'=============incoming comments=============',
incomingComments,
);
for (const comment of incomingComments.edges) {
console.log('============data Object===============:', comment);
comments.push({
comment_id: comment.node.id,
text: comment.node.text,
date: comment.node.created_at,
owner_id: comment.node.owner.id,
owner_username: comment.node.owner.username,
commnet_object: comment.node,
});
}
return {
comments,
cursor: incomingComments.page_info.end_cursor,
hasNextPage: incomingComments.page_info.has_next_page,
};
} catch (err) {
console.log(err);
throw new HttpException(err.message, 500);
}
}
async sendFollowerRequest(client, cursor) { async sendFollowerRequest(client, cursor) {
try { try {
const Infollowers: IFollower[] = new Array<IFollower>(); const Infollowers: IFollower[] = new Array<IFollower>();
...@@ -279,6 +182,26 @@ export class AppService { ...@@ -279,6 +182,26 @@ export class AppService {
} }
} }
getFollowerDateOfFollow(username: string) {
let follower_objectResult: any = 0;
FollowerPrivateData.relationships_followers.forEach((follower_object) => {
if (
follower_object.string_list_data[0]['value'].toString() ===
username.toString()
) {
follower_objectResult =
follower_object.string_list_data[0]['timestamp'];
}
});
if (follower_objectResult === 0) {
console.log(username, 'is not in list');
follower_objectResult = Date.now() / 1000;
} else {
console.log(username, 'is in list');
}
return follower_objectResult;
}
async delay(ms) { async delay(ms) {
// return await for better async stack trace support in case of errors. // return await for better async stack trace support in case of errors.
console.log('delay time:', ms); console.log('delay time:', ms);
...@@ -288,8 +211,8 @@ export class AppService { ...@@ -288,8 +211,8 @@ export class AppService {
async calculateUserScore(owner_username: string) { async calculateUserScore(owner_username: string) {
const foundUserComments = await this.commentModel const foundUserComments = await this.commentModel
.find({ owner_id: owner_username }) .find({ owner_username })
.sort({ createdAt: -1 }); .sort({ createdAt: 1 });
const isUserFollowPage = await this.checkUserFollowingStatus( const isUserFollowPage = await this.checkUserFollowingStatus(
owner_username, owner_username,
); );
...@@ -337,6 +260,7 @@ export class AppService { ...@@ -337,6 +260,7 @@ export class AppService {
const allUserMentions = new Array<UserAllMention>(); const allUserMentions = new Array<UserAllMention>();
for await (const mentionedUser of UseCleanComment.mentions) { for await (const mentionedUser of UseCleanComment.mentions) {
console.log('mentionedUser', mentionedUser);
const newMentionData = new UserAllMention(); const newMentionData = new UserAllMention();
newMentionData.comment_status = new Array<CommentStatus>(); newMentionData.comment_status = new Array<CommentStatus>();
...@@ -361,15 +285,20 @@ export class AppService { ...@@ -361,15 +285,20 @@ export class AppService {
newMentionData.comment_date, newMentionData.comment_date,
newMentionData.page_follow_date, newMentionData.page_follow_date,
); );
if (checkUserFollowedPageBefore == true) { console.log('++++++++++++++++++', checkUserFollowedPageBefore);
if (checkUserFollowedPageBefore === true) {
console.log('followed before');
newMentionData.comment_status.push(CommentStatus.isAFollowerBefore); newMentionData.comment_status.push(CommentStatus.isAFollowerBefore);
} }
console.log('*************', newMentionData.mentioned_username);
const checkUserMentionedBefore = await this.checkUserMentionedBefore( const checkUserMentionedBefore = await this.checkUserMentionedBefore(
newMentionData.mentioned_username, newMentionData.mentioned_username,
newMentionData.comment_date, newMentionData.comment_date,
); );
if (checkUserMentionedBefore == true) { if (
checkUserMentionedBefore == true &&
!newMentionData.comment_status
) {
newMentionData.comment_status.push(CommentStatus.isMentionedBefore); newMentionData.comment_status.push(CommentStatus.isMentionedBefore);
} }
...@@ -388,6 +317,78 @@ export class AppService { ...@@ -388,6 +317,78 @@ export class AppService {
}; };
} }
async getResults() {
const foundUsernames = await this.commentModel.distinct('owner_username');
// console.log('total users for ranking:', foundUsernames.length);
for await (const username of foundUsernames) {
const mentions = await this.calculateUserScore(username);
console.log('--------------', mentions.mentions);
let valid_mentions = 0;
let followed_before = 0;
let mentions_before = 0;
let pending_mentions = 0;
const valid_users = new Array<string>();
const mentions_before_users = new Array<string>();
const followed_before_users = new Array<string>();
const pending_users = new Array<string>();
mentions.mentions.forEach((mention) => {
if (mention.comment_status.includes(CommentStatus.isValid)) {
valid_mentions++;
valid_users.push(mention.mentioned_username);
} else if (
mention.comment_status.includes(CommentStatus.isMentionedBefore)
) {
mentions_before++;
mentions_before_users.push(mention.mentioned_username);
} else if (
mention.comment_status.includes(CommentStatus.isAFollowerBefore)
) {
followed_before++;
followed_before_users.push(mention.mentioned_username);
} else if (mention.comment_status.includes(CommentStatus.notFollower)) {
pending_mentions++;
pending_users.push(mention.mentioned_username);
}
});
await this.delay(_.random(500, 1000));
const foundUser = await this.resultModel.findOne({ username: username });
if (!foundUser) {
await this.resultModel.create({
username: username,
valid_mentions,
mentions_before,
followed_before,
pending_mentions,
score: valid_mentions + 1,
valid_users: valid_users,
followed_before_users: followed_before_users,
mentions_before_users: mentions_before_users,
pending_users: pending_users,
});
} else {
await this.resultModel.updateOne(
{ _id: foundUser._id },
{
username: username,
valid_mentions,
mentions_before,
followed_before,
pending_mentions,
score: valid_mentions + 1,
valid_users: valid_users,
followed_before_users: followed_before_users,
mentions_before_users: mentions_before_users,
pending_users: pending_users,
},
);
}
}
return 'records updated successfully';
}
async checkUserMentionedBefore(username, comment_date) { async checkUserMentionedBefore(username, comment_date) {
const foundCommentsWithThisMention = await this.commentModel.find({ const foundCommentsWithThisMention = await this.commentModel.find({
text: new RegExp(`@${username}`), text: new RegExp(`@${username}`),
...@@ -419,24 +420,168 @@ export class AppService { ...@@ -419,24 +420,168 @@ export class AppService {
} }
} }
getFollowerDateOfFollow(username: string) { async getFinalResults() {
let follower_objectResult: any = 0; const results = await this.resultModel.find().sort({ score: -1 });
FollowerPrivateData.relationships_followers.forEach((follower_object) => { const last_update = await this.resultModel.find().sort({ updatedAt: -1 });
if ( const last_create = await this.resultModel.find().sort({ createdAt: -1 });
follower_object.string_list_data[0]['value'].toString() === let date: number;
username.toString() console.log(last_update[0]['updatedAt']);
) { console.log(last_create[0]['createdAt']);
follower_objectResult = if (last_update[0]['updatedAt'] >= last_create[0]['createdAt']) {
follower_object.string_list_data[0]['timestamp']; date = last_update[0]['updatedAt'];
}
});
if (follower_objectResult === 0) {
console.log(username, 'is not in list');
follower_objectResult = Date.now() / 1000;
} else { } else {
console.log(username, 'is in list'); date = last_create[0]['createdAt'];
}
const finalResult = new Array<ResultResponse>();
for await (const userRes of results) {
const response: ResultResponse = new ResultResponse();
response.users = new Array<any>();
userRes.pending_users.forEach((user) => {
response.users.push({
userId: user,
status: CommentStatus.notFollower,
});
});
userRes.mentions_before_users.forEach((user) => {
response.users.push({
userId: user,
status: CommentStatus.isMentionedBefore,
});
});
userRes.followed_before_users.forEach((user) => {
response.users.push({
userId: user,
status: CommentStatus.isAFollowerBefore,
});
});
userRes.valid_users.forEach((user) => {
response.users.push({ userId: user, status: CommentStatus.isValid });
});
(response.username = userRes.username),
(response.valid_mentions = userRes.valid_mentions),
(response.before_mentions = userRes.mentions_before),
(response.followed_before_mentions = userRes.followed_before),
(response.pending_mentions = userRes.pending_mentions),
(response.score = userRes.score);
finalResult.push(response);
} }
return follower_objectResult; return {
finalResult,
last_update: date,
};
}
async getUserResult(username: string) {
username = username.toLowerCase();
const userRes = await this.resultModel.findOne({ username });
const userIndexs = await this.lotoryResultModel.find({ username });
if (!userRes) return 'User not found';
const response: ResultResponse = new ResultResponse();
response.users = new Array<any>();
response.lottory_chances_codes = new Array<string>();
userRes.pending_users.forEach((user) => {
response.users.push({ userId: user, status: CommentStatus.notFollower });
});
userRes.mentions_before_users.forEach((user) => {
response.users.push({
userId: user,
status: CommentStatus.isMentionedBefore,
});
});
userRes.followed_before_users.forEach((user) => {
response.users.push({
userId: user,
status: CommentStatus.isAFollowerBefore,
});
});
userRes.valid_users.forEach((user) => {
response.users.push({ userId: user, status: CommentStatus.isValid });
});
(response.username = userRes.username),
(response.valid_mentions = userRes.valid_mentions),
(response.before_mentions = userRes.mentions_before),
(response.followed_before_mentions = userRes.followed_before),
(response.pending_mentions = userRes.pending_mentions),
(response.score = userRes.score);
userIndexs.forEach((index) => {
response.lottory_chances_codes.push(index.index.toString());
});
return {
response,
};
}
shuffle(array) {
let currentIndex = array.length,
randomIndex;
// While there remain elements to shuffle...
while (0 !== currentIndex) {
// Pick a remaining element...
randomIndex = Math.floor(Math.random() * currentIndex);
currentIndex--;
// And swap it with the current element.
[array[currentIndex], array[randomIndex]] = [
array[randomIndex],
array[currentIndex],
];
}
return array;
}
async getShuffleData() {
const comptitionArray = new Array<string>();
const foundUsernames = await this.resultModel.find();
console.log(foundUsernames);
let score = 0;
for await (const user of foundUsernames) {
score += user.score;
for (let index = 0; index < user.score; index++) {
comptitionArray.push(user.username);
}
}
const res = this.shuffle(comptitionArray);
console.log('score is', score);
return res;
} }
async addResultsToDB() {
await this.lotoryResultModel.deleteMany();
const comptitionArray = new Array<any>();
const foundUsernames = await this.resultModel.find().sort({ score: -1 });
console.log(foundUsernames);
let index = 1;
for await (const user of foundUsernames) {
for (let u = 0; u < user.score; u++) {
comptitionArray.push({ username: user.username, index });
index++;
}
}
await this.lotoryResultModel.insertMany(comptitionArray);
return 'successfull';
}
async getResultDb() {
return await this.lotoryResultModel
.find()
.select({ username: 1, index: 1 });
}
}
export class ResultResponse {
username: string;
valid_mentions: number;
followed_before_mentions: number;
before_mentions: number;
pending_mentions: number;
score: number;
users?: Array<any>;
lottory_chances_codes: Array<string>;
} }
import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose' import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose';
import { Document, Types } from 'mongoose' import { Document, Types } from 'mongoose';
export type CommentDocument = Comment & Document export type CommentDocument = Comment & Document;
@Schema({timestamps:true}) @Schema({ timestamps: true })
export class Comment { export class Comment {
@Prop() @Prop()
_id: Types.ObjectId _id: Types.ObjectId;
@Prop() @Prop()
comment_id: string comment_id: string;
@Prop() @Prop()
text: string text: string;
@Prop() @Prop()
owner_username : string owner_username: string;
@Prop() @Prop()
owner_id : string owner_id: string;
@Prop() @Prop()
date: number date: number;
@Prop({type:Object})
comment_object : Object
@Prop({ type: Object })
comment_object: Object;
} }
export const CommentSchema = SchemaFactory.createForClass(Comment) export const CommentSchema = SchemaFactory.createForClass(Comment);
...@@ -3,6 +3,6 @@ import { AppModule } from './app.module'; ...@@ -3,6 +3,6 @@ import { AppModule } from './app.module';
async function bootstrap() { async function bootstrap() {
const app = await NestFactory.create(AppModule); const app = await NestFactory.create(AppModule);
await app.listen(3000); await app.listen(4001);
} }
bootstrap(); bootstrap();
import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose' import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose';
import { Document, Types } from 'mongoose' import { Document, Types } from 'mongoose';
export type RequestDocument = Request & Document export type RequestDocument = Request & Document;
@Schema({timestamps:true}) @Schema({ timestamps: true })
export class Request { export class Request {
@Prop() @Prop()
_id: Types.ObjectId _id: Types.ObjectId;
@Prop() @Prop()
cursor: string cursor: string;
@Prop() @Prop()
type: string type: string;
} @Prop()
export const RequestSchema = SchemaFactory.createForClass(Request) post_short_code: string;
@Prop()
account_username: string;
}
export const RequestSchema = SchemaFactory.createForClass(Request);
import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose';
import { Document, Types } from 'mongoose';
export type ResultDocument = Result & Document;
@Schema({ timestamps: true })
export class Result {
@Prop()
username: string;
@Prop()
valid_mentions: number;
@Prop()
mentions_before: number;
@Prop()
followed_before: number;
@Prop()
pending_mentions: number;
@Prop()
score: number;
@Prop()
valid_users: Array<string>;
@Prop()
mentions_before_users: Array<string>;
@Prop()
followed_before_users: Array<string>;
@Prop()
pending_users: Array<string>;
}
export const ResultSchema = SchemaFactory.createForClass(Result);
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment