Skip to content
GitLab
Commit bfd98ea1 authored by soheib's avatar soheib
Browse files

add clean comments

parent 2978e808
Hide whitespace changes
Inline Side-by-side
src/app.controller.ts
View file @ bfd98ea1
......@@ -15,4 +15,10 @@ export class AppController {
async getFollowers() {
return await this.appService.getFollowers();
}
@Get('clean-comments')
async cleanComments() {
return await this.appService.cleanUserComments();
}
}
src/app.module.ts
View file @ bfd98ea1
......@@ -3,17 +3,19 @@ import { MongooseModule } from '@nestjs/mongoose';
import { AccountFollowersSchema } from './account.followers';
import { AppController } from './app.controller';
import { AppService } from './app.service';
import { MentionSchema } from './cleanedcomment.schema';
import { CommentSchema } from './comment.schema';
import { RequestSchema } from './request.schema';
import { UserSchema } from './user.schema';
@Module({
imports: [
MongooseModule.forRoot('mongodb://localhost/test-ig'),
MongooseModule.forRoot('mongodb://localhost/netware'),
MongooseModule.forFeature([{ name: 'User', schema: UserSchema }]),
MongooseModule.forFeature([{ name: 'Request', schema: RequestSchema }]),
MongooseModule.forFeature([{ name: 'Comment', schema: CommentSchema }]),
MongooseModule.forFeature([{ name: 'AccountFollower', schema: AccountFollowersSchema }]),
MongooseModule.forFeature([{ name: 'Mention', schema: MentionSchema }]),
],
controllers: [AppController],
providers: [AppService],
......
src/app.service.ts
View file @ bfd98ea1
......@@ -11,6 +11,7 @@ import { UserDocument } from './user.schema';
const Instagram = require('instagram-web-api')
const { username, password } = process.env
import * as _ from "lodash"
import { MentionDocument } from './cleanedcomment.schema';
@Injectable()
export class AppService {
......@@ -23,7 +24,9 @@ export class AppService {
@InjectModel('Comment')
private commentModel: Model<CommentDocument>,
@InjectModel('AccountFollower')
private followerModel: Model<CommentDocument>
private followerModel: Model<CommentDocument>,
@InjectModel('Mention')
private mentionModel: Model<MentionDocument>
) { }
async getFollowers(postShortCode: string = 'CRWNkkchs2x') {
......@@ -61,8 +64,8 @@ export class AppService {
console.log("nextCursor:", cursor)
console.log("has a next page", hasNextPage)
console.log("object is: ",collectedFollower);
console.log("object is: ", collectedFollower);
for await (const follower of collectedFollower.followers) {
......@@ -97,10 +100,10 @@ export class AppService {
}
}
async getComments(postShortCode: string = 'CRgPRnShKGw') {
async getComments(postShortCode: string = 'CRWNkkchs2x') {
try {
const username = 'shirin.poloo'
const password = "137613761376"
const username = 'jangomangoss'
const password = "kaka7701"
const client = new Instagram({ username, password })
await client.login()
console.log('user logged in...');
......@@ -129,9 +132,7 @@ export class AppService {
cursor = collectedComments.cursor
hasNextPage = collectedComments.hasNextPage
console.log("==================================");
console.log("nextCursor:", cursor)
console.log("has a next page", hasNextPage)
for await (const comment of collectedComments.comments) {
let check = await this.commentModel.findOne({
......@@ -141,19 +142,27 @@ export class AppService {
{ date: comment.date }
]
})
console.log("is this comment imported?",check);
console.log("is this comment imported?", check);
if (!check) {
console.log('adding to database...');
await this.commentModel.create({
let res = await this.commentModel.create({
_id: new Types.ObjectId(),
user_profile: comment.owner_id,
comment: comment.comment_value,
date: comment.date
})
console.log("imported comment", res);
}
}
console.log("==================================");
console.log("nextCursor:", cursor)
console.log("has a next page", hasNextPage)
console.log(collectedComments.comments.length, "comment imported")
commentCount += collectedComments.comments.length
console.log("total added", commentCount)
......@@ -182,8 +191,8 @@ export class AppService {
async sendCommentRequest(client, postShortCode, cursor) {
try {
let comments: IncomingComment[] = new Array<IncomingComment>()
let incomingComments = await client.getMediaComments({ shortcode: postShortCode, after: cursor })
await this.delay(_.random(20,40))
let incomingComments = await client.getMediaComments({ shortcode: postShortCode, first: "49", after: cursor })
await this.delay(_.random(20, 40))
for (const comment of incomingComments.edges) {
console.log(comment);
......@@ -195,6 +204,9 @@ export class AppService {
})
console.log(`${comment.node.text} is pushed.`)
}
console.log("incoming comment is : ", incomingComments);
return {
comments,
cursor: incomingComments.page_info.end_cursor,
......@@ -213,7 +225,7 @@ export class AppService {
const followers = await client.getFollowers({ userId: azadiGoldUser.id, after: cursor })
await this.delay(1000)
for (const user of followers.data) {
Infollowers.push({
......@@ -237,51 +249,59 @@ export class AppService {
throw new HttpException(err.message, 500)
}
}
async delay(ms) {
// return await for better async stack trace support in case of errors.
console.log('delay time:', ms);
return await new Promise(resolve => setTimeout(resolve, ms));
}
}
// const ig = new IgApiClient();
// ig.state.generateDevice('sohe.ibs');
// await ig.simulate.preLoginFlow();
// const loggedInUser = await ig.account.login('sohe.ibs', "kaka1374");
// process.nextTick(async () => await ig.simulate.postLoginFlow());
// let azadiGold = await ig.search.users("azadi.gold")
// console.log(azadiGold);
// const userFeed = ig.feed.user(loggedInUser.pk);
// const myPostsFirstPage = await userFeed.items()
async cleanUserComments() {
let allComments = await this.commentModel.find()
for await (const comment of allComments) {
let rawComment = comment.comment
let commentArray = rawComment.split(' ')
for await (const commentNode of commentArray) {
if (commentNode.includes('@')) {
let foundMention = await this.mentionModel.findOne({
$and: [
{
owner_username: comment.user_profile,
},
{
mentioned_username: commentNode,
}
]
})
if (!foundMention) {
await this.mentionModel.create({
_id: new Types.ObjectId(),
owner_username: comment.user_profile,
mentioned_username: commentNode,
date: comment.date
})
}
}
}
}
}
// const followersFeed = ig.feed.accountFollowers(loggedInUser.pk);
// const wholeResponse = await followersFeed.request();
// console.log("=>",wholeResponse); // You can reach any properties in instagram response
// const items = await followersFeed.items();
// console.log("=>2",items); // Here you can reach items. It's array.
// const thirdPageItems = await followersFeed.items();
// console.log("=>3",thirdPageItems);
// const feedState = followersFeed.serialize(); // You can serialize feed state to have an ability to continue get next pages.
// console.log("=>4",feedState);
// followersFeed.deserialize(feedState);
// const fourthPageItems = await followersFeed.items();
// console.log("=>5",fourthPageItems);
}
// followersFeed.items$.subscribe(
// followers => console.log("=>6",followers),
// error => console.error(error),
// () => console.log('Complete!'),
// );
// let foundUser = await this.userModel.findOne({ username: foundComment.user_profile })
// if (!foundUser) {
// let addUser = await this.userModel.create({
// _id: new Types.ObjectId(),
// username: foundComment.user_profile,
// score: 0
// })
// user = addUser
// }
// else {
// user = foundUser
// }
\ No newline at end of file
src/cleanedcomment.schema.ts 0 → 100644
View file @ bfd98ea1
import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose'
import { Document, Types } from 'mongoose'
export type MentionDocument = Mention & Document
@Schema()
export class Mention {
@Prop()
_id: Types.ObjectId
@Prop()
owner_username: string
@Prop()
mentioned_username : string
@Prop()
date: Date
}
export const MentionSchema = SchemaFactory.createForClass(Mention)
src/user.schema.ts
View file @ bfd98ea1
......@@ -8,6 +8,9 @@ export class User {
@Prop()
_id: Types.ObjectId
@Prop()
username : string
@Prop({ type: [{ type: Types.ObjectId, ref: 'User' }] })
invitedUsers: Types.ObjectId[]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment