diff --git a/.gitignore b/.gitignore index 2714a55..9c2fc89 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ *.png *.prof /src/**/__pycache__ -/*.csv -config.yaml -ig_settings.json +/venv +/__pycache__ +/gui_to_spider.json +/data.csv +/gui.py +/test.py \ No newline at end of file diff --git a/InsWrapper.py b/InsWrapper.py new file mode 100644 index 0000000..6bac0f8 --- /dev/null +++ b/InsWrapper.py @@ -0,0 +1,57 @@ +from ins import Ins +from db.database import DatabaseManager +import config +import datetime + + +# you must have your login cookie +COOKIE = config.cookie + +class InsWrapper(Ins): + def __init__(self): + super().__init__(COOKIE) + self.db = DatabaseManager() + + def get_UserData(self, keywords: list, mode: str, amount: int): + for keyword in keywords: + counter = 0 + username_lst = self.getUsernameBytag(keyword, mode=mode) + for username in username_lst: + if counter >= amount: + break + insert_sql = "insert into users (biography,username,fbid,full_name,user_id,followed_by,follow,noteCount,is_private,is_verified,business_email) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + userInfo = self.get_userInfo(username) + userInfo.pop("avatar", "N/A") + values = list(userInfo.values()) + self.db.execute_insert(insert_sql, values) + counter += 1 + self.randSleep([60,95]) + self.db.remove_duplicate_rows() + + def get_postsByUsername(self, username: str, amount: int): + """Args: + username (str): username + amount (int): user recent posts number + Return: post list + """ + + post_lst = [] + counter = 0 + posts = self.get_userPosts(username) + insert_sql = "insert into posts (code,user_id,comment_count,like_count,introduction,create_time) values (%s, %s, %s, %s, %s, %s)" + for post in posts: + if counter >= amount: + break + post['create_time'] = datetime.datetime.fromtimestamp(int(post['create_time'])).strftime("%Y-%m-%d %H:%M:%S") + + values = list(post.values()) + self.db.execute_insert(insert_sql, values) + post_lst.append(post) + counter+=1 + self.randSleep([60,95]) + return post_lst + + +INS = InsWrapper() +# INS.get_postsByUsername("nasa", 1) +INS.get_UserData(["minecraft"], "top", 10) \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000..7edb547 --- /dev/null +++ b/config.py @@ -0,0 +1,14 @@ +cookie = { + 'ds_user_id': '', + 'dpr': '1.5', + 'ig_nrcb': '1', + 'mid': '', + 'ig_did': '', + 'datr':'', + 'csrftoken': '', + 'sessionid':'' +} + +keyword = ["makeup", "skincare"] + +mode = "recent" \ No newline at end of file diff --git a/config_exmple.yaml b/config_exmple.yaml deleted file mode 100644 index ef56a8c..0000000 --- a/config_exmple.yaml +++ /dev/null @@ -1,6 +0,0 @@ -bot: - Username: root - Password: root - Proxy: http://127.0.0.1:10890 - LoginWithPassWd: 1 - ProxyServer: xxx.com \ No newline at end of file diff --git a/db/__pycache__/database.cpython-311.pyc b/db/__pycache__/database.cpython-311.pyc new file mode 100644 index 0000000..ed7ebcc Binary files /dev/null and b/db/__pycache__/database.cpython-311.pyc differ diff --git a/db/database.py b/db/database.py new file mode 100644 index 0000000..66e4d89 --- /dev/null +++ b/db/database.py @@ -0,0 +1,55 @@ +import pymysql +import db_config + +class DatabaseManager: + def __init__(self): + self.connection = pymysql.connect(host=db_config.host, + user=db_config.user, + password=db_config.password, + database=db_config.database) + + def disconnect(self): + if self.connection: + self.connection.close() + + def execute_query(self, query, params=None): + with self.connection.cursor() as cursor: + cursor.execute(query, params) + result = cursor.fetchall() + self.connection.commit() + return result + + def execute_update(self, query, params=None): + if not self.connection: + self.connect() + with self.connection.cursor() as cursor: + cursor.execute(query, params) + self.connection.commit() + + def execute_insert(self, insert_sql: str, params=None): + with self.connection.cursor() as cursor: + cursor.execute(insert_sql, params) + result = cursor.fetchall() + self.connection.commit() + return result + + def remove_duplicate_rows(self): + try: + with self.connection.cursor() as cursor: + # 构建 SQL 查询，选择每个用户名的最小 ID + select_query = """ + SELECT MIN(ID) AS min_id, username + FROM users + GROUP BY username + """ + cursor.execute(select_query) + results = cursor.fetchall() + delete_query = """ + DELETE FROM users + WHERE ID NOT IN ({}) + """.format(','.join(str(result['min_id']) for result in results)) + cursor.execute(delete_query) + self.connection.commit() + print("Duplicate rows removed successfully.") + except Exception as e: + print("An error occurred:", e) diff --git a/db_config.py b/db_config.py new file mode 100644 index 0000000..485e40c --- /dev/null +++ b/db_config.py @@ -0,0 +1,4 @@ +host = 'localhost' +user = 'root' +password = '123456' +database = 'ins' \ No newline at end of file diff --git a/ig_settings.json b/ig_settings.json deleted file mode 100644 index 15c4216..0000000 --- a/ig_settings.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "uuids": { - "phone_id": "02b09058-e31e-48a9-9530-acaecffa64f0", - "uuid": "796194be-6115-45cc-8b70-3bd7134e1acf", - "client_session_id": "0364d810-982a-4508-a4d2-3b0163707652", - "advertising_id": "f1d7294a-be9c-449b-a40b-7ef0d5cdb7ac", - "android_device_id": "android-c4d5a931245270bc", - "request_id": "226e903b-2924-4432-844c-5a6ea22ee7ff", - "tray_session_id": "d249a2b7-7564-40db-ac4f-b267ebb0090a" - }, - "mid": "ZMB8nAABAAErYv2pbpnwI68PkjPe", - "ig_u_rur": null, - "ig_www_claim": null, - "authorization_data": { - "ds_user_id": "45331931537", - "sessionid": "45331931537%3AvgJFUQ36j0QewY%3A2%3AAYdj97Ee1r8Ny9oako1X5u6mKd2NezOfXO0yA4iEkw" - }, - "cookies": {}, - "last_login": 1690336432.9240289, - "device_settings": { - "app_version": "269.0.0.18.75", - "android_version": 26, - "android_release": "8.0.0", - "dpi": "480dpi", - "resolution": "1080x1920", - "manufacturer": "OnePlus", - "device": "devitron", - "model": "6T Dev", - "cpu": "qcom", - "version_code": "314665256" - }, - "user_agent": "Instagram 194.0.0.36.172 Android (26/8.0.0; 480dpi; 1080x1920; Xiaomi; MI 5s; capricorn; qcom; en_US; 301484483)", - "country": "US", - "country_code": 1, - "locale": "en_US", - "timezone_offset": -25200 -} \ No newline at end of file diff --git a/ins.py b/ins.py new file mode 100644 index 0000000..ecfcbb7 --- /dev/null +++ b/ins.py @@ -0,0 +1,247 @@ +import time +import random +import requests +import re + +PARAMS = r'("app_id":\s*"[^"]+")|("claim":\s*"[^"]+")|("csrf_token":\s*"[^"]+")' + + +class Ins: + def __init__(self, cookies: dict): + self.cookies = cookies + self.session = requests.Session() + self.interval = 0 + self.headers = { + 'authority': 'www.instagram.com', + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'accept-language': 'zh-CN,zh;q=0.9', + 'sec-ch-ua-full-version-list': '"Google Chrome";v="113.0.5672.63", "Chromium";v="113.0.5672.63", "Not-A.Brand";v="24.0.0.0"', + 'sec-fetch-site': 'same-origin', + 'upgrade-insecure-requests': '1', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', + 'viewport-width': '1536', + 'X-Asbd-Id': '129477', + 'X-Csrftoken': '6rXinOpVVHa5jIpWEC7QJjhKW0Y6MQaN', + 'X-Ig-App-Id': '936619743392459', + 'X-Ig-Www-Claim': 'hmac.AR3_sXBhcNJg-F09uRf8g0aPkGKbI70OTolUtBSr0AD5zT6C', + 'X-Requested-With': 'XMLHttpRequest' + } + self.get_Header_params() + + def ajax_request(self, url: str, /, params=None): + """ + do requests, the engine of class + :param url: api url + :param params: api params + :return: json object + """ + for _ in range(5): + try: + resp = self.session.get(url, headers=self.headers, params=params, cookies=self.cookies) + return resp.json() + except requests.exceptions.RequestException: + time.sleep(15) + else: + return None + + def get_Header_params(self): + """ + every time visit ins will change header params, this is to get the header params + :return: None + """ + try: + response = self.session.get('https://www.instagram.com/', cookies=self.cookies, headers=self.headers) + matches = re.findall(PARAMS, response.text) + result = [match[i] for match in matches for i in range(3) if match[i]] + # get app_id + app_id = result[0].split(":")[1].strip().strip('"') + # get claim + claim = result[1].split(":")[1].strip().strip('"') + # get csrf_token, if lose cookies, cannot get this param, also cannot access to other apis + csrf_token = result[2].split(":")[1].strip().strip('"') + # set values to headers + self.headers.update({'x-asbd-id': '198387', 'x-csrftoken': csrf_token, + 'x-ig-app-id': app_id, 'x-ig-www-claim': claim, + 'x-requested-with': 'XMLHttpRequest', }) + except requests.exceptions.RequestException: + raise 'Request error, please try again and check your Internet settings' + + def getUsernameBytag(self, tagName: str, mode: str)->list: + """_summary_ + + Args: + tagName (str): tag name + mode (str): "top" | "rencet" + return username list + """ + params = { + 'tag_name': tagName, + } + username_lst = [] + resp = self.ajax_request('https://www.instagram.com/api/v1/tags/web_info/', params=params) + if resp: + try: + sections = resp['data'][mode]['sections'] + except KeyError: + raise 'could not get tag information...' + + for section_value in sections: + medias = section_value.get('layout_content', {}).get('medias', {}) + for media_value in medias: + username_lst.append(media_value.get('media', {}).get('user', {}).get('username', {})) + return username_lst + + def get_userInfo(self, userName: str): + """ + get user info by username + :param userName: name of user + :return: dict of user info + """ + params = { + 'username': userName, + } + resp = self.ajax_request('https://www.instagram.com/api/v1/users/web_profile_info/', params=params) + + if resp: + try: + # to avoid exception? Internet went wrong may return wrong information + data = resp['data']['user'] + except KeyError: + raise 'Could not get user information...' + return { + 'biography': data.get('biography'), + 'username': data.get('username'), + 'fbid': data.get('fbid'), + 'full_name': data.get('full_name'), + 'id': data.get('id'), + 'followed_by': data.get('edge_followed_by', {}).get('count'), + 'follow': data.get('edge_follow', {}).get('count'), + 'noteCount': data.get('edge_owner_to_timeline_media', {}).get('count'), + 'is_private': data.get('is_private'), + 'is_verified': data.get('is_verified'), + 'business_email': data.get('business_email') + } if data else 'unknown User' + + def randSleep(self, interval: list): + if len(interval) != 2: + raise ValueError("区间只包含两个元素") + self.interval = interval + start, end = self.interval + if not (isinstance(start, (int, float)) and isinstance(end, (int, float))): + raise ValueError("区间元素应该是整数或浮点数。") + + if start >= end: + raise ValueError("区间起始值应该小于结束值。") + + sleep_time = random.uniform(start, end) + time.sleep(sleep_time) + + def get_userPosts(self, userName: str): + """ + get all posts from the username + :param userName: name + :return: generator + """ + continuations = [{ + 'count': '12', + }] + temp = userName + '/username/' + while continuations: + continuation = continuations.pop() + # url will change when second request and later + url = f'https://www.instagram.com/api/v1/feed/user/{temp}' + resp = self.ajax_request(url, params=continuation) + # no such user + if not resp.get('user'): + yield 'checking cookie or unknown/private User: {}'.format(userName) + else: + _items = resp.get('items') + # simulate the mousedown + if resp.get('more_available'): + continuations.append({'count': '12', 'max_id': resp.get('next_max_id')}) + user = resp.get('user') + temp = user.get('pk_id') if user.get('pk_id') else user.get('pk') + yield from self.extract_post(_items) + + def get_comments(self, id): + """ + get comments by given post id + :param id: + :return: generator of comments + """ + continuations = [{ + 'can_support_threading': 'true', + 'permalink_enabled': 'false', + }] + # base url + url = f'https://www.instagram.com/api/v1/media/{id}/comments/' + while continuations: + continuation = continuations.pop() + resp = self.ajax_request(url, params=continuation) + if resp.get('next_min_id'): + continuations.append({ + 'can_support_threading': 'true', + 'min_id': resp.get('next_min_id') + }) + comments = resp.get('comments') + if comments: + for comment in comments: + yield { + 'id': comment.get('pk'), + 'user_name': comment.get('user', {}).get('username'), + 'user_fullname': comment.get('user', {}).get('full_name'), + 'text': comment.get('text'), + 'created_at': comment.get('created_at'), + 'comment_like_count': comment.get('comment_like_count'), + 'reply_count': comment.get('child_comment_count') + } + if comment.get('child_comment_count') > 0: + yield from self.get_child_comment(id, comment.get('pk')) + else: + yield 'no comments or losing login cookies' + + def get_child_comment(self, main_id, id): + """ + get child of the comment by comment_id, only used in function get_comments(). + :param main_id: post id + :param id: comment_id + :return: to comments generator + """ + url = f'https://www.instagram.com/api/v1/media/{main_id}/comments/{id}/child_comments/' + continuations = [{'max_id': ''}] + while continuations: + continuation = continuations.pop() + resp = self.ajax_request(url, params=continuation) + cursor = resp.get('next_max_child_cursor') + if cursor: + continuations.append({'max_id': cursor}) + comments = resp.get('child_comments') + if comments: + for comment in comments: + yield { + 'id': comment.get('pk'), + 'user_name': comment.get('user', {}).get('username'), + 'user_fullname': comment.get('user', {}).get('full_name'), + 'text': comment.get('text'), + 'created_at': comment.get('created_at'), + 'comment_like_count': comment.get('comment_like_count'), + } + + @staticmethod + def extract_post(posts): + """ + to extract a post from a list of posts + :param posts: original instagram posts + :return: dict of posts + """ + for post in posts: + caption = post.get('caption') + item = { + 'code': post.get('code'), + 'user_id':str(post.get('id')).split('_')[1], + 'comment_count': post.get('comment_count'), + 'like_count': post.get('like_count'), + 'introduction': caption.get('text') if caption else None, + 'create_time': caption.get('created_at') if caption else post.get('taken_at'), + } + yield item \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..63d3439 Binary files /dev/null and b/requirements.txt differ diff --git a/src/api/__init__.py b/src/api/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/api/bot.py b/src/api/bot.py deleted file mode 100644 index c5f7fec..0000000 --- a/src/api/bot.py +++ /dev/null @@ -1,170 +0,0 @@ -from instagrapi import Client -from instagrapi.types import Media, User -import logging -from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, ClientError -import yaml -from time import sleep -from functools import wraps, partial -import os -current_dir = os.path.dirname(os.path.abspath(__file__)) -yaml_path = os.path.join(current_dir, '..', '..', 'config.yaml') -UserName = "" -PassWd = "" -Proxy = "http://127.0.0.1:10809" -LoginWithPwd = 0 -with open(yaml_path, "r", encoding='utf-8') as f: - yamlFile = yaml.safe_load(f) - UserName = yamlFile['bot']['Username'] - PassWd = yamlFile['bot']['Password'] - LoginWithPwd = yamlFile['bot']['LoginWithPassWd'] - -Locale = "en_US" -IG_CREDENTIAL_PATH = ".\ig_settings.json" - -settings = { - 'user_agent': 'Instagram 194.0.0.36.172 Android (26/8.0.0; 480dpi; 1080x1920; Xiaomi; MI 5s; capricorn; qcom; en_US; 301484483)', - 'country': 'US', - 'country_code': 1, - 'locale': 'en_US', - 'timezone_offset': -25200 -} - -def retry(times, expections): - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - time = times - while time > 0: - try: - func(*args, **kwargs) - except expections as e: - print(e) - time -= 1 - if time == 0: - raise e - sleep(10) - print("retrying") - - return wrapper - return decorator - - - - -class Bot: - _cl = None - - def __init__(self): - self._cl = Client(settings) - - - self._cl.delay_range = [3,5] - self._cl.set_proxy(Proxy) - self._cl.set_locale() - self._cl.set_timezone_offset(-7 * 60 * 60) - login_via_session = False - login_via_pw = False - if LoginWithPwd == 0: - try: - session = self._cl.load_settings(IG_CREDENTIAL_PATH) - login_via_session = True - except : - pass - if login_via_session == True and session and LoginWithPwd == 0: - try: - logging.info("Login with session") - self._cl.set_settings(session) - self._cl.login(UserName, PassWd) - - # check if session is valid - try: - self._cl.get_timeline_feed() - except LoginRequired: - logging.warning("Session is invalid, need to login via username and password") - - old_session = self._cl.get_settings() - - # use the same device uuids across logins - self._cl.set_settings({}) - self._cl.set_uuids(old_session["uuids"]) - - self._cl.login(UserName, PassWd) - login_via_session = True - except Exception as e: - logging.warning("Couldn't login user using session information: %s" % e) - - if not login_via_session: - try: - logging.warning("Attempting to login via username and password. username: %s" % UserName) - if self._cl.login(UserName, PassWd): - login_via_pw = True - self._cl.dump_settings(IG_CREDENTIAL_PATH) - except Exception as e: - logging.warning("Couldn't login user using username and password: %s" % e) - - if not login_via_pw and not login_via_session: - raise Exception("Couldn't login user with either password or session") - - # @retry(3, Exception) - def getMedia( - self, - hashtags, - ht_type="top", - amount=27, - ): - ht_medias = [] - isFinished = False - while not isFinished: - try: - for hashtag in hashtags: - - # logging.info() - if ht_type == "top": - ht_medias.extend( - self._cl.hashtag_medias_top(name=hashtag, amount=amount if amount <= 9 else 9) - ) - - elif ht_type == "recent": - ht_medias.extend(self._cl.hashtag_medias_recent(name=hashtag, amount=amount)) - logging.info("finish get media list") - isFinished = True - except PleaseWaitFewMinutes: - logging.warning("Detected") - sleep(60) - logging.warning("Start") - # try: - # self._cl.login(UserName, PassWd) - - # # check if session is valid - # try: - # self._cl.get_timeline_feed() - # except LoginRequired: - # logging.warning("Session is invalid, need to login via username and password") - - # old_session = self._cl.get_settings() - - # # use the same device uuids across logins - # self._cl.set_settings({}) - # self._cl.set_uuids(old_session["uuids"]) - - # self._cl.login(UserName, PassWd) - # except Exception as e: - # print(e) - # logging.warning("Couldn't login user using session information: %s" % e) - - return list(dict([(media.pk, media) for media in ht_medias]).values()) - def getUser( - self, - media : Media, - ) -> User: - # return self._cl.media_info_gql(int(media.pk)).dict()["user"]["pk"] - logging.info("") - return self._cl.user_info(media.user.pk) - @staticmethod - def getFollower_count( - user: User - ) -> int: - return user.follower_count - -class StarUser(User): - pass \ No newline at end of file diff --git a/src/flask/__init__.py b/src/flask/__init__.py deleted file mode 100644 index b562c69..0000000 --- a/src/flask/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -import os - -from flask import Flask - - -def create_app(test_config=None): - # create and configure the app - app = Flask(__name__, instance_relative_config=True) - app.config.from_mapping( - SECRET_KEY='dev', - DATABASE=os.path.join(app.instance_path, 'flaskr.sqlite'), - ) - - if test_config is None: - # load the instance config, if it exists, when not testing - app.config.from_pyfile('config.py', silent=True) - else: - # load the test config if passed in - app.config.from_mapping(test_config) - - # ensure the instance folder exists - try: - os.makedirs(app.instance_path) - except OSError: - pass - - # a simple page that says hello - @app.route('/hello') - def hello(): - return 'Hello, World!' - - return app \ No newline at end of file diff --git a/src/flask/app.py b/src/flask/app.py deleted file mode 100644 index 599662f..0000000 --- a/src/flask/app.py +++ /dev/null @@ -1,10 +0,0 @@ -from flask import Flask, render_template -from config import Config -from models import StarUser -app = Flask(import_name=__name__, template_folder='templates') - -@app.route('/') -def render(): - return render_template("index.html", StarUser= ) -if __name__ == '__main__': - app.run(debug=True) \ No newline at end of file diff --git a/src/flask/app_test.py b/src/flask/app_test.py deleted file mode 100644 index 475b21e..0000000 --- a/src/flask/app_test.py +++ /dev/null @@ -1,6 +0,0 @@ -from sqlalchemy import create_engine, text -import sqlite3 -engine = create_engine("sqlite+pysqlite:///:memory:", echo=True) -with engine.connect() as conn: - result = conn.execute(text("select 'hello world'")) - print(result.all()) \ No newline at end of file diff --git a/src/flask/config.py b/src/flask/config.py deleted file mode 100644 index 9d8af35..0000000 --- a/src/flask/config.py +++ /dev/null @@ -1,4 +0,0 @@ -class Config(object): - DEBUG = True - SECRET_KEY = "" - SQLALCHEMY_DATABASE_URI = "" \ No newline at end of file diff --git a/src/flask/models/StarUser.py b/src/flask/models/StarUser.py deleted file mode 100644 index 157fdd0..0000000 --- a/src/flask/models/StarUser.py +++ /dev/null @@ -1,24 +0,0 @@ -import sys -sys.path.append("./") -from flask import Flask -from config import Config -from flask_sqlalchemy import SQLAlchemy -from sqlalchemy import Column, Integer, String -app = Flask(__name__, template_folder='templates') -app.config.from_object(Config) - -db = SQLAlchemy(app) - -class StarUser(db.Model): - __tablename__ = 'tb_staruser' - pk = Column(Integer, primary_key=True) - username = Column(String) - full_name = Column(String) - media_count = Column(Integer) - follower_count = Column(Integer) - following_count = Column(Integer) - external_url = Column(String) - public_email = Column(String) - city = Column(String) - - diff --git a/src/flask/models/__init__.py b/src/flask/models/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/flask/templates/index.html b/src/flask/templates/index.html deleted file mode 100644 index 6074e49..0000000 --- a/src/flask/templates/index.html +++ /dev/null @@ -1,63 +0,0 @@ - - - - - Title - - - - - - - - - - - - - - - - - - - -
usernamefull_namepkmedia_countfollower_countfollowing_countexternal_urlpublic_emailcity
- {% if pagination.has_prev %} - 首 页 - 上一页 - {{ pagination.page-1 }} - {% endif %} - {{ pagination.page }} - {% if pagination.has_next %} - {{ pagination.page+1 }} - 下一页 - 尾 页 - {% endif %} -
- - \ No newline at end of file diff --git a/src/test/getFollowerLst.py b/src/test/getFollowerLst.py deleted file mode 100644 index 7b74777..0000000 --- a/src/test/getFollowerLst.py +++ /dev/null @@ -1,46 +0,0 @@ -import sys, os -current_dir = os.path.dirname(os.path.abspath(__file__)) -api_path = os.path.join(current_dir, '..', 'api') -sys.path.append(api_path) -from bot import Bot, PleaseWaitFewMinutes, ClientError -from requests import HTTPError -import time, csv -# import heartrate -# heartrate.trace(browser=True) -bot = Bot() -print("Finish login") -keyword = ["UGC","Makeup","Skincare"] -print("Getting Media") -lst = bot.getMedia(keyword, amount=40) -print("Finish getMedia") -try: - user_pklst = (bot.getUser(l) for l in lst if Bot.getFollower_count(bot.getUser(l)) > 1000) -except Exception as e: - print(e) -with open("data.csv", "w+", encoding='utf-8') as f: - csvf = csv.writer(f) - csvf.writerow(['index', 'id', 'username', 'full_name', 'follower_count', 'media_count', 'email']) - pos = 0 - while True: - assert user_pklst - try: - print("[Writer]: Start writing") - for _, user_pk in enumerate(user_pklst, pos): - time.sleep(5) - index = pos + 1 - print(f"writing {index}\n") - csvf.writerow([index, user_pk.pk, user_pk.username, user_pk.full_name, user_pk.follower_count, user_pk.media_count, user_pk.public_email]) - pos += 1 - break - - except (PleaseWaitFewMinutes, HTTPError, ClientError) as e: - print(e) - print("Detected pause for 1 min") - time.sleep(60) - print("Start to crawl") - continue - except StopIteration: - break - except Exception as e: - raise e - \ No newline at end of file diff --git a/src/test/test.py b/src/test/test.py deleted file mode 100644 index 9afb6c6..0000000 --- a/src/test/test.py +++ /dev/null @@ -1,20 +0,0 @@ -import sys, os -current_dir = os.path.dirname(os.path.abspath(__file__)) -api_path = os.path.join(current_dir, '..', 'api') -sys.path.append(api_path) -from bot import Bot, PleaseWaitFewMinutes, ClientError -from requests import HTTPError - -bot = Bot() -print("finish login") -keyword = ["UGC","Makeup","Skincare"] -lst = bot.getMedia(keyword, amount=100) -print("finish getMedia") -num = 0 -try: - for l in lst: - if Bot.getFollower_count(bot.getUser(l)) > 100: - num += 1 -except: - pass -print(num) \ No newline at end of file