From 7e97119da5cbca64da6a1b8c252bc16a83a1a54d Mon Sep 17 00:00:00 2001 From: gogoolu Date: Sun, 30 Jul 2023 15:20:40 +0800 Subject: [PATCH 1/4] 1 --- config_exmple.yaml | 2 +- ig_settings.json | 38 +------------------------------------- src/api/bot.py | 40 +++++++++++++++++++++++++++++++++++++++- src/data/users.csv | 0 src/run.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 83 insertions(+), 39 deletions(-) create mode 100644 src/data/users.csv create mode 100644 src/run.py diff --git a/config_exmple.yaml b/config_exmple.yaml index ef56a8c..84ef9fe 100644 --- a/config_exmple.yaml +++ b/config_exmple.yaml @@ -1,6 +1,6 @@ bot: Username: root Password: root - Proxy: http://127.0.0.1:10890 + Proxy: http://127.0.0.1:80 LoginWithPassWd: 1 ProxyServer: xxx.com \ No newline at end of file diff --git a/ig_settings.json b/ig_settings.json index 15c4216..9e26dfe 100644 --- a/ig_settings.json +++ b/ig_settings.json @@ -1,37 +1 @@ -{ - "uuids": { - "phone_id": "02b09058-e31e-48a9-9530-acaecffa64f0", - "uuid": "796194be-6115-45cc-8b70-3bd7134e1acf", - "client_session_id": "0364d810-982a-4508-a4d2-3b0163707652", - "advertising_id": "f1d7294a-be9c-449b-a40b-7ef0d5cdb7ac", - "android_device_id": "android-c4d5a931245270bc", - "request_id": "226e903b-2924-4432-844c-5a6ea22ee7ff", - "tray_session_id": "d249a2b7-7564-40db-ac4f-b267ebb0090a" - }, - "mid": "ZMB8nAABAAErYv2pbpnwI68PkjPe", - "ig_u_rur": null, - "ig_www_claim": null, - "authorization_data": { - "ds_user_id": "45331931537", - "sessionid": "45331931537%3AvgJFUQ36j0QewY%3A2%3AAYdj97Ee1r8Ny9oako1X5u6mKd2NezOfXO0yA4iEkw" - }, - "cookies": {}, - "last_login": 1690336432.9240289, - "device_settings": { - "app_version": "269.0.0.18.75", - "android_version": 26, - "android_release": "8.0.0", - "dpi": "480dpi", - "resolution": "1080x1920", - "manufacturer": "OnePlus", - "device": "devitron", - "model": "6T Dev", - "cpu": "qcom", - "version_code": "314665256" - }, - "user_agent": "Instagram 194.0.0.36.172 Android (26/8.0.0; 480dpi; 1080x1920; Xiaomi; MI 5s; capricorn; qcom; en_US; 301484483)", - "country": "US", - "country_code": 1, - "locale": "en_US", - "timezone_offset": -25200 -} \ No newline at end of file +{} \ No newline at end of file diff --git a/src/api/bot.py b/src/api/bot.py index c5f7fec..980c5a8 100644 --- a/src/api/bot.py +++ b/src/api/bot.py @@ -10,7 +10,7 @@ yaml_path = os.path.join(current_dir, '..', '..', 'config.yaml') UserName = "" PassWd = "" -Proxy = "http://127.0.0.1:10809" +Proxy = "" LoginWithPwd = 0 with open(yaml_path, "r", encoding='utf-8') as f: yamlFile = yaml.safe_load(f) @@ -160,6 +160,44 @@ def getUser( # return self._cl.media_info_gql(int(media.pk)).dict()["user"]["pk"] logging.info("") return self._cl.user_info(media.user.pk) + + + @staticmethod + def getEmail(user: User) -> str: + return user.public_email + + @staticmethod + def getUsername(user: User) -> str: + return user.username + + @staticmethod + def getUserID(user: User) -> int: + return int(user.pk) + + @staticmethod + def getUserLink(user: User) -> str: + return "https://www.instagram.com" + user.username + + @staticmethod + def getFullName(user: User) -> str: + return user.full_name + + @staticmethod + def getMediaTime(media: Media) -> str: + return media.taken_at.strftime("%Y-%m-%d") + + @staticmethod + def getCommentCount(media: Media) -> int: + return media.comment_count + + @staticmethod + def getLikeCount(media: Media) -> int: + return media.like_count + + @staticmethod + def getMediaUrl(media: Media) -> str: + return "https://www.instagram.com/p/" + media.code + @staticmethod def getFollower_count( user: User diff --git a/src/data/users.csv b/src/data/users.csv new file mode 100644 index 0000000..e69de29 diff --git a/src/run.py b/src/run.py new file mode 100644 index 0000000..fe6cb86 --- /dev/null +++ b/src/run.py @@ -0,0 +1,42 @@ +import sys, os +current_dir = os.path.dirname(os.path.abspath(__file__)) +api_path = os.path.join(current_dir, '..', 'api') +sys.path.append(api_path) +from bot import Bot, PleaseWaitFewMinutes, ClientError +from requests import HTTPError +import csv + +CSV_PATH = "./data/users.csv" + +bot = Bot() +print("finish login") + +keyword = ["UGC","Makeup","Skincare"] +lst = bot.getMedia(keyword, amount=100) +print("finish getMedia") + +num = 0 +try: + with open(CSV_PATH, mode="a", newline="") as file: + writer = csv.writer(file) + writer.writerow(["用户名","昵称","邮箱","粉丝数","主页链接"]) + + for l in lst: + user = bot.getUser(l) + if l.getFollower_count(user) > 100: + num += 1 + userName = Bot.getUsername(user) + email = Bot.getEmail(user) + fullName = Bot.getFullName(user) + homeLink = Bot.getUserLink(user) + follower_count = Bot.getFollower_count(user) + + newrow = [userName, fullName, email, follower_count, homeLink] + with open(CSV_PATH, mode="a", newline="") as file: + writer = csv.writer(file) + file.write("\n") + writer.writerow(newrow) +except: + pass + +print(num) \ No newline at end of file From b23a1b2dc10e414d30218eb9fa6c7201ec57eece Mon Sep 17 00:00:00 2001 From: gogoolu Date: Tue, 1 Aug 2023 13:03:59 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E4=BD=BF=E7=94=A8cookies=E7=99=BB=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 6 +- config.py | 10 ++ config_exmple.yaml | 6 - data.json | 1 + gui.py | 104 +++++++++++++ ig_settings.json | 1 - ins.py | 265 +++++++++++++++++++++++++++++++++ requirements.txt | Bin 0 -> 524 bytes run.py | 20 +++ src/api/__init__.py | 0 src/api/bot.py | 208 -------------------------- src/data/users.csv | 0 src/flask/__init__.py | 32 ---- src/flask/app.py | 10 -- src/flask/app_test.py | 6 - src/flask/config.py | 4 - src/flask/models/StarUser.py | 24 --- src/flask/models/__init__.py | 0 src/flask/templates/index.html | 63 -------- src/run.py | 42 ------ src/test/getFollowerLst.py | 46 ------ src/test/test.py | 20 --- 22 files changed, 403 insertions(+), 465 deletions(-) create mode 100644 config.py delete mode 100644 config_exmple.yaml create mode 100644 data.json create mode 100644 gui.py delete mode 100644 ig_settings.json create mode 100644 ins.py create mode 100644 requirements.txt create mode 100644 run.py delete mode 100644 src/api/__init__.py delete mode 100644 src/api/bot.py delete mode 100644 src/data/users.csv delete mode 100644 src/flask/__init__.py delete mode 100644 src/flask/app.py delete mode 100644 src/flask/app_test.py delete mode 100644 src/flask/config.py delete mode 100644 src/flask/models/StarUser.py delete mode 100644 src/flask/models/__init__.py delete mode 100644 src/flask/templates/index.html delete mode 100644 src/run.py delete mode 100644 src/test/getFollowerLst.py delete mode 100644 src/test/test.py diff --git a/.gitignore b/.gitignore index 2714a55..1c9c759 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ *.png *.prof /src/**/__pycache__ -/*.csv -config.yaml -ig_settings.json +/venv +/__pycache__ +/test \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000..0bc4a06 --- /dev/null +++ b/config.py @@ -0,0 +1,10 @@ +cookie = { + 'ds_user_id': '', + 'dpr': '1.25', + 'ig_nrcb': '1', + 'mid': '', + 'ig_did': '', + 'datr':'', + 'csrftoken': '', + 'sessionid':'' +} diff --git a/config_exmple.yaml b/config_exmple.yaml deleted file mode 100644 index 84ef9fe..0000000 --- a/config_exmple.yaml +++ /dev/null @@ -1,6 +0,0 @@ -bot: - Username: root - Password: root - Proxy: http://127.0.0.1:80 - LoginWithPassWd: 1 - ProxyServer: xxx.com \ No newline at end of file diff --git a/data.json b/data.json new file mode 100644 index 0000000..571ab8c --- /dev/null +++ b/data.json @@ -0,0 +1 @@ +{"biography": "@ivan_buhaje \u2727\u02da \u00b7 .\n14/02/22 | spreen me sigue en tiktok\u2661\ufe0e\nno spam \ud83e\ude76....\u00bf50k?\ud83c\udfc1", "username": "spritedmc_", "fbid": "17841451709039812", "full_name": "\ud835\ude57 \ud835\ude5a \ud835\ude61 \ud835\ude6a", "id": "51758499634", "followed_by": 46214, "follow": 74, "avatar": "https://scontent-lax3-2.cdninstagram.com/v/t51.2885-19/357422110_9459682620768919_8869669915496121120_n.jpg?stp=dst-jpg_s320x320&_nc_ht=scontent-lax3-2.cdninstagram.com&_nc_cat=107&_nc_ohc=4niehUuZNJYAX_E37vD&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfBO7X02CEnFu7Go71Kync3jeDB-blgC532b7dE0JXoDxw&oe=64CD82BD&_nc_sid=8b3546", "noteCount": 164, "is_private": false, "is_verified": false} \ No newline at end of file diff --git a/gui.py b/gui.py new file mode 100644 index 0000000..00befc1 --- /dev/null +++ b/gui.py @@ -0,0 +1,104 @@ +import re +import tkinter as tk +from tkinter import ttk + +class GUI: + def __init__(self): + self.window = tk.Tk() + self.window.title("instagram spider") + self.window.geometry("1500x800") + self.cap = None + self.keyword + self.create_widgets() + + def create_widgets(self): + self.left_frame = tk.Frame(self.window) + self.left_frame.pack(side="left",padx=5, pady=5, fill="y") + + self.right_frame = tk.Frame(self.window) + self.right_frame.pack(side="right", padx=5, pady=5, fill="y") + + + + self.log_text = tk.Text(self.right_frame, font="microsoftYahei", width=60, height=20,state="disabled") + self.log_text.pack(side="bottom", anchor="se") + + self.clear_butn = tk.Button(self.right_frame, font="microsoftYahei",height=1, text="clear log", command=self.clear_log) + self.clear_butn.pack(side="bottom",anchor="sw") + + + self.keyword = tk.StringVar() + # self.str_interval.set(str(self.interval/1000)) + vcmd = (self.window.register(self.validate),"%P") + ivcmd = (self.window.register(self.invalidate),) + self.keyword_label = tk.Label(self.left_frame, text="标签(关键词):",font="microsoftYahei") + self.keyword_entry = ttk.Entry(self.left_frame, width=18, font="microsoftYahei", textvariable= self.keyword,validate="focusout",validatecommand=vcmd,invalidcommand=ivcmd)#interval submit entry + self.submit_button = tk.Button(self.left_frame, text="提交数据",height=1,width=11, command=self.submit,font="microsoftYahei") + self.label_error = tk.Label(self.left_frame,fg="red") + + + self.keyword_label.grid(row=0, column=0,sticky=tk.E) + self.keyword_entry.grid(row=0, column=1,sticky=tk.W) + self.label_error.grid(row=1,column=1,sticky=tk.NW) + self.submit_button.grid(row=0, column=2) + + # self.combobox_label = tk.Label(self.left_frame, text="Select camera:", font="microsoftYahei") + # self.combobox_label.grid(row=2, column=0, sticky=tk.E) + + + self.switch_button = tk.Button(self.left_frame,text="open camera",height=1,width=11, command=self.combine_camera, font="microsoftYahei") + self.switch_button.grid(row=2, column=2,sticky=tk.W) + + self.window.mainloop() + + def show_message(self,error='',color="black"): + """ + show the error message + """ + self.label_error['text'] = error + self.keyword_entry['foreground'] = color + + def validate(self,value): + """ + scan interval entry validation rules + """ + pattern = re.compile(r'^[a-zA-Z\u4e00-\u9fff0-9]+$') + if bool(pattern.match(pattern,value)): + return True + self.show_message("ok") + print("show ") + return False + + def invalidate(self): + self.show_message("关键词不能有空格或者特殊字符!","red") + + def submit(self): + self.log_text.focus() + try: + self.keyword = self.keyword_entry.get() + + self.log_text.config(state="normal") + self.log_text.insert("end","System: Scan interval has been changed to "+self.keyword+" s\n") + self.log_text.see("end") + self.log_text.config(state="disabled") + except ValueError: + return False + def clear_log(self): + """ + clear log text + """ + + self.log_text.config(state="normal") + self.log_text.delete("1.0", "end") + self.log_text.config(state="disabled") + + # control camera switch by button + def combine_camera(self): + if self.cap == None: + self.open_camera() + else: + self.close_camera() + + + + \ No newline at end of file diff --git a/ig_settings.json b/ig_settings.json deleted file mode 100644 index 9e26dfe..0000000 --- a/ig_settings.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/ins.py b/ins.py new file mode 100644 index 0000000..4e407e8 --- /dev/null +++ b/ins.py @@ -0,0 +1,265 @@ +import time +import random +import requests +import re + +PARAMS = r'("app_id":\s*"[^"]+")|("claim":\s*"[^"]+")|("csrf_token":\s*"[^"]+")' + + +class Ins: + def __init__(self, cookies: dict): + self.cookies = cookies + self.session = requests.Session() + self.interval = 0 + self.headers = { + 'authority': 'www.instagram.com', + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'accept-language': 'zh-CN,zh;q=0.9', + 'sec-ch-ua-full-version-list': '"Google Chrome";v="113.0.5672.63", "Chromium";v="113.0.5672.63", "Not-A.Brand";v="24.0.0.0"', + 'sec-fetch-site': 'same-origin', + 'upgrade-insecure-requests': '1', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36', + 'viewport-width': '1536', + 'X-Asbd-Id': '129477', + 'X-Csrftoken': '6rXinOpVVHa5jIpWEC7QJjhKW0Y6MQaN', + 'X-Ig-App-Id': '936619743392459', + 'X-Ig-Www-Claim': 'hmac.AR3_sXBhcNJg-F09uRf8g0aPkGKbI70OTolUtBSr0AD5zT6C', + 'X-Requested-With': 'XMLHttpRequest' + } + self.get_Header_params() + + def ajax_request(self, url: str, /, params=None): + """ + do requests, the engine of class + :param url: api url + :param params: api params + :return: json object + """ + for _ in range(5): + try: + resp = self.session.get(url, headers=self.headers, params=params, cookies=self.cookies) + return resp.json() + except requests.exceptions.RequestException: + time.sleep(15) + else: + return None + + def get_Header_params(self): + """ + every time visit ins will change header params, this is to get the header params + :return: None + """ + try: + response = self.session.get('https://www.instagram.com/', cookies=self.cookies, headers=self.headers) + matches = re.findall(PARAMS, response.text) + result = [match[i] for match in matches for i in range(3) if match[i]] + # get app_id + app_id = result[0].split(":")[1].strip().strip('"') + # get claim + claim = result[1].split(":")[1].strip().strip('"') + # get csrf_token, if lose cookies, cannot get this param, also cannot access to other apis + csrf_token = result[2].split(":")[1].strip().strip('"') + # set values to headers + self.headers.update({'x-asbd-id': '198387', 'x-csrftoken': csrf_token, + 'x-ig-app-id': app_id, 'x-ig-www-claim': claim, + 'x-requested-with': 'XMLHttpRequest', }) + except requests.exceptions.RequestException: + raise 'Request error, please try again and check your Internet settings' + + + def getUserBytag(self, tagName: str, mode: str)->list: + """_summary_ + + Args: + tagName (str): tag name + mode (str): "top" | "rencet" + return media list + """ + params = { + 'tag_name': tagName, + } + user_lst = [] + resp = self.ajax_request('https://www.instagram.com/api/v1/tags/web_info/', params=params) + if resp: + try: + sections = resp['data'][mode]['sections'] + except KeyError: + raise 'could not get tag information...' + + for section_value in sections: + medias = section_value.get('layout_content', {}).get('medias', {}) + for media_value in medias: + # print(media_value) + # print(type(media_value)) + user_lst.append(media_value.get('media', {}).get('user', {}).get('username', {})) + return user_lst + + + def get_userInfo(self, userName: str): + """ + get user info by username + :param userName: name of user + :return: dict of user info + """ + params = { + 'username': userName, + } + resp = self.ajax_request('https://www.instagram.com/api/v1/users/web_profile_info/', params=params) + + if resp: + try: + # to avoid exception? Internet went wrong may return wrong information + data = resp['data']['user'] + except KeyError: + raise 'Could not get user information...' + return { + 'biography': data.get('biography'), + 'username': data.get('username'), + 'fbid': data.get('fbid'), + 'full_name': data.get('full_name'), + 'id': data.get('id'), + 'followed_by': data.get('edge_followed_by', {}).get('count'), + 'follow': data.get('edge_follow', {}).get('count'), + 'avatar': data.get('profile_pic_url_hd'), + 'noteCount': data.get('edge_owner_to_timeline_media', {}).get('count'), + 'is_private': data.get('is_private'), + 'is_verified': data.get('is_verified') + } if data else 'unknown User' + + def randSleep(self, interval: list): + if len(interval) != 2: + raise ValueError("区间只包含两个元素") + self.interval = interval + start, end = self.interval + if not (isinstance(start, (int, float)) and isinstance(end, (int, float))): + raise ValueError("区间元素应该是整数或浮点数。") + + if start >= end: + raise ValueError("区间起始值应该小于结束值。") + + sleep_time = random.uniform(start, end) + time.sleep(sleep_time) + + + def get_userPosts(self, userName: str): + """ + get all posts from the username + :param userName: name + :return: generator + """ + continuations = [{ + 'count': '12', + }] + temp = userName + '/username/' + while continuations: + continuation = continuations.pop() + # url will change when second request and later + url = f'https://www.instagram.com/api/v1/feed/user/{temp}' + resp = self.ajax_request(url, params=continuation) + # no such user + if not resp.get('user'): + yield 'checking cookie or unknown/private User: {}'.format(userName) + else: + _items = resp.get('items') + # simulate the mousedown + if resp.get('more_available'): + continuations.append({'count': '12', 'max_id': resp.get('next_max_id')}) + user = resp.get('user') + temp = user.get('pk_id') if user.get('pk_id') else user.get('pk') + yield from self.extract_post(_items) + + def get_comments(self, id): + """ + get comments by given post id + :param id: + :return: generator of comments + """ + continuations = [{ + 'can_support_threading': 'true', + 'permalink_enabled': 'false', + }] + # base url + url = f'https://www.instagram.com/api/v1/media/{id}/comments/' + while continuations: + continuation = continuations.pop() + resp = self.ajax_request(url, params=continuation) + if resp.get('next_min_id'): + continuations.append({ + 'can_support_threading': 'true', + 'min_id': resp.get('next_min_id') + }) + comments = resp.get('comments') + if comments: + for comment in comments: + yield { + 'id': comment.get('pk'), + 'user_name': comment.get('user', {}).get('username'), + 'user_fullname': comment.get('user', {}).get('full_name'), + 'text': comment.get('text'), + 'created_at': comment.get('created_at'), + 'comment_like_count': comment.get('comment_like_count'), + 'reply_count': comment.get('child_comment_count') + } + if comment.get('child_comment_count') > 0: + yield from self.get_child_comment(id, comment.get('pk')) + else: + yield 'no comments or losing login cookies' + + def get_child_comment(self, main_id, id): + """ + get child of the comment by comment_id, only used in function get_comments(). + :param main_id: post id + :param id: comment_id + :return: to comments generator + """ + url = f'https://www.instagram.com/api/v1/media/{main_id}/comments/{id}/child_comments/' + continuations = [{'max_id': ''}] + while continuations: + continuation = continuations.pop() + resp = self.ajax_request(url, params=continuation) + cursor = resp.get('next_max_child_cursor') + if cursor: + continuations.append({'max_id': cursor}) + comments = resp.get('child_comments') + if comments: + for comment in comments: + yield { + 'id': comment.get('pk'), + 'user_name': comment.get('user', {}).get('username'), + 'user_fullname': comment.get('user', {}).get('full_name'), + 'text': comment.get('text'), + 'created_at': comment.get('created_at'), + 'comment_like_count': comment.get('comment_like_count'), + } + + @staticmethod + def extract_post(posts): + """ + to extract a post from a list of posts + :param posts: original instagram posts + :return: dict of posts + """ + for post in posts: + caption = post.get('caption') + item = { + 'code': post.get('code'), + 'id': post.get('pk'), + 'pk_id': post.get('id'), + 'comment_count': post.get('comment_count'), + 'like_count': post.get('like_count'), + 'text': caption.get('text') if caption else None, + 'created_at': caption.get('created_at') if caption else post.get('taken_at'), + } + # other type can be added by yourself + types = post.get('media_type') + item.update({ + 'photo': [_.get('image_versions2', {}).get('candidates', [{}])[0].get('url') for _ in + post.get('carousel_media')] + }) if types == 8 else None + item.update({ + 'video': post.get('video_versions', [{}])[0].get('url') + }) if types == 2 else None + item.update({ + 'photo': post.get('image_versions2', {}).get('candidates', [{}])[0].get('url') + }) if types == 1 else None + yield item \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..d50bb726ad30d900942a70ec43dd7b4c9b06e293 GIT binary patch literal 524 zcmYL`T~5MK5QWdPiA$-Wg+zUD4DYsmG^*Y3~Bw#P36N?46mj_ZrnBXV7Q(m(1ke z+&`f)gw|G9(pzCgw_*m*k#pOl0XgAj-6! zHmEduC*6$juFL3OP(L%#E_Rd=-sm!vvdWuC?v)hYN75E8 K_w#p}zseu)+E2p( literal 0 HcmV?d00001 diff --git a/run.py b/run.py new file mode 100644 index 0000000..d3e082a --- /dev/null +++ b/run.py @@ -0,0 +1,20 @@ +from ins import Ins +import json +import config_my +# from gui import GUI + +# you must have your login cookie +COOKIE = config_my.cookie +JSON_PATH = "./data.json" + + +INS = Ins(COOKIE) +# gui = GUI() +username_lst = INS.getUserBytag("minecraft", "top") +for username in username_lst: + print(INS.get_userInfo(username)) + with open(JSON_PATH, "a") as json_file: + json.dump(INS.get_userInfo(username), json_file) + INS.randSleep([60,95]) + +print("hello world") \ No newline at end of file diff --git a/src/api/__init__.py b/src/api/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/api/bot.py b/src/api/bot.py deleted file mode 100644 index 980c5a8..0000000 --- a/src/api/bot.py +++ /dev/null @@ -1,208 +0,0 @@ -from instagrapi import Client -from instagrapi.types import Media, User -import logging -from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, ClientError -import yaml -from time import sleep -from functools import wraps, partial -import os -current_dir = os.path.dirname(os.path.abspath(__file__)) -yaml_path = os.path.join(current_dir, '..', '..', 'config.yaml') -UserName = "" -PassWd = "" -Proxy = "" -LoginWithPwd = 0 -with open(yaml_path, "r", encoding='utf-8') as f: - yamlFile = yaml.safe_load(f) - UserName = yamlFile['bot']['Username'] - PassWd = yamlFile['bot']['Password'] - LoginWithPwd = yamlFile['bot']['LoginWithPassWd'] - -Locale = "en_US" -IG_CREDENTIAL_PATH = ".\ig_settings.json" - -settings = { - 'user_agent': 'Instagram 194.0.0.36.172 Android (26/8.0.0; 480dpi; 1080x1920; Xiaomi; MI 5s; capricorn; qcom; en_US; 301484483)', - 'country': 'US', - 'country_code': 1, - 'locale': 'en_US', - 'timezone_offset': -25200 -} - -def retry(times, expections): - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - time = times - while time > 0: - try: - func(*args, **kwargs) - except expections as e: - print(e) - time -= 1 - if time == 0: - raise e - sleep(10) - print("retrying") - - return wrapper - return decorator - - - - -class Bot: - _cl = None - - def __init__(self): - self._cl = Client(settings) - - - self._cl.delay_range = [3,5] - self._cl.set_proxy(Proxy) - self._cl.set_locale() - self._cl.set_timezone_offset(-7 * 60 * 60) - login_via_session = False - login_via_pw = False - if LoginWithPwd == 0: - try: - session = self._cl.load_settings(IG_CREDENTIAL_PATH) - login_via_session = True - except : - pass - if login_via_session == True and session and LoginWithPwd == 0: - try: - logging.info("Login with session") - self._cl.set_settings(session) - self._cl.login(UserName, PassWd) - - # check if session is valid - try: - self._cl.get_timeline_feed() - except LoginRequired: - logging.warning("Session is invalid, need to login via username and password") - - old_session = self._cl.get_settings() - - # use the same device uuids across logins - self._cl.set_settings({}) - self._cl.set_uuids(old_session["uuids"]) - - self._cl.login(UserName, PassWd) - login_via_session = True - except Exception as e: - logging.warning("Couldn't login user using session information: %s" % e) - - if not login_via_session: - try: - logging.warning("Attempting to login via username and password. username: %s" % UserName) - if self._cl.login(UserName, PassWd): - login_via_pw = True - self._cl.dump_settings(IG_CREDENTIAL_PATH) - except Exception as e: - logging.warning("Couldn't login user using username and password: %s" % e) - - if not login_via_pw and not login_via_session: - raise Exception("Couldn't login user with either password or session") - - # @retry(3, Exception) - def getMedia( - self, - hashtags, - ht_type="top", - amount=27, - ): - ht_medias = [] - isFinished = False - while not isFinished: - try: - for hashtag in hashtags: - - # logging.info() - if ht_type == "top": - ht_medias.extend( - self._cl.hashtag_medias_top(name=hashtag, amount=amount if amount <= 9 else 9) - ) - - elif ht_type == "recent": - ht_medias.extend(self._cl.hashtag_medias_recent(name=hashtag, amount=amount)) - logging.info("finish get media list") - isFinished = True - except PleaseWaitFewMinutes: - logging.warning("Detected") - sleep(60) - logging.warning("Start") - # try: - # self._cl.login(UserName, PassWd) - - # # check if session is valid - # try: - # self._cl.get_timeline_feed() - # except LoginRequired: - # logging.warning("Session is invalid, need to login via username and password") - - # old_session = self._cl.get_settings() - - # # use the same device uuids across logins - # self._cl.set_settings({}) - # self._cl.set_uuids(old_session["uuids"]) - - # self._cl.login(UserName, PassWd) - # except Exception as e: - # print(e) - # logging.warning("Couldn't login user using session information: %s" % e) - - return list(dict([(media.pk, media) for media in ht_medias]).values()) - def getUser( - self, - media : Media, - ) -> User: - # return self._cl.media_info_gql(int(media.pk)).dict()["user"]["pk"] - logging.info("") - return self._cl.user_info(media.user.pk) - - - @staticmethod - def getEmail(user: User) -> str: - return user.public_email - - @staticmethod - def getUsername(user: User) -> str: - return user.username - - @staticmethod - def getUserID(user: User) -> int: - return int(user.pk) - - @staticmethod - def getUserLink(user: User) -> str: - return "https://www.instagram.com" + user.username - - @staticmethod - def getFullName(user: User) -> str: - return user.full_name - - @staticmethod - def getMediaTime(media: Media) -> str: - return media.taken_at.strftime("%Y-%m-%d") - - @staticmethod - def getCommentCount(media: Media) -> int: - return media.comment_count - - @staticmethod - def getLikeCount(media: Media) -> int: - return media.like_count - - @staticmethod - def getMediaUrl(media: Media) -> str: - return "https://www.instagram.com/p/" + media.code - - @staticmethod - def getFollower_count( - user: User - ) -> int: - return user.follower_count - -class StarUser(User): - pass \ No newline at end of file diff --git a/src/data/users.csv b/src/data/users.csv deleted file mode 100644 index e69de29..0000000 diff --git a/src/flask/__init__.py b/src/flask/__init__.py deleted file mode 100644 index b562c69..0000000 --- a/src/flask/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -import os - -from flask import Flask - - -def create_app(test_config=None): - # create and configure the app - app = Flask(__name__, instance_relative_config=True) - app.config.from_mapping( - SECRET_KEY='dev', - DATABASE=os.path.join(app.instance_path, 'flaskr.sqlite'), - ) - - if test_config is None: - # load the instance config, if it exists, when not testing - app.config.from_pyfile('config.py', silent=True) - else: - # load the test config if passed in - app.config.from_mapping(test_config) - - # ensure the instance folder exists - try: - os.makedirs(app.instance_path) - except OSError: - pass - - # a simple page that says hello - @app.route('/hello') - def hello(): - return 'Hello, World!' - - return app \ No newline at end of file diff --git a/src/flask/app.py b/src/flask/app.py deleted file mode 100644 index 599662f..0000000 --- a/src/flask/app.py +++ /dev/null @@ -1,10 +0,0 @@ -from flask import Flask, render_template -from config import Config -from models import StarUser -app = Flask(import_name=__name__, template_folder='templates') - -@app.route('/') -def render(): - return render_template("index.html", StarUser= ) -if __name__ == '__main__': - app.run(debug=True) \ No newline at end of file diff --git a/src/flask/app_test.py b/src/flask/app_test.py deleted file mode 100644 index 475b21e..0000000 --- a/src/flask/app_test.py +++ /dev/null @@ -1,6 +0,0 @@ -from sqlalchemy import create_engine, text -import sqlite3 -engine = create_engine("sqlite+pysqlite:///:memory:", echo=True) -with engine.connect() as conn: - result = conn.execute(text("select 'hello world'")) - print(result.all()) \ No newline at end of file diff --git a/src/flask/config.py b/src/flask/config.py deleted file mode 100644 index 9d8af35..0000000 --- a/src/flask/config.py +++ /dev/null @@ -1,4 +0,0 @@ -class Config(object): - DEBUG = True - SECRET_KEY = "" - SQLALCHEMY_DATABASE_URI = "" \ No newline at end of file diff --git a/src/flask/models/StarUser.py b/src/flask/models/StarUser.py deleted file mode 100644 index 157fdd0..0000000 --- a/src/flask/models/StarUser.py +++ /dev/null @@ -1,24 +0,0 @@ -import sys -sys.path.append("./") -from flask import Flask -from config import Config -from flask_sqlalchemy import SQLAlchemy -from sqlalchemy import Column, Integer, String -app = Flask(__name__, template_folder='templates') -app.config.from_object(Config) - -db = SQLAlchemy(app) - -class StarUser(db.Model): - __tablename__ = 'tb_staruser' - pk = Column(Integer, primary_key=True) - username = Column(String) - full_name = Column(String) - media_count = Column(Integer) - follower_count = Column(Integer) - following_count = Column(Integer) - external_url = Column(String) - public_email = Column(String) - city = Column(String) - - diff --git a/src/flask/models/__init__.py b/src/flask/models/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/flask/templates/index.html b/src/flask/templates/index.html deleted file mode 100644 index 6074e49..0000000 --- a/src/flask/templates/index.html +++ /dev/null @@ -1,63 +0,0 @@ - - - - - Title - - - - - - - - - - - - - - - - - - - -
usernamefull_namepkmedia_countfollower_countfollowing_countexternal_urlpublic_emailcity
- {% if pagination.has_prev %} - 首 页 - 上一页 - {{ pagination.page-1 }} - {% endif %} - {{ pagination.page }} - {% if pagination.has_next %} - {{ pagination.page+1 }} - 下一页 - 尾 页 - {% endif %} -
- - \ No newline at end of file diff --git a/src/run.py b/src/run.py deleted file mode 100644 index fe6cb86..0000000 --- a/src/run.py +++ /dev/null @@ -1,42 +0,0 @@ -import sys, os -current_dir = os.path.dirname(os.path.abspath(__file__)) -api_path = os.path.join(current_dir, '..', 'api') -sys.path.append(api_path) -from bot import Bot, PleaseWaitFewMinutes, ClientError -from requests import HTTPError -import csv - -CSV_PATH = "./data/users.csv" - -bot = Bot() -print("finish login") - -keyword = ["UGC","Makeup","Skincare"] -lst = bot.getMedia(keyword, amount=100) -print("finish getMedia") - -num = 0 -try: - with open(CSV_PATH, mode="a", newline="") as file: - writer = csv.writer(file) - writer.writerow(["用户名","昵称","邮箱","粉丝数","主页链接"]) - - for l in lst: - user = bot.getUser(l) - if l.getFollower_count(user) > 100: - num += 1 - userName = Bot.getUsername(user) - email = Bot.getEmail(user) - fullName = Bot.getFullName(user) - homeLink = Bot.getUserLink(user) - follower_count = Bot.getFollower_count(user) - - newrow = [userName, fullName, email, follower_count, homeLink] - with open(CSV_PATH, mode="a", newline="") as file: - writer = csv.writer(file) - file.write("\n") - writer.writerow(newrow) -except: - pass - -print(num) \ No newline at end of file diff --git a/src/test/getFollowerLst.py b/src/test/getFollowerLst.py deleted file mode 100644 index 7b74777..0000000 --- a/src/test/getFollowerLst.py +++ /dev/null @@ -1,46 +0,0 @@ -import sys, os -current_dir = os.path.dirname(os.path.abspath(__file__)) -api_path = os.path.join(current_dir, '..', 'api') -sys.path.append(api_path) -from bot import Bot, PleaseWaitFewMinutes, ClientError -from requests import HTTPError -import time, csv -# import heartrate -# heartrate.trace(browser=True) -bot = Bot() -print("Finish login") -keyword = ["UGC","Makeup","Skincare"] -print("Getting Media") -lst = bot.getMedia(keyword, amount=40) -print("Finish getMedia") -try: - user_pklst = (bot.getUser(l) for l in lst if Bot.getFollower_count(bot.getUser(l)) > 1000) -except Exception as e: - print(e) -with open("data.csv", "w+", encoding='utf-8') as f: - csvf = csv.writer(f) - csvf.writerow(['index', 'id', 'username', 'full_name', 'follower_count', 'media_count', 'email']) - pos = 0 - while True: - assert user_pklst - try: - print("[Writer]: Start writing") - for _, user_pk in enumerate(user_pklst, pos): - time.sleep(5) - index = pos + 1 - print(f"writing {index}\n") - csvf.writerow([index, user_pk.pk, user_pk.username, user_pk.full_name, user_pk.follower_count, user_pk.media_count, user_pk.public_email]) - pos += 1 - break - - except (PleaseWaitFewMinutes, HTTPError, ClientError) as e: - print(e) - print("Detected pause for 1 min") - time.sleep(60) - print("Start to crawl") - continue - except StopIteration: - break - except Exception as e: - raise e - \ No newline at end of file diff --git a/src/test/test.py b/src/test/test.py deleted file mode 100644 index 9afb6c6..0000000 --- a/src/test/test.py +++ /dev/null @@ -1,20 +0,0 @@ -import sys, os -current_dir = os.path.dirname(os.path.abspath(__file__)) -api_path = os.path.join(current_dir, '..', 'api') -sys.path.append(api_path) -from bot import Bot, PleaseWaitFewMinutes, ClientError -from requests import HTTPError - -bot = Bot() -print("finish login") -keyword = ["UGC","Makeup","Skincare"] -lst = bot.getMedia(keyword, amount=100) -print("finish getMedia") -num = 0 -try: - for l in lst: - if Bot.getFollower_count(bot.getUser(l)) > 100: - num += 1 -except: - pass -print(num) \ No newline at end of file From 4b0019179ca4ad1e273fc10ccd46f90651141657 Mon Sep 17 00:00:00 2001 From: gogoolu Date: Mon, 7 Aug 2023 22:52:12 +0800 Subject: [PATCH 3/4] 2 --- .gitignore | 4 ++- InsWrapper.py | 61 ++++++++++++++++++++++++++++++++++++++++++ config.py | 16 ++++++----- data.json | 1 - db/database.py | 34 ++++++++++++++++++++++++ db_config.py | 4 +++ doc.md | 17 ++++++++++++ gui.py | 68 ++++++++++++++++++++++++++++++----------------- ins.py | 15 ++++------- requirements.txt | Bin 524 -> 556 bytes run.py | 20 -------------- 11 files changed, 177 insertions(+), 63 deletions(-) create mode 100644 InsWrapper.py delete mode 100644 data.json create mode 100644 db/database.py create mode 100644 db_config.py create mode 100644 doc.md delete mode 100644 run.py diff --git a/.gitignore b/.gitignore index 1c9c759..51acd63 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ /src/**/__pycache__ /venv /__pycache__ -/test \ No newline at end of file +/gui_to_spider.json +/data.csv +/gui.py \ No newline at end of file diff --git a/InsWrapper.py b/InsWrapper.py new file mode 100644 index 0000000..2c1fa3a --- /dev/null +++ b/InsWrapper.py @@ -0,0 +1,61 @@ +from ins import Ins +from db.database import DatabaseManager +import config + + +# you must have your login cookie +COOKIE = config.cookie +DATA_PATH = "./data.csv" +CTRL_PATH = "./gui_to_spider.json" + + +class InsWrapper(Ins): + def __init__(self): + super().__init__(COOKIE) + self.db = DatabaseManager() + + def get_UserData(self, keywords: list, mode: str, amount: int): + for keyword in keywords: + counter = 0 + username_lst = self.getUsernameBytag(keyword, mode=mode) + counter = 0 + for username in username_lst: + if counter >= amount: + break + + insert_sql = "insert into users (biography,username,fbid,full_name,id,followed_by,follow,noteCount,is_private,is_verified) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + userInfo = self.get_userInfo(username) + userInfo.pop("avatar", "N/A") + # print(userInfo) + values = list(userInfo.values()) + self.db.execute_insert(insert_sql, values) + counter += 1 + self.randSleep([60,95]) + + def get_postsByUsername(self, username: str, amount: int): + """Args: + username (str): username + amount (int): user recent posts number + Return: post list + """ + post_lst = [] + counter = 0 + posts = self.get_userPosts(username) + insert_sql = "insert into posts (code, id, pk_id, comment_count, like_count, txt, create_at) values (%s, %s, %s, %s, %s, %s, %s)" + for post in posts: + if counter >= amount: + break + post.pop("photo", "N/A") + post.pop("video", "N/A") + # print(post) + values = list(post.values()) + self.db.execute_insert(insert_sql, values) + post_lst.append(post) + counter+=1 + self.randSleep([60,95]) + return post_lst + + +INS = InsWrapper() +# INS.get_postsByUsername("nasa", 5) +INS.get_UserData(["minecraft"], "top", 3) \ No newline at end of file diff --git a/config.py b/config.py index 0bc4a06..f82b90b 100644 --- a/config.py +++ b/config.py @@ -1,10 +1,14 @@ cookie = { - 'ds_user_id': '', + 'ds_user_id': '60671358732', 'dpr': '1.25', 'ig_nrcb': '1', - 'mid': '', - 'ig_did': '', - 'datr':'', - 'csrftoken': '', - 'sessionid':'' + 'mid': 'ZL5zlQALAAFRD0LKZLsrM9Moep_U', + 'ig_did': '73F1F871-601D-4777-837B-161A1865FE8A', + 'datr':'43u-ZK7MypRIEtSWCkGAtMny', + 'csrftoken': '6rXinOpVVHa5jIpWEC7QJjhKW0Y6MQaN', + 'sessionid':'60671358732%3A3FY10RkDUHDk3W%3A10%3AAYdnC77496N_dQLRUGRPuAbl85mPIGPN3cDBwUBXAg' } + +keyword = ["makeup", "skincare"] + +mode = "recent" \ No newline at end of file diff --git a/data.json b/data.json deleted file mode 100644 index 571ab8c..0000000 --- a/data.json +++ /dev/null @@ -1 +0,0 @@ -{"biography": "@ivan_buhaje \u2727\u02da \u00b7 .\n14/02/22 | spreen me sigue en tiktok\u2661\ufe0e\nno spam \ud83e\ude76....\u00bf50k?\ud83c\udfc1", "username": "spritedmc_", "fbid": "17841451709039812", "full_name": "\ud835\ude57 \ud835\ude5a \ud835\ude61 \ud835\ude6a", "id": "51758499634", "followed_by": 46214, "follow": 74, "avatar": "https://scontent-lax3-2.cdninstagram.com/v/t51.2885-19/357422110_9459682620768919_8869669915496121120_n.jpg?stp=dst-jpg_s320x320&_nc_ht=scontent-lax3-2.cdninstagram.com&_nc_cat=107&_nc_ohc=4niehUuZNJYAX_E37vD&edm=AOQ1c0wBAAAA&ccb=7-5&oh=00_AfBO7X02CEnFu7Go71Kync3jeDB-blgC532b7dE0JXoDxw&oe=64CD82BD&_nc_sid=8b3546", "noteCount": 164, "is_private": false, "is_verified": false} \ No newline at end of file diff --git a/db/database.py b/db/database.py new file mode 100644 index 0000000..b0604f8 --- /dev/null +++ b/db/database.py @@ -0,0 +1,34 @@ +import pymysql +import db_config + +class DatabaseManager: + def __init__(self): + self.connection = pymysql.connect(host=db_config.host, + user=db_config.user, + password=db_config.password, + database=db_config.database) + + def disconnect(self): + if self.connection: + self.connection.close() + + def execute_query(self, query, params=None): + with self.connection.cursor() as cursor: + cursor.execute(query, params) + result = cursor.fetchall() + self.connection.commit() + return result + + def execute_update(self, query, params=None): + if not self.connection: + self.connect() + with self.connection.cursor() as cursor: + cursor.execute(query, params) + self.connection.commit() + + def execute_insert(self, insert_sql: str, params=None): + with self.connection.cursor() as cursor: + cursor.execute(insert_sql, params) + result = cursor.fetchall() + self.connection.commit() + return result \ No newline at end of file diff --git a/db_config.py b/db_config.py new file mode 100644 index 0000000..485e40c --- /dev/null +++ b/db_config.py @@ -0,0 +1,4 @@ +host = 'localhost' +user = 'root' +password = '123456' +database = 'ins' \ No newline at end of file diff --git a/doc.md b/doc.md new file mode 100644 index 0000000..c63ff31 --- /dev/null +++ b/doc.md @@ -0,0 +1,17 @@ +# 用户信息字段 + ++ ##### biography: 用户简介 + ++ ##### username: 用户名（ins唯一标识） + ++ ##### fbid: facebook id（facebook唯一标识） + ++ ##### full_name: 昵称 + ++ ##### id: instagram id （ins唯一标识） + ++ ##### followed_by: 粉丝数 + ++ ##### noteCount: 贴子数 + + \ No newline at end of file diff --git a/gui.py b/gui.py index 00befc1..9209ca8 100644 --- a/gui.py +++ b/gui.py @@ -1,14 +1,20 @@ import re import tkinter as tk from tkinter import ttk +import json +import os + +DATA_PATH = "./gui_to_spider.json" class GUI: def __init__(self): self.window = tk.Tk() self.window.title("instagram spider") self.window.geometry("1500x800") - self.cap = None - self.keyword + self.amount = 0 + self.keyword = "" + self.mode = "recent" + self.mode_list = ["top", "recent"] self.create_widgets() def create_widgets(self): @@ -18,7 +24,6 @@ def create_widgets(self): self.right_frame = tk.Frame(self.window) self.right_frame.pack(side="right", padx=5, pady=5, fill="y") - self.log_text = tk.Text(self.right_frame, font="microsoftYahei", width=60, height=20,state="disabled") self.log_text.pack(side="bottom", anchor="se") @@ -27,7 +32,6 @@ def create_widgets(self): self.clear_butn.pack(side="bottom",anchor="sw") - self.keyword = tk.StringVar() # self.str_interval.set(str(self.interval/1000)) vcmd = (self.window.register(self.validate),"%P") ivcmd = (self.window.register(self.invalidate),) @@ -36,20 +40,30 @@ def create_widgets(self): self.submit_button = tk.Button(self.left_frame, text="提交数据",height=1,width=11, command=self.submit,font="microsoftYahei") self.label_error = tk.Label(self.left_frame,fg="red") - self.keyword_label.grid(row=0, column=0,sticky=tk.E) self.keyword_entry.grid(row=0, column=1,sticky=tk.W) self.label_error.grid(row=1,column=1,sticky=tk.NW) self.submit_button.grid(row=0, column=2) - # self.combobox_label = tk.Label(self.left_frame, text="Select camera:", font="microsoftYahei") - # self.combobox_label.grid(row=2, column=0, sticky=tk.E) + self.combobox_label = tk.Label(self.left_frame, text="获取数据模式:", font="microsoftYahei") + self.combobox_label.grid(row=2, column=0, sticky=tk.E) + self.mode_combobox = ttk.Combobox(self.left_frame, values=self.mode_list, font="microsoftYahei",state="readonly",width=18) + self.mode_combobox.current(0) + self.mode_combobox.grid(row=2,column=1,sticky=tk.W) - self.switch_button = tk.Button(self.left_frame,text="open camera",height=1,width=11, command=self.combine_camera, font="microsoftYahei") - self.switch_button.grid(row=2, column=2,sticky=tk.W) + self.amount_label = tk.Label(self.left_frame, text="查找数目:", font="microsoftYahei") + self.amount_label.grid(row=3, column=0, sticky=tk.E) + self.amount_entry = tk.Entry(self.left_frame, font="microsoftYahei", width=18) + self.amount_entry.grid(row=3, column=1, sticky=tk.W) + + self.launch_button = tk.Button(self.left_frame, text="启动", font="microsoftYahei", command=self.launch, height=1, width=11) + self.launch_button.grid(row=4, column=1, sticky=tk.E) + + self.data_change() self.window.mainloop() + return self.window def show_message(self,error='',color="black"): """ @@ -58,15 +72,15 @@ def show_message(self,error='',color="black"): self.label_error['text'] = error self.keyword_entry['foreground'] = color - def validate(self,value): + def validate(self, value): """ scan interval entry validation rules """ pattern = re.compile(r'^[a-zA-Z\u4e00-\u9fff0-9]+$') - if bool(pattern.match(pattern,value)): + if pattern.match(value) is not None: return True self.show_message("ok") - print("show ") + print(value) return False def invalidate(self): @@ -76,29 +90,33 @@ def submit(self): self.log_text.focus() try: self.keyword = self.keyword_entry.get() - + self.mode = self.mode_combobox.get() + self.amount = self.amount_entry.get() self.log_text.config(state="normal") - self.log_text.insert("end","System: Scan interval has been changed to "+self.keyword+" s\n") + self.log_text.insert("end","keyword: "+self.keyword+" s\n") self.log_text.see("end") self.log_text.config(state="disabled") except ValueError: return False + + def launch(self): + os.system("python bot_ctrl.py") + def clear_log(self): """ clear log text """ - self.log_text.config(state="normal") self.log_text.delete("1.0", "end") self.log_text.config(state="disabled") - # control camera switch by button - def combine_camera(self): - if self.cap == None: - self.open_camera() - else: - self.close_camera() - - - - \ No newline at end of file + def data_change(self): + with open(DATA_PATH, "r") as file: + data = json.load(file) + data['keyword'] = self.keyword + data['amount'] = self.amount + data['mode'] = self.mode + + with open(DATA_PATH, "w") as file: + json.dump(data, file) + \ No newline at end of file diff --git a/ins.py b/ins.py index 4e407e8..1380182 100644 --- a/ins.py +++ b/ins.py @@ -66,19 +66,18 @@ def get_Header_params(self): except requests.exceptions.RequestException: raise 'Request error, please try again and check your Internet settings' - - def getUserBytag(self, tagName: str, mode: str)->list: + def getUsernameBytag(self, tagName: str, mode: str)->list: """_summary_ Args: tagName (str): tag name mode (str): "top" | "rencet" - return media list + return username list """ params = { 'tag_name': tagName, } - user_lst = [] + username_lst = [] resp = self.ajax_request('https://www.instagram.com/api/v1/tags/web_info/', params=params) if resp: try: @@ -89,11 +88,8 @@ def getUserBytag(self, tagName: str, mode: str)->list: for section_value in sections: medias = section_value.get('layout_content', {}).get('medias', {}) for media_value in medias: - # print(media_value) - # print(type(media_value)) - user_lst.append(media_value.get('media', {}).get('user', {}).get('username', {})) - return user_lst - + username_lst.append(media_value.get('media', {}).get('user', {}).get('username', {})) + return username_lst def get_userInfo(self, userName: str): """ @@ -140,7 +136,6 @@ def randSleep(self, interval: list): sleep_time = random.uniform(start, end) time.sleep(sleep_time) - def get_userPosts(self, userName: str): """ get all posts from the username diff --git a/requirements.txt b/requirements.txt index d50bb726ad30d900942a70ec43dd7b4c9b06e293..63d3439f23214c14889bd7b2457fac8373d7700d 100644 GIT binary patch delta 35 lcmeBSS;MlSf>GR;p^_n(A&|j`!4?P&8T7!|U~)d=TmXz@2R{G+ delta 11 ScmZ3((!;W$f^qT;#wh?8@&qaX diff --git a/run.py b/run.py deleted file mode 100644 index d3e082a..0000000 --- a/run.py +++ /dev/null @@ -1,20 +0,0 @@ -from ins import Ins -import json -import config_my -# from gui import GUI - -# you must have your login cookie -COOKIE = config_my.cookie -JSON_PATH = "./data.json" - - -INS = Ins(COOKIE) -# gui = GUI() -username_lst = INS.getUserBytag("minecraft", "top") -for username in username_lst: - print(INS.get_userInfo(username)) - with open(JSON_PATH, "a") as json_file: - json.dump(INS.get_userInfo(username), json_file) - INS.randSleep([60,95]) - -print("hello world") \ No newline at end of file From aba332e58f0790812afc85cd25177c7e21b06f51 Mon Sep 17 00:00:00 2001 From: gogoolu Date: Tue, 8 Aug 2023 23:54:37 +0800 Subject: [PATCH 4/4] database --- .gitignore | 3 +- InsWrapper.py | 22 ++--- config.py | 14 +-- db/__pycache__/database.cpython-311.pyc | Bin 0 -> 4699 bytes db/database.py | 23 ++++- doc.md | 17 ---- gui.py | 122 ------------------------ ins.py | 23 +---- 8 files changed, 45 insertions(+), 179 deletions(-) create mode 100644 db/__pycache__/database.cpython-311.pyc delete mode 100644 doc.md delete mode 100644 gui.py diff --git a/.gitignore b/.gitignore index 51acd63..9c2fc89 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ /__pycache__ /gui_to_spider.json /data.csv -/gui.py \ No newline at end of file +/gui.py +/test.py \ No newline at end of file diff --git a/InsWrapper.py b/InsWrapper.py index 2c1fa3a..6bac0f8 100644 --- a/InsWrapper.py +++ b/InsWrapper.py @@ -1,13 +1,11 @@ from ins import Ins from db.database import DatabaseManager import config +import datetime # you must have your login cookie COOKIE = config.cookie -DATA_PATH = "./data.csv" -CTRL_PATH = "./gui_to_spider.json" - class InsWrapper(Ins): def __init__(self): @@ -18,19 +16,17 @@ def get_UserData(self, keywords: list, mode: str, amount: int): for keyword in keywords: counter = 0 username_lst = self.getUsernameBytag(keyword, mode=mode) - counter = 0 for username in username_lst: if counter >= amount: break - - insert_sql = "insert into users (biography,username,fbid,full_name,id,followed_by,follow,noteCount,is_private,is_verified) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + insert_sql = "insert into users (biography,username,fbid,full_name,user_id,followed_by,follow,noteCount,is_private,is_verified,business_email) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" userInfo = self.get_userInfo(username) userInfo.pop("avatar", "N/A") - # print(userInfo) values = list(userInfo.values()) self.db.execute_insert(insert_sql, values) counter += 1 self.randSleep([60,95]) + self.db.remove_duplicate_rows() def get_postsByUsername(self, username: str, amount: int): """Args: @@ -38,16 +34,16 @@ def get_postsByUsername(self, username: str, amount: int): amount (int): user recent posts number Return: post list """ + post_lst = [] counter = 0 posts = self.get_userPosts(username) - insert_sql = "insert into posts (code, id, pk_id, comment_count, like_count, txt, create_at) values (%s, %s, %s, %s, %s, %s, %s)" + insert_sql = "insert into posts (code,user_id,comment_count,like_count,introduction,create_time) values (%s, %s, %s, %s, %s, %s)" for post in posts: if counter >= amount: break - post.pop("photo", "N/A") - post.pop("video", "N/A") - # print(post) + post['create_time'] = datetime.datetime.fromtimestamp(int(post['create_time'])).strftime("%Y-%m-%d %H:%M:%S") + values = list(post.values()) self.db.execute_insert(insert_sql, values) post_lst.append(post) @@ -57,5 +53,5 @@ def get_postsByUsername(self, username: str, amount: int): INS = InsWrapper() -# INS.get_postsByUsername("nasa", 5) -INS.get_UserData(["minecraft"], "top", 3) \ No newline at end of file +# INS.get_postsByUsername("nasa", 1) +INS.get_UserData(["minecraft"], "top", 10) \ No newline at end of file diff --git a/config.py b/config.py index f82b90b..7edb547 100644 --- a/config.py +++ b/config.py @@ -1,12 +1,12 @@ cookie = { - 'ds_user_id': '60671358732', - 'dpr': '1.25', + 'ds_user_id': '', + 'dpr': '1.5', 'ig_nrcb': '1', - 'mid': 'ZL5zlQALAAFRD0LKZLsrM9Moep_U', - 'ig_did': '73F1F871-601D-4777-837B-161A1865FE8A', - 'datr':'43u-ZK7MypRIEtSWCkGAtMny', - 'csrftoken': '6rXinOpVVHa5jIpWEC7QJjhKW0Y6MQaN', - 'sessionid':'60671358732%3A3FY10RkDUHDk3W%3A10%3AAYdnC77496N_dQLRUGRPuAbl85mPIGPN3cDBwUBXAg' + 'mid': '', + 'ig_did': '', + 'datr':'', + 'csrftoken': '', + 'sessionid':'' } keyword = ["makeup", "skincare"] diff --git a/db/__pycache__/database.cpython-311.pyc b/db/__pycache__/database.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ed7ebccd4a698793d74a3fa55c5c1e895af4e21f GIT binary patch literal 4699 zcmd^DT}&I<6~5z{@gHLt6U+k6&rk^9z>>64R@pS6Wr1wdCJr02TLsa&j%Ua^GPd`Q zy?|@BsESl}#cHD{5>Z-dd1*?N-N#COP4m)68(G2{6{%8R+Ly`*s-mb5J!i%qdyLug z(x=`r-<-MU-gEEV`^`D$@;i^mO`!a0W>fyNgOI=CL#1$KVfP#eYeXh86C=~cotb8A z7+Yc1p9RQPUKRc?gi;d=^=Fg0V@LB`>B zAL>J;YIOHaIaPT@Xmh_ZIbx|bTUcs&=%5!%Esm*|ub5}h?}gbnOG2=?Mv)0j-WCjS z8uwvBO=(W05XiI9dlW~b zXqGw=TmYK8+%B4kKW8*fRbsO$zOD57#Q3F|NJ3U-l8ZCZxH==>ov92ooLtnLp-?m) zO@%_4j=EJHu4wR`)$;($WV@^TH)p?I{OXf!-^s^6-R}HWp>xo{l7scXxkCW<5LUR= zmKC^mp&PO657I^7pC5v-MzZ8SHqi_9*FLSAyH(NS{TckdFe?}!<&^&SoCyYO)B_rg zkHiwNw9OzJYd}S;cFR$GxTjJn@hO54Jys3wLo2cBpZGpW*<{5_EAq-rWJu#VW8VA8@A9`?8gtmd=IdANU zJ(8wsf@%(BL5ZYOisqbEQjrhCv6yC$B1D`cuu9ASZAk- znvSJZB)Oe7d)p+#OCVXXfFz_ zUj#o3KDn^omlpyBA&?USMW25y^eB|~^%Z=5D_2WAwA7pHu!9}<5UO>Jr2YNz@W!gDb%XEMr_}8nX<2 z$Qd+Yzz#}#=3`Y+C+W1RbgJt9;uH>$3U~!S6bamUM344~$Qu%OUX z2;xm+hW(#mTQZ;es%n z6NZcJ$JY{%68ZN2LVLgAk_OPb7=SKm09{f|T~%9^$}Z_ZtHWIq?S}bUA~~iYdL%DF zq+TN~Yos_r^agLj$jp*0%N!EpY|p^T3)2$#E9OWM4pPc>{JswqnO!$CxLPHoo)A69 z>mr$TbnDX+cj&A+Ys)$>Yp)%*B3r#ILsA&8%0=eu?KSqQ=nET1a6^;AET~+13H2}W zmQn0kzCI(k16jfn{|Rs8SSZai^Tfyw0*+wjb2mEB`Z6_fePZmkbmLlZ;M#aV8l95n zqVZ5v9+EKOjfdxyhTX2-x_Luyp*B>%d+X-=lhQkP4mXndS;LOwFrC{I(tk1k;~z}i znvkxIOTnABrE5WH;O9$$1Lj^cLzWK!sf2m)?)L$H1gR2+;h%IaF|~(|ESz*OjWLRK zD?cD>3`17C^mCINP z_*e0~S#fj}0biFtMSax4)E$JCKwv*&lfo_ZGGZ*VP>T+bx@{xJL7u?_piLSf)y zzW41y@7qRoE$T^brgywzUqbtmF{&)7=}1IT)!B3`wm6(=9gRy0r3oq}B9H)5MZOep z8DSb1#E8&-kTm;jg3g8ENOV6Djq5>MDheOV4-X9 zp`&DDrM7?BiO>J&>XY+dUVM7dSnEGI;GXZ9ECBi^3;xM`+k1t!_eu`pJyCKtFe|n_ zZrKw1HpRZYI8YD=a^k?Y=*zWV&Wl$H;+33urF07BrcY3Z9h~YOLbdKtNe>p^)O7!! z5)nHw1sSZuw&2aR_H8hE;Y>j|lWX(?!~2;048mRgE-ddaJI6%w>v#NPEc?f|nm~TV za$^q1D+i0Rn?bormqqv31@={!IChSGb&dzwNCVw?phX`O3Tf_8Xf7eAV<-zDAVoN4 zXej1E6mvK#A@m>&BJ?4=1@N5cOxIaMF_h?lg8C`IvU!zk9RFsCm>}4AI1ZZN9&b literal 0 HcmV?d00001 diff --git a/db/database.py b/db/database.py index b0604f8..66e4d89 100644 --- a/db/database.py +++ b/db/database.py @@ -31,4 +31,25 @@ def execute_insert(self, insert_sql: str, params=None): cursor.execute(insert_sql, params) result = cursor.fetchall() self.connection.commit() - return result \ No newline at end of file + return result + + def remove_duplicate_rows(self): + try: + with self.connection.cursor() as cursor: + # 构建 SQL 查询，选择每个用户名的最小 ID + select_query = """ + SELECT MIN(ID) AS min_id, username + FROM users + GROUP BY username + """ + cursor.execute(select_query) + results = cursor.fetchall() + delete_query = """ + DELETE FROM users + WHERE ID NOT IN ({}) + """.format(','.join(str(result['min_id']) for result in results)) + cursor.execute(delete_query) + self.connection.commit() + print("Duplicate rows removed successfully.") + except Exception as e: + print("An error occurred:", e) diff --git a/doc.md b/doc.md deleted file mode 100644 index c63ff31..0000000 --- a/doc.md +++ /dev/null @@ -1,17 +0,0 @@ -# 用户信息字段 - -+ ##### biography: 用户简介 - -+ ##### username: 用户名（ins唯一标识） - -+ ##### fbid: facebook id（facebook唯一标识） - -+ ##### full_name: 昵称 - -+ ##### id: instagram id （ins唯一标识） - -+ ##### followed_by: 粉丝数 - -+ ##### noteCount: 贴子数 - - \ No newline at end of file diff --git a/gui.py b/gui.py deleted file mode 100644 index 9209ca8..0000000 --- a/gui.py +++ /dev/null @@ -1,122 +0,0 @@ -import re -import tkinter as tk -from tkinter import ttk -import json -import os - -DATA_PATH = "./gui_to_spider.json" - -class GUI: - def __init__(self): - self.window = tk.Tk() - self.window.title("instagram spider") - self.window.geometry("1500x800") - self.amount = 0 - self.keyword = "" - self.mode = "recent" - self.mode_list = ["top", "recent"] - self.create_widgets() - - def create_widgets(self): - self.left_frame = tk.Frame(self.window) - self.left_frame.pack(side="left",padx=5, pady=5, fill="y") - - self.right_frame = tk.Frame(self.window) - self.right_frame.pack(side="right", padx=5, pady=5, fill="y") - - - self.log_text = tk.Text(self.right_frame, font="microsoftYahei", width=60, height=20,state="disabled") - self.log_text.pack(side="bottom", anchor="se") - - self.clear_butn = tk.Button(self.right_frame, font="microsoftYahei",height=1, text="clear log", command=self.clear_log) - self.clear_butn.pack(side="bottom",anchor="sw") - - - # self.str_interval.set(str(self.interval/1000)) - vcmd = (self.window.register(self.validate),"%P") - ivcmd = (self.window.register(self.invalidate),) - self.keyword_label = tk.Label(self.left_frame, text="标签(关键词):",font="microsoftYahei") - self.keyword_entry = ttk.Entry(self.left_frame, width=18, font="microsoftYahei", textvariable= self.keyword,validate="focusout",validatecommand=vcmd,invalidcommand=ivcmd)#interval submit entry - self.submit_button = tk.Button(self.left_frame, text="提交数据",height=1,width=11, command=self.submit,font="microsoftYahei") - self.label_error = tk.Label(self.left_frame,fg="red") - - self.keyword_label.grid(row=0, column=0,sticky=tk.E) - self.keyword_entry.grid(row=0, column=1,sticky=tk.W) - self.label_error.grid(row=1,column=1,sticky=tk.NW) - self.submit_button.grid(row=0, column=2) - - self.combobox_label = tk.Label(self.left_frame, text="获取数据模式:", font="microsoftYahei") - self.combobox_label.grid(row=2, column=0, sticky=tk.E) - - self.mode_combobox = ttk.Combobox(self.left_frame, values=self.mode_list, font="microsoftYahei",state="readonly",width=18) - self.mode_combobox.current(0) - self.mode_combobox.grid(row=2,column=1,sticky=tk.W) - - self.amount_label = tk.Label(self.left_frame, text="查找数目:", font="microsoftYahei") - self.amount_label.grid(row=3, column=0, sticky=tk.E) - - self.amount_entry = tk.Entry(self.left_frame, font="microsoftYahei", width=18) - self.amount_entry.grid(row=3, column=1, sticky=tk.W) - - self.launch_button = tk.Button(self.left_frame, text="启动", font="microsoftYahei", command=self.launch, height=1, width=11) - self.launch_button.grid(row=4, column=1, sticky=tk.E) - - self.data_change() - self.window.mainloop() - return self.window - - def show_message(self,error='',color="black"): - """ - show the error message - """ - self.label_error['text'] = error - self.keyword_entry['foreground'] = color - - def validate(self, value): - """ - scan interval entry validation rules - """ - pattern = re.compile(r'^[a-zA-Z\u4e00-\u9fff0-9]+$') - if pattern.match(value) is not None: - return True - self.show_message("ok") - print(value) - return False - - def invalidate(self): - self.show_message("关键词不能有空格或者特殊字符!","red") - - def submit(self): - self.log_text.focus() - try: - self.keyword = self.keyword_entry.get() - self.mode = self.mode_combobox.get() - self.amount = self.amount_entry.get() - self.log_text.config(state="normal") - self.log_text.insert("end","keyword: "+self.keyword+" s\n") - self.log_text.see("end") - self.log_text.config(state="disabled") - except ValueError: - return False - - def launch(self): - os.system("python bot_ctrl.py") - - def clear_log(self): - """ - clear log text - """ - self.log_text.config(state="normal") - self.log_text.delete("1.0", "end") - self.log_text.config(state="disabled") - - def data_change(self): - with open(DATA_PATH, "r") as file: - data = json.load(file) - data['keyword'] = self.keyword - data['amount'] = self.amount - data['mode'] = self.mode - - with open(DATA_PATH, "w") as file: - json.dump(data, file) - \ No newline at end of file diff --git a/ins.py b/ins.py index 1380182..ecfcbb7 100644 --- a/ins.py +++ b/ins.py @@ -116,10 +116,10 @@ def get_userInfo(self, userName: str): 'id': data.get('id'), 'followed_by': data.get('edge_followed_by', {}).get('count'), 'follow': data.get('edge_follow', {}).get('count'), - 'avatar': data.get('profile_pic_url_hd'), 'noteCount': data.get('edge_owner_to_timeline_media', {}).get('count'), 'is_private': data.get('is_private'), - 'is_verified': data.get('is_verified') + 'is_verified': data.get('is_verified'), + 'business_email': data.get('business_email') } if data else 'unknown User' def randSleep(self, interval: list): @@ -238,23 +238,10 @@ def extract_post(posts): caption = post.get('caption') item = { 'code': post.get('code'), - 'id': post.get('pk'), - 'pk_id': post.get('id'), + 'user_id':str(post.get('id')).split('_')[1], 'comment_count': post.get('comment_count'), 'like_count': post.get('like_count'), - 'text': caption.get('text') if caption else None, - 'created_at': caption.get('created_at') if caption else post.get('taken_at'), + 'introduction': caption.get('text') if caption else None, + 'create_time': caption.get('created_at') if caption else post.get('taken_at'), } - # other type can be added by yourself - types = post.get('media_type') - item.update({ - 'photo': [_.get('image_versions2', {}).get('candidates', [{}])[0].get('url') for _ in - post.get('carousel_media')] - }) if types == 8 else None - item.update({ - 'video': post.get('video_versions', [{}])[0].get('url') - }) if types == 2 else None - item.update({ - 'photo': post.get('image_versions2', {}).get('candidates', [{}])[0].get('url') - }) if types == 1 else None yield item \ No newline at end of file