一个帅气的py爬虫模块 环境windows11+Ubuntu+"python3.10+"+playwright

# coding: utf-8 """ @Time : 11/7/2022 13:42 @Author: fff @File: test2.py @Software: PyCharm """ import asyncio,Class_function,random from playwright.async_api import async_playwright class spider: def __init__(self): self.Core_Function = Class_function.Class_function() self.page_result_list=[] self.request_list = [] self.response_list = [] self.list_url = [] self.HTML_list=[] async def click_function(self,page): ''' ;模拟遍历点击 :return: ''' try: num222 = await page.evaluate(''' window.stop(); num222=document.querySelectorAll('*').length; num222; ''') # self.Core_Function.callback_logging.info(num222) #print(num222) await page.evaluate(''' window.scrollBy(1920, 50); treeWalker = document.createTreeWalker(document); num111=0 while (treeWalker.nextNode() && num111<1500) { console.log("[*] processing node " + treeWalker.currentNode.tagName + ' ' + treeWalker.currentNode.id); if (treeWalker.currentNode.click) { treeWalker.currentNode.target=''; treeWalker.currentNode.click(); num111=num111+1; } } ''') flag_num=0 while await page.evaluate('num111') < 666: #self.Core_Function.callback_logging().info('while') await asyncio.sleep(0.5) flag_num=flag_num+1 if flag_num<3: break except Exception as e: self.Core_Function.callback_logging().error(e) async def hook_requset(self,route): ''' # hook 请求包 :param route: :return: ''' # print(route.request.url) if route.request.url == self.target_request['url']: if self.target_request['body'] != 'Null': await route.continue_(headers=self.target_request['headers'], method=self.target_request['method'], post_data=self.target_request['body']) elif self.target_request['method'] == 'GET': await route.continue_(headers=self.target_request['headers'], method=self.target_request['method']) else: await route.continue_() elif route.request.url != 'about:blank' and route.request.is_navigation_request(): # print(route.request.url) if route.request.method == 'GET': request = {"headers": route.request.headers, "method": route.request.method, "url": route.request.url, "body": "Null"} self.request_list.append(request) elif self.target_request['body'] != 'Null': request = {"headers": route.request.headers, "method": route.request.method, "url": route.request.url, "body": route.request.post_data} self.request_list.append(request) # await route.continue_() await route.abort(error_code='aborted') else: await route.continue_() async def handle_popup(self, page): ''' # 关闭click打开的新窗口 :param page: :return: ''' await page.close() async def handle_dialog(self, dialog): ''' # 处理alert之类 :param dialog: :return: ''' await dialog.dismiss() async def handle_network_http_request(self, request): ''' ; 获取页面http请求 :param request: :return: ''' try: if request.resource_type not in ['image', 'stylesheet', 'websocket', 'media', 'font']: request_data = {} # print(request.url) if request.post_data == None: request_data['body'] = 'Null' else: request_data['body'] = request.post_data request_data['url'] = request.url request_data['headers'] = request.headers request_data['method'] = request.method request_data['time'] = self.Core_Function.callback_time(0) request_data['describe'] = 'Null' request_data['status'] = 0 #print(request_data) self.request_list.append(request_data) except Exception as e: self.Core_Function.callback_logging().error(e) async def handle_http_response(self, response): ''' # 处理http响应 ''' try: # print(response.request.url) response_data = {} html_data={} if response.request.url == self.target_request['url']: if response.status in [200, 301, 302, 404, 500]: response_data['body'] = self.target_request['body'] response_data['url'] = self.target_request['url'] response_data['headers'] = self.target_request['headers'] response_data['method'] = self.target_request['method'] response_data['http_status_code'] = response.status response_data['headers_response'] = response.headers html=await response.text() html_md5=self.Core_Function.md5_convert(html) html_data['html']=html html_data['html_md5'] = html_md5 html_data['time'] = self.Core_Function.callback_time(0) html_data['status']=0 response_data['html_md5']=html_md5 response_data['time'] = self.Core_Function.callback_time(0) response_data['describe'] = 'Null' response_data['status'] = 0 # print(response_data) self.response_list.append(response_data) self.HTML_list.append(html_data) except Exception as e: self.Core_Function.callback_logging().error(e) async def page_data(self,page,request): ''' ;页面信息获取 :param page: :param request: :return: ''' html=await page.content() html_md5 = self.Core_Function.md5_convert(html) html_data={} html_data['html'] = html html_data['html_md5'] = html_md5 html_data['time'] = self.Core_Function.callback_time(0) html_data['status'] = 0 request['status'] = 0 request['html_md5'] = html_md5 request['title'] = await page.title() print(request['title']) await page.evaluate(''' list_href=[] window.open = function(url) { console.log("new link: " + url);list_href.push(url); }; window.close = function () { return false; }; ''') await page.evaluate(''' list_href=[] for(i=0;i<document.getElementsByTagName("a").length;i++){ list_href.push(document.getElementsByTagName("a")[i].href); //输出该页面的所有链接。 } ''') # print(await page.content()) filename_img = "%s/%s.jpeg" % (self.Core_Function.create_image_path(), ''.join( random.sample('ABCDEFGHIJKLMNOPQRSTUVWXYZ012345678zyxwvutsrqponmlkjihgfedcba', 15))) try: await page.screenshot(path=filename_img, type='jpeg', quality=15) except Exception as error: filename_img