基于selenium模拟天眼查登录并爬取企业工商信息的python爬虫

#!/usr/bin/python # -*- coding:utf-8 -*- # author: Jola # datetime:2018/4/20 17:15 # software-version: python 3.5 import time from selenium import webdriver from selenium.webdriver import Firefox class GetCompanyInfo(object): """ 爬取天眼查下的企业的信息 """ def __init__(self): """ 初始化爬虫执行代理，使用firefox访问 """ self.username = '13160676288' self.password = 'panjie19970620' self.options = webdriver.FirefoxOptions() self.options.add_argument('-headless') # 无头参数 self.geckodriver = r'geckodriver.exe' self.driver = Firefox(executable_path=self.geckodriver, firefox_options=self.options) self.start_url = 'https://www.tianyancha.com' def test(self): """ 调试专用 :return: """ start_url = 'http://y2.twenteen.cn/Home/Index' self.driver.get(start_url) cookies = { 'ASP.NET_SessionId': 'v3gnz3zsx0l2vxqmszhzat4w', 'Hm_lvt_ddd605dfec122be0f190ebb874331df1': '1524279814', 'Hm_lpvt_ddd605dfec122be0f190ebb874331df1': '152428022' } for k, v in cookies.items(): self.driver.add_cookie({ 'name': k, 'value': v }) time.sleep(1) print(self.driver.page_source) self.driver.close() def login(self): """ 登录并检查状态 :return: """ try: self.driver.get(self.start_url) print(self.driver.get_cookies()) username = self.index_login() username_pattern = username[:3] + ' **** ' + username[-4:] print(username_pattern) page = self.driver.page_source is_login = page.find(username_pattern) print(is_login) if is_login != -1: print('登录成功') except Exception as e: print(e) def index_login(self): """ 主页下的登录模式 :return: """ get_login = self.driver.find_elements_by_xpath('//a[@class="media_port"]')[0] # 登录/注册 print(get_login.text) # url为login的input get_login.click() login_by_pwd = self.driver.find_element_by_xpath('//div[@class="bgContent"]/div[2]/div[2]/div') # 切换到手机登录 print(login_by_pwd.text) login_by_pwd.click() input1 = self.driver.find_element_by_xpath('//div[@class="bgContent"]/div[2]/div/div[2]/input') # 手机号码 input2 = self.driver.find_element_by_xpath('//div[@class="bgContent"]/div[2]/div/div[3]/input') # 密码 print(input1.get_attribute('placeholder')) print(input2.get_attribute('placeholder')) username, password = self._check_user_pass() input1.send_keys(username) input2.send_keys(password) login_button = self.driver.find_element_by_xpath('//div[@class="bgContent"]/div[2]/div/div[5]') # 点击登录 print(login_button.text) time.sleep(1) # 必须等待否则鉴别是爬虫 login_button.click() return username def _check_user_pass(self): """ 检查是否有帐号密码 :return: """ if self.username and self.password: return self.username, self.password else: username = input('输入您的手机号码\n') password = input('输入您的密码\n') return username, password def login_page_login(self): """ url：www.tianyancha.com/login 在这个url下的登录模式 :return: """ input1 = self.driver.find_element_by_xpath('//div[contains(@class,"in-block")' ' and contains(@class, "vertical-top")' ' and contains(@class, "float-right")' ' and contains(@class, "right_content")' ' and contains(@class, "mt50")' ' and contains(@class, "mr5")' ' and contains(@class, "mb5")' ']/div[2]/div[2]/div[2]/input') input2 = self.driver.find_element_by_xpath('//div[contains(@class,"in-block")' ' and contains(@class, "vertical-top")' ' and contains(@class, "float-right")' ' and contains(@class, "right_content")' ' and contains(@class, "mt50")' ' and contains(@class, "mr5")' ' and contains(@class, "mb5")' ']/div[2]/div[2]/div[3]/input') print(input1.get_attribute('placeholder')) input1.send_keys("13160676288") print(input2.get_attribute('placeholder')) input2.send_keys('panjie19970620') login_button = self.driver.find_element_by_xpath('//div[contains(@class,"in-block")' ' and contains(@class, "vertical-top")' ' and contains(@class, "float-right")' ' and contains(@class, "right_content")' ' and contains(@class, "mt50")' ' and contains(@class, "mr5")' ' and contains(@class, "mb5")' ']/div[2]/div[2]/div[5]') print(login_button.text) time.sleep(1) login_button.click() def get_company_info(self, company_name, company_onwer): """ 获取想要的公司信息 :param company_name: :param company_onwer: :return: """ try: time.sleep(1) index_input_company = self.driver.find_element_by_xpath('//input[@id="home-main-search"]') # 主页搜索框 index_input_company.send_keys(company_name) self.driver.find_element_by_xpath('//div[contains(@class, "input-group-addon")' ' and contains(@class, "search_button")' ' and contains(@class, " white-btn")' ']').click() # 点击搜索 # button_name = find_company_button.find_element_by_xpath('//span').text # span中的文本应该为【天眼一下】 # print(button_name) # time.sleep(1) company_list = self.driver.find_elements_by_xpath('//div[contains(@class, "b-c-white")' ' and contains(@class, "search_result_container")' ']/div') # 获取当前页面所有公司的div company_info = list() for each_company in company_list: company_name_from_web = each_company.find_element_by_tag_name('img').get_attribute('alt') company_url = each_company.find_element_by_tag_name('a').get_attribute('href') company_reg_money = each_company.\ find_element_by_css_selector('div .search_row_new.pt20 div div:nth