python学习抓取头条首页新闻资源-CSDN下载

共3个文件

py：3个

python爬虫

python3

解析网页新闻

需积分: 49 199 浏览量 2018-09-26 10:47:21 上传评论收藏 26KB ZIP 举报

资源详情

资源评论

资源推荐

收起资源包目录

学习抓取头条首页新闻.zip （3个子文件）

newsInfo.py 524B

ptt.py 167KB

parseNews.py 7KB

# -*- coding: utf-8 -*- # 引入模拟浏览器框架支持库 from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.keys import Keys from selenium.webdriver.common.desired_capabilities import DesiredCapabilities from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC # 引入ActionChains鼠标操作类支持库 from selenium.webdriver.common.action_chains import ActionChains # xpath解析支持库 from lxml import etree # 正则表达式支持库 import re # py代码中直接调用js方法支持库 import execjs import execjs.runtime_names # 发网络请求支持库 import requests # 自定义的新闻结构体 from newsInfo import NewsInfo # 自定义解析html结构的实现类 from parseNews import PaseNews class ParseTouTiao(object): """ 构造函数，初始化资源 """ def __init__(self): self.__firefox_options = webdriver.FirefoxOptions() self.__firefox_options.add_argument('--headless') self.__firefox_options.add_argument('--disable-gpu') self.__browser = webdriver.Firefox(firefox_options=self.__firefox_options) """ 析构函数，释放资源 """ def __del__(self): if self.__browser: try: self.__browser.close() self.__browser.quit() except Exception as ex: print(ex) """ 获取头条首页内容 """ def __getTouTiaoHtml(self, url): # 简单的入参校验 if url and '' != url and url.startswith("http"): # 浏览器打开页面 self.__browser.get(url) try: # 此处等到我们所需的热文元素加载出来了再进行下一步，避免页面还没加载完成就去解析内容导致内容为空 element = WebDriverWait(self.__browser, 10).until( EC.presence_of_element_located((By.XPATH, "//ul/li/div[@ga_event=article_item_click]")) ) except Exception as ex: print(ex) finally: pass resHtml = self.__browser.page_source return resHtml resHtml = """ <html> <head> <script mod_name="tanxssp-probe" async="" src="https://atanx.alicdn.com/t/tanxssp/probe.js" charset="utf-8"></script> <style class="vjs-styles-defaults"> .video-js { width: 300px; height: 150px; } .vjs-fluid { padding-top: 56.25% } </style> <meta charset="utf-8"> <title>今日头条</title> <meta http-equiv="x-dns-prefetch-control" content="on"> <meta name="renderer" content="webkit"> <link rel="dns-prefetch" href="//s3.pstatp.com/"> <link rel="dns-prefetch" href="//s3a.pstatp.com/"> <link rel="dns-prefetch" href="//s3b.pstatp.com"> <link rel="dns-prefetch" href="//p1.pstatp.com/"> <link rel="dns-prefetch" href="//p3.pstatp.com/"> <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"> <meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,minimum-scale=1,user-scalable=no,minimal-ui"> <meta name="360-site-verification" content="b96e1758dfc9156a410a4fb9520c5956"> <meta name="360_ssp_verify" content="2ae4ad39552c45425bddb738efda3dbb"> <meta name="google-site-verification" content="3PYTTW0s7IAfkReV8wAECfjIdKY-bQeSkVTyJNZpBKE"> <meta name="shenma-site-verification" content="34c05607e2a9430ad4249ed48faaf7cb_1432711730"> <meta name="baidu_union_verify" content="b88dd3920f970845bad8ad9f90d687f7"> <meta name="domain_verify" content="pmrgi33nmfuw4ir2ej2g65lunfqw6ltdn5wselbcm52wszbchirdqyztge3tenrsgq3dknjume2tayrvmqytemlfmiydimddgu4gcnzcfqrhi2lnmvjwc5tfei5dcnbwhazdcobuhe2dqobrpu"> <meta name="keywords" content="今日头条，头条，头条网，头条新闻，今日头条官网"> <meta name="description" content="《今日头条》(www.toutiao.com)是一款基于数据挖掘的推荐引擎产品，它为用户推荐有价值的、个性化的信息，提供连接人与信息的新型服务，是国内移动互联网领域成长最快的产品服务之一。"> <link rel="alternate" media="only screen and (max-width: 640px)" href="//m.toutiao.com/"> <link rel="shortcut icon" href="//s3a.pstatp.com/toutiao/resource/ntoutiao_web/static/image/favicon_8e9c9c7.ico" type="image/x-icon"> <link rel="stylesheet" href="//s3.pstatp.com/toutiao/player/dist/pc_vue.css" media="screen" title="no title">  <script src="//s3.pstatp.com/toutiao/monitor/sdk/slardar.js?ver=20171221_1" crossorigin="anonymous"></script> <script>window.Slardar && window.Slardar.install({ sampleRate: 1, bid: 'toutiao_pc', pid: 'index_new', ignoreAjax: [/\/action_log\//, /\/stream\/widget\//], ignoreStatic: [/\.tanx\.com\//, /\.alicdn\.com\//, /\.mediav\.com/] });</script> <link rel="stylesheet" href="//s3a.pstatp.com/toutiao/static/css/page/index_node/index.31d54c35426ed45610edf7c5e7ac8b74.css"> <script>!function (e) { function t(a) { if (o[a]) return o[a].exports; var r = o[a] = { exports: {}, id: a, loaded: !1 }; return e[a].call(r.exports, r, r.exports, t), r.loaded = !0, r.exports } var a = window.webpackJsonp; window.webpackJsonp = function (n, c) { for (var p, s, l = 0, i = []; l < n.length; l++)s = n[l], r[s] && i.push.apply(i, r[s]), r[s] = 0; for (p in c) Object.prototype.hasOwnProperty.call(c, p) && (e[p] = c[p]); for (a && a(n, c); i.length;)i.shift().call(null, t); if (c[0]) return o[0] = 0, t(0) }; var o = {}, r = { 0: 0 }; t.e = function (e, a) { if (0 === r[e]) return a.call(null, t); if (void 0 !== r[e]) r[e].push(a); else { r[e] = [a]; var o = document.getElementsByTagName("head")[0], n = document.createElement("script"); n.type = "text/javascript", n.charset = "utf-8", n.async = !0, n.src = t.p + "static/js/" + e + "." + { 1: "24063eb49e6e579544d4", 2: "4cfa0b7d433696569909", 3: "f1ff2831f57d148eec40", 4: "cce2c7d4326515e8702e" }[e] + ".js", o.appendChild(n) } }, t.m = e, t.c = o, t.p = "/toutiao/", t.p = "//s3.pstatp.com/toutiao/" }([]);</script> <style type="text/css"> @keyframes resizeanim { from { opacity: 0; } to { opacity: 0; } } .resize-triggers { animation: 1ms resizeanim; visibility: hidden; opacity: 0; } .resize-triggers, .resize-triggers>div, .contract-trigger:before { content: " "; display: block; position: absolute; top: 0; left: 0; height: 100%; width: 100%; overflow: hidden; } .resize-triggers>div { background: #eee; overflow: auto; } .contract-trigger:before { width: 200%; height: 200%; } </style> <script src="https://show-g.mediav.com/s?jsonp=_qihu_jsonpFun_&type=1&of=4&newf=1&showid=P5AcFE&ref=toutiao.com&uid=15378869247912674181990789027651&scheme=https&impct=5&time=ts_1537886924791" async=""></script> <style id="5e59571">.imga_wrapper{position:relative;display:inline-block;}.imga_flag{display: inline-block;position: absolute;bottom: 8px;left: 8px;border: 1px solid #fff;color: #ff