phantomjs 截取twitter的网页(动态生成的页面)
// This example shows how to render pages that perform AJAX calls
// upon page load.
//
// Instead of waiting a fixed amount of time before doing the render,
// we are keeping track of every resource that is loaded.
//
// Once all resources are loaded, we wait a small amount of time
// (resourceWait) in case these resources load other resources.
//
// The page is rendered after a maximum amount of time (maxRenderTime)
// or if no new resources are loaded.
//page.onResourceRequested =
function
(requestData, networkRequest) {}
//page的所要加载的资源在发起请求时,都可以回调该函数
//page.onResourceReceived =
function
(response) {}
//page的所要加载的资源在加载过程中,每加载一个相关资源,都会在此先做出响应,它相当于
var fs = require("fs");
var resourceWait = 300,
maxRenderWait = 10000,
url='https://twitter.com/CXA10';
var page = require('webpage').create(),
count = 0,
forcedRenderTimeout,
renderTimeout;
page.viewportSize = { width: 1280, height : 1024 };
function doRender() {
page.render('twitter.png');
fs.write('1.html', page.content, 'w');
phantom.exit();
}
page.onResourceRequested = function (req) {
count += 1;
console.log('> ' + req.id + ' - ' + req.url);
clearTimeout(renderTimeout);
};
page.onResourceReceived = function (res) {
if (!res.stage || res.stage === 'end') {
count -= 1;
console.log(res.id + ' ' + res.status + ' - ' + res.url);
if (count === 0) {
renderTimeout = setTimeout(doRender, resourceWait);
}
}
};
page.open(url, function (status) {
if (status !== "success") {
console.log('Unable to load url');
// phantom.exit();
} else {
forcedRenderTimeout = setTimeout(function () {
console.log(count);
doRender();
}, maxRenderWait);
}
});
相关文章
- 网页基础
- 如何下载网页中的视频成mp4格式
- 使用Python爬取网页的相关内容(图片,文字,链接等等)
- 网页中显示xml,直接显示xml格式的文件
- 第三百五十节,Python分布式爬虫打造搜索引擎Scrapy精讲—selenium模块是一个python操作浏览器软件的一个模块,可以实现js动态网页请求
- EasyNVR摄像机网页直播中,推流组件EasyRTMP推送RTMP扩展支持HEVC(H.265)的方案
- [转]用chrome模拟微信浏览器访问需要OAuth2.0网页授权的页面
- HTML+CSS网页制作:一个简易美观的登录页面实例代码
- TF之DNN:对DNN神经网络进行Tensorboard可视化(得到events.out.tfevents本地服务器输出到网页可视化)
- 网页特效——金钱蓝图(.html)【超级棒的财商思维】
- 写【Python折线图】的一百个技巧(一、生成折线图网页)