利用NodeJS和PhantomJS抓取网站页面信息以及网站截图

phantomjs-node成功将PhantomJS作为NodeJS的一个模块来使用，但我们看看作者的原理解释：

Iwillanswerthatquestionwithaquestion.Howdoyoucommunicatewithaprocessthatdoesn"tsupportsharedmemory,sockets,FIFOs,orstandardinput?

Well,there"sonethingPhantomJSdoessupport,andthat"sopeningwebpages.Infact,it"sreallygoodatopeningwebpages.SowecommunicatewithPhantomJSbyspinningupaninstanceofExpressJS,openingPhantominasubprocess,andpointingitataspecialwebpagethatturnssocket.iomessagesinto alert()calls.Those alert() callsarepickedupbyPhantomandthereyougo!

ThecommunicationitselfhappensviaJamesHalliday"sfantastic dnode library,whichfortunatelyworkswellenoughwhencombinedwith browserify torunstraightoutofPhantomJS"spidginJavascriptenvironment.

实际上phantomjs-node使用的也是HTTP或者Websocket来进行通讯，不过其依赖庞大，我们只想做一个简单的东西，暂时还是不考虑这个东东吧。

让我们开始吧
我们在第一版中选用HTTP进行实现。

首先利用cluster进行简单的进程守护（index.js）：

复制代码代码如下:
module.exports=(function(){
 "usestrict"
 varcluster=require("cluster")
   ,fs=require("fs");

 if(!fs.existsSync("./snapshot")){
   fs.mkdirSync("./snapshot");
 }

 if(cluster.isMaster){
   cluster.fork();

   cluster.on("exit",function(worker){
     console.log("Worker"+worker.id+"died:(");
     process.nextTick(function(){
       cluster.fork();
     });
   })
 }else{
   require("./extract.js");
 }
})();

然后利用connect做我们的对外API（extract.js）：

复制代码代码如下:
module.exports=(function(){
 "usestrict"
 varconnect=require("connect")
   ,fs=require("fs")
   ,spawn=require("child_process").spawn
   ,jobMan=require("./lib/jobMan.js")
   ,bridge=require("./lib/bridge.js")
   ,pkg=JSON.parse(fs.readFileSync("./package.json"));

 varapp=connect()
   .use(connect.logger("dev"))
   .use("/snapshot",connect.static(__dirname+"/snapshot",{maxAge:pkg.maxAge}))
   .use(connect.bodyParser())
   .use("/bridge",bridge)
   .use("/api",function(req,res,next){
     if(req.method!=="POST"||!req.body.campaignId)returnnext();
     if(!req.body.urls||!req.body.urls.length)returnjobMan.watch(req.body.campaignId,req,res,next);

     varcampaignId=req.body.campaignId
       ,imagesPath="./snapshot/"+campaignId+"/"
       ,urls=[]
       ,url
       ,imagePath;

     function_deal(id,url,imagePath){
       //justpushintourlslist
       urls.push({
         id:id,
         url:url,
         imagePath:imagePath
       });
     }

     for(vari=req.body.urls.length;i--;){
       url=req.body.urls[i];
       imagePath=imagesPath+i+".png";
       _deal(i,url,imagePath);
     }

     jobMan.register(campaignId,urls,req,res,next);
     varsnapshot=spawn("phantomjs",["snapshot.js",campaignId]);
     snapshot.stdout.on("data",function(data){
       console.log("stdout:"+data);
     });
     snapshot.stderr.on("data",function(data){
       console.log("stderr:"+data);
     });
     snapshot.on("close",function(code){
       console.log("snapshotexitedwithcode"+code);
     });

   })
   .use(connect.static(__dirname+"/html",{maxAge:pkg.maxAge}))
   .listen(pkg.port,function(){console.log("listen:"+"http://localhost:"+pkg.port);});

})();

这里我们引用了两个模块bridge和jobMan。

其中bridge是HTTP通讯桥梁，jobMan是工作管理器。我们通过campaignId来对应一个job，然后将job和response委托给jobMan管理。然后启动PhantomJS进行处理。

通讯桥梁负责接受或者返回job的相关信息，并交给jobMan（bridge.js）：

复制代码代码如下:
module.exports=(function(){
 "usestrict"
 varjobMan=require("./jobMan.js")
   ,fs=require("fs")
   ,pkg=JSON.parse(fs.readFileSync("./package.json"));

 returnfunction(req,res,next){
     if(req.headers.secret!==pkg.secret)returnnext();
     //SnapshotAPPcanposturlinformation
     if(req.method==="POST"){
       varbody=JSON.parse(JSON.stringify(req.body));
       jobMan.fire(body);
       res.end("");
     //SnapshotAPPcangettheurlsshouldextract
     }else{
       varurls=jobMan.getUrls(req.url.match(/campaignId=([^&]*)(\s|&|$)/)[1]);
       res.writeHead(200,{"Content-Type":"application/json"});
       res.statuCode=200;
       res.end(JSON.stringify({urls:urls}));
     }
 };

})();

如果requestmethod为POST，则我们认为PhantomJS正在给我们推送job的相关信息。而为GET时，则认为其要获取job的信息。

jobMan负责管理job，并发送目前得到的job信息通过response返回给client（jobMan.js）：

复制代码代码如下:
module.exports=(function(){
 "usestrict"
 varfs=require("fs")
   ,fetch=require("./fetch.js")
   ,_jobs={};

 function_send(campaignId){
   varjob=_jobs[campaignId];
   if(!job)return;
   if(job.waiting){
     job.waiting=false;
     clearTimeout(job.timeout);
     varfinished=(job.urlsNum===job.finishNum)
       ,data={
       campaignId:campaignId,
       urls:job.urls,
       finished:finished
     };
     job.urls=[];
     varres=job.res;
     if(finished){
       _jobs[campaignId]=null;
       delete_jobs[campaignId]
     }
     res.writeHead(200,{"Content-Type":"application/json"});
     res.statuCode=200;
     res.end(JSON.stringify(data));
   }
 }

 functionregister(campaignId,urls,req,res,next){
   _jobs[campaignId]={
     urlsNum:urls.length,
     finishNum:0,
     urls:[],
     cacheUrls:urls,
     res:null,
     waiting:false,
     timeout:null
   };
   watch(campaignId,req,res,next);
 }

 functionwatch(campaignId,req,res,next){
   _jobs[campaignId].res=res;
   //20stimeout
   _jobs[campaignId].timeout=setTimeout(function(){
     _send(campaignId);
   },20000);
 }

 functionfire(opts){
   varcampaignId=opts.campaignId
     ,job=_jobs[campaignId]
     ,fetchObj=fetch(opts.html);

   if(job){
     if(+opts.status&&fetchObj.title){
       job.urls.push({
         id:opts.id,
         url:opts.url,
         image:opts.image,
         title:fetchObj.title,
         description:fetchObj.description,
         status:+opts.status
       });
     }else{
       job.urls.push({
         id:opts.id,
         url:opts.url,
         status:+opts.status
       });
     }

     if(!job.waiting){
       job.waiting=true;
       setTimeout(function(){
         _send(campaignId);
       },500);
     }
     job.finishNum++;
   }else{
     console.log("jobcannotfound!");
   }
 }

 functiongetUrls(campaignId){
   varjob=_jobs[campaignId];
   if(job)returnjob.cacheUrls;
 }

 return{
   register:register,
   watch:watch,
   fire:fire,
   getUrls:getUrls
 };

})();

这里我们用到fetch对html进行抓取其title和description，fetch实现比较简单（fetch.js）：

复制代码代码如下:
module.exports=(function(){
 "usestrict"

 returnfunction(html){
   if(!html)return{title:false,description:false};

   vartitle=html.match(/\<title\>(.*?)\<\/title\>/)
     ,meta=html.match(/\<meta\s(.*?)\/?\>/g)
     ,description;

   if(meta){
     for(vari=meta.length;i--;){
       if(meta[i].indexOf("name="description"")>-1||meta[i].indexOf("name="Description"")>-1){
         description=meta[i].match(/content\=\"(.*?)\"/)[1];
       }
     }
   }

   (title&&title[1]!=="")?(title=title[1]):(title="NoTitle");
   description||(description="NoDescription");

   return{
     title:title,
     description:description
   };
 };

})();

最后是PhantomJS运行的源代码，其启动后通过HTTP向bridge获取job信息，然后每完成job的其中一个url就通过HTTP返回给bridge（snapshot.js）：

复制代码代码如下:
varwebpage=require("webpage")
 ,args=require("system").args
 ,fs=require("fs")
 ,campaignId=args[1]
 ,pkg=JSON.parse(fs.read("./package.json"));

functionsnapshot(id,url,imagePath){
 varpage=webpage.create()
   ,send
   ,begin
   ,save
   ,end;
 page.viewportSize={width:1024,height:800};
 page.clipRect={top:0,left:0,width:1024,height:800};
 page.settings={
   javascriptEnabled:false,
   loadImages:true,
   userAgent:"Mozilla/5.0(WindowsNT6.1)AppleWebKit/537.31(KHTML,likeGecko)PhantomJS/1.9.0"
 };
 page.open(url,function(status){
   vardata;
   if(status==="fail"){
     data=[
       "campaignId=",
       campaignId,
       "&url=",
       encodeURIComponent(url),
       "&id=",
       id,
       "&status=",
     ].join("");
     postPage.open("http://localhost:"+pkg.port+"/bridge","POST",data,function(){});
   }else{
     page.render(imagePath);
     varhtml=page.content;
     //callbackNodeJS
     data=[
       "campaignId=",
       campaignId,
       "&html=",
       encodeURIComponent(html),
       "&url=",
       encodeURIComponent(url),
       "&image=",
       encodeURIComponent(imagePath),
       "&id=",
       id,
       "&status=",
     ].join("");
     postMan.post(data);
   }
   //releasethememory
   page.close();
 });
}

varpostMan={
 postPage:null,
 posting:false,
 datas:[],
 len:0,
 currentNum:0,
 init:function(snapshot){
   varpostPage=webpage.create();
   postPage.customHeaders={
     "secret":pkg.secret
   };
   postPage.open("http://localhost:"+pkg.port+"/bridge?campaignId="+campaignId,function(){
     varurls=JSON.parse(postPage.plainText).urls
       ,url;

     this.len=urls.length;

     if(this.len){
       for(vari=this.len;i--;){
         url=urls[i];
         snapshot(url.id,url.url,url.imagePath);
       }
     }
   });
   this.postPage=postPage;
 },
 post:function(data){
   this.datas.push(data);
   if(!this.posting){
     this.posting=true;
     this.fire();
   }
 },
 fire:function(){
   if(this.datas.length){
     vardata=this.datas.shift()
       ,that=this;
     this.postPage.open("http://localhost:"+pkg.port+"/bridge","POST",data,function(){
       that.fire();
       //killchildprocess
       setTimeout(function(){
         if(++this.currentNum===this.len){
           that.postPage.close();
           phantom.exit();
         }
       },500);
     });
   }else{
     this.posting=false;
   }
 }
};
postMan.init(snapshot);

本文地址： 利用NodeJS和PhantomJS抓取网站页面信息以及网站截图

相关文章

• Linux 安装nodejs_pe安装linux
• ES6转ES5_nodejs支持es6吗
• Scrapy爬虫框架_nodejs爬虫框架对比
• Nodejs相关ORM框架分析_2023-02-27
• nodejs实现jwt_2023-03-01
• 盘点那些将nodejs作为后端的大公司
• CentOS7 安装 NodeJS
• Nodejs开发规范
• Nodejs书写爬虫工具详解编程语言
• Nodejs经验谈详解编程语言
• NodeJS stream 一：Buffer详解编程语言
• Nodejs快速操作Oracle（node操作oracle）
• NodeJS中Buffer模块详解
• 基于NodeJS的前后端分离的思考与实践（一）全栈式开发
• 基于NodeJS的前后端分离的思考与实践（四）安全问题解决方案