.net爬虫了解一下-.net core 爬虫
直接上代码
using System; //添加selenium的引用 using OpenQA.Selenium.PhantomJS; using OpenQA.Selenium.Chrome;
using OpenQA.Selenium.Support.UI; using OpenQA.Selenium; using System.IO; using System.Collections.Generic; using System.Text; using System.Threading; using System.Net; using HtmlAgilityPack; namespace ConsoleApplication1 { class requests {
public static string UserAgent = “Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36”;
public static string AcceptLang = “zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3″;
public static bool AddProxy = false;
public static bool ProxyPort = false; public static string ProxyHost = null; private static string Status; private static string ResponseUri; private static HtmlDocument Document=new HtmlDocument();
static PhantomJSDriver driver { get; set; }
static void Main(string[] args) { //PhantomJSDriverService driverService = PhantomJSDriverService.CreateDefaultService(); ////driverService.IgnoreSslErrors = true; //”any” also works //driverService.HideCommandPromptWindow = true; //driverService.ProxyType = “none”; //driverService.SslProtocol = “any”; //driverService.MaxDiskCacheSize = 1000; //driverService.DiskCache = true; //driverService.WebSecurity = false; //PhantomJSOptions phs = new PhantomJSOptions(); //phs.AddAdditionalCapability(“phantomjs.page.settings.userAgent”, “Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36”); var req = get(new Uri(“http://www.baidu.com”)); var resp = response(req); var root = GetResult(resp); var nodelist=parse(root,”//div”); Console.WriteLine(root.DocumentNode.InnerHtml); ChromeOptions options = new ChromeOptions(); options.AddArgument(“–headless”); options.AddArgument(“–nogpu”); List<String> tagNmaeList = new List<string>(); ; //using (driver = new PhantomJSDriver(driverService, phs)) //{ using (var driver = new ChromeDriver(options)) { driver.Manage().Window.Maximize(); driver.Navigate().GoToUrl(@”https://qiita.com/ttake/items/2e470462f8d90e76e996″); Thread.Sleep(5000); string source = driver.PageSource; Console.WriteLine(driver.Url); var html = driver.FindElement(By.TagName(“html”)); try { var div = driver.FindElement(By.Id(“global-nav”));
} catch (NoSuchElementException) { Console.WriteLine(“找不到该元素”); ; } IJavaScriptExecutor js = (IJavaScriptExecutor)driver;
var divnode = driver.FindElement(By.ClassName(“highlight”));
var bkgimg = js.ExecuteScript(“arguments[0].style.getPropertyValue(‘background-image’)”, divnode); if (bkgimg == null) { var outerhtml = divnode.GetAttribute(“outerHTML”); //outerhtml var outerText = divnode.GetAttribute(“outerText”); var innerText = divnode.GetAttribute(“innerText”); var children = divnode.GetAttribute(“children”); var textContent = divnode.GetAttribute(“textContent”); var childElementCount = divnode.GetAttribute(“childElementCount”); }
var image = driver.FindElement(By.ClassName(“it-Header_authorImage”));
var width = image.Size.Width; var height = image.Size.Height; var elem = js.ExecuteScript(“return document.defaultView.getComputedStyle(arguments[0])”, image);
Screenshot screenshot = ((ITakesScreenshot)driver).GetScreenshot(); var dirpath = @”D: emp”; if (!Directory.Exists(dirpath)) {
Directory.CreateDirectory(dirpath);
} screenshot.SaveAsFile(@”D: empseleniumScreenshot.png”, ScreenshotImageFormat.Png); } } private static HttpWebRequest get(Uri uri) { HttpWebRequest httpWebRequest = WebRequest.Create(uri) as HttpWebRequest;
if (httpWebRequest != null) { httpWebRequest.AllowAutoRedirect = true; httpWebRequest.AutomaticDecompression = (DecompressionMethods.GZip | DecompressionMethods.Deflate); httpWebRequest.UserAgent = UserAgent; if (!string.IsNullOrEmpty(AcceptLang)) { httpWebRequest.Headers.Add(“Accept-Language”, AcceptLang); } if (AddProxy) { WebProxy proxy = new WebProxy(ProxyHost, ProxyPort); httpWebRequest.Proxy = proxy; } } return httpWebRequest; } private static HttpWebResponse response(WebRequest request) { return (HttpWebResponse)request.GetResponse(); } private static HtmlDocument GetResult(HttpWebResponse response) { if (response != null) { Status = response.StatusCode.ToString().ToUpper(); ResponseUri = response.ResponseUri.AbsoluteUri; using (Stream responseStream = response.GetResponseStream()) { byte[] array = new byte[1024]; using (MemoryStream memoryStream = new MemoryStream()) { int num; do { num = responseStream.Read(array, 0, array.Length); if (num > 0) { memoryStream.Write(array, 0, num); } } while (num > 0); memoryStream.Seek(0L, SeekOrigin.Begin); Document.Load(memoryStream,Encoding.UTF8); } } } return Document; } private static HtmlNodeCollection parse(HtmlDocument doc,string xpath) {
var nodelist=doc.DocumentNode.SelectNodes(xpath); return nodelist;
}
}
}
相关文章
- WPF开发入门尝试
- 微软公告:ASP.NET曝漏洞 Win7等均中招
- WCF初接触实作之服务发布和使用
- WCF初次操作实践
- .NET程序员快来Visual Studio刷“成就”
- .NET垃圾回收机制知识点整理
- ASP.NET_正则表达式_匹配HTML中的一行或多行
- 利用WCF实现将服务器端的错误信息返回
- 怎样做一个快乐的ASP.NET程序员
- 为什么我们不要.NET程序员
- DDD落地,如何持久化聚合
- 迁移 dotnet 6 提示必须将目标平台设置为 Windows 平台
- C++|I/O|基于缓冲区的位级IO流
- 006: Django ajax请求
- 012:Django高级请求
- 开发者调查:C#在云应用中失去了地位 在游戏中依然大行其道
- 彻底理解 IO 多路复用实现机制
- 微软.NET 5.1迎来RC1版本 开发者已可上线尝鲜
- 速度和性能狂卷,.NET 7有多少人买单
- 风云再续:他抖任他抖,IO诊断在我手