使用C#+Selenium+ChromeDriver爬取网页，模拟真实用户浏览行为，转载请联系UP技控公众号。背景Selenium是一种用于Web应用程序测试的工具。Selenium测试直接在浏览器中运行，就像真实用户一样。对于爬虫来说，利用Selenium来操纵浏览器来爬取在线数据绝对是爬虫中的杀手锏。这里介绍下selenium+GoogleChrome的一般使用。需求在正常的爬虫开发中，有时候一个网页就是一堆JS代码，涉及到大量的异步计算。如果是普通的http控制台请求，那么拿到的源文件就是一堆JS，需要自己组装数据，很费力；但是使用Selenium+ChromeDriver可以达到所见即所得的完美效果。实现方法项目结构：为了方便使用，使用了winform程序，附上nuget包。下面是form1.cs的代码，这里只放关键方法代码。需要安装最新的chrome浏览器+代码中使用的chromedriver是v2.9.248315privatevoidcrawlingWebFunc(){SetText("\r\n开始尝试...");Listsurls=newList();stringpath=System.Environment.CurrentDirectory+"\\图片url\\";DirectoryInforoot=newDirectoryInfo(path);DirectoryInfo[]dics=root.GetDirectories();foreach(varitemdicindics){stringtxt="";StreamReadersr=newStreamReader(itemdic.FullName+"\\data.txt");while(!sr.EndOfStream){stringstr=sr.ReadLine();txt+=str;//+"\n";}sr.Close();surls.Add(newtestfold(){key=itemdic.FullName,picurl=txt});}ChromeDriverServiceservice=ChromeDriverService.CreateDefaultService(System.Environment.CurrentDirectory);//service.HideCommandPromptWindow=true;ChromeOptionsoptions=newChromeOptions();options.AddArguments("--test-type","--ignore-certificate-errors");options.AddArgument("enable-automation");//options.AddArgument("headless");//options.AddArguments("--proxy-server=http://user:password@yourProxyServer.com:8080");使用(IWebDriverdriver=newOpenQA.Selenium.Chrome.ChromeDriver(service,options,TimeSpan.FromSeconds(120))){driver.Url="https://www.1688.com/";Thread.Sleep(200);try{inta=1;foreach(varitemsurlsinsurls){SetText("\r\nth"+a.ToString()+"a");driver.Navigate().GoToUrl(itemsurls.picurl);//登录if(driver.Url.Contains("login.1688.com")){SetText("\r\n需要登录,开始尝试...");trylogin(driver);//尝试登录完成//再次尝试driver.Navigate().GoToUrl("https://s.1688.com/youyuan/index.htm?tab=imageSearch&imageType=oss&imageAddress=cbuimgsearch/eWXC7XHHPN1607529600000&spm=");if(driver.Url.Contains("login.1688.com")){//无法退出SetText("\r\n退出，换ip再试。..");return;}}//鼠标放上去的内容是因为页面本身只能显示一种原因，没办法全部显示，只能通过其他方式下载//varelements=document.getElementsByClassName('hover-container');//Array.prototype.forEach.call(elements,function(element){//element.style.display="block";//console.log(element);//});//IJavaScriptExecutorjs=(IJavaScriptExecutor)driver;//varsss=js.ExecuteScript("varelements=document.getElementsByClassName('hover-container');Array.prototype.forEach.call(elements,function(element){console.log(element);element.setAttribute(\"class\",\"测试标题\");element.style.display=\"block\";console.log(element);});");Thread.Sleep(500);varresponseModel=Write(itemsurls.key,driver.PageSource,Pagetypeenum.列表);Thread.Sleep(500);inti=1;foreach(varofferinresponseModel?.data?.offerList??newList()){driver.Navigate().GoToUrl(offer.information.detailUrl);stringresponseDatadetail=driver.PageSource;Write(itemsurls.key,driver.PageSource,Pagetypeenum.详情);SetText("\r\n第"+a.ToString()+"-"+i.ToString()+"个");Thread.Sleep(500);i++;}}}catch(Exceptionex){CloseChromeDriver(driver);throw;}}}#region异常退出chromedriver[DllImport("user32.dll",EntryPoint="FindWindow")]privateexternstaticIntPtrFindWindow(stringlpClassName,stringlpWindowName);[DllImport("user32.dll",EntryPoint="SendMessage")]publicstaticexternintSendMessage(IntPtrhWnd,intMsg,intwParam,intlParam);publicconstintSW_HIDE=0;publicconstintSW_SHOW=5;[DllImport("user32.dll",EntryPoint="ShowWindow")]publicstaticexternintShowWindow(IntPtrhwnd,intnCmdShow);//////获取窗口句柄//////publicIntPtrGetWindowHandle(){stringname=(Environment.CurrentDirectory+"\\chromedriver.exe");IntPtrhwd=FindWindow(null,name);returnhwd;}//////关闭chromedriver窗口///publicvoidCloseWindow(){try{IntPtrhwd=GetWindowHandle();SendMessage(hwd,0x10,0,0);}catch{}}//////退出chromedriver//////publicvoidCloseChromeDriver(IWebDriverdriver){try{driver.Quit();driver.Dispose();}catch{}CloseWindow();}#endregionchromedriver异常退出效果总结说说思路：1.跳转到指定网页driver.Navigate().GoToUrl2.判断数据源，从driver.PageSource中读取数据3.解析html数据
