htmlunit爬取js异步加载后的页面-白红宇

htmlunit爬取js异步加载后的页面

阅读量：4179 次

发布时间：2019-05-26

本文共 2220 字，大约阅读时间需要 7 分钟。

直接上代码：

一、 index.html

调用后台请求获取content中的内容。

	Hello World!

二、TestController.java

/test/testList接口从后台数据库获取数据。

package com.everhomes.proxy.controller;import javax.annotation.Resource;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import org.springframework.web.bind.annotation.ExceptionHandler;import org.springframework.web.bind.annotation.RequestMapping;import org.springframework.web.bind.annotation.RestController;import com.everhomes.proxy.mapper.TestMapper;@RestController@RequestMapping("/test")public class TestController {	private static final Logger logger = LoggerFactory.getLogger(TestController.class);		@Resource	private TestMapper testMapper;		@RequestMapping("testList")	public Object testList(){		return testMapper.testList();	};		@ExceptionHandler(Exception.class)	public Object exception(Exception e){		logger.error("error: ", e);		return "error: " + e.toString();	}}

三、Crawler.java

package com.everhomes.generate;import java.io.IOException;import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;import com.gargoylesoftware.htmlunit.WebClient;import com.gargoylesoftware.htmlunit.html.HtmlPage;public class Crawler {	public static void main(String[] args) throws IOException, InterruptedException {		WebClient webClient = new WebClient(BrowserVersion.CHROME);  		    webClient.getOptions().setJavaScriptEnabled(true);		    webClient.getOptions().setCssEnabled(false);		    webClient.getOptions().setRedirectEnabled(true);		    webClient.getOptions().setThrowExceptionOnScriptError(false);		    webClient.getOptions().setTimeout(50000);		    HtmlPage rootPage = webClient.getPage("http://localhost:8080/evh/index.html");  		    webClient.waitForBackgroundJavaScript(10000);		 		    FileUtils.createFile(DIRECTORY+"cc.html", rootPage.asXml());		    webClient.close();	}}

四、pom.xml

添加相关依赖。


   		
    
     commons-lang
    		
    
     commons-lang
    		
    
     2.6
    	
   	
    	        
    
     net.sourceforge.htmlunit
     	        
    
     htmlunit-core-js
     	        
    
     2.23
         
        
    	        
    
     net.sourceforge.htmlunit
     	        
    
     htmlunit
     	        
    
     2.25