java 爬虫--爬取阿里指数网站的每个分类的top50 …
2018-07-25 13:05:46来源：博客园 阅读 ()
主题：java 爬虫--爬取阿里指数网站的每个分类的top50 相关数据；
工具：使用selenium 爬取指定谷歌浏览器的内容；可进行可视化操作；
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.bcel.generic.Select;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.phantomjs.PhantomJSDriver;
import org.openqa.selenium.phantomjs.PhantomJSDriverService;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.ProxyConfig;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebClientOptions;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlButton;
import com.gargoylesoftware.htmlunit.html.HtmlForm;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSubmitInput;
import com.gargoylesoftware.htmlunit.html.HtmlTextInput;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.text.SimpleDateFormat;
public class AliZhiShu {
//日期格式
private static SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
//PHANTOMJS驱动路劲
private static String PHANTOMJS_DRIVER = "D:/phantomjs-2.1.1-windows/bin/phantomjs.exe";
//文件位置
private static String path = null;
//文件位置
private static String path2 = null;
private static WebDriver driver = null;
private static int companyNameNum = 0;
//谷歌驱动器
private static String CHROME_DRIVER = "C:/Program Files (x86)/Google/Chrome/Application/chromedriver.exe";
//爬虫时模拟的浏览器请求头
private static String[] userAgent = {
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; 360SE)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1",
"Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148 Safari/6533.18.5",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10",
"Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; SMJB; rv:11.0) like Gecko",
"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:36.0) Gecko/20100101 Firefox/36.0"};
//主方法
public static void main(String[] args) throws UnsupportedEncodingException{
String url=null;
String urlWhole=null;
String category2=null;
WebElement search = null;
List<String> arr2=new ArrayList<>();
List<String> arrName2=new ArrayList<>();
List<String> arrName23=new ArrayList<>();
List<String> arr3=new ArrayList<>();
List<String> arrName3=new ArrayList<>();
Document doc = new AliZhiShu().connect("http://index.1688.com/alizs/top.htm?curType=offer&userType=purchaser&cat=7");
//将二级分类文件取出
Elements categoryTwo = doc.select("[class=selector cate-select fd-clr]").select("[data-key=-1]").select("a[data-key]");
int i=0;
for(Element e:categoryTwo){
//将二级分类名称取出
String categoryTwoName =categoryTwo.get(i).text();
// System.out.println("name:"+categoryTwoName);
//将二级分类id取出
String categoryTwoId = categoryTwo.get(i).attr("data-key");
// System.out.println("http://index.1688.com/alizs/top.htm?curType=offer&userType=purchaser&cat=7"+","+categoryTwoId);
url="http://index.1688.com/alizs/top.htm?curType=offer&userType=purchaser&cat=7"+","+categoryTwoId;
arr2.add(url);
arrName2.add(categoryTwoName);
i++;
}
//将三级分类取出
for (int j = 0; j < arr2.size(); j++) {
// System.out.println(arr2.get(j)); //.get(index)
Document doc1 = new AliZhiShu().connect(arr2.get(j));
Elements categoryThird = doc1.select("[class=selector cate-select fd-clr]").select("[data-key=-1]").select("a[data-key]");
for(int n=1; n < categoryThird.size();n++){
String categoryThirdName = categoryThird.get(n).text();
String categoryThirdId = categoryThird.get(n).attr("data-key");
// System.out.println(arr2.get(j)+","+categoryThirdId);
urlWhole= arr2.get(j)+","+categoryThirdId;
arrName23.add(arrName2.get(j));
arr3.add(urlWhole);
arrName3.add(categoryThirdName);
}
}
//输入每一个三级分类标签，爬虫
for (int j = 0; j < arr3.size(); j++) {
// System.out.println(arr3.get(j));
//搜索'最近30天'按钮，并点击
System.setProperty("webdriver.chrome.driver",CHROME_DRIVER);
driver = new ChromeDriver();
driver.get(arr3.get(j));
driver.findElement(By.xpath(".//*/input[@value='month']")).click();
//爬虫
new AliZhiShu().crawler(arr3.get(j),arrName23.get(j),arrName3.get(j));
//wait
waitTime(2000);
// //关闭并退出浏览器
driver.quit();
// break;
//爬虫
// new AliZhiShu().crawler("http://index.1688.com/alizs/top.htm?curType=offer&userType=purchaser&cat=7,729,1037149","笔记本电脑配件","笔记本电池");
}
}
//爬虫
public void crawler(String url,String arrName2,String arrName3) throws UnsupportedEncodingException{
String[] p = new String[10];
/**小区参数数组：p
* p[0]:三级分类链接
* p[1]:一级分类名称
* p[2]:二级分类名称
* p[3]:三级分类名称
* p[4]:产品链接
* p[5]:产品名称
* p[6]:价格
* p[7]:产品所属公司
* p[8]:交易指数
* p[9]:评价
**/
p[0] = url;
p[1] = "数码、电脑";
p[2] = arrName2;
p[3] = arrName3;
Elements location = null;
Matcher matcher = null;
WebElement search = null;
// p[0] ="http://index.1688.com/alizs/top.htm?curType=offer&userType=purchaser&cat=7"+","+category1+","+category2;
Document docDetail = new AliZhiShu().connect(url);
// exportFile(docDetail.toString(),new File("C:/Users/91911/Desktop/liyanjie.html"));
if(docDetail!=null){
Elements elementEach = docDetail.select("div[class=unit hot unfolded]").select(".item.fd-clr");
// System.out.println(elementEach);
for(int i=0;i<elementEach.size();i++){
String elementUrl = elementEach.select("div[class=detail each]").select("p[class=title]").select("a").get(i).attr("href");
p[4] = elementUrl;
String elementName = elementEach.select("div[class=detail each]").select("p[class=title]").select("a").get(i).text();
p[5] = elementName;
String elementPrice = elementEach.select("div[class=detail each]").select("em[class=money]").get(i).text();
p[6] = elementPrice;
String elementCompany = elementEach.select("div[class=detail each]").select("p[class=com]").get(i).text();
p[7] = elementCompany;
String elementTrade = elementEach.select("p[class=first each]").get(i).text();
p[8] = elementTrade;
String elementEstimate = elementEach.select("p[class=second each]").get(i).text();
p[9] = elementEstimate;
// System.out.println("url:"+elementUrl);
// System.out.println("name:"+elementName);
// System.out.println("price:"+elementPrice);
// System.out.println("company:"+elementCompany);
exportFile(p[0] + "||"
+ p[1] + "||"
+ p[2] + "||"
+ p[3] + "||"
+ p[4] + "||"
+ p[5] + "||"
+ p[6] + "||"
+ p[7] + "||"
+ p[8] + "||"
+ p[9] ,new File("C:/Users/91911/Desktop/jieguo_alizhishu_new_20180724.txt"));
}
}
//
}
public static Document connect(String url){
if(url==null || url.equals("")) return null;
WebDriver webDriver = null;
try{
System.setProperty("webdriver.chrome.driver","C:/Program Files (x86)/Google/Chrome/Application/chromedriver.exe");
webDriver = new ChromeDriver();
webDriver.get(url);
//开始打开网页，等待输入元素出现
WebDriverWait wait = new WebDriverWait(webDriver,5);
wait.until(ExpectedConditions.presenceOfElementLocated(By.name("keywords")));
int tried = 5;
while(webDriver==null && tried>0){
waitTime(2000);
tried--;
}
}catch(Exception e){
//e.printStackTrace();
System.out.println("！连接网站超时");
}
Document doc = null;
if(webDriver!=null) {
doc = Jsoup.parse(webDriver.getPageSource());
}
webDriver.quit();
return doc;
}
//等待
static public void waitTime(int time){
try {
Thread.sleep(time);
}catch(InterruptedException e) {
e.printStackTrace();
System.out.println("！进程被打断");
}
}
//写文件
public static void exportFile(String content,File file){
try {
FileWriter out = new FileWriter(file,true);
out.write(content + "\r\n");
out.flush();
out.close();
}catch (IOException e){
System.out.println("！IO异常，写文件异常");
}
}
爬取阿里指数页面的数码电脑分类下的每个分类（三级分类的）top50条内容；
标签：
版权申明：本站文章部分自网络，如有侵权，请联系：west999com@outlook.com
特别注意：本站所有转载文章言论不代表本站观点，本站所提供的摄影照片，插画，设计作品，如需使用，请与原作者联系，版权归原作者所有
- 国外程序员整理的Java资源大全（全部是干货） 2020-06-12
- 2020年深圳中国平安各部门Java中级面试真题合集（附答案） 2020-06-11
- 2020年java就业前景 2020-06-11
- 04.Java基础语法 2020-06-11
- Java--反射(框架设计的灵魂)案例 2020-06-11
IDC资讯： 主机资讯 注册资讯 托管资讯 vps资讯 网站建设
网站运营： 建站经验 策划盈利 搜索优化 网站推广 免费资源
网络编程： Asp.Net编程 Asp编程 Php编程 Xml编程 Access Mssql Mysql 其它
服务器技术： Web服务器 Ftp服务器 Mail服务器 Dns服务器 安全防护
软件技巧： 其它软件 Word Excel Powerpoint Ghost Vista QQ空间 QQ FlashGet 迅雷
网页制作： FrontPages Dreamweaver Javascript css photoshop fireworks Flash
