import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;

public class WebContent {
	
	public static String AREA_LIST = "\\[list\\]";
	public static String AREA_TITLE ="\\[title\\]";
	public static String AREA_URL = "\\[url\\]";
	public static String AREA_MESSAGE = "\\[message\\]";
	public static String PAGE_URL_NUM = "\\[num\\]";
	
	public static int PAGE_STYLE_LIST = 1;
	public static int PAGE_STYLE_NEXT = 2;

	/**
	 * 给定一个列表url，得到本url对象的html,然后其它操作在这个基础上做
	 * @param url
	 * @param encoding
	 * @return
	 * @throws Exception
	 */
	public String getHtml(String url, String encoding) throws Exception {
		String value = null;
		HttpClient httpclient = new DefaultHttpClient();        
        HttpGet httpget = new HttpGet(url);
       
        // 以下这条如果不加会发现无论你设置Accept-Charset为gbk还是utf-8，他都会默认返回gb2312（本例针对google.cn来说）
        httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.2)");
        // 用逗号分隔显示可以同时接受多种编码
        httpget.setHeader("Accept-Language", "zh-cn,zh;q=0.5");
        httpget.setHeader("Accept-Charset", "GB2312,utf-8;q=0.7,*;q=0.7");        
        HttpResponse response = httpclient.execute(httpget);     

        // 判断页面返回状态判断是否进行转向抓取新链接
        int statusCode = response.getStatusLine().getStatusCode();
        if ((statusCode == HttpStatus.SC_MOVED_PERMANENTLY) ||
                (statusCode == HttpStatus.SC_MOVED_TEMPORARILY) ||
                (statusCode == HttpStatus.SC_SEE_OTHER) ||
                (statusCode == HttpStatus.SC_TEMPORARY_REDIRECT)) {
            // 此处重定向处理  此处还未验证
            String newUri = response.getLastHeader("Location").getValue();
            httpclient = new DefaultHttpClient();
            httpget = new HttpGet(newUri);
            response = httpclient.execute(httpget);
        }

        // Get hold of the response entity
        HttpEntity entity = response.getEntity();        
        // If the response does not enclose an entity, there is no need
        // to bother about connection release
        if (entity != null) {
            // 将源码流保存在一个byte数组当中，因为可能需要两次用到该流，
            byte[] bytes = EntityUtils.toByteArray(entity);
            // 如果头部Content-Type中包含了编码信息，那么我们可以直接在此处获取
            
            if(encoding != null && !"".equals(encoding.trim())){
            	value = new String(bytes, encoding);
            }else{
            	String charSet = EntityUtils.getContentCharSet(entity);
            	// 如果头部中没有，那么我们需要 查看页面源码，这个方法虽然不能说完全正确，因为有些粗糙的网页编码者没有在页面中写头部编码信息
	            if (charSet == null || "".equals(charSet.trim())) {
	                String regEx="(?=<meta).*?(?<=charset=[\\'|\\\"]?)([[a-z]|[A-Z]|[0-9]|-]*)";
	                Pattern p=Pattern.compile(regEx, Pattern.CASE_INSENSITIVE);
	                Matcher m=p.matcher(new String(bytes));  // 默认编码转成字符串，因为我们的匹配中无中文，所以串中可能的乱码对我们没有影响
	                boolean result=m.find();
	                if (m.groupCount() == 1) {
	                    charSet = m.group(1);
	                }
	            }
	            value = new String(bytes, charSet);
            }       
        }

        httpclient.getConnectionManager().shutdown();   
        return value;
    }

	
	/**
	 * 列表区域识别规则
	 * @param html 原始的html
	 * @param areaRegex 所提取区域的正则表达式
	 * @param area 正则表达式中需要提取的区域，取值是：AREA_LIST，AREA_TITLE等等
	 * @return
	 */
	public String getArea(String html, String areaRegex ,String area) {
		String valueHtml=null;
		String regex = areaRegex.replaceFirst(area, "([\\\\s\\\\S]*?)");
//		System.out.println(regex);
		Pattern pattern = Pattern.compile(regex,Pattern.MULTILINE); 
		Matcher matcher = pattern.matcher(html); 
		while(matcher.find()) {
			valueHtml = matcher.group(1).trim();
			break;
		} 
		return valueHtml;
	}
	
	public List<String> getListUrl(String areaHtml,String regex,String withPrefix){
		return this.getListUrl(areaHtml, regex, withPrefix, false, null);
	}
	
	/**
	 * 提取给定区域的文章列表地址
	 * @param areaHtml 
	 * @param regex 
	 * @param withPrefix 返回的地址是否需要加一个前缀，如21cn的新闻，提取到后是没有前缀的，得加一个
	 * @param hasOmit  提取文章内分页时，是否有分页是否有省略号，如淘宝的就有。
	 * @param pageNumRegex  提取文章内的分页正则表达式
	 * @return
	 */
	public List<String> getListUrl(String areaHtml,String regex,String withPrefix,boolean hasOmit,String pageNumRegex){
		List<String> list = new ArrayList<String>();
		regex = regex.replaceFirst(WebContent.AREA_URL, "(.*?)");
		Pattern pa = Pattern.compile(regex, Pattern.MULTILINE);
		Matcher ma = pa.matcher(areaHtml);
		while (ma.find()) {
			if(withPrefix !=null && !"".equals(withPrefix.trim())){
				list.add(withPrefix+ma.group(1).trim());
			}else{
				list.add(ma.group(1).trim());
			}
		}
		
		if(hasOmit && list.size()>1){
			String lasturl = list.get(list.size()-1);
			list.clear(); 
			regex = "(.*?)"+pageNumRegex.replaceFirst(WebContent.PAGE_URL_NUM, "(-?\\\\d+)");			
			pa = Pattern.compile(regex);
			ma = pa.matcher(lasturl);
			if(ma.find()){
				int num = Integer.parseInt(ma.group(2));
				String prefix = ma.group(1).trim();
				String baseurl = prefix+pageNumRegex;
				
				for(int i = 2; i <= num; i ++)
				list.add(baseurl.replaceFirst(WebContent.PAGE_URL_NUM, i+""));
			}			
		}
		
		return list;
	}
	
	/**
	 * 这个仅用于淘宝中的url替换成我的搜索推广
	 * @param oriMessage
	 * @param regex
	 * @return
	 */
	public String messageUrlReplease(String oriMessage){
		String regex = "<a(.*?);\">(.*?)\\<";
		Pattern pa = Pattern.compile(regex,Pattern.MULTILINE);
		Matcher ma = pa.matcher(oriMessage);
		String searchStr = null;
		if(ma.find()){
			try {
				searchStr= URLEncoder.encode(ma.group(2), "GBK");
//				System.out.println(ma.group(2));
			} catch (UnsupportedEncodingException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		
		regex = "href=\"http://search8.taobao.com(.*?)\"";
		pa = Pattern.compile(regex,Pattern.MULTILINE);
		ma = pa.matcher(oriMessage);
		String repleaceStr = "http://z.alimama.com/tksEncrypt.php?q="+searchStr+"&cat=16&pid=mm_16462935_0_0&unid=&commend=all&search_type=auction&user_action=initiative&f=D9_5_1&at_topsearch=1&sid=%286e3f806637f4a4de24b5b7a43a41aefc%29&sort=&spercent=0&st=0";
		while(ma.find()){
			oriMessage = ma.replaceAll("href=\""+repleaceStr+"\"");
		}
		
		return oriMessage;
	}
	
	public static void main(String[] args){
		//21cn
//		String url ="http://news.21cn.com/world/guojisaomiao/list1.shtml";
//		String prefixUrl = "http://news.21cn.com";
//		String encoding = "gb2312";
//		String arearegex = "<table width=\"100%\" border=\"0\" cellpadding=\"0\" cellspacing=\"1\" bgcolor=\"#FFFFFF\" class=\"block\">[list]<td height=\"20\" colspan=\"3\"><table width=\"100%\" border=\"0\" cellspacing=\"0\" cellpadding=\"5\">";
//		String urlregex = "<a href=\"[url]\" target=\"_blank\">";
//		String titleRegex = "<h2>[title]</h2>";
//		String messageRegex = "<!-- google_ad_section_start\\(name=content\\) -->[message]<!-- google_ad_section_end\\(name=content\\) -->";
//		boolean iSpagination = false;
		
		//taobao
		String url ="http://info.taobao.com/list/lady/23/30/2330b6ed-16ce-4d0e-b7fa-58a411e1871a_1.php";
		String prefixUrl = "";
		String encoding = "GBK";
		String arearegex = "<ul id=\"art-list\" class=\"dot-front-gruy\">[list]<div class=\"page\" id=\"page-down\">";
		String urlregex = "<a target=\"_blank\" href=\"[url]\">";
		String titleRegex = "<h1>[title]</h1>";
		String messageRegex = "<div class=\"info-main-text\" id=\"art-content\">[message]</div>";
		boolean iSpagination = true;
		boolean listHasOmit = true;
		boolean needReSeacheUrl = true;
		String pageNumRegex = "_[num].php";
		
		String pageAreaRegex = "<div id=\"J_pageNav\" class=\"page-navigation\">[list]class=\"page-down\">下一页</a>";
		String pageUrlRegex = "<a href=\"[url]\">";
		
		WebContent wc = new WebContent();
		try {
			String html = wc.getHtml(url, encoding);
			String listarea = wc.getArea(html, arearegex, WebContent.AREA_LIST);
			List<String> urllist = wc.getListUrl(listarea, urlregex, prefixUrl);
			System.out.println(urllist.size());
			long s = System.currentTimeMillis();
			
//			String url2 = "http://info.taobao.com/detail/lady/a2/ef/a2ef0063-a1fc-46fe-bd43-23c7b8dc8158_1.php";
			for(String url2 : urllist){
				System.out.println(url2);
				String subHtml = wc.getHtml( url2, encoding);
				String title = wc.getArea(subHtml, titleRegex, WebContent.AREA_TITLE);
				String message = wc.getArea(subHtml, messageRegex, WebContent.AREA_MESSAGE);
				if(needReSeacheUrl)
					message = wc.messageUrlReplease(message);
				if(iSpagination){
					//先取出分页的url
					String pageAreaHtml = wc.getArea(subHtml, pageAreaRegex, WebContent.AREA_LIST);
					List pageUrlList = wc.getListUrl(pageAreaHtml, pageUrlRegex, prefixUrl,listHasOmit,pageNumRegex);				
					StringBuffer allMessage = new StringBuffer(message);
					allMessage.append("<!--more--><!--pagetitle:分页1--><!--nextpage-->\r\n");
					
					Iterator<String > pageurls = pageUrlList.iterator();
					int pageSize = pageUrlList.size(), i = 0;
					while(pageurls.hasNext()){
						i ++;
						subHtml = wc.getHtml( pageurls.next(), encoding);
						message = wc.getArea(subHtml, messageRegex, WebContent.AREA_MESSAGE);
						if(needReSeacheUrl)
							message = wc.messageUrlReplease(message);
						allMessage.append(message);
						if( pageSize > i )
							allMessage.append("<!--pagetitle:分页"+(i+1)+"--><!--nextpage-->\r\n");
					}
					message = allMessage.toString();		
				}
				
				Wordpress.post(title, message);
			}
			System.out.println((System.currentTimeMillis()-s));
			
//			System.out.println(message);
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
}