使用CURL和火车头软件采集搜狐文章

直接上代码:

//参数1：访问的URL，参数2：post数据(不填则为GET)，参数3：提交的$cookies,参数4：是否返回$cookies
function curl_request($url,$post='',$cookie='', $returnCookie=0){
	$curl = curl_init();
	curl_setopt($curl, CURLOPT_URL, $url);
	curl_setopt($curl, CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)');
	curl_setopt($curl, CURLOPT_FOLLOWLOCATION, 1);
	curl_setopt($curl, CURLOPT_AUTOREFERER, 1);
	curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
	curl_setopt($curl, CURLOPT_REFERER, "http://www.baidu.com/");
	if($post) {
		curl_setopt($curl, CURLOPT_POST, 1);
		curl_setopt($curl, CURLOPT_POSTFIELDS, http_build_query($post));
	}
	if($cookie) {
		curl_setopt($curl, CURLOPT_COOKIE, $cookie);
	}
	curl_setopt($curl, CURLOPT_HEADER, $returnCookie);
	curl_setopt($curl, CURLOPT_TIMEOUT, 10);
	curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
	$data = curl_exec($curl);
	if (curl_errno($curl)) {
		return curl_error($curl);
	}
	curl_close($curl);
	if($returnCookie){
		list($header, $body) = explode("\r\n\r\n", $data, 2);
		preg_match_all("/Set\-Cookie:([^;]*);/", $header, $matches);
		$info['cookie']  = substr($matches[1][0], 1);
		$info['content'] = $body;
		return $info;
	}else{
		return $data;
	}
}

$caiji_set = [];
$caiji_url = '';
for($i=0;$i<85;$i++){
	// page-100
	//$url = "http://mp.sohu.com/apiV2/profile/newsListAjax?xpt=NTYzOTU5NjY1OUBzaW5hLnNvaHUuY29t&pageNumber=".$i."&pageSize=10&categoryId=&_=1541053659128";
	// page-85
	$url = "http://mp.sohu.com/apiV2/profile/newsListAjax?xpt=cHBhZzU5MTM5NjA2NmVlM0Bzb2h1LmNvbQ==&pageNumber=".$i."&pageSize=10&categoryId=&_=1541122188390";
	$detail_url = curl_request($url,'GET', '');
	$detail_data = json_decode($detail_url);

	$result = stripslashes(html_entity_decode($detail_data)); //传递过来的json字符串
	$result = json_decode($result, TRUE);

	if($result['msg'] == 'succes'){
		foreach($result['data'] as $k=>$item){
			$caiji_set[$k][] = array(
				"brief" => urldecode($item['brief']),
				"thumbnail" => $item["thumbnail"],
				"title"=>urldecode(title),
				"url"=>"http:".$item["url"]
			);
			$caiji_url .= "http:".$item["url"].'<br/>';
		}
	}
}
if(!empty($caiji_url)){
	//file_put_contents('./gougou.txt', $caiji_url, FILE_APPEND);
}
var_export($caiji_url);exit;