curl模拟ip和来源进行网站采集的实现方法

对于限制了ip和来源的网站，使用正常的采集方式是不行的。这里说我的一种方法吧，使用php的curl类实现模拟ip和来源，可以实现采集限制ip和来源的网站。

1.设置页面限制ip和来源访问
比如服务端的server.php

<?php

$client_ip = getip();
$referer = getreferer();

$allow_ip = '192.168.0.100';
$allow_referer = 'http://www.xxx.cn';

if($client_ip==$allow_ip && strpos($referer, $allow_referer)===0){
  echo 'allow access';
}else{
  echo 'deny access';
} 

// 获取访问者ip
function getip(){
  if(!empty($_SERVER['HTTP_CLIENT_IP'])){
    $cip = $_SERVER['HTTP_CLIENT_IP'];
  }elseif(!empty($_SERVER['HTTP_X_FORWARDED_FOR'])){
    $cip = $_SERVER['HTTP_X_FORWARDED_FOR'];
  }elseif(!empty($_SERVER['REMOTE_ADDR'])){
    $cip = $_SERVER['REMOTE_ADDR'];
  }else{
    $cip = '';
  }
  return $cip;
}

// 获取访问者来源
function getreferer(){
  if(isset($_SERVER['HTTP_REFERER'])){
    return $_SERVER['HTTP_REFERER'];
  }
  return '';
}

?>

使用curl正常访问

<?php
function doCurl($url, $data=array(), $header=array(), $timeout=30){

  $ch = curl_init();
  curl_setopt($ch, CURLOPT_URL, $url);
  curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
  curl_setopt($ch, CURLOPT_POST, true);
  curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data));
  curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
  curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);

  $response = curl_exec($ch);

  if($error=curl_error($ch)){
    die($error);
  }

  curl_close($ch);

  return $response;

}

// 调用
$url = 'http://www.xxx.cn/server.php';
$response = doCurl($url);

echo $response;
?>

使用curl模拟ip和来源进行访问

模拟来源

curl_setopt($ch, CURLOPT_REFERER, '来源');

模拟ip

curl_setopt($ch, CURLOPT_HTTPHEADER, array('CLIENT-IP: 模拟ip','X-FORWARDED-FOR: 模拟ip'));

完整代码如下：

<?php
function doCurl($url, $data=array(), $header=array(), $referer='', $timeout=30){

  $ch = curl_init();
  curl_setopt($ch, CURLOPT_URL, $url);
  curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
  curl_setopt($ch, CURLOPT_POST, true);
  curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($data));
  curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
  curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);

  // 模拟来源
  curl_setopt($ch, CURLOPT_REFERER, $referer);

  $response = curl_exec($ch);

  if($error=curl_error($ch)){
    die($error);
  }

  curl_close($ch);

  return $response;

}

// 调用
$url = 'http://www.example.cn/server.php';//外部采集网站
$data = array();

// 设置IP
$header = array(
  'CLIENT-IP: 192.168.0.100',
  'X-FORWARDED-FOR: 192.168.0.100'
);

// 设置来源
$referer = 'http://www.xxx.cn/';

$response = doCurl($url, $data, $header, $referer, 5);

echo $response;
?>