@ 2014-11-06T02:41:57Z <?php
//爬虫程序,统计成功,失败
/*
$res=fetch("http://map.baidu.com/place/bd180e5585e0f428bd45de9a");
file_put_contents("log.txt", $res["header"]["http_code"]." ".$res["header"]["total_time"]." ".$res["content"]);
exit;
*/
$MENU_COUNT = array( "1"=>225,"2"=>45,"3"=>65,"4"=>99,"5"=>17,"6"=>168,"7"=>91,
"8"=>57,"9"=>46,"10"=>92,"11"=>72,"12"=>66,"13"=>72,"14"=>45,
"15"=>15 );
$EVERY = 50 * 50 ;
$RAND = 40 ;
//list,detail(all_cost,pre_cost,all,defeate,success,success_cost,defeate_cost,success_pre,defeate_pre)
$ALL_RESULT = array();
$handle = fopen("get_page.txt",'a');
fwrite($handle , "URL KIND COST FLAG(0失败,1)" );
for( $i=0; $i < $RAND; $i++ ){
$have_url = 0;
$k = 0;
$m = rand(1, 15);
$n = rand(1, $MENU_COUNT[$m]);
$url = "http://map.baidu.com/place/beijing/".$m."/".$n;
$url_list_handle_w = fopen("get_page_url.txt",'w');
$url_list_handle_r = fopen("get_page_url.txt",'r');
$res = getPage($url, "list", &$have_url, z$EVERY ,$url_list_handle_w);
initRes($res, $handle, &$ALL_RESULT, $url);
if(!$res["flag"] || !$res['no_url'] ){
//如果失败,或者成功但是页面中没有 列表或详情页链接
$i--;
@fclose($url_list_handle_w);
@fclose($url_list_handle_r);
continue;
}
while (!feof($url_list_handle_r)) {
$buffer = fgets($url_list_handle_r);
if(!$buffer || $k>= $EVERY ){
break;
}
$k++;
$buffer=explode(" ", $buffer);
$res = getPage($buffer[1], $buffer[0] , &$have_url, $EVERY ,$url_list_handle_w);
initRes($res,$handle,&$ALL_RESULT, rtrim($buffer[2]) );
}
@fclose($url_list_handle_w);
@fclose($url_list_handle_r);
}
$ALL_RESULT['list']['pre_cost'] = $ALL_RESULT['list']["all_cost"]/$ALL_RESULT['list']["all"];
$ALL_RESULT['list']['success_pre'] = $ALL_RESULT['list']["success_cost"]/$ALL_RESULT['list']["success"];
if( $ALL_RESULT['list']["defeate"] ){
$ALL_RESULT['list']['defeate_pre'] = $ALL_RESULT['list']["defeate_cost"]/$ALL_RESULT['list']["defeate"];
}
$ALL_RESULT['detail']['pre_cost'] = $ALL_RESULT['detail']["all_cost"]/$ALL_RESULT['detail']["all"];
$ALL_RESULT['detail']['success_pre'] = $ALL_RESULT['detail']["success_cost"]/$ALL_RESULT['detail']["success"];
if($ALL_RESULT['detail']["defeate"]){
$ALL_RESULT['detail']['defeate_pre'] = $ALL_RESULT['detail']["defeate_cost"]/$ALL_RESULT['detail']["defeate"];
}
$str="\nKIND 302 ALL ALL_COST PER SUCCESS DEFEATE SUCCESS_COST DEFEATE_COST SUCCESS_PRE DETEATE_PRE";
fwrite( $handle,$str );
foreach ($ALL_RESULT as $key => $value) {
$tem=array($key,$value['302'],$value['all'],$value['all_cost'],$value['pre_cost'],$value['success'],intval($value['defeate']),$value['success_cost'],$value['defeate_cost'],$value['success_pre'],$value['defeate_pre']);
fwrite($handle,"\n".join(" ",$tem));
}
fclose($handle);
function initRes($res,$handle,$allres,$from){
fwrite($handle,"\n".$res['url']." ".$from." ".$res['kind']." ".$res['header']["total_time"]." ".$res['flag']." ".$res["http_code"]." ".$res["no_url"]);
if($res["http_code"] == "302"){
$allres[$res['kind']]['302']++;
return;
}
$allres[$res['kind']]['all']++;
$allres[$res['kind']]['all_cost'] += $res['header']["total_time"];
if(!$res["flag"]){
$allres[$res['kind']]['defeate']++;
$allres[$res['kind']]['defeate_cost'] += $res['header']["total_time"];
}else{
$allres[$res['kind']]['success']++;
$allres[$res['kind']]['success_cost'] += $res['header']["total_time"];
}
}
function getPage($url,$kind,$have,$all,$handle){
$res = fetch($url);
$result['header'] = $res["header"];
if( $res["header"]["url"] != $url ){
$result["http_code"] = "302";
}else{
$result["http_code"] = $res["header"]["http_code"];
}
$result['kind'] = $kind;
$result['url'] = $url;
if( $res == -500 ){
$result['flag'] = 0;
return $result;
}
$result['flag'] = 1;
if( $have >= $all ){
@fclose($handle);
$result['no_url'] = 2;
return $result;
}
$result['url_list'] = array();
preg_match_all("/<a.*?href=[\"\'](.*?)[\"\'].*?>/", $res['content'], $matches, PREG_PATTERN_ORDER);
for($i=0 ,$k=0 ; $i < count($matches[1]) && $k < 20; $i++){
$matches[1][$i]=substr($matches[1][$i],stripos($matches[1][$i],'/place')+6);
$tem_url="http://map.baidu.com/place".$matches[1][$i];
if($tem_url==$url){
continue;
}
if(preg_match("/^\/\w{24}$/i", $matches[1][$i]) != 0 ){
$k++;
$result['url_list'][]="detail http://map.baidu.com/place".$matches[1][$i]." ".$url;
}else if(preg_match("/^\/[^\/]{1,20}\/\d{1,10}\/\d{1,10}/i", $matches[1][$i]) != 0 ){
$k++;
$result['url_list'][]="list http://map.baidu.com/place".$matches[1][$i]." ".$url;
}
}
$k = count($result['url_list']);
$result['no_url'] = 1;
if( $k <= 0 ){
$result['no_url'] = 0;
}else{
fwrite($handle,join("\n" , $result['url_list'])."\n");
}
$have += $k;
return $result;
}
function fetch($url) {
$c = curl_init($url);
if (!$c)
{
return -500;
}
curl_setopt($c, CURLOPT_MAXREDIRS, 3);
curl_redir_exec($c);
curl_setopt($c, CURLOPT_RETURNTRANSFER, true);
curl_setopt($c,CURLOPT_USERAGENT,"baiduspider");
curl_setopt($c, CURLOPT_AUTOREFERER, true);
curl_setopt($c, CURLOPT_CONNECTTIMEOUT, 50);
curl_setopt($c, CURLOPT_TIMEOUT, 50);
$rst_str = curl_exec($c);
$header = curl_getinfo($c);
curl_close($c);
if ($rst_str == false)
{
return -500;
}
return array('content' => $rst_str , "header" => $header);
}
function curl_redir_exec($ch,$debug = "")
{
static $curl_loops = 0;
static $curl_max_loops = 20;
if ($curl_loops++ >= $curl_max_loops)
{
$curl_loops = 0;
return FALSE;
}
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$data = curl_exec($ch);
$debbbb = $data;
list($header, $data) = explode("\n\n", $data, 2);
$http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if ($http_code == 301 || $http_code == 302) {
$matches = array();
preg_match('/Location:(.*?)\n/', $header, $matches);
$url = @parse_url(trim(array_pop($matches)));
if (!$url)
{
$curl_loops = 0;
return $data;
}
$last_url = parse_url(curl_getinfo($ch, CURLINFO_EFFECTIVE_URL));
$new_url = $url['scheme'] . '://' . $url['host'] . $url['path'] . ($url['query']?'?'.$url['query']:'');
@curl_setopt($ch, CURLOPT_URL, $new_url);
return curl_redir_exec($ch);
} else {
$curl_loops=0;
return $debbbb;
}
}
?>
Enable javascript to submit You have javascript disabled. You will not be able to edit any code.
Here you find the average performance (time & memory) of each version. A grayed out version indicates it didn't complete successfully (based on exit-code).
Version System time (s) User time (s) Memory (MiB) 5.4.34 0.010 0.035 11.99 5.4.32 0.009 0.036 12.48 5.4.31 0.005 0.040 12.47 5.4.30 0.007 0.037 12.48 5.4.29 0.008 0.037 12.48 5.4.28 0.009 0.036 12.37 5.4.27 0.005 0.040 12.37 5.4.26 0.007 0.038 12.37 5.4.25 0.011 0.040 12.37 5.4.24 0.005 0.036 12.37 5.4.23 0.006 0.045 12.36 5.4.22 0.008 0.034 12.36 5.4.21 0.007 0.051 12.36 5.4.20 0.061 0.052 12.36 5.4.19 0.057 0.037 12.36 5.4.18 0.011 0.038 12.36 5.4.17 0.007 0.037 12.37 5.4.16 0.012 0.034 12.37 5.4.15 0.005 0.037 12.36 5.4.14 0.012 0.046 12.05 5.4.13 0.008 0.034 12.03 5.4.12 0.007 0.038 11.99 5.4.11 0.008 0.034 11.99 5.4.10 0.006 0.035 11.99 5.4.9 0.010 0.040 11.98 5.4.8 0.004 0.040 11.98 5.4.7 0.004 0.036 11.98 5.4.6 0.005 0.034 11.98 5.4.5 0.008 0.032 11.98 5.4.4 0.004 0.035 11.97 5.4.3 0.005 0.037 11.96 5.4.2 0.009 0.041 11.96 5.4.1 0.006 0.034 11.96 5.4.0 0.005 0.036 11.45 5.3.29 0.009 0.039 12.80 5.3.28 0.041 0.038 12.71 5.3.27 0.027 0.038 12.72 5.3.26 0.042 0.039 12.72 5.3.25 0.058 0.045 12.72 5.3.24 0.029 0.038 12.72 5.3.23 0.029 0.038 12.70 5.3.22 0.011 0.044 12.68 5.3.21 0.010 0.039 12.68 5.3.20 0.007 0.036 12.68 5.3.19 0.004 0.040 12.67 5.3.18 0.008 0.034 12.67 5.3.17 0.005 0.037 12.67 5.3.16 0.005 0.037 12.67 5.3.15 0.007 0.036 12.67 5.3.14 0.005 0.037 12.66 5.3.13 0.007 0.039 12.65 5.3.12 0.004 0.041 12.66 5.3.11 0.005 0.040 12.65 5.3.10 0.006 0.036 12.12 5.3.9 0.008 0.035 12.08 5.3.8 0.005 0.039 12.08 5.3.7 0.007 0.035 12.07 5.3.6 0.004 0.038 12.06 5.3.5 0.006 0.036 12.00 5.3.4 0.006 0.036 12.00 5.3.3 0.007 0.034 11.96 5.3.2 0.008 0.033 11.73 5.3.1 0.006 0.034 11.70 5.3.0 0.008 0.033 11.68 5.2.17 0.005 0.030 9.20 5.2.16 0.006 0.027 9.20 5.2.15 0.004 0.030 9.20 5.2.14 0.005 0.030 9.19 5.2.13 0.007 0.026 9.15 5.2.12 0.005 0.028 9.16 5.2.11 0.001 0.032 9.16 5.2.10 0.004 0.029 9.15 5.2.9 0.007 0.026 9.15 5.2.8 0.004 0.030 9.15 5.2.7 0.004 0.030 9.15 5.2.6 0.004 0.030 9.10 5.2.5 0.028 0.034 9.07 5.2.4 0.024 0.032 9.05 5.2.3 0.031 0.037 9.03 5.2.2 0.024 0.033 9.01 5.2.1 0.020 0.034 8.94 5.2.0 0.025 0.034 8.80 5.1.6 0.019 0.030 8.08 5.1.5 0.018 0.035 8.08 5.1.4 0.030 0.022 8.07 5.1.3 0.024 0.026 8.41 5.1.2 0.027 0.028 8.43 5.1.1 0.019 0.025 8.16 5.1.0 0.019 0.027 8.16 5.0.5 0.024 0.018 6.64 5.0.4 0.023 0.019 6.51 5.0.3 0.022 0.041 6.32 5.0.2 0.020 0.019 6.29 5.0.1 0.016 0.021 6.26 5.0.0 0.030 0.027 6.26 4.4.9 0.015 0.016 4.78 4.4.8 0.014 0.014 4.75 4.4.7 0.016 0.016 4.76 4.4.6 0.015 0.016 4.75 4.4.5 0.008 0.016 4.77 4.4.4 0.025 0.024 4.71 4.4.3 0.018 0.015 4.76 4.4.2 0.005 0.015 4.84 4.4.1 0.003 0.015 4.85 4.4.0 0.021 0.022 4.76 4.3.11 0.011 0.019 4.67 4.3.10 0.020 0.019 4.67 4.3.9 0.019 0.014 4.63 4.3.8 0.018 0.027 4.58 4.3.7 0.015 0.014 4.63 4.3.6 0.011 0.016 4.63 4.3.5 0.015 0.012 4.63 4.3.4 0.012 0.025 4.54 4.3.3 0.012 0.016 3.30 4.3.2 0.015 0.014 3.28 4.3.1 0.009 0.019 3.24 4.3.0 0.003 0.020 6.85
preferences:dark mode live preview
142.22 ms | 1394 KiB | 7 Q