Online PHP editor | output for SfEFp

<?php //爬虫程序，统计成功，失败 /* $res=fetch("http://map.baidu.com/place/bd180e5585e0f428bd45de9a"); file_put_contents("log.txt", $res["header"]["http_code"]." ".$res["header"]["total_time"]." ".$res["content"]); exit; */ $MENU_COUNT = array( "1"=>225,"2"=>45,"3"=>65,"4"=>99,"5"=>17,"6"=>168,"7"=>91, "8"=>57,"9"=>46,"10"=>92,"11"=>72,"12"=>66,"13"=>72,"14"=>45, "15"=>15 ); $EVERY = 50 * 50 ; $RAND = 40 ; //list,detail(all_cost,pre_cost,all,defeate,success,success_cost,defeate_cost,success_pre,defeate_pre) $ALL_RESULT = array(); $handle = fopen("get_page.txt",'a'); fwrite($handle , "URL KIND COST FLAG(0失败,1)" ); for( $i=0; $i < $RAND; $i++ ){ $have_url = 0; $k = 0; $m = rand(1, 15); $n = rand(1, $MENU_COUNT[$m]); $url = "http://map.baidu.com/place/beijing/".$m."/".$n; $url_list_handle_w = fopen("get_page_url.txt",'w'); $url_list_handle_r = fopen("get_page_url.txt",'r'); $res = getPage($url, "list", &$have_url, z$EVERY ,$url_list_handle_w); initRes($res, $handle, &$ALL_RESULT, $url); if(!$res["flag"] || !$res['no_url'] ){ //如果失败，或者成功但是页面中没有列表或详情页链接 $i--; @fclose($url_list_handle_w); @fclose($url_list_handle_r); continue; } while (!feof($url_list_handle_r)) { $buffer = fgets($url_list_handle_r); if(!$buffer || $k>= $EVERY ){ break; } $k++; $buffer=explode(" ", $buffer); $res = getPage($buffer[1], $buffer[0] , &$have_url, $EVERY ,$url_list_handle_w); initRes($res,$handle,&$ALL_RESULT, rtrim($buffer[2]) ); } @fclose($url_list_handle_w); @fclose($url_list_handle_r); } $ALL_RESULT['list']['pre_cost'] = $ALL_RESULT['list']["all_cost"]/$ALL_RESULT['list']["all"]; $ALL_RESULT['list']['success_pre'] = $ALL_RESULT['list']["success_cost"]/$ALL_RESULT['list']["success"]; if( $ALL_RESULT['list']["defeate"] ){ $ALL_RESULT['list']['defeate_pre'] = $ALL_RESULT['list']["defeate_cost"]/$ALL_RESULT['list']["defeate"]; } $ALL_RESULT['detail']['pre_cost'] = $ALL_RESULT['detail']["all_cost"]/$ALL_RESULT['detail']["all"]; $ALL_RESULT['detail']['success_pre'] = $ALL_RESULT['detail']["success_cost"]/$ALL_RESULT['detail']["success"]; if($ALL_RESULT['detail']["defeate"]){ $ALL_RESULT['detail']['defeate_pre'] = $ALL_RESULT['detail']["defeate_cost"]/$ALL_RESULT['detail']["defeate"]; } $str="\nKIND 302 ALL ALL_COST PER SUCCESS DEFEATE SUCCESS_COST DEFEATE_COST SUCCESS_PRE DETEATE_PRE"; fwrite( $handle,$str ); foreach ($ALL_RESULT as $key => $value) { $tem=array($key,$value['302'],$value['all'],$value['all_cost'],$value['pre_cost'],$value['success'],intval($value['defeate']),$value['success_cost'],$value['defeate_cost'],$value['success_pre'],$value['defeate_pre']); fwrite($handle,"\n".join(" ",$tem)); } fclose($handle); function initRes($res,$handle,$allres,$from){ fwrite($handle,"\n".$res['url']." ".$from." ".$res['kind']." ".$res['header']["total_time"]." ".$res['flag']." ".$res["http_code"]." ".$res["no_url"]); if($res["http_code"] == "302"){ $allres[$res['kind']]['302']++; return; } $allres[$res['kind']]['all']++; $allres[$res['kind']]['all_cost'] += $res['header']["total_time"]; if(!$res["flag"]){ $allres[$res['kind']]['defeate']++; $allres[$res['kind']]['defeate_cost'] += $res['header']["total_time"]; }else{ $allres[$res['kind']]['success']++; $allres[$res['kind']]['success_cost'] += $res['header']["total_time"]; } } function getPage($url,$kind,$have,$all,$handle){ $res = fetch($url); $result['header'] = $res["header"]; if( $res["header"]["url"] != $url ){ $result["http_code"] = "302"; }else{ $result["http_code"] = $res["header"]["http_code"]; } $result['kind'] = $kind; $result['url'] = $url; if( $res == -500 ){ $result['flag'] = 0; return $result; } $result['flag'] = 1; if( $have >= $all ){ @fclose($handle); $result['no_url'] = 2; return $result; } $result['url_list'] = array(); preg_match_all("/<a.*?href=[\"\'](.*?)[\"\'].*?>/", $res['content'], $matches, PREG_PATTERN_ORDER); for($i=0 ,$k=0 ; $i < count($matches[1]) && $k < 20; $i++){ $matches[1][$i]=substr($matches[1][$i],stripos($matches[1][$i],'/place')+6); $tem_url="http://map.baidu.com/place".$matches[1][$i]; if($tem_url==$url){ continue; } if(preg_match("/^\/\w{24}$/i", $matches[1][$i]) != 0 ){ $k++; $result['url_list'][]="detail http://map.baidu.com/place".$matches[1][$i]." ".$url; }else if(preg_match("/^\/[^\/]{1,20}\/\d{1,10}\/\d{1,10}/i", $matches[1][$i]) != 0 ){ $k++; $result['url_list'][]="list http://map.baidu.com/place".$matches[1][$i]." ".$url; } } $k = count($result['url_list']); $result['no_url'] = 1; if( $k <= 0 ){ $result['no_url'] = 0; }else{ fwrite($handle,join("\n" , $result['url_list'])."\n"); } $have += $k; return $result; } function fetch($url) { $c = curl_init($url); if (!$c) { return -500; } curl_setopt($c, CURLOPT_MAXREDIRS, 3); curl_redir_exec($c); curl_setopt($c, CURLOPT_RETURNTRANSFER, true); curl_setopt($c,CURLOPT_USERAGENT,"baiduspider"); curl_setopt($c, CURLOPT_AUTOREFERER, true); curl_setopt($c, CURLOPT_CONNECTTIMEOUT, 50); curl_setopt($c, CURLOPT_TIMEOUT, 50); $rst_str = curl_exec($c); $header = curl_getinfo($c); curl_close($c); if ($rst_str == false) { return -500; } return array('content' => $rst_str , "header" => $header); } function curl_redir_exec($ch,$debug = "") { static $curl_loops = 0; static $curl_max_loops = 20; if ($curl_loops++ >= $curl_max_loops) { $curl_loops = 0; return FALSE; } curl_setopt($ch, CURLOPT_HEADER, true); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); $data = curl_exec($ch); $debbbb = $data; list($header, $data) = explode("\n\n", $data, 2); $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE); if ($http_code == 301 || $http_code == 302) { $matches = array(); preg_match('/Location:(.*?)\n/', $header, $matches); $url = @parse_url(trim(array_pop($matches))); if (!$url) { $curl_loops = 0; return $data; } $last_url = parse_url(curl_getinfo($ch, CURLINFO_EFFECTIVE_URL)); $new_url = $url['scheme'] . '://' . $url['host'] . $url['path'] . ($url['query']?'?'.$url['query']:''); @curl_setopt($ch, CURLOPT_URL, $new_url); return curl_redir_exec($ch); } else { $curl_loops=0; return $debbbb; } } ?>