PHP CURL抓取数据简单操作

gacvd

　　无聊中看到php中curl模块可以抓取数据，简单实现以下：
　　需求分析：
抓取大众点评数据
住区内容
1，地区
2,分类
3,店铺详细信息，店铺名称，店铺招牌，地址， 电话， 营业时间，人均消费，其他分店（关联其他分店），环境图片
http://m.dianping.com/citylist
　　
1，定义的简单的curl类库：
<?php
namespace getdp;
class CURL {
private $ch;
private $flag_if_have_run;

public function __construct($url) {
      $this->ch = curl_init($url);
      curl_setopt($this->ch, CURLOPT_RETURNTRANSFER , 1 );
}

public function close() {
      curl_close($this->ch);
}

public function __destruct() {
      $this->close();
}

public function set_time_out($timeout) {
      curl_setopt($this->ch, CURLOPT_TIMEOUT, intval($timeout));
      return $this;
}

public function set_referer($referer) {
if (!empty($referer))
     curl_setopt($this->ch, CURLOPT_REFERER , $referer);
     return $this;
}

public function load_cookie($cookie_file) {
     curl_setopt($this->ch, CURLOPT_COOKIEFILE , $cookie_file);
return $this;
}

public function save_cookie($cookie_file="") {
if(empty($cookie_file))
     $cookie_file = tempnam('./', 'cookie');
     curl_setopt($this->ch, CURLOPT_COOKIEJAR , $cookie_file);
     return $this;
}

public function exec () {
     $str = curl_exec($this->ch);
     $this->flag_if_have_run = true;
     return $str;
}

public function post ($post) {
curl_setopt($this->ch, CURLOPT_POST , 1);
curl_setopt($this->ch, CURLOPT_POSTFIELDS , $post );
return $this;
}

public function get_info() {
if($this->flag_if_have_run == true )
return curl_getinfo($this->ch);
else
throw new Exception("aaaaa");
}

public function set_proxy($proxy) {
curl_setopt($this->ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
curl_setopt($this->ch, CURLOPT_PROXY,$proxy);
return $this;
}

public function set_ip($ip) {
if(!empty($ip))
curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("X-FORWARDED-FOR:$ip", "CLIENT-IP:$ip"));
return $ip;
}

public function set_browser($user_agent, $language) {
curl_setopt ($this->ch , CURLOPT_HTTPHEADER, array ("User-Agent: $user_agent","Accept-Language: $language"));
return $this;
}
}


2，使用pdo操作数据库，将解析获取的数据插入数据库
<?php
require 'curl.class.php';
$pdo = new \PDO('mysql:host=localhost;dbname=getdazong', 'root', '');
$pdo->query("set names utf8");

require 'functions.php';

3，functions.php,简单的pdo数据库操作方法
<?php
function getCitys() {
global $pdo;
return $pdo->query("select * from city")->fetchAll();
}

function getCityById($id) {
global $pdo;
return $pdo->query("select * from city where id = $id")->fetch();
}

function getShops(){
global $pdo;
return $pdo->query("select id from shop group by id")->fetchAll();
}

创建的数据库表：
1，city
CREATE TABLE `city` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `city` varchar(64) NOT NULL,
   PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=2506 DEFAULT CHARSET=utf8

2,category
CREATE TABLE `category` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `pid` int(11) NOT NULL,
   `name` varchar(255) DEFAULT NULL,
   PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=32743 DEFAULT CHARSET=utf8

3,店铺表
CREATE TABLE `shop` (
   `id` int(32) unsigned NOT NULL,
   `name` varchar(255) NOT NULL COMMENT '店铺名称',
   `subname` varchar(255) DEFAULT NULL COMMENT '分店名称',
   `area` varchar(64) NOT NULL COMMENT '店铺区域',
   `address` varchar(255) DEFAULT NULL COMMENT '店铺地址',
   `mobile` varchar(32) DEFAULT NULL COMMENT '联系电话',
   `per_consumption` varchar(12) DEFAULT NULL COMMENT '消费',
   PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
4，店铺环境图片表
CREATE TABLE `shop_image` (
   `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
   `shop_id` int(11) NOT NULL,
   `image_path` varchar(255) NOT NULL,
   `type` tinyint(1) NOT NULL DEFAULT '0' COMMENT '1:店铺招牌，0，店铺环境图片',
   PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=526 DEFAULT CHARSET=utf8 CHECKSUM=1 DELAY_KEY_WRITE=1 ROW_FORMAT=DYNAMIC

现在我们开始抓取数据啦：
1:抓取http://m.dianping.com/citylist所有城市数据
<?php
require 'common.php';
$homePage = "http://m.dianping.com/citylist";
$curl = new \getdp\CURL($homePage);
$homePageContent = $curl->exec();
$hrefRegex = "/<a onclick=\"_gaq\.push\(.*\)\" href=\"javascript:window.location.href='\/c([0-9]{1,})\.csf'\" title=\".*\">(.*)<\/a>/i";
preg_match_all($hrefRegex, $homePageContent, $hrefs);

foreach ($hrefs[0] as $k => $v) {
$pdo->query("insert into city values (" . $hrefs[1][$k] .  ", " . "'". $hrefs[2][$k] . "') on duplicate key update id = id");
}

2，获取店铺种类的数据
<?php
set_time_limit(0);
require 'common.php';
$url = "http://m.dianping.com";
$curl = new \getdp\CURL($url);
$curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();

for ($i = 1; $i <= 30000; $i++) {
$url = "http://m.dianping.com/getchildrencategory?categoryid=$i";
$curl = new \getdp\CURL($url);
$content = $curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->exec();
$content = json_decode($content, true);
$message = $content['message'];
if (!empty($message['category']) && is_array($message['category'])) {
foreach ($message['category'] as $category) {
$pdo->query("insert into category values ({$category['categoryId']}, $i, '{$category['categoryName']}') on duplicate key update id = id");
}
}
}

3，获取所有地区的店铺基本信息
<?php
set_time_limit(0);
require 'common.php';

$url = "http://m.dianping.com";
$curl = new \getdp\CURL($url);
$curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();

$cityList = getCitys();
foreach ($cityList as $item){
$index = 1;
while(true){
$url = "http://m.dianping.com/shoplist/{$item['id']}?reqType=ajax&page=$index";
$curl = new \getdp\CURL($url);
$homePageContent = $curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();
$hrefRegex = "/href=\"\/shop\/(.*)\">.*<div class=\"intro Fix\">.*<span>(.*)<\/span>/Us";
preg_match_all($hrefRegex, $homePageContent, $hrefs);
foreach ($hrefs[0] as $k => $v) {
//get shop basic information
$url = "http://m.dianping.com/shop/".$hrefs[1][$k];
$area = $hrefs[2][$k];
$curl = new \getdp\CURL($url);
$homePageContent = $curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();
$regex ="/<div class=\"details-mode shop-info\">.*<img src=\"(.*)\">.*<span class=\"shop-name\">(.*)<\/span>.*<span class=\"price\">.*[人均|消费|费用]:(.*)<\/span>/Us";
preg_match($regex, $homePageContent,$matches);
$sign_image = !empty($matches[1]) ? $matches[1]:'';
$temp_name = !empty($matches[2]) ? $matches[2] :'';
$per_consumption = !empty($matches[3]) ? trim($matches[3]):'';
$regex = "/(.*)\((.*)\)/Us";
preg_match($regex, $temp_name,$temp_matches);
if(count($temp_matches)>0){
$name = $temp_matches[1];
$subname = $temp_matches[2];
}else{
$name = $temp_name;
$subname = '';
}
$regex = "/<i class=\"icon-address\"><\/i>(.*)<i class=\"arrowent\"><\/i>.*href=\"tel:(.*)\"/Us";
preg_match($regex, $homePageContent,$_matches);
$address = !empty($_matches[1])?$_matches[1]:'';
$mobile = !empty($_matches[2])? $_matches[2]:'';
$pdo->query("INSERT INTO shop(id,name,subname,area,address,mobile,per_consumption) VALUES(".$hrefs[1][$k].",'".$name."','".$subname."','".$area."','".$address."','".$mobile."','".$per_consumption."')");
$pdo->query("INSERT INTO shop_image(shop_id,image_path,TYPE) VALUES(".$hrefs[1][$k].",'".$sign_image."',1)");
}
if(count($hrefs[0])<25) break;
$index++;
}
}


4，获取所有店铺的环境图片
<?php
set_time_limit(0);
require 'common.php';

$url = "http://m.dianping.com";
$curl = new \getdp\CURL($url);
$curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();
$shopList = getShops();
foreach ($shopList as $item){
$existedCityids = file_get_contents('./a.txt');
if(is_numeric(strpos($existedCityids,$item['id']))) continue;
$index = 1;
while (true) {
$url = "http://m.dianping.com/shop/{$item['id']}/photos?reqType=ajax&page=$index";
$curl = new \getdp\CURL($url);
$homePageContent = $curl->save_cookie('./cookie.txt')->load_cookie('./cookie.txt')->set_time_out('30')->exec();
    $regex= "/<img src=\"(.*)\" onerror=\"DP\.prior\.nofind\(\)\">/";
    preg_match_all($regex, $homePageContent, $matches);
    foreach ($matches[0] as $k => $v){
    $pdo->query("INSERT INTO shop_image(shop_id,image_path,TYPE) VALUES(".$item['id'].",'".$matches[1][$k]."',0)");
    }
if(count($matches[0])<15) break;
$index++;
}
file_put_contents('./a.txt', $item['id']."\n",FILE_APPEND);
}