Perl爬虫动手(抓首页)
下面的代码目前只实现了抓取首页，后面会改下，要求抓2级到3级页面。#!/usr/bin/perl
use HTML::LinkExtor;
use LWP::Simple;
sub sparit{
local($base_url)= shift;
# 抓取URL
$parser = HTML::LinkExtor->new(undef,$base_url);
$parser->parse(get($base_url))->eof;
@links = $parser->links;
foreach $linkarray (@links){
my @element = @$linkarray;
my $elt_type = shift @element;
while( @element){
my ($attr_name , $attr_value) = splice(@element, 0, 2);
$seen{$attr_value}++;
}
}
# 排序/去除非同站URL
for (sort keys %seen){
if($_=~m#^$base_url#){
push(@tmp_url,"$_\n");
}
}
# 返回匹配的URL
return @tmp_url;
}
页:
[1]