perl 爬ku6视频
# cat new_hash_ku6.pl#!/usr/bin/perl
use strict;
use warnings;
use LWP::Simple;
use HTML::TreeBuilder;
use JSON qw(decode_json);
my $html = get ("http://www.ku6.com");
my $root = HTML::TreeBuilder->new_from_content($html);
my %images = ();
my @urls;
my $sub_html;
my $sub_root;
my %urls;
my @sub_urls;
foreach my $node ($root->find_by_tag_name('a')){
$images{$node->attr('href')}++;
}
foreach my $pic (sort keys %images){
if ( ($pic =~/http:\/\/v\.ku6\.com.*/s) &&($pic =~ m{/show/})){
push(@urls, $pic);
$urls{$pic}++;
}
}
上面代码是把ku6首页的视频连接都抓出来；
子程序get_sub_url 是逐个请求视频连接，顺便把每打开一个视频网页，那页上面的推广视频连接也抓下来保存到数组；
sub get_url {
foreach my $url (@urls){
$sub_html = get ($url);
$sub_root = HTML::TreeBuilder->new_from_content($sub_html);
my %href = ();
for my $sub_node ($sub_root->find_by_tag_name('a')){
$href{$sub_node->attr('href')}++;
}
for my $sub_pic (sort keys %href){
if (($sub_pic =~ /http:\/\/v\.ku6\.com.*/s) &&($sub_pic =~ m{/show/})){
if (exists $urls{$sub_pic}){next;}
else { push(@sub_urls,$sub_pic) };
}
}
get_mail_url(@sub_urls);
}
}
子程序get_mail_url 是模拟用户去访问ku6的调度服务器，下载视频内容；
这样就相当于主动把ku6的热点文件下载到缓存服务器，复用了，凌晨的空闲
带宽
sub get_mail_url {
foreach my $get_mail (@sub_urls){
my $url_mail ="$get_mail";
my $lwp =
new LWP::UserAgent( agent =>
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727; CIBA)'
);
my $request = HTTP::Request->new( GET => $url_mail );
$lwp->max_redirect;
$request->header( Accept => 'text/html' );
my $response = $lwp->request($request);
my $html = $response->decoded_content;
my ($flv_download_link) = ( $html=~ /"f":"([^\"]+)"/s );
my $json = '{ "url":"' . $flv_download_link. '"}';
$flv_download_link = decode_json($json)->{url};
system("wget $flv_download_link -O /dev/null");
}
}
get_url (@urls);
页:
[1]