SHELL爬虫实例

3r211

#!/bin/bash
#

filename=$1
a="a.$filename"
b="b.$filename"
c="c.$filename"
d="d.$filename"
e="e.$filename"
ee="https://www.sunsky.com/"
eee="eee.$filename"

while read x1;do
url="https://www.sunsky.com/model/YMMTSelects.cfc?method=getMakes&PassYear=$x1"
curl -s $url | awk 'BEGIN{RS="<"}{print $0}'|awk -F'>' '{print $2}'|sed '1,9d'|sed '$d'|grep -v '^$' > $a
#echo "注释 $(date +%F-%T) "$x1
  while read x2;do
  #echo "注释 $(date +%F-%T) "$x1======$x2
  x2=`echo $x2|sed 's# #%20#g'`
  url="https://www.sunsky.com/model/YMMTSelects.cfc?method=getModel&PassYear=$x1&PassMakeName=$x2"
  curl -s $url|awk 'BEGIN{RS="<"}{print $0}'|awk -F'>' '{print $2}'|sed '1,6d'|sed '$d'|grep -v '^$' > $b
    while read x3;do
    #echo "注释 $(date +%F-%T) "$x1======$x2======$x3
    x3=`echo $x3|sed 's# #%20#g'`
    url="https://www.sunsky.com/model/YMMTSelects.cfc?method=getTrim&PassYear=$x1&PassMakeName=$x2&PassModel=$x3"
    curl -s $url|sed 's#[[:space:]]##g'|awk 'BEGIN{RS="<|=|>"}{print $0}'|egrep '^[0-9]+$' > $c
      while read x4;do
      #echo "注释 $(date +%F-%T) "$x1======$x2======$x3======$x4
      x4=`echo $x4|sed 's# #%20#g'`
      url="${ee}model/YMMTSelects.cfc?method=getTrimSections&PassYear=$x1&PassMakeNam=$x2&PassModel=$x3&PassVehicleID=$x4"
      curl -s $url|awk 'BEGIN{RS="<"}{print $0}'| awk -F'[>;]' '{print $2}'|sed -e '1,3d' -e '$d' -e '/^$/d' > $d
        while read x5;do
        #echo "注释 $(date +%F-%T) "$x1======$x2======$x3======$x4======$x5
        x5=`echo $x5|sed 's# #%20#g'`
        url="${ee}index.cfm?fuseaction=store.sectionSearch&YMMTyears=$x1&YMMTmakenames=$x2&YMMTmodelnames=$x3&YMMTtrimnames=$x4&YMMTsectionnames=$x5"
        curl -s $url|grep 'a href="index.cfm?fuseaction=store.PartInfo&PartNumbe'|cut -d'"' -f2 > $e
          while read x6;do
          ee="${ee}${x6}"
          echo $ee
          curl -s $ee > $eee
          LABEL=`grep 'diagram-label' $eee|awk -F'[<>]' '{print $5}'`
          GIF=`grep -B 10 partInfo $eee|grep -o "https.*gif"|awk '{a=$0}END{print a}'`
          PRO=`grep 'productID' $eee|awk -F'[<>]' '{print $3}'`
          GIFDIR=`dirname $(echo ${GIF#*com})`
          GIFILE=`echo ${GIF##*/}`
          GIFIMG="/dataimg$GIFDIR/$GIFILE"
          if test -d $GIFDIRmkdir;then
            echo | wget -q -P $GIFIMG $GIF
          else
        mkdir -p /dataimg$GIFDIR && echo | wget -q -P $GIFIMG $GIF
          fi
          echo "资源记录 ===>类目一是"$LABEL",类目二是"$PRO,"类目三是"$GIFIMG
          ee="https://www.sunsky.com/"
          done < $e
        done < $d
      done < $c
    done < $b
  done < $a
done < 888