有些服务器配置，禁止一些命令的url抓取，wget是使用不了的，不过我们可以模拟一个userage绕过

命令方法为:

wget http://img001.*********.cn/public/images/27/b5/ed/b7928ebe55c59f1a990007b45908cf58a533a9da,jpg --user-agent="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36"

#!/bin/bash

mysql_host=1.2.3.4

mysql_user=zhangsan

mysql_pass=123456

mysql_port=33060

DB_HYD_20=unkonw

DB_HYD_30=unkonw_new

imported_sql=/home/hask_product_pic

hgj_prefix="http://img001.*******.cn"

mth_prefix="http://img001.******.com"

imported_file_name=`date +%Y%m%d%H`

saved_file_name=`date +%Y%m%d%H%M%S`

download_pic_save=/webdata/webdir/public/ffffff_product_common

##清理远程保存的文件

ssh $mysql_host "rm -rf $imported_sql/*"

mysql -h $mysql_host -u $mysql_user -P $mysql_port -p$mysql_pass 2>/dev/null <<EOF

select 

concat(barcode,'#',image_oss_url,'#',image)

into outfile "$imported_sql/$imported_file_name.csv"

       fields terminated by ',' 

       lines terminated by '\n' 

from $DB_HYD_20.huigujia_item_lib_v4 where barcode in (

SELECT

barcode

FROM

$DB_HYD_30.sysitem_product_common

WHERE

image_default_id = ''

);

EOF

#复制到本地

scp $mysql_host:$imported_sql/* $imported_sql

for pic_path in `cat $imported_sql/$imported_file_name.csv`;do

barcode=`awk -F '#' '{print $1}' <<<"$pic_path"`

hgj_pic_url=`awk -F '#' '{print $2}' <<<"$pic_path"`

mth_pic_url=`awk -F '#' '{print $3}' <<<"$pic_path"`

 code=`curl -I -m 10 -o /dev/null -s -w %{http_code}"\n" $hgj_prefix/$hgj_pic_url`

   if [ "$code" = "200" ];then

            hgj_pic_file=`basename $hgj_pic_url`

            hgj_pic_suffix=`awk -F '.' '{print $2}' <<<"$hgj_pic_file"`

       wget $hgj_prefix/$hgj_pic_url -O   $download_pic_save/$barcode.$hgj_pic_suffix

mysql -h $mysql_host -u $mysql_user -P $mysql_port -p$mysql_pass 2>/dev/null <<EOF

update $DB_HYD_30.sysitem_product_common set image_default_id ="http://cdn.********/hyd30_product_common/$barcode.$hgj_pic_suffix" where barcode=$barcode;

EOF

else

 code=`curl -I -m 10 -o /dev/null -s -w %{http_code}"\n" $mth_prefix/$mth_pic_url`

  if [ "$code" = "200" ];then

            mth_pic_file=`basename $mth_pic_url`

            mth_pic_suffix=`awk -F '.' '{print $2}' <<<"$mth_pic_file"`

       wget $mth_prefix/$mth_pic_url -O   $download_pic_save/$barcode.$mth_pic_suffix

      mysql -h $mysql_host -u $mysql_user -P $mysql_port -p$mysql_pass 2>/dev/null <<EOF

      update $DB_HYD_30.sysitem_product_common set image_default_id ="http://cdn.********/hyd30_product_common/$barcode.$mth_pic_suffix" where barcode=$barcode;

EOF

    fi

    fi

done