虫言虫语 虫言虫语

Python编写单线程爬虫

in Pythonread (28807) 文章转载请注明来源!
    #!/usr/bin/env/ python3
import re
import urllib.request
import os

def picurl(url,path):
    if os.path.exists(path):
        print(path + 'exist')
    else:
        os.makedirs(path)
    while True:
        html = loadurl(url)
        if html == '':
            print('load' + url + 'error')
            continue
        else:
            break
    pic_list(re_pic(url),path)

def save_pic(url,path):
    searchname = '.*/(.*?.jpg)|(.*?.gif)'
    name = re.findall(searchname,url)
    filename = path + "/" + name[0][0]
    print(filename + ':start')
    tryTimes = 3
    while tryTimes != 0:
        tryTimes -= 1
        if os.path.exists(filename):
            print(filename + 'exists,skip')
            return True
        else:
            print(filename)
            open(filename,'wb')
        if download(url,filename):
            break
    if tryTimes != 0:
        print(filename + ': Over')
    else:
        print(url + ': Failed to download')

def download(url,filename):
    req = urllib.request.Request(url)
    req.add_header('User-Agent','Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:36.0) Gecko/20100101 Firefox/36.0')
    try:
        conn = urllib.request.urlopen(req,timeout=5)
        with open(filename,'wb') as f:
            f.write(conn.read())
            f.close()
        return True
    except urllib.error.URLError:
        print('load' + url + 'error')
        return False
    except Exception:
        print('unkown exception in conn.read()')
        return ''
def pic_list(picList,path):
    a = ' '.join(picList)
    b = re.findall('(\w+://[^\s]+)',a)
    for picurl in b:
        save_pic(picurl,path)

def re_pic(url):
    name_third = []
    name_fourth = []
    searchname = '<img alt=\"([^\"]*)\"( class="scrollLoading")?(  style="width:100%;")?\s*src=\"(\w+://[^\s]+)\"( title=\"([^\"]*)\")? />'
    name = re.findall(searchname,loadurl(url))
    for i in range(len(name)):
        name_first = name[i]
        for e in range(len(name_first)):
            name_second = name_first[e]
            if 'class' in name_second or 'style' in name_second or 'title' in name_second:
                continue
            else:
                name_third.append(name_second)
    name_fourth = sorted(set(name_third),key=name_third.index)
    return(name_fourth)

def loadurl(url):
    req = urllib.request.Request(url)
    req.add_header('User-Agent','Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:36.0) Gecko/20100101 Firefox/36.0')
    try:
        conn = urllib.request.urlopen(req)
        html = conn.read()
        html = html.decode('GBK')
        return html
    except urllib.error.URLError:
        return ''
    except Exception:
        print('unkown exception in conn.read()')
        return ''

if __name__ == '__main__':
    url = 'http://www.meizitu.com/a/454.html'
    picurl(url,'D:/WW')
jrotty WeChat Pay

微信打赏

jrotty Alipay

支付宝打赏

文章二维码

扫描二维码,在手机上阅读!

发表新评论
已有 7564 条评论
  1. smagmevagype
    smagmevagype
    回复

    mint credit score credit scoring does forbearance affect credit score credit score range

  2. crapyragacoup

    transunion credit report free http://creditcheckweb.com/ - free credit score report my free credit report how to dispute credit report free credit report

  3. Amitsflormror

    transunion credit score http://checkcreditps.com/ - how to increase credit score government free credit report excellent credit score experian credit score

  4. Amitsflormror

    annual credit report free http://checkcreditps.com/ - annualcreditreport com get my credit report credit score companies

  5. smagmevagype
    smagmevagype
    回复

    http://checkcredithistorynx.com/ experian plc anual credit report annualcreditreport com freeze credit report

  6. smagmevagype
    smagmevagype
    回复

    free annual credit report official site free yearly credit report check credit score for free

  7. swimarumTauff

    http://creditcheckmax.com/ how to unfreeze credit report credit score ratings chart how to check credit score anual credit report

  8. Amitsflormror

    national credit report http://checkcreditps.com/ credit score free my annual credit report what is considered a good credit score

  9. swimarumTauff

    dispute credit report http://creditcheckmax.com/ - credit karma check my credit score check credit score free dispute credit report

  10. crapyragacoup

    credit bureaus how to unfreeze credit report annual credit report official site business credit report

  11. hbbrdbleme
    hbbrdbleme
    回复

    cialis
    cialis generic online
    https://paydailoanz.com/cialis.php - viagra vs cialis
    tadalafil cialis
    cheap tadalafil
    https://paydailoanz.com/tadalafil.php - tadalafil 5 mg
    levitra
    levitra pills
    https://paydailoanz.com/levitra.php - levitra online
    levitra pills
    online levitra
    https://cbdoilnumber1.com/levitra/ - viagra vs cialis
    cialis buy online
    cialis pills
    https://cbdoilnumber1.com/cialis/ - cialis price

  12. JudyDes
    JudyDes
    回复

    buy indocin 75 mg baclofen drug amoxicillin buy tadalafil 5 mg buy toradol cymbalta drug seroquel generic buy diclofenac anafranil 25 tadalafil 20

  13. smagmevagype
    smagmevagype
    回复

    how to increase credit score consumer credit report freecreditreport

  14. crapyragacoup

    credit karma credit score free check my credit score credit karma equifax credit report free annual free experian credit report

  15. Amitsflormror

    how to improve credit score http://checkcreditps.com/ - credit score free get my credit score consumer credit report

  16. Amitsflormror

    credit score ranges http://checkcreditps.com/ - credit history check my credit score perfect credit score credit score rating

  17. smagmevagype
    smagmevagype
    回复

    http://checkcredithistorynx.com/ karma credit report dispute credit report how to get credit report free credit report equifax

  18. smagmevagype
    smagmevagype
    回复

    http://checkcredithistorynx.com/ fico credit score http://checkcredithistorynx.com/ - how to dispute credit report social credit score

  19. swimarumTauff

    highest credit score http://creditcheckmax.com/ credit reporting agency annual credit score annualcreditreport com official website

  20. Amitsflormror

    http://checkcreditps.com/ equifax credit report http://checkcreditps.com/ - free transunion credit report check credit score free

博客已萌萌哒运行
© 2020 由 Typecho 强力驱动.Theme by Yodu
前篇 后篇
雷姆
拉姆