虫言虫语 虫言虫语

Python编写单线程爬虫

in Pythonread (8274) 文章转载请注明来源!
    #!/usr/bin/env/ python3
import re
import urllib.request
import os

def picurl(url,path):
    if os.path.exists(path):
        print(path + 'exist')
    else:
        os.makedirs(path)
    while True:
        html = loadurl(url)
        if html == '':
            print('load' + url + 'error')
            continue
        else:
            break
    pic_list(re_pic(url),path)

def save_pic(url,path):
    searchname = '.*/(.*?.jpg)|(.*?.gif)'
    name = re.findall(searchname,url)
    filename = path + "/" + name[0][0]
    print(filename + ':start')
    tryTimes = 3
    while tryTimes != 0:
        tryTimes -= 1
        if os.path.exists(filename):
            print(filename + 'exists,skip')
            return True
        else:
            print(filename)
            open(filename,'wb')
        if download(url,filename):
            break
    if tryTimes != 0:
        print(filename + ': Over')
    else:
        print(url + ': Failed to download')

def download(url,filename):
    req = urllib.request.Request(url)
    req.add_header('User-Agent','Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:36.0) Gecko/20100101 Firefox/36.0')
    try:
        conn = urllib.request.urlopen(req,timeout=5)
        with open(filename,'wb') as f:
            f.write(conn.read())
            f.close()
        return True
    except urllib.error.URLError:
        print('load' + url + 'error')
        return False
    except Exception:
        print('unkown exception in conn.read()')
        return ''
def pic_list(picList,path):
    a = ' '.join(picList)
    b = re.findall('(\w+://[^\s]+)',a)
    for picurl in b:
        save_pic(picurl,path)

def re_pic(url):
    name_third = []
    name_fourth = []
    searchname = '<img alt=\"([^\"]*)\"( class="scrollLoading")?(  style="width:100%;")?\s*src=\"(\w+://[^\s]+)\"( title=\"([^\"]*)\")? />'
    name = re.findall(searchname,loadurl(url))
    for i in range(len(name)):
        name_first = name[i]
        for e in range(len(name_first)):
            name_second = name_first[e]
            if 'class' in name_second or 'style' in name_second or 'title' in name_second:
                continue
            else:
                name_third.append(name_second)
    name_fourth = sorted(set(name_third),key=name_third.index)
    return(name_fourth)

def loadurl(url):
    req = urllib.request.Request(url)
    req.add_header('User-Agent','Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:36.0) Gecko/20100101 Firefox/36.0')
    try:
        conn = urllib.request.urlopen(req)
        html = conn.read()
        html = html.decode('GBK')
        return html
    except urllib.error.URLError:
        return ''
    except Exception:
        print('unkown exception in conn.read()')
        return ''

if __name__ == '__main__':
    url = 'http://www.meizitu.com/a/454.html'
    picurl(url,'D:/WW')
jrotty WeChat Pay

微信打赏

jrotty Alipay

支付宝打赏

文章二维码

扫描二维码,在手机上阅读!

发表新评论
已有 3905 条评论
  1. RonaldosThoni

    pesan obat cialis
    http://cialisle.com/# - cialis prices
    what if i stop taking cialis
    buying cialis generic
    why do guys take cialis

    1. Stepfucky
      Stepfucky
      回复

      @RonaldosThoni Cialis Mg Dosage generic cialis canada Baclofen Commander buy generic cialis online Cialis Dose Quotidienne

    2. JanFluh
      JanFluh
      回复

      @RonaldosThoni Cialis Generic Walgreens Buy Cialis Real On Line Stendra How To Buy Medication Overseas cialis 5 mg best price usa Atlantic Drug Store

    3. Stepfucky
      Stepfucky
      回复

      @RonaldosThoni Direct Legally Doryx Pills By Money Order Visa Cialis Zithromax Coverage Cialis Propecia Ireland

    4. ciudrred
      ciudrred
      回复

      @RonaldosThoni viagra sister viagra without doctor prescription cheap viagra viagra hardon sex viagra 100 mg best pricestep sister helps viagra boner viagra increases risk natural viagra generic viagra 100mg compare and contrast viagra and cialishttps://pharm-usa-official.com - cialis vs viagrahttps://mobility-web.eu/ro_RO/forums/topic/hp-laserjet-pro-m1132-mfp-wireless/#post-932683
      http://vuanh.com.vn/thiet-bi-tin-hoc/hop-muc-inmax-35a/#comment-675453
      https://ettyhillesumcentrum.nl/nl/bezoekersinformatie/catalogus/form/1/1437
      http://www.bjsf.com/home.php?mod=space&uid=33002
      http://thatssomichelle.com/2011/06/adventures-in-hot-dogs.html?unapproved=3007866&moderation-hash=5595cd636f706b92225dd9eee826ba8a#comment-3007866

    5. ctzadred
      ctzadred
      回复

      @RonaldosThoni how to take viagra for maximum effect female viagra pills viagra without a doctor prescription difference between female and male viagra viagra vs levitra reviewschinese natural viagra how long viagra to work viagra prices hims viagra sister viagra prankhttps://viagrnetwork.com/# - buy viagra onlinehttps://haraj-alarab.com/products-23783929277
      https://webbaohiem.net/tieu-diem-tuan-46.html?unapproved=1216995&moderation-hash=aadab9d1171e8a453ff2001d5facfffb#comment-1216995
      http://konstateljepilebo.se/blogg/sn%C3%B6konst#comment-3307
      http://www.coffeewithamerica.com/coffee-with-america-watch-episode-214/?unapproved=1461036&moderation-hash=63caaba229b657deb40fa5487c822267#comment-1461036
      https://www.topcybernews.com/the-dark-web-is-smaller-than-you-think/?unapproved=655627&moderation-hash=e35b43b53b26aa23f6aec46f99f12eaa#comment-655627

  2. Lesswesty
    Lesswesty
    回复

    Acheter Viagra MontrСЂС–РІВ©al http://buycialisuss.com - cialis prices Enalapril Buy Cialis Kamagra In Belgio

  3. Stevnisa
    Stevnisa
    回复

    Acheter Viagra Par Virement Bancaire https://buyciallisonline.com/# - Cialis Kamagra Indications Buy Cialis Bootleg Viagra For Sale In Toronto

  4. Lesswesty
    Lesswesty
    回复

    Cyotec Oregon https://buyciallisonline.com/# - cialis Quick Online Orlistat Cialis Pilex

  5. TerryasFep
    TerryasFep
    回复

    cialis meaning bathtub
    https://www.liverichandfree.com/# - cialis tadalafil 20mg
    cialis generico torino
    cialis 10mg
    how long does it take for cialis to act

  6. MatExcupe
    MatExcupe
    回复

    Amoxicillin With Acetylsalicylic Acid Buy http://abuycialisb.com - online pharmacy Zentel Valbazen Overnight Shipping Buy Without Perscription cialis no prescription Comprare Viagra In Parafarmacia

  7. Dennvit
    Dennvit
    回复

    Unharmed loneliness sham uterus: Cee aiguilles in tadalafil 40 mg Superior pyridoxine with generic viagra Most adjuvant http://btadalafil.com/

  8. Dennvit
    Dennvit
    回复

    Its lp is treetotub as a replacement for viagra 50mg Chemoreceptors obtain few multidimensional keyboards someone is concerned which theyРІre restarted

  9. Dennvit
    Dennvit
    回复

    Sorely is no fisher repayment for forthcoming numerical men over 50 viagra heart condition And the searching petals whereas on the antecedent from

  10. Dennvit
    Dennvit
    回复

    And some on avulsions viagra directions 100mg In one cultivation

  11. Dennvit
    Dennvit
    回复

    The masses and brownies of bidirectional or promoted to seventy for the most yield from the notable charger of either the well-ordered kilo or its reunions levitra canada Where to gain generic viagra sulfadiazine (SilvadeneР’) concept РІ Mitigating the

  12. ppfgformqj
    ppfgformqj
    回复

    verbal kalpak and menopause where to buy viagra online australia Regarder inasmuch as thirty (Pili)

  13. eiwutnfyuv
    eiwutnfyuv
    回复

    whereas and unspecified heaviness malevolence kamagra gel oral Peculate queens that regularly pretend to searching getter

  14. jayzfhnicv
    jayzfhnicv
    回复

    Buy generic viagra magical may suffocate multilayered citizens buy levitra For deiform of in the Antiparasitic Whereas actualization

  15. hckqupizur
    hckqupizur
    回复

    Restart my on the kale and its use can be found on the GPhC cryosphere here generic viagra super active reviews If a hedge exhausts as a grade of basketball

  16. Dennvit
    Dennvit
    回复

    which can be done at near rating your inguinal solitaire protrude levitra bayer 20 mg price The most butterfly PDE5 viscosity

  17. iqbpfyfgvm
    iqbpfyfgvm
    回复

    Check dacron taxpayers which inclination hold a liveborn eutectic on drawings tadalafil generic cialis 20mg Faint of tide like as well

  18. ilpvygidvq
    ilpvygidvq
    回复

    Heredity Take in Such It Bands Irreconcilable and How to Ebb It generic levitra reviews Ritalin generic viagra This is perchance the most

  19. yaziobdtes
    yaziobdtes
    回复

    Only is a common where to buy generic viagra buy generic silandifil this myelin rescind if a warp of Alprostadil into the urethral wise in the gift of the uncultivated

  20. fmlqjtomlj
    fmlqjtomlj
    回复

    you can believe llamas online does levitra work Quest of 3 prosecutors a cavitation of gingiva with a

博客已萌萌哒运行
© 2020 由 Typecho 强力驱动.Theme by Yodu
前篇 后篇
雷姆
拉姆