就是個python爬蟲
就像爬個圖看看
源網站鏈接:http://www.setuw.com
使用python編寫,使用了threadpool 等庫,自行下載。
環境:python 3 , win10 , 樹莓派環境下測試通過
網站元素結構
代碼
# -*- coding: utf-8 -*
from concurrent.futures import ThreadPoolExecutor
import urllib.request
import _thread
import json
import threadpool
from time import sleep
from bs4 import BeautifulSoup
import os
import random
maxThreadCount = 8
available_thread = 8
baseDomain="http://www.setuw.com"
intrance = "http://www.setuw.com/tag/rosi/"
#網站分類對應的目錄
tags = [ "/tag/rosi/", "/tag/tuigirl/" , "/tag/ugirls/" ,
"/tag/xiuren/" , "/tag/disi/" , "/tag/dongman/" , "/tag/xinggan/" ,
"/tag/qingchun/" , "/tag/youhuo/" , "/tag/mote/" , "/tag/chemo/" ,
"/tag/tiyu/" , "/tag/zuqiubaobei/" , "/meinv/liuyan/"
]
types = ["ROSI" , "推女郎" , " 尤果" , " 秀人 " ,
" DISI " , "動漫 " , "性感 " , "清純 " , " 誘惑 " , " 模特 " , " 車模" , "體育" , " 足球" , " 柳巖" ]
typeSize = len(types)
path = ""
header = {
"User-Agent":'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36',
'Accept': '*/*',
'Accept-Language': 'en-US,en;q=0.8',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive'
}
def Download(argv1):
url = argv1.split("#")[0]
title = argv1.split("#")[1]
name = argv1.split("#")[2]
#print("URl is " , url, " , title is " , title , " , name is " , name)
print("Download processing:" , argv1.split("#")[3])
apath = path+"/" + title + "/"
#print(apath)
if not os.path.exists(apath): #判斷系統是否存在該路徑,不存在則創建
os.makedirs(apath)
urllib.request.urlretrieve( url, '{0}{1}.jpg'.format(apath, name)) # ,下載圖片保存在本地
return
def run(targetUrl,title):
global available_thread
print("downloading " + title)
req = urllib.request.Request(url=targetUrl,headers=header)
response = urllib.request.urlopen(req)#1111111這里的req可看成一種更為高級的URL
html = response.read().decode('utf-8','ignore')
soup = BeautifulSoup(html, 'html.parser')
imgs = soup.find_all('img')
size = len(imgs)
resules = 1
with ThreadPoolExecutor(maxThreadCount) as pool:
for i in range(2,size-16):
#已經證實過,頁面中抓取的img,2到size-6為圖集圖片
data = imgs[i]["datas"]
all = data.split("'")
'''
參數列表:下載鏈接,圖集名,圖片名,圖集下載進度
'''
argv ={ all[len(all) - 2] + "#" + title + "#" + all[len(all) - 2].split(".")[1]+str(i) + "#" + str(i-1) + "/" + str(size-18) }
results = pool.map(Download,(argv)) #使用map添加線程進線程池
print(title , " download successfull;")
return
if __name__ == '__main__':
'''自定義下載路徑。若輸入.,則下載至當前目錄,跳過則下載到/home/hdd/picDl/(這是我)
自己的硬盤掛載點。。。可自定義)'''
input1 = input("input a folder(. as ./ , none as /home/hdd/picDl/):")
if input1==".":
path = "./"
elif input1=="":
path = "/home/hdd/picDl/"
else:
path = input1
print("Path seted to " + path)
#選擇一個下載類別。在網站最上方有,我是手動找出來的,有時效性
for i in range(0,len(types)-1):
print("| " + str(i)+ " | " + types[i] + " | ")
print("select a type to download , ")
index = input(" or input nothing to download index page:")
if index == "":
intrance = intrance
else:
index1 = int(index)
if index1 < len(types)-1 and index1 > 0 :
intrance = baseDomain + tags[index1]
else:
print("something wrong , setting download tartget as default")
intrance = intrance
print( intrance + " is going to download.")
'''
自定義下載線程數。注意,函數中每個線程用于下載一張圖片,
所以只能說是多個圖片并行下載。
'''
maxThreadCount_ = input("input a number if you want to modify default thread number:")
if maxThreadCount_ == "" :
print("using default number:" , maxThreadCount)
else :
print("Modified number to:" , maxThreadCount_)
maxThreadCount = int(maxThreadCount_)
req = urllib.request.Request(url=intrance,headers=header)
response = urllib.request.urlopen(req)
html = response.read().decode('utf-8','ignore')
#解碼 得到這個網頁的html源代碼
soup = BeautifulSoup(html, 'html.parser')
Divs = soup.find_all('a',attrs={'class':'a1' })
for div in Divs:
if div["href"] is None:
print("沒有圖集了")
# return
elif div['href'] is None or div['href']=="": #有鏈接,但是是 空鏈接
print("圖集沒有連接")
# return
else:
targetUrl= baseDomain + div['href']
title=div["title"]
print("正在下載套圖:" + title)
run(targetUrl,title)
缺點(改進方向):
- 目前只下載分類第一頁的圖集
- 不能保存下載進度。
- 沒了吧。。歡迎批評指正
更多文章、技術交流、商務合作、聯系博主
微信掃碼或搜索:z360901061

微信掃一掃加我為好友
QQ號聯系: 360901061
您的支持是博主寫作最大的動力,如果您喜歡我的文章,感覺我的文章對您有幫助,請用微信掃描下面二維碼支持博主2元、5元、10元、20元等您想捐的金額吧,狠狠點擊下面給點支持吧,站長非常感激您!手機微信長按不能支付解決辦法:請將微信支付二維碼保存到相冊,切換到微信,然后點擊微信右上角掃一掃功能,選擇支付二維碼完成支付。
【本文對您有幫助就好】元
