Python + pyspider某小说站的爬虫,入数据库,火车头发布,资源下载到本地,另可写爬虫!
0 y" Y9 L7 l. `3 h; H- #!/usr/bin/env python& f' w, d- S0 b7 o- y
- # -*- encoding: utf-8 -*-5 J: @' d; t" h( ]
- # Created on 2019-05-05 21:43:11
- c: j6 E2 y! g" Q. v7 b - # Project: XiaoShuo
: |6 [3 J- N) [$ a - & c& r3 K- h( Z& a" {
- from pyspider.libs.base_handler import *: o4 O0 S+ K. R t* \3 G
- import pymysql# Z: E3 R9 x D a! E
- import random
0 _' `6 v1 A4 z' u, X - import datetime; J4 c/ a. V( ?: R7 x# A8 E1 u
- import urllib2,HTMLParser,re+ t- Y+ i6 C, l) @5 e
- import os- v2 A, o& C0 q# c/ L
- import sys
6 m" G6 d# K! y" M% L - import re/ s. }) u2 a; W7 Z1 A
- import codecs
8 D! |& ?7 ^$ u6 ^ V2 F l - import requests
/ j: Z- m1 t/ n - import json
5 d6 L! Q$ \. _4 q1 H- v6 O8 @ -
7 D$ p* d+ o" N% W9 g/ b - class Handler(BaseHandler):
; E2 Q' |% Q! b+ o - global Datos: V, c* ^7 w8 h
- global P_dir 3 ~" ^9 h9 @4 o9 J4 f0 F0 W
- P_dir = '/Tools/Debug/' #采集时候图片保持到本地的路径6 N) R0 b' B; S
- global Datos
1 G/ v) s7 G) H7 [1 }5 j% V g! D - Datos = {}
& [: z+ c! Z3 ?! O$ E, |4 A - headers= {
5 @2 N( a7 R, v/ `: h, l' {( q - 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',5 k. L; k! m) Y" `/ v7 q, K
- 'Accept-Encoding':'gzip, deflate, sdch',! ]3 m6 o- x" k" T2 `* p4 D
- 'Accept-Language':'zh-CN,zh;q=0.8',
9 O U) d7 |1 @, @9 y - 'Cache-Control':'max-age=0',4 T$ D+ l( v& H
- 'Connection':'keep-alive',$ h2 E# h, s/ d r7 b7 }, ]* \
- 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'
- H% Y' ]* _" n. m- }( a% r7 c - }) _7 l/ E! g7 f; Y& _( ?
- crawl_config = {7 M- C3 }5 ]; Y. K [" ^9 I
- 'headers' : headers,* C0 m; @/ t9 j& p) g$ }1 j
- 'timeout' : 3000 V& c& j h# N8 I9 {* s+ B
- }
( [0 e/ ~2 }, K8 |( e( B4 j - def add_BookFile(self,Bookname, BookIDs, img, Locaimg, Book_Dates):
5 g: H' d. a' o+ T - db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")
; Z- [5 h+ @) j t7 t# m - try:
0 m2 s, U) [0 `+ S% H5 O - cursor = db.cursor()$ K# y6 G0 l- S! o
- #注意此处字符串的占位符要加双引号"%s"
( B8 G) v P" c; _9 U8 f) H/ w - sql = 'insert into BookFile(Bookname, BookID, img, Locaimg, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, BookIDs, img, Locaimg, Book_Dates);2 t) B0 H+ B, w
- # print(sql)
! U% e2 u+ o3 K, o( m - cursor.execute(sql)
9 d+ y9 g8 Y6 K' p4 U% z - ) `! T# e1 o8 ^$ v& S
- #qid = cursor.lastrowid B) k2 Q4 S* B% ^, H
- #print(qid)
8 d' i* ~# }( B% G( M - 7 Y& ?5 ]# B; Z0 F
- db.commit()
/ P1 L) I9 I8 }2 x) b$ [5 | - except Exception as err:7 ?- x z$ T" W
- print("Error %s for execute sql: %s" % (err, sql))
6 }, t8 {- N4 I+ _3 i0 r9 R - db.rollback(). m6 k) q9 G* p3 O9 X
- def add_comment(self,Bookname, Booktitle, BookID, Titleid, Book_Date):* c9 A4 n% n; ?' h+ Y! K
- db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")
: Z3 j5 d8 N! S9 J6 m/ r% M - try:4 c/ |, |- Z9 a9 ^( Y
- cursor = db.cursor()
. {8 v2 b) S- u1 R& W - #注意此处字符串的占位符要加双引号"%s"4 I' W6 s/ m* b2 ?
- sql = 'insert into BookTitle(Bookname, Booktitle, BookID, Titleid, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, Booktitle, BookID, Titleid, Book_Date);4 B9 l0 I- i; q: j. {- S
- # print(sql)
& M3 b1 F5 K8 X, c: v - cursor.execute(sql); G/ j& B% Y. K5 I* W, ~# m. ~
- 0 M. s! Z/ d. `& q# I* I* L
- #qid = cursor.lastrowid
: o% u) k6 V2 l2 U" k0 i$ z - #print(qid)3 {7 J6 Y0 G5 S
-
7 |( b6 \" R8 x) b - db.commit()( x: V! A8 Y3 {" V
- except Exception as err:* f& ~0 M# g& ?" j; s, w7 `
- print("Error %s for execute sql: %s" % (err, sql))
! x& |0 ^; p9 i% h u; [9 s - db.rollback()
6 q1 m; j# C' @& N1 u! J - def add_question(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date):
" [: J" O7 {2 ~5 l# T v - db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")
, \* s. y( C2 R, S$ Y, V6 M, m - try:
5 h7 a- H* O; N; P% C/ [/ O4 P - cursor = db.cursor()
4 t- n) L8 l& h- X* U - #注意此处字符串的占位符要加双引号"%s"
" M# J U0 u8 {6 x* X - sql = ' INSERT INTO BookConte (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) VALUES("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")' % (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date);
; h; `4 T* h3 h$ m4 X9 O4 M - print(sql)/ O- r: E& D7 _2 R* [; h% L5 L
- cursor.execute(sql)
& w/ A" r3 e& q' T; H% c - print(cursor.lastrowid)5 ?# }3 `. a# Q! V
- db.commit(): s7 D2 O, _9 c& |
- except Exception as err: g7 n& }' v, e( m" m
- # except:
5 l: M8 n- k$ J, G" s - # print('Failed')3 ~* ^' L. @; [5 q& p
- print("Error %s for execute sql: %s" % (err, sql))
) e4 L' Q$ W5 Q" c% _$ ] - db.rollback()5 h: Q; \0 h- S+ p3 D
- 9 h! s6 [: \* u4 W8 F; n' ~
- def add_locoy(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover):
7 r2 ]8 ~1 F9 w6 H - reload(sys)
8 ]9 o! U9 b q7 _3 p7 \ - sys.setdefaultencoding("gbk")
, \) F( F2 y8 |" p3 n - locoy_url = 'http://www.******.net/locoy/?my=book' #697火车头发接口地址, n: T$ q2 F2 w; i, K: R8 n
- locoy_data = {
0 y: [/ b* D# g5 S6 F6 \0 j - 'my_u':'用户名', #后台用户名
5 s4 d4 d: ?6 C( O8 W' e. X3 h - 'my_p':'密码', #后台密码0 ]( {( r+ o8 ~, |4 I) ~ f
- 'subject_669977_net':Bookname.encode('gbk', 'ignore'),' k8 l0 H' P" o; y2 n# w, j
- 'caid':Cater_Name.encode('gbk', 'ignore'),
5 q s+ A1 j2 A6 Y6 ]" L& p3 V - 'title_669977_net':Booktitle.encode('gbk', 'ignore'),% p5 C0 z$ `* m3 l
- 'article':BookConte.encode('gbk', 'ignore'),6 t/ o' z! L8 Q, e0 c2 n
- 'author':Book_author.encode('gbk', 'ignore'),6 d$ z, n: ~1 G* Q/ C
- 'ready_1':Book_Palabras.encode('gbk', 'ignore'),
( X$ e& a/ r7 X9 [ - 'thumb':Book_img,. Y! `1 p7 a, `2 w. I% U, u5 Q+ {; F
- 'content':Book_Introduction.encode('gbk', 'ignore'),
c) _; m- i$ G4 w0 M `7 ? - 'abover':abover.encode('gbk', 'ignore') # y2 q- C, D0 A0 ^
- }4 F4 k/ R7 Z3 {
- res = requests.post(locoy_url, data=locoy_data)6 Y8 u) K+ y0 c; f+ B* A4 w
- print res.text5 J+ V% A" X# u6 m- S* l
- print res.content
0 k, \: x0 L. D" {. o$ ~1 V - # print Dsd
) K& b# q9 m# j# F8 G - return res# V3 F J9 K) v7 s: R1 u
-
* H/ G) C' t6 U& n( Y }& ~ - def __init__(self):
( b* |1 |& B5 H - self.base_url1 = 'https://www.****.cc/'. o' s4 }1 e8 Z1 T1 s8 Y! u
- self.base_url2 = '/'
$ H# @1 h, O* i( \5 Q" O - self.CaterId = []
& L" @7 v0 S0 T - self.CaterIds = ['xuanhuan', 'wuxia', 'yanqing', 'lishi', 'kehuan', 'kongbu', 'nvsheng']
# V1 `* g! X& U @6 a - self.page_num = 1/ S; I, a _8 n/ ^. J7 R) J
- self.total_num = 200 ) M) ?4 y/ n1 f. `1 I
-
+ j% p) C8 _5 x9 `9 \) ] - @every(minutes=8 * 60)
' D) i8 b9 @6 X; Q1 n- @ - def on_start(self):# t4 U0 h `5 p
- global Cater_Name
" k& O) m: f9 \5 W! i" S. O, _ - Cater_Name = []+ H+ ^- p z1 Z( ^7 K8 ^; j
- while self.page_num <= self.total_num:
2 r( C8 ?2 g* m; _) q2 S - for self.CaterId in self.CaterIds:1 ?( J' o9 d7 L* g& g: w+ O
- if self.CaterId == 'xuanhuan':
. \0 W6 u0 y3 U' L; f; q - Cater_Name = '玄幻'
3 O% {7 D. j2 t9 w0 [# P - if self.CaterId == 'wuxia':/ I2 j' K I: m( d" f
- Cater_Name = '武侠'
9 \- S7 r0 ?6 I0 d6 H$ G - if self.CaterId == 'lishi':
1 V4 w5 g+ u' v9 B - Cater_Name = '历史' " V3 G) U% T) P6 |8 [+ h6 @
- if self.CaterId == 'yanqing':
% A( [+ V) J6 {' S7 y& t I2 V! c- J - Cater_Name = '都市'
1 |( b% u7 j& \# o+ L3 d W3 ] - if self.CaterId == 'nvsheng':
; G: A! L5 E( r# y - Cater_Name = '都市'
6 J: l. m: i/ x! Q6 S - if self.CaterId == 'kehuan':
6 \/ m6 z+ T7 Q0 m0 h2 \ - Cater_Name = '科幻'
' J$ Y/ E3 x6 a: B3 C' c - if self.CaterId == 'kongbu':; K3 j4 M0 a- y7 b% a* Y
- Cater_Name = '游戏'
. j2 v4 l3 J3 x - print self.CaterId) L, H! d- _ @! c k+ ~( L0 _
- url = self.base_url1 + str(self.CaterId) + self.base_url2 + str(self.page_num) + "/"
4 N4 H) o% j* P ? - self.crawl(url, callback=self.list_Caterg,save=Cater_Name)
$ I$ | g/ J+ p1 K, _ - self.page_num += 1 3 e) w E" N' [+ v1 z
- # Y+ z+ d S) p) Y6 K8 H
- def list_Caterg(self, response):: E; n# ~, v8 Z4 e' V6 @$ u
- Cater_Name = response.save
7 @9 G# P2 M9 ^, b9 c - for each in response.doc('.pic-list a[href^="http"]').items():
7 E6 F. h P2 Y. u, U - self.crawl(each.attr.href, callback=self.list_Caterg_detail,save=Cater_Name)( H6 P& I+ r' i' @: [
- + A5 b) S+ l# k
- def list_Caterg_detail(self, response):7 u, N. \+ X* V; G) b
- Cater_Name = response.save% Y% p- M7 ^& I9 E
- # print Cater_Name
5 u* e2 F4 o# V: J( P, P5 O! L - Bookname = response.doc('h1').text()% S+ ~& M& J! K1 ^) e! V
- print Bookname
5 w% p- T) X2 | - Book_author = response.doc('.authorname > a').text()% K, {- @, P. `! {- n
- # print Book_author" ^ o2 Z8 y* }1 Z7 C0 U
- Book_Introduction = response.doc('.book-intro > div').text()
& I J1 }% w q, @ e( D - # print Book_Introduction
8 M- \) l4 f; r% C - Book_Synopsis = response.doc('b').eq(1).text()+ L6 x+ ?, N; y+ [' I8 \' h' \. _
- # print Book_Synopsis
: G$ [/ G+ y# V+ h4 ]4 r4 G7 P - Book_Palabras = response.doc('.booktitle p').text().split(' ')[1].split('|')[0]
1 e; G" |" X7 }; N) _! [9 w - # print Book_Palabras4 ~+ e. U5 u' f3 ~3 }* m, P
- BookIDs = response.url.split("xiaoshuo/")[-1].split("/")[0] #小说ID
2 w6 `' n/ s7 V% P& H3 |/ B - # print BookIDs( ^/ f& M2 W( ?
- Book_Dates = str(datetime.datetime.now()) + Q; l/ e. R" v C! V$ e% M
- for imgs in response.doc('.bigpic > img[src^="http"]').items():
$ d U7 \) F T- d7 g, s5 e f - img = imgs.attr.src
1 o$ f! B: v0 ^ - print img5 w" D9 S+ H8 N/ n8 r
- #小说封面下载/ R0 Z; M- Y4 A# V3 \
- extension = self.getExtension(img)
* R; K5 ]& C# b8 v% ?) q - name = self.getname(img)
' B* t- I2 h! ]) t1 i: K - file_name = name + "." + extension
! |: g9 B& n& O - imgDir = P_dir + name2 x- l& W- _! Q" {# `9 E# C4 ]
- Locaimg = imgDir + "/" + file_name
% R. n; n2 G+ M' y - print Locaimg
% B3 H M0 q" }4 n* t - if(self.download(P_dir, imgDir, file_name, img)): #这2行可注译,图片下载到本地
) t E4 ^$ g( B3 i - print('attachment url is ' + img) #( l2 G! _- j0 @
- Datos = {
9 p5 d1 u, e- @2 Q - "Cater_Name":Cater_Name,
% U/ e) p5 y3 z - "Book_author":Book_author,( P! b5 M+ O& v- d- z
- "Book_Introduction":Book_Introduction,
- A$ y; v- Q6 J* F; u - "Book_Synopsis":Book_Synopsis,$ Z3 _: A1 K& W% P
- "Book_Palabras":Book_Palabras,( z$ k4 e0 f+ v
- "img":img,# x" {% F$ |- s
- }& l9 k3 w/ y8 t' b, l
- self.add_BookFile(Bookname, BookIDs, img, Locaimg, Book_Dates) #这行可注译,数据库发布接口,方便其他系统的发布
- m1 s; C* Z- y& c7 X - for each in response.doc('div[class="bookbtn-txt"] a[class="catalogbtn"]').items():
0 o* H) u- G7 U - self.crawl(each.attr.href, callback=self.index_page,save=Datos)3 X7 A/ r6 {) j( P5 H
-
" \9 J4 R; w: N4 a6 z! X - @config(age=8 * 60 * 60) 0 R& d7 ?& t* f! S/ E
- def index_page(self, response): & I) A: H4 r2 G+ ?
- Datos = {
6 P! p2 k2 B; ~- A# X% W) o* h8 z - "Cater_Name":response.save['Cater_Name'],
; ?0 ?! _+ \2 z& s) E' q/ X4 Z - "Book_author":response.save['Book_author'],
) o7 h v1 p, ~& s& |6 N - "Book_Introduction":response.save['Book_Introduction'],1 P9 o0 N0 m$ \
- "Book_Synopsis":response.save['Book_Synopsis'],+ j- e; N, |/ t, i
- "Book_Palabras":response.save['Book_Palabras'],
1 ]! @2 b) `# A4 |8 n" ` - "img":response.save['img'],6 U" E n& `6 J% C+ J
- }9 j' D2 k, F1 Q! v
- for each in response.doc('.chapter-list li:first-child a[href^="http"]').items():
% U4 N, ?9 H( K; x - # for each in response.doc('.chapter-list a[href^="http"]').items():
( j7 _ n5 u. o. \5 b" L& s - self.crawl(each.attr.href, callback=self.detail_page,save=Datos)
6 f. R4 C$ p! M7 Z2 l - @config(priority=2)
6 M' W: t* m4 v: a - @catch_status_code_error
/ t/ ^7 m: A1 |# v7 Z) ?7 v' p; V - def detail_page(self, response): " G6 p p% d0 m- U
- NewRe1 = u'哈书' n3 Q, j, b. d5 H
- NewRe2 = u'huhjsd.CC'
" o" Z4 C# @7 y - NewRe3 = r'^\\n\\n'/ |7 i' [) i3 f0 l" Q$ k* v M
- NewRe5 = u'小说网'' v; P9 O8 T. b
- NewRe6 = u'fgdfgf'+ ?% X4 ?$ B; g* V7 _. I
- NewRe7 = u'fgfgf'1 c& Y. C9 Y5 w( l6 c4 K8 w
- NewRe8 = u'ffhgf'
/ F5 D+ E1 Q# `( k - NewRe4 = r'[\f\t\v+\.\{\(\)\}\!\/_,$%^*(+"\')]+|[+——()?【】“”!,。?、~@#¥%……&*()]+'
/ j! T8 j8 e, x/ E# M' _* V0 B# } - ReC1 = u'静思'
/ V# b f- g( H: H' @1 I2 L! Y - ReC2 = u'aghgf.com', ~0 U0 X0 B* Z5 }
- ReC3 = u'aghgfh.com'# ?+ Y( R0 K/ S
- ReC4 = u''* Q& \8 W: M2 p, S9 M+ i8 ~% l: G
- ReC5 = u'文学网'
2 Q: C3 ^7 ^) P: ]2 n - ReC6 = r'<BR>'6 F$ h+ B. P4 A& |- h) |2 s
- Bookname = response.doc('.readlocation a').eq(2).text() #小说名称2 F, [; z4 ~5 J0 z" w
- print Bookname
R9 H. a. Q; N$ J( l s" B - Cater_Name = response.save['Cater_Name'] # 小说分类
% D G) r9 G" H/ p, I - Book_author = response.save['Book_author'] #小说作者/ y: T- l! [) m# B; j& B5 A) m1 `& p
- Book_Introduction1 = response.save['Book_Introduction'] #小说简介/ z2 h: G1 x/ m* Z E
- Book_Synopsis = response.save['Book_Synopsis'] #最近更新
/ g9 k/ d' F) t7 q3 S: s - Book_Palabras = response.save['Book_Palabras'] #小说字数
2 D ? n7 ?6 I6 R* H/ r' p# O. B - Bookurl = response.url #小说网址& ~% C5 X; F* d% X" g) `$ B% ~' }" r
- Booktitle = response.doc('.article-title').text() #章节名称
: T- a) ~) y7 b" e. _( ^) ?/ j - BookID = response.doc('.readset-r span').text() #小说ID
0 e; n$ Y9 r' }2 ~6 E% O. g/ R. b* V - BookConte1 = response.doc('.article-con').text() #小说章节内容
0 M8 F7 G) N# t/ _0 _3 i - abover = response.doc('.article-title').text() + response.save['Book_Synopsis'] + response.save['Book_Palabras'] + response.save['Book_Introduction'] #小说状态(连载还是完成)4 W+ X- @; U9 I8 a: W: U7 z2 w7 @
- Book_Date = str(datetime.datetime.now()) # 采集时间7 U z* g% k2 Y. e" \7 a& I, F
- BookConte2 = BookConte1.replace(NewRe1 , ReC1)
0 T8 `- |2 ]( \5 |1 ]1 b - BookConte3 = BookConte2.replace(NewRe2 , ReC2)0 y5 e( x& Z( q% J
- BookConte5 = BookConte3.replace(NewRe5 , ReC5)' k3 C5 o1 A3 K/ J1 k# L
- BookConte6 = BookConte5.replace(NewRe6 , ReC2); c3 w' H# O$ T4 c& r. E& r
- BookConte7 = BookConte6.replace(NewRe7 , ReC2)5 H' q: j+ w: }
- BookConte8 = BookConte7.replace(NewRe3 , ReC6), L6 T( S5 M( h" k. P- @
- BookConte4 = re.sub(NewRe4 , ReC4 , BookConte8)
r1 F. A8 s) C - BookConte = BookConte4.replace("\n\n","<br>"). T" Q3 ^) |- c( [) [3 P6 c2 r
- print BookConte
6 h: N( u3 f& B' @ - Book_Introduction2 = Book_Introduction1.replace(NewRe1 , ReC1)
/ u% W, f$ C! Q h& q; B. x - Book_Introduction3 = Book_Introduction2.replace(NewRe2 , ReC2)
: @6 K$ U/ o$ o4 W! G( V9 F# n - Book_Introduction4 = Book_Introduction3.replace(NewRe3 , ReC3)( h2 o" s+ O- [& m
- Book_Introduction = re.sub(NewRe4 , ReC4 , Book_Introduction4)/ M# F f7 ]. I( E2 D+ l f
- Titleid = response.url.split(BookID + "/")[-1].split("/")[0]
/ x: Y! G0 u( F) O9 u - Book_img = response.save['img'], #小说图片
3 o7 J: l3 i5 F$ w - " I) @7 K5 @# @2 G2 w- @( |! H0 E
- #insert into MySQL 小说入库
H& ^3 W1 y) {' C - self.add_question(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布
7 R+ z- b3 N# x+ I" K - self.add_comment(Bookname, Booktitle, BookID, Titleid, Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布0 q5 O- }" E6 k Z- p- i
- #post提交发布/ }; y' `) a, t5 A2 k
- self.add_locoy(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover) #这行可注译,火车头发布接口,不需要可取消- |& X+ Q) C0 l! t& ?1 k
- Datos = {+ p+ v9 Q4 {% ]9 O5 b8 |. \6 Z
- "Cater_Name":response.save['Cater_Name'],4 X4 s7 j! f7 K: U6 P: g
- "Book_author":response.save['Book_author'],
1 P* i4 ?+ k8 C - "Book_Introduction":response.save['Book_Introduction'],- q$ b' v* t6 y0 [% B
- "Book_Synopsis":response.save['Book_Synopsis'],
5 C) w* w- c3 V2 o8 n% t% \" Y+ I - "Book_Palabras":response.save['Book_Palabras'],
% o/ a# D8 e5 B0 S/ C" M - "img":response.save['img'],
7 N& |0 ]' v' {) F Z9 D% } - }
$ K; r: q: v' z8 u - for each in response.doc('.articlebtn > a:nth-child(4)[href*="/xiaoshuo"]').items():" I) s& n: j7 u
- self.crawl(each.attr.href, callback=self.detail_page,save=Datos) 4 U1 l! p! e- M1 x" ^
- return {1 k/ Q4 G' X, b6 n% ~! J
- "Cater_Name":Cater_Name,4 s; e6 B, _; E* L* J: s
- "Bookname":Bookname,
+ x) ]& J# z# s; \8 y) @3 y3 g - "Book_author":Book_author,( U- E5 d" ^- K" I) Z6 C
- "Book_Introduction":Book_Introduction,
7 z2 _* \! E. m3 I - "Book_Synopsis":Book_Synopsis,
, p- {, G' L+ G4 ^ - "Book_Palabras":Book_Palabras,
; h7 w4 x2 C8 X3 o& W - "Book_img":Book_img,. x1 T0 p1 Y* r7 E3 \4 X
- "Bookurl": response.url,
' ]% p+ I% S/ I* A% | - "Booktitle": Booktitle,
& Z$ v; [" L6 H* j - "BookID": BookID,
% Q h+ o" j" e6 }) c/ ?+ _ - "BookConte": BookConte,
2 P7 s- v( g3 M9 ?( O. S2 Q' k - "Titleid": Titleid," R* F _. u& E1 x: _2 m, ~" u
- "abover":abover," W Q. T( X# l" b1 {
- # "Book_Date" = str(datetime.datetime.now()),
# E1 V9 l# t' T4 u$ O - }+ O7 ], W# @. {! U* X/ o5 a
- def download(self, P_dir, imgDir, file_name, Book_img):4 C! [5 Q8 H+ q9 q
- if not os.path.exists(imgDir): ; B1 q3 U. T' p# ?9 N0 \4 l
- os.makedirs(imgDir)
* Q( w' s9 ], R' { - file = imgDir + "/" + file_name
$ F3 e: {; G. _1 K9 m. {/ Q - # print file
: [% A3 Q' C- } - f = open(file, 'wb+')) e5 u1 m0 [1 r* U8 M
- imag = requests.get(Book_img)
+ C2 |. V# O ] - f.write(imag.content)8 S* ~% Q0 A% W8 b0 _
- f.close()
8 \5 o6 K& J- B - #保存图片前+ t y! n$ [3 L `% _9 o
- def save_imgs(self,response):3 p' E0 {+ k: T' O
- content = response.content7 ~% @ q" [5 y' j# ]3 l( w
- file_name = response.save["file_name"]# ~6 D! ~1 l3 Z, [6 n$ e+ n
- imgDir = response.save["imgDir"]
* L; s# g6 R3 J( T4 C' [% k! |# P+ [ - file_path = imgDir + file_name' B9 [- e5 D( p
- self.save_img(content,imgDir,file_path)
, s6 `. C) S) i7 M6 A - #保存图片
$ T7 V6 L5 J0 @& \ - def save_img(self,content,imgDir,path):( P$ V9 p' k2 ]* H
- if not os.path.exists(imgDir): 4 i( g6 \7 g* G
- os.makedirs(imgDir)
! `; _. U! k* o$ [! d# V - f = open(path,"wb" )
2 F( L% j3 Q$ ?- Y+ _$ } - f.write(content)7 f/ L I2 T- d
- f.close()2 M+ S# Z2 R: t9 u" M
- #获取url后缀名. h* z# x. X0 J O
- def getExtension(self,url): 0 _6 S, P! y1 ?2 x
- extension = url.split(".")[-1]# @6 J" Y4 a$ n: ]( P$ r# y1 R5 b
- return extension
6 N6 z+ h, {% i6 q4 Z& P+ _ { - ( w6 H$ X) h$ k3 g5 ]- C
- #获取图片名
( w3 a! C9 f$ |6 w! f9 X( g8 X) m - def getname(self,url):
% A. Z* t! g1 ?" H - name=url.split("/")[-1].split(".")[0]# m) D3 j$ e6 g& N! g
- return name
复制代码 0 M' t X+ k- {, B% B- t4 }
# R. d1 R# Z# }% \6 z! J |