Python + pyspider某小说站的爬虫,入数据库,火车头发布,资源下载到本地,另可写爬虫!& t* Q8 C! G- C( J' d. B/ N) ?
- #!/usr/bin/env python$ l% O+ S" R! V
- # -*- encoding: utf-8 -*-% a8 n0 ~0 ?0 U$ Z( P. h% G( ?4 \
- # Created on 2019-05-05 21:43:118 p0 _3 r) K% t
- # Project: XiaoShuo. e! H* r/ ]% {* o0 Q. k4 T0 [) l4 h3 @
- : o. N% Z! s% e2 o% O$ r
- from pyspider.libs.base_handler import *
R. ] x- s2 e% Y/ e! R - import pymysql
5 C, [# |% B- L; _9 \4 r1 H/ n - import random7 l+ a0 v1 {7 c, g! l0 q6 e9 x% @
- import datetime4 ]5 o& u/ q2 G$ X& N8 F7 F- m
- import urllib2,HTMLParser,re6 D9 C8 g5 ~: t. x) K& ~3 M% ]
- import os
c k3 @! l" Q - import sys' C% S* | G# m0 p0 O, J
- import re
0 w! f. c& P: ]/ G8 n6 z - import codecs
- ^! g' c1 s0 m4 I4 D - import requests2 e, g7 m: S8 T0 I1 [2 d
- import json
7 C) w7 x- b, H2 {- P -
, a/ z5 ?5 _) t, {8 P - class Handler(BaseHandler):
7 R% l% Q0 x4 I6 p+ z - global Datos" ~2 @8 e) A5 Z& c
- global P_dir
- S# |/ o! H# ~) E - P_dir = '/Tools/Debug/' #采集时候图片保持到本地的路径: F% d& M3 @. e% X
- global Datos
+ K. G, @& W S% w3 V: ~* W7 x1 y8 K - Datos = {}7 K" n9 B& M' O2 q2 e
- headers= {
, v# D" e5 x" D* L- q - 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
" ~2 x6 r. k: Q# W - 'Accept-Encoding':'gzip, deflate, sdch',
A/ ~. \7 ]0 A I: I. L! ~6 J - 'Accept-Language':'zh-CN,zh;q=0.8',1 y6 N( U% ? ~- }
- 'Cache-Control':'max-age=0',
9 r2 G8 r$ W% J2 ^) S; g. R+ X/ B - 'Connection':'keep-alive',
+ b: J* K5 P, C; X( I' w, O - 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'4 j! A5 u P5 v4 S
- }5 u; ~0 V( H' ^
- crawl_config = {
" b1 H- B' }: B# \5 Z# V: X+ J - 'headers' : headers,$ O. D2 |1 ]8 D4 K& k4 n" c" K3 ~
- 'timeout' : 300: l: u _/ ]/ t3 B. q
- }
r$ _0 ^, i6 k. e O - def add_BookFile(self,Bookname, BookIDs, img, Locaimg, Book_Dates):
5 |( H5 V) T. |7 p! }7 R - db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")# \# J4 m9 X7 z# a$ H- [, w
- try:. S8 {0 J: F- b* f8 z% Y
- cursor = db.cursor() {4 g4 w/ V& J4 [1 H
- #注意此处字符串的占位符要加双引号"%s"0 |+ ~* z7 o( a" Z9 J9 T2 l q
- sql = 'insert into BookFile(Bookname, BookID, img, Locaimg, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, BookIDs, img, Locaimg, Book_Dates);3 r1 y+ j* d/ h: e8 E
- # print(sql)9 N1 S3 ?0 _. @6 j. \
- cursor.execute(sql)! M/ v' e4 O( y+ A( X5 _$ A. P. z2 s
-
/ @; W v8 R% Z2 W) D' \2 I0 j& ~ - #qid = cursor.lastrowid; A6 Y9 T; C* S" D
- #print(qid)
5 C6 s* d2 v ^7 c* r - # l6 K3 v* A. l- X8 M4 c3 Y2 a, T
- db.commit()- t( S" g, e; \/ `% o+ T1 E
- except Exception as err:
" M, J( h, X# U% ]! y8 p. [- i - print("Error %s for execute sql: %s" % (err, sql)). _7 [8 e. k" \+ F
- db.rollback()
& N% \" Q+ s1 E& s) Z4 S - def add_comment(self,Bookname, Booktitle, BookID, Titleid, Book_Date):
1 ^5 G9 M9 T$ j: }' y) ?+ N - db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")# Q; V# o `; s+ i9 x( _+ E5 o+ Y
- try:; U! _2 s* H- D6 w$ N
- cursor = db.cursor() [1 \- N1 s- y$ k" z
- #注意此处字符串的占位符要加双引号"%s"
3 ~: ]" z0 L6 l" |. {) h, Z - sql = 'insert into BookTitle(Bookname, Booktitle, BookID, Titleid, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, Booktitle, BookID, Titleid, Book_Date);6 r* t) F5 M" P0 f0 j( y2 V5 A
- # print(sql)
' B. M K4 Y# ~9 L6 S - cursor.execute(sql)7 v T! S" q8 @* Y& r
- ( |# D8 x3 m& K+ M& s3 X1 n- q
- #qid = cursor.lastrowid
0 S4 w$ N+ x! m7 R+ K7 D - #print(qid)0 ?6 u8 H5 C% m, |
-
9 }5 e( }- h1 D1 F* Q. S - db.commit()
% O- D; W( ~: I2 L" j - except Exception as err:) n k1 } o8 E% ]$ t+ {7 ^
- print("Error %s for execute sql: %s" % (err, sql)) _/ L7 W9 x( M
- db.rollback()1 c- j- r8 Q( A' `
- def add_question(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date):+ g+ ]% F# k; u+ a9 o
- db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")
8 y: s3 Z3 S* H" h' A9 Q0 N - try:$ y$ ?8 D/ f4 q- w! n4 U' x; k: I
- cursor = db.cursor()
% ?7 ~. }' f$ |, d$ c7 y% V+ N% @ - #注意此处字符串的占位符要加双引号"%s"
7 t2 L4 s( h; u! b; z) ~2 N+ ? - sql = ' INSERT INTO BookConte (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) VALUES("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")' % (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date);
' b4 U% m) I5 |2 ^$ w - print(sql)
+ D: T1 D& \8 L+ ^2 p9 ?) x: e - cursor.execute(sql)) I6 ]% N3 s& H! S% B3 ~
- print(cursor.lastrowid)' w0 R, E1 ^8 k1 H y1 }$ t, Y
- db.commit()
0 a; e3 }1 f# w7 ~8 a9 s6 A - except Exception as err:
g! M" ^7 T+ l8 Q# b - # except:
3 U" K+ G" M! a$ Q+ c1 ]/ d - # print('Failed')3 B7 h6 m4 q; ~( K' V" ?
- print("Error %s for execute sql: %s" % (err, sql)): ^0 s, T: G2 j
- db.rollback()* S2 L- I+ @3 c. E2 d R$ n
-
$ N7 N8 p+ @6 C - def add_locoy(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover):
# k; m) `8 o$ ]/ l2 H9 `, _) ~9 d' ?' [ - reload(sys)
+ t0 U H( x* E5 K% D; ?" p - sys.setdefaultencoding("gbk")7 \1 p' X& I' L$ R' D# B* }- J* F d
- locoy_url = 'http://www.******.net/locoy/?my=book' #697火车头发接口地址
: N$ D" b d8 C+ W - locoy_data = {" F5 P0 a8 U4 y6 }2 f
- 'my_u':'用户名', #后台用户名( l) F2 b @7 [$ K
- 'my_p':'密码', #后台密码
" n. c$ Z, K4 t; `( |" M: J [ - 'subject_669977_net':Bookname.encode('gbk', 'ignore'),
3 t# Q6 O( H1 T5 G& z* |9 w' { - 'caid':Cater_Name.encode('gbk', 'ignore'),
1 q. D8 e, T; c. z - 'title_669977_net':Booktitle.encode('gbk', 'ignore'),
) w: P. P7 o8 @6 b! i3 a - 'article':BookConte.encode('gbk', 'ignore'),
- B, }7 N. R/ E, g3 j' m7 I1 Y - 'author':Book_author.encode('gbk', 'ignore'),3 {* {5 l, H$ p0 I& ?& p
- 'ready_1':Book_Palabras.encode('gbk', 'ignore'),
. b- _# P$ e' }4 }/ m8 P5 A) J! J, c - 'thumb':Book_img,
' D2 A! d! i" ^& ] - 'content':Book_Introduction.encode('gbk', 'ignore'),
7 o8 k) n, \5 b* R - 'abover':abover.encode('gbk', 'ignore') ) ?' @6 a. ]' E
- }
5 q6 z5 C, |" @. A, h3 { - res = requests.post(locoy_url, data=locoy_data)
8 {/ a- w- t+ t! b2 o. L - print res.text
7 r& p# J0 J! u( O - print res.content
( ^9 @1 M' @7 H& n2 _ - # print Dsd
5 J0 R" g0 ^( t# T: o - return res
/ r, X8 I, o# R+ d - ) o T9 c% d+ Z, Y2 d# k6 h% {0 b
- def __init__(self):* X1 T6 i+ Z/ w6 r3 ?% t5 v
- self.base_url1 = 'https://www.****.cc/'
6 I, k7 }! U+ Z! _0 C! h - self.base_url2 = '/'5 v( _$ n$ C5 e: C( g. f8 F
- self.CaterId = []& T: |5 \5 O1 \( ^( d2 ?
- self.CaterIds = ['xuanhuan', 'wuxia', 'yanqing', 'lishi', 'kehuan', 'kongbu', 'nvsheng']
+ w- F" O% M% C; P1 |: _) N. @' z: b - self.page_num = 18 N3 h) O7 {4 K {, f
- self.total_num = 200 7 R5 `4 R" n7 N* z; h* E
- % r+ c7 q) e$ {+ u* s8 Y
- @every(minutes=8 * 60)
: L- I, F+ L$ B! S - def on_start(self):
. S+ x; U+ P0 x - global Cater_Name( g' M4 [- }; a, m9 B2 _. z4 O+ A
- Cater_Name = []
' G+ c1 O& |1 X c) } F* a - while self.page_num <= self.total_num: " c5 h+ _' V/ b
- for self.CaterId in self.CaterIds:. G1 }6 Q+ r# w: L$ B+ u
- if self.CaterId == 'xuanhuan':
' g; g6 U% v; R, U) t+ A- Y - Cater_Name = '玄幻'( M& L1 X9 i# J6 a3 ^& h) a0 Q( |) {
- if self.CaterId == 'wuxia':
: ?2 `% p) k8 M - Cater_Name = '武侠'6 `( Q+ k* K' \, U7 l) S: D$ K; {
- if self.CaterId == 'lishi':& w" H$ @* K% o5 M/ i
- Cater_Name = '历史' , i7 j# F" G# p
- if self.CaterId == 'yanqing':
3 M, N8 H4 X% @& y - Cater_Name = '都市' , N5 @5 P: j' p I4 {
- if self.CaterId == 'nvsheng':
" h- n+ V+ q3 f4 D3 |. W - Cater_Name = '都市'
7 v. u3 d1 T1 X, q - if self.CaterId == 'kehuan':# \# |8 Z7 R; j' v7 i6 {
- Cater_Name = '科幻'
+ O4 Q" |0 I2 f" y9 ~% e - if self.CaterId == 'kongbu':
$ Z2 z! r. W/ M" }' x - Cater_Name = '游戏'
! U3 Z" @; h. E4 i4 C* @5 f; R - print self.CaterId% P7 y$ j; H& D: B/ j) K1 D* @
- url = self.base_url1 + str(self.CaterId) + self.base_url2 + str(self.page_num) + "/"
/ h8 B" K4 ]3 q; U! J7 { - self.crawl(url, callback=self.list_Caterg,save=Cater_Name)+ ~7 U$ N$ H6 h2 ?3 W+ i) H1 [9 P
- self.page_num += 1
8 ?& @$ B( f( @$ j - % x4 N, e' s/ h9 j
- def list_Caterg(self, response):5 k" ^* a1 N7 x( P5 ~9 F
- Cater_Name = response.save
0 R& _: \& Y6 q, t" ? - for each in response.doc('.pic-list a[href^="http"]').items():
! r. O e8 G4 C$ }; |5 B - self.crawl(each.attr.href, callback=self.list_Caterg_detail,save=Cater_Name)
7 f6 r- i$ C+ k - n" u ], R0 S ^) C$ r- q
- def list_Caterg_detail(self, response):$ V' P: ?# M' \ a4 L1 {
- Cater_Name = response.save9 \" l, y( R+ e `3 m1 \9 \9 o
- # print Cater_Name9 F8 ~' u4 F6 G# \" O
- Bookname = response.doc('h1').text()
& h! J$ d# m, U9 }7 m: H - print Bookname7 ]* U* ?, M6 Q& A- j$ A# U
- Book_author = response.doc('.authorname > a').text()
0 N. l b k8 p9 c& L - # print Book_author
% G7 C3 ?& g% Y7 X$ e% w6 D' \ - Book_Introduction = response.doc('.book-intro > div').text()8 i1 n& j/ f( i
- # print Book_Introduction
( G5 H# C# r) Z- |6 ? - Book_Synopsis = response.doc('b').eq(1).text()
. I% s+ L0 v+ M/ { - # print Book_Synopsis
, f4 a L' `/ H8 E2 K - Book_Palabras = response.doc('.booktitle p').text().split(' ')[1].split('|')[0]3 z+ j5 l$ ] G9 ~5 p
- # print Book_Palabras
* J% F2 u* I2 H# J4 V" a9 b) C W# L - BookIDs = response.url.split("xiaoshuo/")[-1].split("/")[0] #小说ID8 c& K! @: B0 Y
- # print BookIDs
, D/ T3 F" _) o: m% v - Book_Dates = str(datetime.datetime.now()) , U8 V) g0 U# n- l! }/ a! G7 e
- for imgs in response.doc('.bigpic > img[src^="http"]').items():
4 C9 D* i' H P; [( o0 [9 ? - img = imgs.attr.src P1 Z; m C6 t, p* j
- print img) i1 Q( Q5 o8 z! F5 W0 O7 ^0 I" ]
- #小说封面下载3 g. K+ @5 B8 d5 a3 |" [- A
- extension = self.getExtension(img): r5 }2 N5 {5 |8 k2 C- b/ Q
- name = self.getname(img)
& m7 F% ~; B9 e( ~8 `# M. x2 I - file_name = name + "." + extension
5 V! t; d) @7 u6 g2 W - imgDir = P_dir + name7 v) n! B3 z% ?* x, i0 u- @" i6 x& s
- Locaimg = imgDir + "/" + file_name# _1 i+ a6 J/ t+ \# _: @ Y
- print Locaimg6 Z( c U, d3 q1 Z& x; B
- if(self.download(P_dir, imgDir, file_name, img)): #这2行可注译,图片下载到本地' K0 s6 O% R$ F/ g5 e/ I# U) a* C
- print('attachment url is ' + img) #, p8 {' J0 u# E! U4 i
- Datos = {4 x0 }8 S% t2 T& A' `" j" G5 c
- "Cater_Name":Cater_Name,
$ m) |* _% X6 \ - "Book_author":Book_author,
- s* e' M, |6 f - "Book_Introduction":Book_Introduction,5 Q7 Q4 G8 G1 @! v
- "Book_Synopsis":Book_Synopsis,
6 x) R1 h0 R" Q/ B, p1 V+ S3 x - "Book_Palabras":Book_Palabras,
. j0 M0 ?5 i" L" Q) x6 l+ T5 W; n - "img":img,
/ U# W- H% o2 T% `/ Q - }
/ u) x; f& e5 Q - self.add_BookFile(Bookname, BookIDs, img, Locaimg, Book_Dates) #这行可注译,数据库发布接口,方便其他系统的发布
. w1 p8 P; Q1 n# o* Q+ l% P5 ^ - for each in response.doc('div[class="bookbtn-txt"] a[class="catalogbtn"]').items():5 K% q; d4 }. z( E
- self.crawl(each.attr.href, callback=self.index_page,save=Datos)
: n _. a: L3 U7 t - " f7 f4 \) W- M+ p* [+ |( W
- @config(age=8 * 60 * 60)
0 T' Z7 x; S$ n; o" O# [) G - def index_page(self, response): 1 k1 K0 A% Q6 {; Q5 b$ [
- Datos = {3 q5 M( [ d/ b5 _
- "Cater_Name":response.save['Cater_Name'],) j4 O/ b2 @! C% C
- "Book_author":response.save['Book_author'],
0 I- ?7 w2 Q7 P - "Book_Introduction":response.save['Book_Introduction'],- y, ^: y+ ?! }0 I! ~5 d2 a4 l
- "Book_Synopsis":response.save['Book_Synopsis'],
% q6 g$ } X- I; A% |- b - "Book_Palabras":response.save['Book_Palabras'],* V7 Z2 f$ w7 c- N$ F7 A. G
- "img":response.save['img'],' C; |( G/ D7 Z7 ?9 n
- }
/ J3 j# J3 M% K+ w: X" j* r: m - for each in response.doc('.chapter-list li:first-child a[href^="http"]').items():
, {% }/ I: T$ a* c1 R0 R5 G; i4 J - # for each in response.doc('.chapter-list a[href^="http"]').items():
) y3 v! P l" i8 w; P - self.crawl(each.attr.href, callback=self.detail_page,save=Datos)
$ V. b! S, H- g' f, x: K( i9 t - @config(priority=2)
" g* C& s4 F$ _ - @catch_status_code_error9 q, w/ _8 N7 S6 @ Y; g
- def detail_page(self, response):
! a* a! L D" X, w4 } - NewRe1 = u'哈书'( I* W: o3 L9 \+ G( y
- NewRe2 = u'huhjsd.CC'
( w6 E+ a" y/ R O8 J9 w - NewRe3 = r'^\\n\\n'+ a( K0 q* |/ Q% B: `
- NewRe5 = u'小说网'
& K. a& s; [! i# u4 x; M - NewRe6 = u'fgdfgf'- ?# g% j, G9 d/ B' T
- NewRe7 = u'fgfgf') D0 f' n4 ^" L' ~7 ?0 S7 Z# b
- NewRe8 = u'ffhgf'% q4 ?8 H' V, Y& v: {
- NewRe4 = r'[\f\t\v+\.\{\(\)\}\!\/_,$%^*(+"\')]+|[+——()?【】“”!,。?、~@#¥%……&*()]+'
: ?. s# B8 \: [+ v - ReC1 = u'静思'$ M v" r/ a. x
- ReC2 = u'aghgf.com'6 H3 b# L3 i+ n0 t
- ReC3 = u'aghgfh.com'
+ L0 q& U1 t6 R0 g' r - ReC4 = u''
# k8 h+ ^, ~3 r0 L3 I3 a - ReC5 = u'文学网'* S5 g6 l/ X8 F) c) e
- ReC6 = r'<BR>'
) E5 J& z2 [, W& J2 k6 w; l( E - Bookname = response.doc('.readlocation a').eq(2).text() #小说名称* c0 B& h/ g) M, E
- print Bookname
" R" o, U$ l' g3 e - Cater_Name = response.save['Cater_Name'] # 小说分类
) R! q* j1 F0 K! E# ? - Book_author = response.save['Book_author'] #小说作者
2 z+ d# p2 c6 O6 h/ d" F1 E - Book_Introduction1 = response.save['Book_Introduction'] #小说简介
' `. h1 S5 z& L - Book_Synopsis = response.save['Book_Synopsis'] #最近更新
6 X: u+ F& k( f' L% b5 f, ~( x( y - Book_Palabras = response.save['Book_Palabras'] #小说字数
# C! g5 S, A5 q% n- P - Bookurl = response.url #小说网址
% p, ?6 W5 [) |" u4 u, Z; P( Q - Booktitle = response.doc('.article-title').text() #章节名称8 P7 M/ D& ?! u+ g( K( J- |) Y
- BookID = response.doc('.readset-r span').text() #小说ID+ ~1 y, Y9 X5 q) u2 }% u
- BookConte1 = response.doc('.article-con').text() #小说章节内容+ n A3 k5 J8 y& W7 U
- abover = response.doc('.article-title').text() + response.save['Book_Synopsis'] + response.save['Book_Palabras'] + response.save['Book_Introduction'] #小说状态(连载还是完成)
6 Y& {; `& t( u1 ~' S - Book_Date = str(datetime.datetime.now()) # 采集时间
( Y) O7 d# L) G; W - BookConte2 = BookConte1.replace(NewRe1 , ReC1)
1 i/ X1 s# H* Z P2 Y5 H9 [ - BookConte3 = BookConte2.replace(NewRe2 , ReC2)
4 L, I& }/ m% _/ c4 q( u F2 | - BookConte5 = BookConte3.replace(NewRe5 , ReC5)
9 Q( M" t) r9 Z( `4 V. Q3 f* e; F - BookConte6 = BookConte5.replace(NewRe6 , ReC2)) D9 z% V. k1 o7 l
- BookConte7 = BookConte6.replace(NewRe7 , ReC2)5 Q# B7 y/ Q9 ?0 I! C
- BookConte8 = BookConte7.replace(NewRe3 , ReC6)
- w/ Q4 B. p5 P9 c4 y, W9 _. N m. U - BookConte4 = re.sub(NewRe4 , ReC4 , BookConte8)6 c0 ^9 ]% G# C/ _
- BookConte = BookConte4.replace("\n\n","<br>")
- p M) m( }6 z* h. ` - print BookConte
9 a+ j- [- Q* d' V0 F - Book_Introduction2 = Book_Introduction1.replace(NewRe1 , ReC1)
( m4 s8 s7 c. A2 c! K5 L$ T - Book_Introduction3 = Book_Introduction2.replace(NewRe2 , ReC2)
% c- }8 z2 u5 w( K4 y+ { z( s& @$ z - Book_Introduction4 = Book_Introduction3.replace(NewRe3 , ReC3)2 [0 Z7 y! D1 m; f1 p" {+ k5 s( p
- Book_Introduction = re.sub(NewRe4 , ReC4 , Book_Introduction4)1 g2 n% W8 w: V' i
- Titleid = response.url.split(BookID + "/")[-1].split("/")[0] & Z0 Z* r Z- S4 S
- Book_img = response.save['img'], #小说图片
( R/ r! ^$ f4 u$ c5 y X -
8 z% j K/ J! a" H8 |/ U; @ - #insert into MySQL 小说入库
: ?" g7 V. X6 s) [ - self.add_question(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布$ e9 v0 ~! v. O$ A! M x$ o1 e1 Z
- self.add_comment(Bookname, Booktitle, BookID, Titleid, Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布! ~! ?# r8 i2 m. H4 N+ k
- #post提交发布; v0 }0 |. B7 H- b, T( Z) P. z2 A
- self.add_locoy(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover) #这行可注译,火车头发布接口,不需要可取消
" L) d$ @3 k& F; ]: c+ d - Datos = {3 L- a6 J2 p _3 ?
- "Cater_Name":response.save['Cater_Name'],
; r& @; _9 ~7 @! | - "Book_author":response.save['Book_author'],$ N% m$ `0 _5 B3 y" r& e
- "Book_Introduction":response.save['Book_Introduction'],
: R! n; @# U: P4 r. P2 j - "Book_Synopsis":response.save['Book_Synopsis'],
$ D2 e& |7 Y/ d N - "Book_Palabras":response.save['Book_Palabras'],3 Z4 q; @+ z. N* m8 e: U/ N
- "img":response.save['img'],; ]) E# |* q2 F# d; q+ x _& }- a
- }" C8 x2 z7 w( V* G+ X
- for each in response.doc('.articlebtn > a:nth-child(4)[href*="/xiaoshuo"]').items():
" L1 l/ m; I/ P' k - self.crawl(each.attr.href, callback=self.detail_page,save=Datos)
: X7 B/ `/ x! G - return {
6 s( m' D7 [3 G. A5 m) V! E3 B - "Cater_Name":Cater_Name,: D( ~1 v; }7 V B0 E, N2 X
- "Bookname":Bookname,
! X9 N7 r, E$ O' A - "Book_author":Book_author,
5 g* X' J' f. n7 S4 q - "Book_Introduction":Book_Introduction,
( R: i2 t# P, e# Y0 b( [0 i - "Book_Synopsis":Book_Synopsis,9 r( I' i; Q. y# |
- "Book_Palabras":Book_Palabras,
1 S2 ?0 ~; b; W: U - "Book_img":Book_img,
8 t# Z+ k \2 [; h# e - "Bookurl": response.url,
c: p" u3 H* ^8 d' X - "Booktitle": Booktitle,
6 p7 v8 M- Y7 I0 Z2 b - "BookID": BookID,7 Y4 N) \3 \/ l
- "BookConte": BookConte," W, m$ u+ @( c5 N1 m+ b
- "Titleid": Titleid,
" d# k9 d% V1 T* Q - "abover":abover,1 t2 H+ l z p$ v4 m
- # "Book_Date" = str(datetime.datetime.now()),) R6 n' N1 n" Q6 I" }( {
- }6 U4 I, v2 f8 r3 n- p
- def download(self, P_dir, imgDir, file_name, Book_img):
% i% d* k h4 _! M - if not os.path.exists(imgDir): - j/ D+ Q6 t; v3 b3 W3 R0 W$ ] }
- os.makedirs(imgDir)
9 g5 a- P2 X. s: @8 ? - file = imgDir + "/" + file_name" D0 K! P/ F ~ A3 ^
- # print file
3 _; v) i2 k. S, `. K% e - f = open(file, 'wb+')/ r n5 o- n$ j' j. \* O ~
- imag = requests.get(Book_img)
6 }( Y) b) m1 j" @1 u& l; j - f.write(imag.content)
( U. S7 l7 ]' d' O! E) F - f.close()
- A& d) j3 n( ~6 M6 J/ P; b8 J - #保存图片前
7 |: j7 }; M* q! \ - def save_imgs(self,response):
7 d3 O& R3 T' z* l' p - content = response.content4 ]" Q4 _/ u. X
- file_name = response.save["file_name"]
' Z- ]9 C3 W, t+ M( |$ E - imgDir = response.save["imgDir"]! T9 y8 @# z& n
- file_path = imgDir + file_name
. f8 }( B# f5 V& D$ j) w - self.save_img(content,imgDir,file_path)- _, O1 O7 T3 P0 e* s$ F9 Z
- #保存图片, l1 Z. Q. i- k
- def save_img(self,content,imgDir,path):
3 h. w! ]. v& B( M) S) r - if not os.path.exists(imgDir):
' i' A2 M( E) m9 _! R - os.makedirs(imgDir)% R. [& S/ p9 ^. N+ @4 S& y
- f = open(path,"wb" )# X2 a ^1 t4 v" U$ X
- f.write(content)
7 q p2 S2 Y0 Y - f.close()) v1 u0 h4 M) o$ y7 j
- #获取url后缀名4 I% ^; j, e$ E9 `! X+ }
- def getExtension(self,url): - X% l* B% N _/ u' k
- extension = url.split(".")[-1]+ F; b- k+ O9 S
- return extension
1 @3 a; u: N$ w' K' t, x: c/ [ -
+ e2 W0 ?* U# F% ^# O) s9 U; o - #获取图片名5 B! B, ^" i1 G' {* k2 E
- def getname(self,url):
/ m0 T; l% B/ n - name=url.split("/")[-1].split(".")[0]
3 @- x4 X3 B/ I2 I* k# H$ K' N! | - return name
复制代码
" N/ l) L5 I& `! i( j) Q' T
& t" n* Q5 V, }- ^& G- Z |