Python + pyspider某小说站的爬虫,入数据库,火车头发布,资源下载到本地,另可写爬虫!
8 U5 ~9 T( M! l/ o' E- #!/usr/bin/env python
/ B, ]* }) U9 o - # -*- encoding: utf-8 -*-
, k1 Q4 W* C+ w/ V1 H - # Created on 2019-05-05 21:43:11* a0 }. m7 w# Z
- # Project: XiaoShuo" O4 ~7 Z6 y* v& E
- + m( L/ S3 y: X0 y) A& H
- from pyspider.libs.base_handler import *% ` R; h0 y# V% Y/ d, c3 E, F
- import pymysql
# N3 G8 Z9 r5 i/ v6 F - import random5 V$ h- z6 Y, P% }
- import datetime
( c. ] m) `2 `. ?7 x6 K! w8 D - import urllib2,HTMLParser,re
3 t+ H' z. e/ p - import os8 L- M: {) [4 e2 }3 {$ h" u
- import sys- T& ?% d' j' @1 n2 o0 v$ Y. m1 r
- import re; o$ ^+ [! c- X; H; I: \
- import codecs
6 g! n3 s" ?$ w4 o - import requests
3 z( r E. Z A" N% X - import json
, p4 Y- v. y& y- Q7 M/ _ -
# _( r0 i' a* b9 w/ U% k1 y - class Handler(BaseHandler):
) a% a8 I7 V7 U. ], g9 B5 }1 z - global Datos
& c* y& q/ X9 z! q2 { - global P_dir
/ Z8 W: e3 J, ]( X# [ - P_dir = '/Tools/Debug/' #采集时候图片保持到本地的路径
) A! f8 ]; t( l' k U4 T( Q, V - global Datos- F: [, I0 z% U4 G2 g0 u! v- x
- Datos = {}# H* W. I2 W4 L: ]' ^
- headers= {
' k- m9 M6 V3 J$ _# } - 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',; D7 Q/ t( B$ G r7 t9 u7 W
- 'Accept-Encoding':'gzip, deflate, sdch',
1 V& C. U0 L6 }9 k, j' u ]0 ~ - 'Accept-Language':'zh-CN,zh;q=0.8',2 C7 a& I; p' Q9 x0 W9 w) z
- 'Cache-Control':'max-age=0',# \: ?+ [3 n/ U
- 'Connection':'keep-alive',
' m6 A# y; \: W. R$ n' k+ n( { - 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'0 Q; y N* p0 b! {
- }& b. d3 |1 K9 ?
- crawl_config = {7 y5 n& l+ e7 d9 f- {* G! n
- 'headers' : headers,
% R3 Q9 D5 ~+ y- W) a; Y0 @! y0 G - 'timeout' : 300. H" P0 p: E% d% l H( G
- }
& h* A6 @5 r: k( T6 y# | - def add_BookFile(self,Bookname, BookIDs, img, Locaimg, Book_Dates):
2 R4 i1 C8 S B/ v1 g - db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")5 \9 \/ C6 }: U* F
- try:
% \) v9 X \6 ?( F" ]* C9 H - cursor = db.cursor()# m$ r0 x7 @4 y: J: W! E
- #注意此处字符串的占位符要加双引号"%s"+ q. k q9 F; x2 G) |( z. ~
- sql = 'insert into BookFile(Bookname, BookID, img, Locaimg, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, BookIDs, img, Locaimg, Book_Dates);# V+ i7 F/ {8 y) ^2 ~& E
- # print(sql)
3 `, o+ l% M7 k% T l4 @" p - cursor.execute(sql)
) _% @4 [- i. w -
/ e1 \1 u1 N2 E+ \' Q% G - #qid = cursor.lastrowid
7 G4 U9 l, c3 {! f$ u - #print(qid)- X) m& }) i9 s: U4 g3 O
-
v# y1 K& z1 K6 w+ |; X) G# [$ O - db.commit()8 ~7 ?$ ~$ m' ]8 n
- except Exception as err:
1 [' R1 G% y% e9 v - print("Error %s for execute sql: %s" % (err, sql))
; v0 H, T) }5 O7 v* h. y - db.rollback()
. [3 Q0 t2 O# Z' U3 O6 b - def add_comment(self,Bookname, Booktitle, BookID, Titleid, Book_Date):; E* ^' x$ [. q: i
- db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")2 O6 I( {8 H3 y6 u
- try:
( J m e$ n- ?8 |. g+ D - cursor = db.cursor()1 I$ L: a9 O c5 E0 C
- #注意此处字符串的占位符要加双引号"%s"' U1 x1 T2 P0 G- E; g. U) L& _$ \" ?
- sql = 'insert into BookTitle(Bookname, Booktitle, BookID, Titleid, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, Booktitle, BookID, Titleid, Book_Date);
7 p/ |% z+ M6 [- P6 i( } - # print(sql)* {, G9 ~: _9 h; [8 K
- cursor.execute(sql)9 m1 {1 p& C! L5 U( \
- : |, ^/ W' v& J! f3 T0 g2 o
- #qid = cursor.lastrowid
; G; W* ]& {$ I$ d, o4 G - #print(qid)- O3 q+ d) |( \. h: j
- ) d3 Z; ^% Q- R: P0 Z& U6 F8 s3 y
- db.commit()+ ?) ]+ M+ @3 [. c8 [7 h A
- except Exception as err:7 X5 n: d, u* |; L6 N7 f
- print("Error %s for execute sql: %s" % (err, sql)); p* p: w7 c p# h# h0 [* Q( k4 G
- db.rollback()
) Z9 u% W: q H8 L5 K8 @. t - def add_question(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date):
- `) U3 G' D8 H6 _ - db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")6 ?: ^% i' w& N' V4 e. g
- try:7 B4 }! R3 W+ s5 Z6 X: W
- cursor = db.cursor()
2 f; V" H1 ]. o' K6 Q) z: K/ C - #注意此处字符串的占位符要加双引号"%s") _/ @3 h3 U, A* Z
- sql = ' INSERT INTO BookConte (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) VALUES("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")' % (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date);
( E* j: A: a# Y( j - print(sql)
" G, ~4 v; V u h" l: J - cursor.execute(sql)3 @4 W {, E& k- G# C- M
- print(cursor.lastrowid)
H, {9 @! ?/ B - db.commit()/ `% \: ?. C; z; j# P
- except Exception as err:
* i' K4 S9 l$ z6 T4 ?) _/ ~7 h - # except:
1 E0 t; ^% Y5 w - # print('Failed')
( @3 x$ Z* n1 J; P2 O - print("Error %s for execute sql: %s" % (err, sql)) N7 n% Q7 L; k9 ]3 A# }# d
- db.rollback(): c. g# H7 c' ~8 u
- 4 n) t! ~2 \. @# x8 Q
- def add_locoy(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover):
) E9 r# z- o/ }8 v2 e# P - reload(sys)& Q) w% f7 _' r* t! e3 @8 E
- sys.setdefaultencoding("gbk")% n/ s# M& z' j* H9 I
- locoy_url = 'http://www.******.net/locoy/?my=book' #697火车头发接口地址
; e9 i# `# I! O% Z+ `! ?: a - locoy_data = {* K5 m8 K/ z: P- i* V
- 'my_u':'用户名', #后台用户名, U4 q( y; v/ ?
- 'my_p':'密码', #后台密码
, I9 b# s) w6 Y! o! S8 \$ r - 'subject_669977_net':Bookname.encode('gbk', 'ignore'),; m9 G$ P/ K8 F/ R P0 D
- 'caid':Cater_Name.encode('gbk', 'ignore'),
. F p6 l' u9 w - 'title_669977_net':Booktitle.encode('gbk', 'ignore'),
+ A e$ u M+ m1 |$ J0 ~% }9 e& ~ - 'article':BookConte.encode('gbk', 'ignore'),
8 N' q- j: u, Y9 F- Z# J - 'author':Book_author.encode('gbk', 'ignore'),
- f$ i5 [, p1 H, s - 'ready_1':Book_Palabras.encode('gbk', 'ignore'),
1 q. S9 k( b/ W6 w - 'thumb':Book_img,. a3 W( C; H0 b4 \7 J# w, d
- 'content':Book_Introduction.encode('gbk', 'ignore'),) T9 @4 p9 e9 ^, i! U% Z
- 'abover':abover.encode('gbk', 'ignore') # e( z3 t+ `' F# m
- }
, c/ R! g6 ?+ z' p - res = requests.post(locoy_url, data=locoy_data)
4 ^2 ~3 H0 G# i2 e0 H6 @ - print res.text7 D4 E3 h: z8 ], t2 P' S) e, \* a, R
- print res.content/ t+ m9 p# o2 r0 [
- # print Dsd
+ x# a) e. Q. ]2 g( g7 d" b - return res* ]9 F- o F" r: Y
-
1 i( M: ^& Y1 }. |' F1 w% f - def __init__(self):+ @5 u4 `* @7 V% y! ]2 x
- self.base_url1 = 'https://www.****.cc/'1 x8 \" A. n2 w; G) h7 F* J
- self.base_url2 = '/', E; t5 y( N0 A5 q( y+ ]1 v$ h
- self.CaterId = []4 C }2 b& D0 L8 C5 E/ o/ M
- self.CaterIds = ['xuanhuan', 'wuxia', 'yanqing', 'lishi', 'kehuan', 'kongbu', 'nvsheng']: [- C6 x: j8 C9 {/ t3 l
- self.page_num = 1
( U& p8 S, |5 E( v2 g& \; d - self.total_num = 200 ' z& F) v0 ^ ~* g! c) ]* z
- 5 Z4 a2 y2 U* x8 ?
- @every(minutes=8 * 60)7 i6 g9 D& c# X7 m
- def on_start(self):
& e8 @7 a; P- x" a6 W i. S3 a - global Cater_Name3 A m" v; s/ S! Y/ g/ d" U) a9 o
- Cater_Name = []' w6 l8 W$ ^, H& X
- while self.page_num <= self.total_num: 9 f; \, X* v( y' B/ q$ `' m
- for self.CaterId in self.CaterIds:
" B9 m7 v- Y3 F9 j+ P/ [& I4 X3 w* x - if self.CaterId == 'xuanhuan':
4 i* X! T" A* m - Cater_Name = '玄幻', O4 [6 l! C) y$ D7 S% P5 i) O E# q
- if self.CaterId == 'wuxia':6 z# G$ A+ ^4 L6 o+ ]$ Q8 r' _, G
- Cater_Name = '武侠'* Q- s4 i# h8 Q1 H, w- p
- if self.CaterId == 'lishi':
' C8 ~1 A: k4 U& C* P& @ - Cater_Name = '历史' ! z+ l# w% R8 c* D5 ~' W
- if self.CaterId == 'yanqing':
, V2 s, P3 Q; i4 o/ D8 ~) C7 l - Cater_Name = '都市' 5 _/ g# U: l9 s
- if self.CaterId == 'nvsheng':
1 t9 H6 H$ @* r5 f6 L# @ - Cater_Name = '都市'
* u- O5 _7 ^" L0 W - if self.CaterId == 'kehuan':
1 ?; G! r( t7 [( d' Q - Cater_Name = '科幻'
4 f n( |7 y9 A1 c6 z- q - if self.CaterId == 'kongbu':
" k/ H$ q( y0 w7 a5 ` - Cater_Name = '游戏' 8 F/ `; r. D8 U- v/ ~ Z3 P/ M9 Q
- print self.CaterId) Z9 T. _! \; Y5 Y e a
- url = self.base_url1 + str(self.CaterId) + self.base_url2 + str(self.page_num) + "/" q* ~) t$ N! ?0 B$ m" ` u4 b
- self.crawl(url, callback=self.list_Caterg,save=Cater_Name)0 S% L- K/ N* x# b- K0 g- R
- self.page_num += 1 # Y5 B4 F' \/ ]# K
-
( c5 s0 ^4 W; R' p& y8 } - def list_Caterg(self, response):
S/ Z4 H6 r& ~2 e P3 ] - Cater_Name = response.save
, p2 p* c, d0 X5 ] - for each in response.doc('.pic-list a[href^="http"]').items():
+ v, H1 {5 l/ ^% w: [! x ?5 P; g - self.crawl(each.attr.href, callback=self.list_Caterg_detail,save=Cater_Name)
. N8 R. F" u9 n1 `. p! h -
8 G) ~' ~5 w' Z6 S# p" B, Z5 m - def list_Caterg_detail(self, response):
7 \7 c; D) s& C5 q C - Cater_Name = response.save
! B1 C% ~( u2 C! ?3 B - # print Cater_Name1 W8 a6 Y+ ]7 M3 u; a7 s$ c! }
- Bookname = response.doc('h1').text()/ w+ Q2 v: w: [: G: l
- print Bookname# P' A( U9 t+ o& ^' `9 K" v! A A
- Book_author = response.doc('.authorname > a').text()/ H9 Z0 g( Q; k. R. f4 Q
- # print Book_author
: P) F- Y9 ~% w3 I8 k9 R - Book_Introduction = response.doc('.book-intro > div').text()8 c! F: t/ g* c2 z' b- X6 ^' Y
- # print Book_Introduction
, }8 r' Q$ v2 h* B% Q - Book_Synopsis = response.doc('b').eq(1).text()
$ B0 x- N- I% ^; H) Q4 L7 _ - # print Book_Synopsis
8 ^8 R& N" x& o - Book_Palabras = response.doc('.booktitle p').text().split(' ')[1].split('|')[0]
" W+ K5 f6 a$ k- T' o }' \6 t - # print Book_Palabras
% N5 z8 G2 m* S8 M- q; o: g& r - BookIDs = response.url.split("xiaoshuo/")[-1].split("/")[0] #小说ID7 A z4 E& q$ r7 r1 L
- # print BookIDs
+ a3 F8 _! L2 Y& [ - Book_Dates = str(datetime.datetime.now()) 5 {! S- N- }. M& T
- for imgs in response.doc('.bigpic > img[src^="http"]').items():. G# j( A3 I' N" ^
- img = imgs.attr.src7 P# f% O5 h, _" N& Q- f7 n3 G$ f& Y
- print img
' c0 U* i3 C& E4 { - #小说封面下载
- h& @. L+ Q3 ?2 S2 X - extension = self.getExtension(img)/ C, D) Z- E+ L6 X) [/ v
- name = self.getname(img)
1 Z+ k; v9 b4 @/ y. m$ l - file_name = name + "." + extension
; U7 s8 ~0 o, ]% T - imgDir = P_dir + name
( c4 G, e7 p+ D( L - Locaimg = imgDir + "/" + file_name
8 ]8 u6 ?6 E" {5 e/ O2 _ - print Locaimg
6 @& P' J7 \9 p9 b8 i - if(self.download(P_dir, imgDir, file_name, img)): #这2行可注译,图片下载到本地0 u# S% R- s& ]
- print('attachment url is ' + img) #8 k* z9 {) [0 s' v
- Datos = {* g1 b3 [6 x R( K. ]7 X3 {; [
- "Cater_Name":Cater_Name,
2 d; U, ?. _3 ~: Z* H - "Book_author":Book_author,
( l2 O" [" L- B - "Book_Introduction":Book_Introduction,8 r# h2 K' Y: ~" \0 ]
- "Book_Synopsis":Book_Synopsis,5 {/ H6 Z! ?& q0 A) {
- "Book_Palabras":Book_Palabras,: J. @/ |: C5 O
- "img":img,% N5 C0 e4 w+ Z/ x& P* b
- }5 A+ {$ R* {) Y" h2 |9 l4 M
- self.add_BookFile(Bookname, BookIDs, img, Locaimg, Book_Dates) #这行可注译,数据库发布接口,方便其他系统的发布
+ n3 l M d4 D - for each in response.doc('div[class="bookbtn-txt"] a[class="catalogbtn"]').items():) e: O+ W3 j+ B7 x9 q9 U
- self.crawl(each.attr.href, callback=self.index_page,save=Datos)
- ]8 ~. ]0 z& i& p0 }( y$ R& F -
1 C& o5 [9 M+ |2 Q1 K3 b9 X - @config(age=8 * 60 * 60) 1 H) q# M% J. I+ g- h
- def index_page(self, response):
6 U$ r& P; n3 U- C - Datos = {
, ]* [( q5 o4 q1 u- B# H8 p) s - "Cater_Name":response.save['Cater_Name'],, \# }2 [) R) n! X- l
- "Book_author":response.save['Book_author'],' P: W4 m6 ^- |# N! c" }
- "Book_Introduction":response.save['Book_Introduction'],
7 L* ~: I7 z+ R# B - "Book_Synopsis":response.save['Book_Synopsis'],3 \: J' `, J* }- @1 ~% V, _
- "Book_Palabras":response.save['Book_Palabras'],
1 z6 u3 \' X, w& Q& _) \7 r - "img":response.save['img'],; j, l5 ?, a J: w5 f! v3 T
- }
0 }. |; c: l v - for each in response.doc('.chapter-list li:first-child a[href^="http"]').items():) a) o T. b4 P* }0 s" x" G! V8 d% ]! s
- # for each in response.doc('.chapter-list a[href^="http"]').items():
! Y% p$ j. c( M* P4 O - self.crawl(each.attr.href, callback=self.detail_page,save=Datos)0 ~; g- j& V" b1 y- p% v
- @config(priority=2): C8 s9 p4 @$ L- C% ]# j6 L: n
- @catch_status_code_error" \9 y* C- t" U
- def detail_page(self, response):
# [' l9 N* n/ J: @ x; a - NewRe1 = u'哈书'9 e0 t+ e; s4 a% g) ]* @- R# O
- NewRe2 = u'huhjsd.CC'5 H* X2 Q6 K0 c- N1 n+ y. K
- NewRe3 = r'^\\n\\n'
/ a4 o1 h) y% b5 \8 w/ M9 k - NewRe5 = u'小说网'
- h, c: S( D* l$ d$ x - NewRe6 = u'fgdfgf'3 F1 k, n/ N9 J7 H. r/ k! A
- NewRe7 = u'fgfgf'/ k8 f! C9 I$ N+ L
- NewRe8 = u'ffhgf'* G# I; \2 \9 @/ v
- NewRe4 = r'[\f\t\v+\.\{\(\)\}\!\/_,$%^*(+"\')]+|[+——()?【】“”!,。?、~@#¥%……&*()]+'
% U: \2 [5 U, x - ReC1 = u'静思'
9 a" U F1 S0 P3 R: r! K" M- S - ReC2 = u'aghgf.com'6 ^ h; }% [' ~! f
- ReC3 = u'aghgfh.com'9 k: e; G7 X5 Z
- ReC4 = u''
/ z0 w$ v# t& F% r% P3 M9 B- Q1 p - ReC5 = u'文学网'
) R- U) z1 i2 V - ReC6 = r'<BR>'" Y) y. ~+ n' c& ^( @
- Bookname = response.doc('.readlocation a').eq(2).text() #小说名称1 n3 g: @7 ^" U, g* M
- print Bookname" @ e$ \3 U* a
- Cater_Name = response.save['Cater_Name'] # 小说分类
5 j! ], [, ^; K7 L- ?* o - Book_author = response.save['Book_author'] #小说作者
( d. ^8 I' J/ F( Q1 l5 F - Book_Introduction1 = response.save['Book_Introduction'] #小说简介
8 @ U; x; [3 p. C - Book_Synopsis = response.save['Book_Synopsis'] #最近更新# [5 U9 n; p$ q
- Book_Palabras = response.save['Book_Palabras'] #小说字数
- V& c/ I" e' u$ ^$ C; h2 ^ - Bookurl = response.url #小说网址
& u. S4 u! V" F% v* b( T3 s - Booktitle = response.doc('.article-title').text() #章节名称
$ ^2 w; g- a2 { - BookID = response.doc('.readset-r span').text() #小说ID
2 r: h; S, E: N2 _1 v - BookConte1 = response.doc('.article-con').text() #小说章节内容; [; I3 }2 X2 u- q( T
- abover = response.doc('.article-title').text() + response.save['Book_Synopsis'] + response.save['Book_Palabras'] + response.save['Book_Introduction'] #小说状态(连载还是完成)* \+ S1 o- l0 S
- Book_Date = str(datetime.datetime.now()) # 采集时间* U% S9 e" O$ ^; _
- BookConte2 = BookConte1.replace(NewRe1 , ReC1)
- o& Q8 |: i: ]# _$ E$ B5 w - BookConte3 = BookConte2.replace(NewRe2 , ReC2)
4 F6 e" n# `) B5 ^- N$ ` ? - BookConte5 = BookConte3.replace(NewRe5 , ReC5)7 O- t: W( A9 ~# f& R& X+ K1 M9 x* f
- BookConte6 = BookConte5.replace(NewRe6 , ReC2)2 N( w! a+ H( ^! y
- BookConte7 = BookConte6.replace(NewRe7 , ReC2)
4 V, J9 E4 K) S \1 @ - BookConte8 = BookConte7.replace(NewRe3 , ReC6)
4 \* Z# v$ q$ G) k - BookConte4 = re.sub(NewRe4 , ReC4 , BookConte8)6 v5 D% M- p& t" k7 q
- BookConte = BookConte4.replace("\n\n","<br>")
! a- ]! u, h# T4 { - print BookConte
! l8 f7 w) y, |3 Y* \3 j0 X - Book_Introduction2 = Book_Introduction1.replace(NewRe1 , ReC1)
% Y$ I, o. M& j- l+ E, b! h4 Q1 | - Book_Introduction3 = Book_Introduction2.replace(NewRe2 , ReC2)% i5 E. [9 ~- U! W
- Book_Introduction4 = Book_Introduction3.replace(NewRe3 , ReC3)8 B- l) E2 I( H, g
- Book_Introduction = re.sub(NewRe4 , ReC4 , Book_Introduction4)
" y' l# G8 [/ ~$ a' n8 M0 T - Titleid = response.url.split(BookID + "/")[-1].split("/")[0]
: j5 N- b8 F- M/ {, c - Book_img = response.save['img'], #小说图片: _ s) q8 J0 g
-
5 E% O. o3 Z- n) M- \7 U - #insert into MySQL 小说入库
, G9 D5 K( y" {" H - self.add_question(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布
& E5 B: c9 `: E: j - self.add_comment(Bookname, Booktitle, BookID, Titleid, Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布
/ H: h( O+ |# a# K3 j& }4 R - #post提交发布
8 N' ~0 A$ \, ^( p S - self.add_locoy(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover) #这行可注译,火车头发布接口,不需要可取消5 w) u4 r% K6 E, k
- Datos = {" m& \2 X$ B5 E, _! g U2 J6 s
- "Cater_Name":response.save['Cater_Name'],- O1 t6 T4 _- M' C( n5 p
- "Book_author":response.save['Book_author'],
/ Q8 i; Y- @' ~$ L% J) R - "Book_Introduction":response.save['Book_Introduction'],
6 w5 i" l* Q! K5 ^; O* ^ - "Book_Synopsis":response.save['Book_Synopsis'],
+ A6 g$ M. U. }7 {- m- I9 @ - "Book_Palabras":response.save['Book_Palabras'],
, o2 N6 G8 @: g6 @' n - "img":response.save['img'],
: _: `5 c4 R: P - }* q3 M) r' E# T. p6 p$ |
- for each in response.doc('.articlebtn > a:nth-child(4)[href*="/xiaoshuo"]').items():
6 R( ]. A; E# m+ V - self.crawl(each.attr.href, callback=self.detail_page,save=Datos) 1 e7 L) Y e2 ?: d, \8 J8 q
- return {! p0 }# }% Y5 t8 Z
- "Cater_Name":Cater_Name,! R9 U. v% d/ s h3 B/ W
- "Bookname":Bookname,
/ F% [6 u' G* I; c/ j# t3 C" F - "Book_author":Book_author,
' s' s6 Q8 g2 X- N6 n3 @2 l - "Book_Introduction":Book_Introduction,+ v( R5 }/ V! U' O0 g
- "Book_Synopsis":Book_Synopsis,
+ d) O6 p7 j0 a/ l$ H$ v - "Book_Palabras":Book_Palabras,
* e* u7 `$ f, u6 J - "Book_img":Book_img,, c8 l$ I/ A/ v- o
- "Bookurl": response.url,
; r- x9 Y2 P+ v, H% g+ I1 j$ P - "Booktitle": Booktitle,5 U4 j( F7 T1 B* D, S) g/ @8 q
- "BookID": BookID,5 |. x$ G( u, i
- "BookConte": BookConte,1 i& w: j$ o2 [" c1 I7 Z
- "Titleid": Titleid,: w9 g4 a, `# T, P
- "abover":abover,+ u3 @5 g8 L. O# L. x
- # "Book_Date" = str(datetime.datetime.now()),) |* u; i( J. U
- }' O1 O0 q2 X+ |: y5 Q) [
- def download(self, P_dir, imgDir, file_name, Book_img):' Q1 G/ Y% D7 g4 [3 q4 i7 ^7 \
- if not os.path.exists(imgDir): , N- v, _" I/ c& M/ n0 _
- os.makedirs(imgDir)6 s k8 ^ Z* X- a S
- file = imgDir + "/" + file_name
4 b$ N! {" F4 W - # print file
7 p; Z. s$ \9 z2 G( Y - f = open(file, 'wb+')) y" E$ o" O; @: i
- imag = requests.get(Book_img)
5 o, h4 O8 ]: D6 y$ N* v5 \ - f.write(imag.content)! q/ s2 H: [% C' V. Y
- f.close()
* x8 K6 ~4 f3 S% S6 }& r - #保存图片前( _" V8 g7 s5 g. F/ l
- def save_imgs(self,response):
. Z& O8 `2 @1 O8 u7 E0 P, ~( L" G - content = response.content
; ~7 K& O) [! M. k - file_name = response.save["file_name"]
+ r% O$ O$ T" q2 Q0 z1 _ - imgDir = response.save["imgDir"]/ B0 h1 q* `6 E# P' s ?; u
- file_path = imgDir + file_name
. g3 @+ ?2 L/ h - self.save_img(content,imgDir,file_path)
& @$ E( F) I3 O. i% L" x, l* a - #保存图片
+ `* O0 |8 @9 F1 k7 n/ m! I - def save_img(self,content,imgDir,path):
8 C0 y# p0 h/ H( o' a: ?! @ - if not os.path.exists(imgDir):
" F, ?/ a* P! { i: }; Y" U9 N - os.makedirs(imgDir); G' V2 T! I# I8 s1 |, r
- f = open(path,"wb" )% d! R! q$ q s9 k/ |6 H0 ~
- f.write(content): g' a6 \( s: E+ h# A
- f.close()# \% _8 T/ I; {$ h
- #获取url后缀名% b, {3 L* D$ C* J* [* C) M j% j2 r
- def getExtension(self,url): ( ^& n7 I( A0 o% U# `
- extension = url.split(".")[-1]
* m! l7 W7 [8 x' p1 S - return extension + k7 S0 {2 L' Y, t: @: E8 u1 i
-
) d% w3 |& W3 A5 [3 m% K4 C - #获取图片名6 L$ V2 h$ t+ P9 H6 x
- def getname(self,url):0 O9 ?: h6 ~8 v" Y' g1 S
- name=url.split("/")[-1].split(".")[0] s1 u) y3 q( V7 E! I
- return name
复制代码 - [( U+ D: h; k
6 x7 c. H1 n! U7 ^
|