Python + pyspider某小说站的爬虫,入数据库,火车头发布,资源下载到本地,另可写爬虫!# O( }! I- r0 ^# s% P2 O
- #!/usr/bin/env python
" X' A2 _! T5 Z" I8 M0 }+ [ - # -*- encoding: utf-8 -*-6 T, Q$ g6 R8 K k
- # Created on 2019-05-05 21:43:119 t, P7 ?3 h% z% k, j
- # Project: XiaoShuo
8 E! s# w& n& P8 w4 A - ' o: t3 [4 J- _$ T* y! z: }
- from pyspider.libs.base_handler import *
/ o/ D# r B. ? - import pymysql
; j# X% \7 ^1 m4 R+ f* v& y - import random1 _& Q% G; `; d( c+ h
- import datetime
8 \( |5 X' f& N' K5 \- \5 [ - import urllib2,HTMLParser,re1 m L* g, k3 f4 S" y2 {& N$ F
- import os$ c( F- m; I* C) g
- import sys
& }- a! D- P# B# o - import re7 f( C4 v! ~- {1 @) N/ [' W
- import codecs
! T* ]% n/ P: |( @. p! y& L - import requests5 C8 B1 _; G% {8 l1 S& j2 D
- import json
1 [0 Y/ P& r v5 ^3 V6 U) e" e -
5 _" X1 j# Q* O) h - class Handler(BaseHandler):, l. n* z! T2 [( a V
- global Datos
6 o- J9 m8 n/ R' `' T - global P_dir
& [! n* B' F. x# v% l) f: E - P_dir = '/Tools/Debug/' #采集时候图片保持到本地的路径
/ O6 |" D4 B) L! x - global Datos6 L; ?7 m: F8 j! P- K) l
- Datos = {}7 \$ Z& t' d/ ^" k7 s2 _
- headers= {9 J& W) {, C; t5 Y5 h; B8 e7 M5 l- c; X
- 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
8 C* S; n1 L( P0 a - 'Accept-Encoding':'gzip, deflate, sdch',
& E9 p1 Y3 `2 f8 C0 ]8 h - 'Accept-Language':'zh-CN,zh;q=0.8',
" S. F6 n1 d% u" {% \" X - 'Cache-Control':'max-age=0',
, S( h. Q* Z/ ]$ [' z5 ^5 U, t - 'Connection':'keep-alive',
5 Z, t# d7 ^; M - 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'
2 K: |' U6 q7 B& _( r- k" _ - }
8 a5 {% V) } j- d8 K* t - crawl_config = {2 b1 ?7 q2 \0 }! V% N! I- o
- 'headers' : headers,
6 `8 M1 d' u5 Y - 'timeout' : 300
" b$ @' U) [3 u& Y0 B - }9 T* M/ a; q) ]" f
- def add_BookFile(self,Bookname, BookIDs, img, Locaimg, Book_Dates):
* E: [4 b& N9 l# \* C - db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")
; x! E1 r; r" P+ o& E5 i - try:
' z, i2 k# ~+ ^' h7 a - cursor = db.cursor()! M* p) ~6 H9 R {! k3 L& k
- #注意此处字符串的占位符要加双引号"%s"
* F3 L5 B5 Z% n5 B' @5 ~+ j) T$ A - sql = 'insert into BookFile(Bookname, BookID, img, Locaimg, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, BookIDs, img, Locaimg, Book_Dates);
* i- d: p, `4 ?# t8 O - # print(sql)
6 P6 o( f! r( H- V" Y7 N - cursor.execute(sql)
, A! F4 Y# B9 o -
5 [3 f2 V6 s$ p& T( l$ P k3 a - #qid = cursor.lastrowid
8 K' a5 V$ |9 B" _- O - #print(qid)
4 {# N3 p' E# i -
O+ X# V6 X K+ o! o! Y0 g" B - db.commit()2 | [+ n2 w; @0 {+ U" u- Y
- except Exception as err:7 `* i2 d2 S0 B
- print("Error %s for execute sql: %s" % (err, sql))6 S, ]! w+ d+ y, D
- db.rollback()
/ o, ?9 q P a - def add_comment(self,Bookname, Booktitle, BookID, Titleid, Book_Date):$ l) z, X' Q/ i4 i* R2 p- g% q
- db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")
1 f6 g5 {0 X; \2 k: V - try:8 G( U+ M2 N6 J9 @5 t
- cursor = db.cursor()
0 X+ p) V4 ?) h7 E' {1 q - #注意此处字符串的占位符要加双引号"%s") z+ h6 v- y# V% S2 {
- sql = 'insert into BookTitle(Bookname, Booktitle, BookID, Titleid, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, Booktitle, BookID, Titleid, Book_Date);
7 ]; e+ [/ L; k# w! C; j - # print(sql)
* C1 z* {$ u* `/ }9 u - cursor.execute(sql)3 p1 \ N; m$ m4 f' P2 S
-
7 H/ x$ m9 L3 o/ z - #qid = cursor.lastrowid5 _+ o; [* J. ^; {, ~( z
- #print(qid)
. G# L/ l* ]+ n* |3 V( l0 `: m - 4 a9 \) |8 t; }
- db.commit()) ?. `' d4 W" X3 e$ m3 E! r
- except Exception as err:3 [- m' X) X' I4 ~
- print("Error %s for execute sql: %s" % (err, sql))+ ^$ m( U( @, Q8 H
- db.rollback()' g) J5 t7 S+ G
- def add_question(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date):0 W; K( z" I+ l9 U
- db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")
9 K2 g, ~0 u! C( S, a5 S - try:& i, C! l3 a/ p4 O1 C6 s
- cursor = db.cursor()7 T6 g9 z- c0 `6 D4 }* v8 x
- #注意此处字符串的占位符要加双引号"%s"7 }; w5 N) \6 k7 z
- sql = ' INSERT INTO BookConte (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) VALUES("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")' % (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date);7 W% ~: d( v& @" p. a
- print(sql)1 i% l! S4 Q, ~. O
- cursor.execute(sql)
0 N0 f: q/ p8 b2 b - print(cursor.lastrowid)6 R& m6 s6 z' g, S
- db.commit(). j" ~1 N5 U( l7 t4 ]( o
- except Exception as err:
' ~# s h8 L9 m% K/ W7 S! r' }& f - # except:7 d& F" e- Q) O8 s2 d, S; s
- # print('Failed')# l4 `6 z6 v5 m0 L# l$ p
- print("Error %s for execute sql: %s" % (err, sql))
" z2 E$ h/ ?, f5 W! \1 K+ i9 W - db.rollback()
7 T) q' t9 @% c7 }+ } -
0 N# F: V0 X$ G4 N% K1 B7 x - def add_locoy(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover): 7 f1 B. B, |* L1 Z4 l: C2 _8 A
- reload(sys)- x: ?" }0 N5 J9 B
- sys.setdefaultencoding("gbk")
3 L& @) b. `6 n+ B9 O5 k: V. R, V - locoy_url = 'http://www.******.net/locoy/?my=book' #697火车头发接口地址# |- p6 f1 V5 V
- locoy_data = {
6 s7 R/ Z. e5 b& D - 'my_u':'用户名', #后台用户名
E4 h) R6 K) ~6 V - 'my_p':'密码', #后台密码
; [% b0 \" G3 I6 q5 V - 'subject_669977_net':Bookname.encode('gbk', 'ignore'),
" G. i$ P0 F1 E: g, Y# g - 'caid':Cater_Name.encode('gbk', 'ignore'),
7 h% O3 S4 c. j' [' m - 'title_669977_net':Booktitle.encode('gbk', 'ignore'),0 q( e5 e9 N) J% m8 S
- 'article':BookConte.encode('gbk', 'ignore'),% }9 P: h8 T! N8 y" h' v
- 'author':Book_author.encode('gbk', 'ignore'),+ r+ S4 m3 ?8 k7 ?
- 'ready_1':Book_Palabras.encode('gbk', 'ignore'),0 Z1 ]# x) v/ w' M* f+ {
- 'thumb':Book_img,
- c s2 N" Z/ X - 'content':Book_Introduction.encode('gbk', 'ignore'),
' I, A: C/ |! a' Q) h/ g - 'abover':abover.encode('gbk', 'ignore') 9 N, K& U \$ U3 [( z
- }& m4 x9 r! ~9 E8 f# i8 E
- res = requests.post(locoy_url, data=locoy_data)- C) ], t9 j5 M3 K1 q
- print res.text) b" `1 o$ R. h! S& s
- print res.content5 Q7 ]* ?) P0 W" `- d
- # print Dsd
* o7 _: g, @) c - return res" V' U" H! R4 k4 s
-
# A! Z% ^$ k" ~, K - def __init__(self):
$ V1 I% X p6 ` - self.base_url1 = 'https://www.****.cc/'6 @& }5 Q, Y- _; B% X
- self.base_url2 = '/': H/ G# V- h' z; H! }+ @
- self.CaterId = []
$ l' k5 U( T6 G i O9 D - self.CaterIds = ['xuanhuan', 'wuxia', 'yanqing', 'lishi', 'kehuan', 'kongbu', 'nvsheng']& ^1 S# a: i/ ]0 k# ^
- self.page_num = 17 T2 ~( s8 x3 O( H' K8 ~+ r
- self.total_num = 200 - e4 I4 H- b8 e' f3 @( B1 ^/ C8 V
-
9 c" S7 z T. u% \6 L+ M" Q - @every(minutes=8 * 60); y$ _$ f0 @! J5 z
- def on_start(self):. ^) }2 r% ?$ s6 i: ~) x! C3 s+ H: l
- global Cater_Name3 ?4 U$ K s2 C
- Cater_Name = []
Q& ^, [7 } N/ p0 B - while self.page_num <= self.total_num:
( `) q* g: m6 d0 l0 \/ M& d - for self.CaterId in self.CaterIds:
3 `5 V: y! m& w( ] - if self.CaterId == 'xuanhuan':/ O H; o# h# ?9 v
- Cater_Name = '玄幻'
) m' h4 W3 Q: G, | - if self.CaterId == 'wuxia':; n V7 `$ Q; z
- Cater_Name = '武侠'3 A* v" r1 \6 m! a
- if self.CaterId == 'lishi':7 p. y; w% \$ R+ _5 R( n
- Cater_Name = '历史'
' ?' E) C, O( ]7 [) R - if self.CaterId == 'yanqing':$ @1 b( f7 @$ i( C) b: B" T6 B7 k
- Cater_Name = '都市'
3 n# o8 ]. D Y0 V5 I; c - if self.CaterId == 'nvsheng':0 e$ O' p1 _( X9 a) z# p' `
- Cater_Name = '都市' 4 t2 k( G8 E- t: G# f7 z
- if self.CaterId == 'kehuan':
/ i/ i! O) Q5 H - Cater_Name = '科幻' " s( M4 u m$ T/ k
- if self.CaterId == 'kongbu':
' ?" }, p0 x |; R - Cater_Name = '游戏' , v; d5 T4 Z7 I# l
- print self.CaterId& L* m0 W3 v% ~3 h& v
- url = self.base_url1 + str(self.CaterId) + self.base_url2 + str(self.page_num) + "/"
2 \1 _: T! u- v$ { - self.crawl(url, callback=self.list_Caterg,save=Cater_Name)
/ t0 l3 ]7 F) G1 R) y; L - self.page_num += 1 8 z2 S1 v# \# V7 D5 X, q
- $ Q; o0 d& D8 J1 w( A! L( i
- def list_Caterg(self, response):2 m1 ~. X1 L# V
- Cater_Name = response.save7 K% ?+ N$ {- y5 V4 a
- for each in response.doc('.pic-list a[href^="http"]').items():3 s& |, O) q- i* E/ P, q
- self.crawl(each.attr.href, callback=self.list_Caterg_detail,save=Cater_Name)
, G S6 L& D4 U. D# a \9 |2 R -
9 L+ \! O# \3 a% H3 W9 B - def list_Caterg_detail(self, response):
- ]! ^# V) p3 [9 m( ?1 M - Cater_Name = response.save
) E' n+ T" k+ g0 L: n( P$ _ - # print Cater_Name: h0 y% x6 x; O! \
- Bookname = response.doc('h1').text()& m9 ?7 U/ N# F2 X0 [+ K
- print Bookname
/ T! Y1 q& d0 x! J0 E - Book_author = response.doc('.authorname > a').text()
- m1 J \) R- ? - # print Book_author
. Y& `5 z# f- |- [ - Book_Introduction = response.doc('.book-intro > div').text()
( W+ Z# m7 \3 X) W4 `8 W- z, L - # print Book_Introduction
) _4 p4 S4 j6 u% N1 G5 J. T - Book_Synopsis = response.doc('b').eq(1).text()! r' l4 Z6 V8 o' H8 v/ @* y
- # print Book_Synopsis
' G3 }2 ^# N# C5 u8 U - Book_Palabras = response.doc('.booktitle p').text().split(' ')[1].split('|')[0]8 j: H2 G, B7 e7 d! L2 h- q
- # print Book_Palabras
1 l' i; |% v; `, H) Y4 ` - BookIDs = response.url.split("xiaoshuo/")[-1].split("/")[0] #小说ID
' _, c, }4 K* d1 d - # print BookIDs
5 p9 x/ V$ M+ I - Book_Dates = str(datetime.datetime.now())
, t! O! f8 p4 p - for imgs in response.doc('.bigpic > img[src^="http"]').items():
+ y& _0 p. {* e W. L0 O0 y2 \ - img = imgs.attr.src
. P/ x/ X0 n, x - print img8 Z9 A0 X* K7 L+ Z# M- v, k" n0 y" L
- #小说封面下载
0 u( c9 U5 T1 H, e - extension = self.getExtension(img)
& i5 ]) q! j4 h8 M6 b$ Q1 X+ } - name = self.getname(img)/ O: B2 \, I% Q; v/ t, e/ z/ p8 L
- file_name = name + "." + extension
9 |0 y/ E4 i( J$ i7 h W& k1 B - imgDir = P_dir + name
! u% D; [8 ]$ e& { - Locaimg = imgDir + "/" + file_name
. L3 c" O/ ~3 r1 `% n: S8 F9 E - print Locaimg6 U, d9 ^0 |# x/ ]
- if(self.download(P_dir, imgDir, file_name, img)): #这2行可注译,图片下载到本地
* K* A; F0 I+ }" @( R& A - print('attachment url is ' + img) #
6 a2 d$ P o: ]: U9 e - Datos = {; P4 U2 K* u* M
- "Cater_Name":Cater_Name,
2 p' h; B! i; k8 E1 a% s$ ~ - "Book_author":Book_author,
1 b; h* J' r# V- S - "Book_Introduction":Book_Introduction,9 e5 ?4 J( \7 Y
- "Book_Synopsis":Book_Synopsis,
; i+ j* y/ X2 e* ~. Q/ F - "Book_Palabras":Book_Palabras,
4 O- P, {& F2 ]* O' M - "img":img,0 @" i8 k$ e3 f/ `9 q
- }
3 D3 Z A& o/ @8 V: x: A4 G2 ~/ P) f - self.add_BookFile(Bookname, BookIDs, img, Locaimg, Book_Dates) #这行可注译,数据库发布接口,方便其他系统的发布4 o, p6 O; @3 x2 D
- for each in response.doc('div[class="bookbtn-txt"] a[class="catalogbtn"]').items():
; k( j% w y, q5 L/ ^! B - self.crawl(each.attr.href, callback=self.index_page,save=Datos)! m# _) u9 Y. [+ j
- 1 H- J. e- @6 d
- @config(age=8 * 60 * 60)
% {# _3 A5 C( A. T+ l9 f - def index_page(self, response):
/ s7 K J8 i5 y# m8 E: O4 @. ~$ X - Datos = {
4 i! }. C9 ^, I2 g& P4 U" ` - "Cater_Name":response.save['Cater_Name'],
. V! a8 D: j, j# }, ?, `6 C/ Y" c. w) [ - "Book_author":response.save['Book_author'],
6 A2 ]- t/ G5 M - "Book_Introduction":response.save['Book_Introduction'],7 j9 K7 n5 u/ R5 [% A( o
- "Book_Synopsis":response.save['Book_Synopsis'],2 D( p! L4 \) H7 ^, G0 B
- "Book_Palabras":response.save['Book_Palabras'],
s& ?4 P4 w" ^/ v4 P - "img":response.save['img'],% _5 T0 \' H. F
- }) i; R5 i, b, p: Q7 M9 }, ^
- for each in response.doc('.chapter-list li:first-child a[href^="http"]').items():9 ^/ j* f E [$ X$ b
- # for each in response.doc('.chapter-list a[href^="http"]').items(): 8 d* F( r! D2 O
- self.crawl(each.attr.href, callback=self.detail_page,save=Datos)$ l8 B" Z! H3 G1 c. M+ Y
- @config(priority=2)6 P4 ~- p/ i* d! g& }) |
- @catch_status_code_error4 u; f* B& n/ \& o$ }8 {1 g. m
- def detail_page(self, response): # v+ e% O& A2 t. T- o
- NewRe1 = u'哈书'
, }7 [1 Y. p( r: P0 m6 c' P" B - NewRe2 = u'huhjsd.CC'
1 W* F9 R( B3 C8 S* K - NewRe3 = r'^\\n\\n'
4 j( M# j) U7 l6 u - NewRe5 = u'小说网'6 m5 @' O4 K& p9 ~& H* B
- NewRe6 = u'fgdfgf'
7 S2 p9 Q- X& W1 a8 j2 l - NewRe7 = u'fgfgf'
4 e5 L7 v" P' w6 l; ^ - NewRe8 = u'ffhgf'
0 `- ~8 \( i& e1 }9 T6 e" X$ U9 [; |# x - NewRe4 = r'[\f\t\v+\.\{\(\)\}\!\/_,$%^*(+"\')]+|[+——()?【】“”!,。?、~@#¥%……&*()]+'0 }7 T! c2 T* q9 B
- ReC1 = u'静思'9 h9 s+ K. |5 l
- ReC2 = u'aghgf.com'
& l& a' f: Y' Z( _7 z - ReC3 = u'aghgfh.com'
" X w* F6 a3 t6 T - ReC4 = u''( z# L$ s9 R# }3 w. o! c5 t
- ReC5 = u'文学网'
( k! i0 a8 A' D. K4 c - ReC6 = r'<BR>'
u* z. C* R" x+ C1 m8 }" T - Bookname = response.doc('.readlocation a').eq(2).text() #小说名称
j: z0 ], X! I; q - print Bookname
( t, x- z8 }9 C4 c. ?8 R0 Y( P O" t2 r - Cater_Name = response.save['Cater_Name'] # 小说分类6 w+ U# ^" l5 d. [8 G* e- ]
- Book_author = response.save['Book_author'] #小说作者
2 t% V D4 D6 Y - Book_Introduction1 = response.save['Book_Introduction'] #小说简介+ ]- l: z$ M) L8 e7 |
- Book_Synopsis = response.save['Book_Synopsis'] #最近更新
, Q2 X$ s2 [* [* S3 _3 E - Book_Palabras = response.save['Book_Palabras'] #小说字数
X) _' H# h/ L+ h - Bookurl = response.url #小说网址. C' p, e: n9 g
- Booktitle = response.doc('.article-title').text() #章节名称
: r# v( V5 t9 Q' {8 |. j5 C - BookID = response.doc('.readset-r span').text() #小说ID3 d) Z% x3 r" q, i- P3 I
- BookConte1 = response.doc('.article-con').text() #小说章节内容
C: v* {8 D9 o0 K5 S - abover = response.doc('.article-title').text() + response.save['Book_Synopsis'] + response.save['Book_Palabras'] + response.save['Book_Introduction'] #小说状态(连载还是完成)! e6 D; W; b3 n1 g m% p
- Book_Date = str(datetime.datetime.now()) # 采集时间
R; U, Y. Q9 R" D - BookConte2 = BookConte1.replace(NewRe1 , ReC1)
( K! B" T0 H6 O' w y1 r G2 } - BookConte3 = BookConte2.replace(NewRe2 , ReC2)
+ e+ E# r; ^: P( G8 O$ V - BookConte5 = BookConte3.replace(NewRe5 , ReC5)- k' ~! _3 ?% O" i% n/ ?# _1 r
- BookConte6 = BookConte5.replace(NewRe6 , ReC2)% G+ S6 ^, Q) `5 ^
- BookConte7 = BookConte6.replace(NewRe7 , ReC2)
- y# @/ H! w, ^+ E6 [7 z! p) ]5 N - BookConte8 = BookConte7.replace(NewRe3 , ReC6)
- ^6 j7 F! o9 r& A l$ D - BookConte4 = re.sub(NewRe4 , ReC4 , BookConte8)
" d; a, h' N. e* ~ P) e - BookConte = BookConte4.replace("\n\n","<br>")
4 `2 q$ O8 T* @) h: ^9 n - print BookConte5 F) \( \# U, Q8 I* O
- Book_Introduction2 = Book_Introduction1.replace(NewRe1 , ReC1)
6 t1 I3 i8 i8 B4 e - Book_Introduction3 = Book_Introduction2.replace(NewRe2 , ReC2)
3 o( N7 P8 A; T- c! K7 X+ G d) j - Book_Introduction4 = Book_Introduction3.replace(NewRe3 , ReC3): |: L1 C; N# `8 `) E
- Book_Introduction = re.sub(NewRe4 , ReC4 , Book_Introduction4)7 s- x* k: q+ g8 g9 w
- Titleid = response.url.split(BookID + "/")[-1].split("/")[0] ) n# G, [' D# o, C/ ^4 A
- Book_img = response.save['img'], #小说图片
3 j& n2 X+ s4 c" V: w% j$ {5 N -
7 m1 _ o4 B3 P- e - #insert into MySQL 小说入库4 L/ e5 @) t9 ^
- self.add_question(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布
' L. p4 Z" {3 W* d3 k8 `/ l& A - self.add_comment(Bookname, Booktitle, BookID, Titleid, Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布9 e9 P! a* b, {
- #post提交发布3 t$ G' p, ^* G) g3 t
- self.add_locoy(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover) #这行可注译,火车头发布接口,不需要可取消
- u* q4 v' _" \: d( A - Datos = {" j# U; F G6 d! C2 r
- "Cater_Name":response.save['Cater_Name'],
. P+ m! r- F5 e) B - "Book_author":response.save['Book_author'],
7 j. y: @' k* q8 P6 `) h - "Book_Introduction":response.save['Book_Introduction'],0 `- o) `9 ^: [: F
- "Book_Synopsis":response.save['Book_Synopsis'],
! \+ a0 z2 _) N! L$ s - "Book_Palabras":response.save['Book_Palabras'],
" Q/ v# N( N9 z* u# }0 @" t3 V: e; F' U - "img":response.save['img'],( r8 s3 ^; R/ c4 d
- }
# V7 t7 a* o" k8 k - for each in response.doc('.articlebtn > a:nth-child(4)[href*="/xiaoshuo"]').items():
& w" i- _7 ?; O9 V- g* u - self.crawl(each.attr.href, callback=self.detail_page,save=Datos)
! Z' w: C8 @# R/ ^ - return {% @4 C* b8 s6 `9 H& R" m
- "Cater_Name":Cater_Name,
- N9 c2 p* \7 {) |, ^ - "Bookname":Bookname,
J9 p$ _ i2 T* \ - "Book_author":Book_author,) R8 S5 T& t% g/ x
- "Book_Introduction":Book_Introduction,% v- r4 s, L& \2 e& h4 d- X
- "Book_Synopsis":Book_Synopsis,
/ q$ O9 Z) Y2 a6 H! E8 p/ b5 |& f2 O - "Book_Palabras":Book_Palabras,3 q3 V) }: N' E6 A( {
- "Book_img":Book_img,
2 w: t) k7 X" C - "Bookurl": response.url,' Q7 T9 I, t7 ?; n
- "Booktitle": Booktitle,
8 R/ W1 X8 `+ Q/ f# t - "BookID": BookID,# P. I+ U2 d0 p8 V5 a- \5 j
- "BookConte": BookConte,. T5 Z9 E/ r \' F% Q
- "Titleid": Titleid,
$ b7 q% k8 R1 u) e3 H U9 o - "abover":abover,
) K: y1 i! ]9 V; _( Z: ?3 x3 B - # "Book_Date" = str(datetime.datetime.now()),
5 C" m' j/ [& b' p/ Q - }
" l8 k/ v0 V/ I: f) l- I. U. x - def download(self, P_dir, imgDir, file_name, Book_img):$ l$ d! S: L9 { U1 b
- if not os.path.exists(imgDir):
( A4 ]) e; r( h' R - os.makedirs(imgDir)4 I) w6 O4 U$ D# W. ~
- file = imgDir + "/" + file_name! M& c4 M) J% r' j& e- ^' n
- # print file- H% Z2 @: G1 Z( ~" t
- f = open(file, 'wb+')6 `/ j: H4 O6 c1 [
- imag = requests.get(Book_img) , K& P' D B' L0 l; U
- f.write(imag.content)' c- @5 s8 b2 P8 @5 m: c, x' {
- f.close()
7 o, N! ]( [1 W. f% l! J - #保存图片前1 \0 g! e& I, [& F3 X' d! |, m3 X
- def save_imgs(self,response):: _# k7 ~9 X0 }, c
- content = response.content# A# r" z: _; h( D
- file_name = response.save["file_name"]" @" m P& y' ]/ p
- imgDir = response.save["imgDir"]
; n# A; [! B8 u - file_path = imgDir + file_name& k; i5 V! G' y9 j! H
- self.save_img(content,imgDir,file_path)
& v# Q6 |) ^! ]3 H1 P- M4 c - #保存图片/ n% A+ m% N* v4 z# ^
- def save_img(self,content,imgDir,path):
5 F+ |. _5 C( {6 T8 ` - if not os.path.exists(imgDir):
+ c! p/ D2 v# k - os.makedirs(imgDir)
+ q2 W( [! Z% O' d; y; U - f = open(path,"wb" )
; ]2 Y; V9 L9 ^; n/ F - f.write(content)/ M& q' @! o- c, z5 `3 g
- f.close()* Q6 Q0 V% W, h
- #获取url后缀名
2 b2 o# L* X2 N% \ - def getExtension(self,url): 4 z( o' [5 @2 @) L2 \* I
- extension = url.split(".")[-1], J& S6 }% M0 J( T
- return extension ) v. R" q9 a! H( ~) m
- " x) t# x4 z+ x, w) ]
- #获取图片名( G& h- R' k9 _4 u6 B1 P
- def getname(self,url):
7 w' x$ M( N4 X- i - name=url.split("/")[-1].split(".")[0]$ D" G6 \- g$ ~$ Q# `
- return name
复制代码
' @& U5 N3 V! x' l% C
) F$ q+ v9 P# _/ S+ {% ? |