# spider文件
# -*- coding: utf-8 -*-
import scrapy
import re
from scrapy import Request
import json
import string
import random
from xpc.items import PostItem, CommentItem, CopyItem # 多个itemdef strip(s):
# s存在就去空，不存在就返回空
if s:
return s.strip()
return ""# 使用scrapy.Request和scrapy.FormRequest发送请求的时候，默认会把cookies保存下来
# 模拟登录的时候不用scrapy框架，直接使用request模块
cookies = dict(
Authorization='4F635191B0602B5D3B06024483B0602AAF8B06023C2F6259656D'
)
# 上面的cookies是网站返回的，需要先登陆的一下把这个cookies找到# 生成26个字母+数字
def gen_sessionid():
return "".join(random.choices(string.ascii_lowercase + string.digits, k=26))class XinpianchangSpider(scrapy.Spider):
name = 'XinPianChang'
allowed_domains = ['xinpianchang.com', 'openapi-vtom.vmovier.com']
start_urls = ['https://www.xinpianchang.com/channel/index/sort-like?from=tabArticle']
# 假设从21页开始访问，这里就需要带上cookies，这时候最开始设置的cookies就不能用了，网站会返回4个cookies。需要从写start_requests函数
# start_urls = ['https://www.xinpianchang.com/channel/index/type-/sort-like/duration_type-0/resolution_type-/page-21']
page_count = 0 # 重写父类中的 start_requests方法，该方法默认对start_urls中的url发get请求
# def start_requests(self):
# for url in self.start_urls:
# # data = {
# # "kw": "cat"
# # }
# # post请求发送,使用FormRequest
# # yield scrapy.FormRequest(url=url, callback=self.parse, formdata=data)
#
# c = cookies.copy()
# c.update(PHPSESSID=gen_sessionid(),
# SERVER_ID='b52601c8-285bdd26',
# channel_page='apU%3D')
# yield Request(url, cookies=c, dont_filter=True) def parse(self, response):
# from scrapy.shell import inspect_response
# inspect_response(response, self)
self.page_count += 1
if self.page_count >= 100:
cookies.update(PHPSESSID=gen_sessionid())
self.page_count = 0 url_list = response.xpath('//ul[@class="video-list"]/li/@data-articleid').extract()
for pid in url_list:
detail_url = 'https://www.xinpianchang.com/a{}?from=ArticleList'.format(pid)
# print(detail_url)
request = response.follow(detail_url, callback=self.parse_post)
request.meta['pid'] = pid
yield request # 进入作品的详情页请求 pages = response.xpath('//div[@class="page"]/a/@href').extract()
for page_url in pages:
# print("列表页翻页url", page_url) # page_url是一个相对路径，不完整的
yield response.follow(page_url, self.parse, cookies=cookies) def parse_post(self, response):
pid = response.meta['pid']
post = PostItem()
post['pid'] = pid
post['title'] = response.xpath('//div[@class="title-wrap"]/h3/text()').get()
# video_url = 'https://openapi-vtom.vmovier.com/v3/video/5E34203E92450?expand=resource&usage=xpc_web'
# response.text拿到网页返回的源码
vid = re.findall('vid: "(.*?)",', response.text)[0]
# print(vid)
video_url = 'https://openapi-vtom.vmovier.com/v3/video/{}?expand=resource&usage=xpc_web'.format(vid)
cates = response.xpath('//span[@class="cate v-center"]/a/text()').extract()
post['category'] = ''.join([cate.strip() for cate in cates])
post['create_time'] = response.xpath('//span[contains(@class,"update-time")]/i/text()').get()
post['play_count'] = response.xpath('//i[contains(@class,"play-counts")]/text()').get()
desc_lst = response.xpath('//p[contains(@class,"desc")]//text()').extract()
post['desc'] = ' '.join([i.strip() for i in desc_lst]) # 请求这个video_url, 多了一步这个注意一下
request = Request(video_url, callback=self.parse_video)
# 把之前获取到的post通过meta传到下一个函数中. 这个post是请求传参
request.meta['post'] = post
yield request # 获取评论链接‘https://app.xinpianchang.com/comments?resource_id=10664352&type=article&page=1&per_page=24’
comment_url = "https://app.xinpianchang.com/comments?resource_id={}&type=article&page=1&per_page=24".format(
pid)
request = Request(comment_url, callback=self.parse_comment)
# 把之前获取到的post通过meta传到下一个函数中
request.meta['pid'] = pid
yield request # 获取作者页链接
creator_list = response.xpath('//div[@class="filmplay-creator right-section"]/ul[@class="creator-list"]/li')
composer_url = 'https://www.xinpianchang.com/u{}?from=articleList'
# cid = response.xpath('//div[@class="filmplay-creator right-section"]/ul[@class="creator-list"]/li/a/@data-userid')
for creator in creator_list:
cid = creator.xpath('./a/@data-userid').get()
composer_url = 'https://www.xinpianchang.com/u{}?from=articleList'.format(cid)
request = response.follow(composer_url, self.parse_composer)
request.meta['cid'] = cid
# 避免在cookies更新之后，不断的添加到请求头里面，避免请求头里带有一串cookies
request.meta['dont_merge_cookies'] = True
yield request # 作者和视频的对应关系
cr = CopyItem()
cr['pid'] = pid
cr['cid'] = cid
cr['pcid'] = pid + cid
cr['role'] = creator.xpath('./div[@class="creator-info"]/span/text()').get()
# print("cr", cr)
yield cr def parse_video(self, response): # 这个response是json格式
post = response.meta['post']
# 先把返回的json转化一下， 注意一下
result = json.loads(response.text)
post['video_url'] = result['data']['resource']['default']['url']
# 直接返回给管道了
yield post def parse_comment(self, response):
result = json.loads(response.text)
for c in result['data']['list']:
comment = CommentItem()
comment['uname'] = c['userInfo']['username']
comment['user_id'] = c['userInfo']['id']
# comment['user_page'] = c['userInfo']['web_url']
comment['content'] = c['content']
comment['content_id'] = c['id']
print(comment)
yield comment # 如果有下一页
if result['data']['next_page_url']:
next_page = 'https://app.xinpianchang.com' + result['data']['next_page_url']
# print("next_page", next_page)
yield response.follow(next_page, self.parse_comment) def parse_composer(self, response):
pass
# settings文件
# -*- coding: utf-8 -*-# Scrapy settings for xpc project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.htmlBOT_NAME = 'xpc'SPIDER_MODULES = ['xpc.spiders']
NEWSPIDER_MODULE = 'xpc.spiders'# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'xpc (+http://www.yourdomain.com)'# Obey robots.txt rules
ROBOTSTXT_OBEY = False# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16# Disable cookies (enabled by default)
# 如果使用自定义cookie就把COOKIES_ENABLED设置为True
# 如果使用settings的cookie就把COOKIES_ENABLED设置为False
COOKIES_ENABLED = True
COOKIES_DEBUG = True # 可以打印出来详细的cookies信息# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)',
}# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = {
# 'xpc.middlewares.XpcSpiderMiddleware': 543,
# }# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
# 'xpc.middlewares.XpcDownloaderMiddleware': 543,
# }# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
# }# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'xpc.pipelines.XpcPipeline': 300, # 优先级高
# 'xpc.pipelines.MysqlPipeline': 301,
# 'xpc.pipelines.RedisPipeline': 302,
}# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
# AUTOTHROTTLE_ENABLED = True
# The initial download delay
# AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
# AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
# AUTOTHROTTLE_DEBUG = False# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
HTTPCACHE_ENABLED = False # True缓存访问过的网页，不会真实的发请求
# HTTPCACHE_ENABLED = True# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'# 日志类型： INFO DEBUG ERROR
LOG_LEVEL = 'DEBUG'
# item文件
# -*- coding: utf-8 -*-
import scrapyclass PostItem(scrapy.Item):
# 保存视频信息
# 自定义字段，有多个表的时候需要写个table_name
table_name = 'posts' # 下面的是数据字段
pid = scrapy.Field()
title = scrapy.Field()
category = scrapy.Field()
create_time = scrapy.Field()
play_count = scrapy.Field()
desc = scrapy.Field()
video_url = scrapy.Field()class CommentItem(scrapy.Item):
# 保存评论信息
table_name = 'comments'
content_id = scrapy.Field()
pid = scrapy.Field()
cid = scrapy.Field()
uname = scrapy.Field()
user_id = scrapy.Field()
content = scrapy.Field()
user_page = scrapy.Field()class CopyItem(scrapy.Item):
table_name = 'copyrights'
pcid = scrapy.Field() # 表的主键
pid = scrapy.Field()
cid = scrapy.Field()
role = scrapy.Field()
# pipeline文件
# -*- coding: utf-8 -*-import csv
from xpc.items import PostItem, CommentItem, CopyItem
import pymysql
from redis import Redis
import osclass XpcPipeline(object):
def __init__(self):
# 当前文件的上一级
store_file = os.path.dirname(__file__) + '/xpc.csv'
# 打开文件
self.file = open(store_file, 'w', newline="")
# csv 写法
self.writer = csv.writer(self.file) def open_spider(self, spider):
print("pipeline 开始爬虫......")
# 执行多个不同的item时
def process_item(self, item, spider):
if isinstance(item, PostItem):
print("这是发布信息：", item)
elif isinstance(item, CommentItem):
print("这是评论信息：", item)
elif isinstance(item, CopyItem):
print("这是版权信息：", item)
return item # 返回给下一个要执行的管道类 def close_spider(self, spider):
print("pipeline 结束爬虫......")# 连接数据库
class MysqlPipeline(object):
conn = None
cursor = None def open_spider(self, spider):
self.conn = pymysql.Connect(
host='127.0.0.1',
port=3306,
user='root',
password='',
db='test_db',
charset='utf8'
)
print("数据库连接成功") def process_item(self, item, spider):
self.cursor = self.conn.cursor()
try:
self.cursor.execute('insert into test_db values("%s", "%s")' % (item['author'], item['content']))
self.conn.commit()
except Exception as e:
print("数据库插入异常：", e)
print("数据库执行回滚")
self.conn.rollback()
return item def close_spider(self, spider):
print("断开数据库连接")
self.cursor.close()
self.conn.close()# 连接数据库
class RedisPipeline(object):
conn = None
cursor = None def open_spider(self, spider):
self.conn = Redis(
host='127.0.0.1',
port=6379
)
print("数据库连接成功") def process_item(self, item, spider):
dic = {
"author": item["author"],
"content": item["content"]
}
self.conn.lpush("队列名字", dic) def close_spider(self, spider):
print("断开数据库连接")
self.cursor.close()
self.conn.close()
