亚马逊是国际知名的电商平台，而国内访问的话是cn国内站点，不同的地区有不同的站点，每个商品有一个id号，不同地区商品是存在差异的！
亚马逊amazon商品数据采集有点类似于采集百度搜索结果信息，协议头非常重要，除了ua之外，cookies头需要携带，要不然不能访问，国内国外站点一样！
感觉网站存在反爬，尤其是英文站点，因为网页源码价格信息与实际前端页面看到的价格信息存在差异！
输入商品id号采集商品相关信息！
国内站（cn）采集
采集效果
附源码
代码语言：javascript复制#国内亚马逊商品爬虫
#20200213 by微信：huguo00289
# -*- coding=utf-8 -*-
import requests
from fake_useragent import UserAgent
import re,os,time
from lxml import etree
def ua():
ua=UserAgent()
headers={
'User-Agent':ua.random,
'Cookie': 'x-wl-uid=1eZRN4GNhENdZSGdOrvzQEy2WvlxT/sXztd0uB1drNz9lanSFUVkDtpyWsVQQfwSjhXmvZLrY67w=; session-id=459-1321777-5720413; ubid-acbcn=459-5647010-5360714; lc-acbcn=zh_CN; i18n-prefs=CNY; session-token=g6hxLDDoHhzZLHWxd7FnNbtphW7mG7zCPY29lJB7vwUfa73azlZ8jPh8iS6M c/4mKa3c/d/Pzgiv61e7sJx858blgOf pmyxOtu55z5AlVE2nRoPAyWFMeG4OKmZQI3Lg5/MNhcN71PW9x2OkQWWLOeqcikSKmxqaEQL9qGyYcnTbrYggdlInP0pROsR8oz; session-id-time=2082787201l; csm-hit=tb:s-KV6TYQQV77AQ5HHBPD94|1581595664859&t:1581595666568&adb:adblk_yes'
}
return headers
#保存txt
def tx(id,text,path):
print(f"正在保存商品数据..")
with open(f'{path}{id}.txt','w') as f:
f.write(text)
print(f">>>保存商品数据成功！")
#下载图片
def down(img_url,img_name,path):
ua = UserAgent()
headers = {'User-Agent': ua.random}
print(f"下载{img_name}图片..")
r=requests.get(img_url,headers=headers,timeout=10)
with open(f'{path}{img_name}','wb') as f:
f.write(r.content)
print(f">>>下载{img_name}图片完成！")
time.sleep(2)
def get_shopping(id):
url=f"https://www.amazon.cn/dp/{id}"
html=requests.get(url,headers=ua(),timeout=10).content.decode('utf-8')
time.sleep(3)
title=re.findall(r'"立即购买：(. ?)",',html,re.S)[0] #B&O PLAY by Bang & Olufsen Beoplay P6 便携式扬声器1140026 黑色
print(title)
path=f'{title}/'
os.makedirs(path,exist_ok=True) #创建目录
req=etree.HTML(html)
price=req.xpath('//span[@id="priceblock_ourprice"]/text()')[0]
print(price)
text='%s%s%s%s%s'%(url,'n',title,'n',price)
tx(id, text, path)
imgs=req.xpath('//span[@class="a-button-text"]/img/@src')
for img in imgs:
if 'jpg' in img:
imgurl=img.split('._')[0]
img_url =f'{imgurl}.jpg'
img_name = img_url.split('/')[-1]
print(img_url,img_name)
down(img_url, img_name, path)
if __name__=='__main__':
id=input("请输入要采集的商品id（比如：B00C3YADSK）：")
get_shopping(id)美国站（com）采集
我的展示页面
什么鬼我的配送不到？无价格显示？
不管了，价格pass
采集效果
附上源码参考：
代码语言：javascript复制#国外亚马逊商品爬虫
#20200213
#https://www.amazon.com/dp/B07S3659V2
# -*- coding=utf-8 -*-
import requests
from fake_useragent import UserAgent
import re,os,time,random
from lxml import etree
def ua():
ua=UserAgent()
sjs = random.randint(1111111, 9999999)
#print(sjs)
sj = str(sjs)
headers={
'User-Agent':ua.random,
'Cookie': f'x-wl-uid=1eZRN4GNhENdZSGdOrvzQEy2WvlxT/sXztd0uB1drNz9lanSFUVkDtpyWsVQQfwSjhXmvZLrY67w=; session-id=459-1321777-{sj}; ubid-acbcn=459-5647010-{sj}; lc-acbcn=zh_CN; i18n-prefs=CNY; session-token=g6hxLDDoHhzZLHWxd7FnNbtphW7mG7zCPY29lJB7vwUfa73azlZ8jPh8iS6M c/4mKa3c/d/Pzgiv61e7sJx858blgOf pmyxOtu55z5AlVE2nRoPAyWFMeG4OKmZQI3Lg5/MNhcN71PW9x2OkQWWLOeqcikSKmxqaEQL9qGyYcnTbrYggdlInP0pROsR8oz; session-id-time=2082787201l; csm-hit=tb:s-KV6TYQQV77AQ5HHBPD94|1581595664859&t:1581595666568&adb:adblk_yes'
}
return headers
#保存txt
def tx(id,text,path):
print(f"正在保存商品数据..")
with open(f'{path}{id}.txt','w',encoding='utf-8') as f:
f.write(text)
print(f">>>保存商品数据成功！")
#下载图片
def down(img_url,img_name,path):
ua = UserAgent()
headers = {'User-Agent': ua.random}
print(f"下载{img_name}图片..")
r=requests.get(img_url,headers=headers,timeout=10)
with open(f'{path}{img_name}','wb') as f:
f.write(r.content)
print(f">>>下载{img_name}图片完成！")
time.sleep(1)
def get_shopping(id):
#id="B07S3659V2"
#url="https://www.amazon.com/dp/B07S3659V2"
url=f"https://www.amazon.com/dp/{id}"
html=requests.get(url,headers=ua(),timeout=10).content.decode('utf-8')
#print(html)
time.sleep(2)
req = etree.HTML(html)
title=re.findall(r'<title>Amazon.com: (. ?)</title>',html,re.S)[0] #B&O PLAY by Bang & Olufsen Beoplay P6 便携式扬声器1140026 黑色
print(title)
path=f'{id}/'
os.makedirs(path,exist_ok=True) #创建目录
price=re.findall(r'"isPreorder":. ?,"price":(. ?),"doesMAPPolicyApply":. ?',html,re.S)[0]
price =f'${price}'
'''
try:
price=req.xpath('//span[@id="priceblock_saleprice"]/text()')[0]
except:
price = req.xpath('//span[@id="priceblock_ourprice"]/text()')[0]
'''
print(price)
productdescriptions=req.xpath('//div[@id="productDescription"]//text()')
productdescription='n'.join(productdescriptions)
text='%s%s%s%s%s%s%s'%(url,'n',title,'n',price,'n',productdescription)
tx(id, text, path)
imgs=req.xpath('//span[@class="a-button-text"]/img/@src')
for img in imgs:
if 'jpg' in img:
imgurl=img.split('._')[0]
img_url =f'{imgurl}.jpg'
img_name = img_url.split('/')[-1]
print(img_url,img_name)
down(img_url, img_name, path)
print(f">>>下载图片完毕！")
if __name__=='__main__':
#id="B07XR5TRSZ"
id=input("请输入要采集的商品id（比如：B07GJ2MWTZ）：")
get_shopping(id)
