大家好，又见面了，我是你们的朋友全栈君。
代码语言：javascript复制笔者公司是一家区块链门户网站，该网站的很多资讯，快讯，视频等数据都是通过爬虫爬取得第三方网站获得的，需要从很多网站要爬取数据，如果每个数据源网站都需要单独写个接口去爬的话，工作量无疑是巨大的，因为笔者想到了通过动态代理实现一套爬虫机制，每次要爬取新的数据源，只要在数据库里增加一条数据源即可，无需修改代码。 废话不多说，下贴出数据库表结构
DROP TABLE IF EXISTS `yiyi_crawler_website`;
CREATE TABLE `yiyi_crawler_website` ( `id` bigint(16) NOT NULL AUTO_INCREMENT COMMENT '自增ID', `gmt_create` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', `url` varchar(255) DEFAULT NULL COMMENT '网站链接（抓取内容的接口）', `interval` bigint(16) DEFAULT '0' COMMENT '抓取时间间隔（以毫秒为单位）', `website_type` tinyint(20) DEFAULT NULL COMMENT '网站类型(1、快讯)', `website_name` varchar(32) DEFAULT NULL COMMENT '网站名', `source_link` varchar(255) DEFAULT NULL COMMENT '来源链接', `data_field` varchar(32) DEFAULT NULL COMMENT '数据所在字段，如果没有，为空则直接取数(多级以.连接，如果：data.items表示data下面的items为内容列表)', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;
DROP TABLE IF EXISTS `yiyi_crawler_website_content`;
CREATE TABLE `yiyi_crawler_website_content` ( `id` bigint(16) NOT NULL AUTO_INCREMENT, `gmt_create` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', `gmt_modified` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', `website_id` bigint(16) DEFAULT NULL COMMENT '网站ID', `content_name` varchar(16) DEFAULT NULL COMMENT '内容名', `table_name` varchar(32) DEFAULT NULL COMMENT '所属表名', `column_name` varchar(32) DEFAULT NULL COMMENT '所属字段名', `return_field` varchar(32) DEFAULT NULL COMMENT '当前要抓取的字段所返回的字段', `field_type` tinyint(2) DEFAULT '2' COMMENT '字段类型（1、日期2、数值0、其他）', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;代码语言：javascript复制下面贴出爬虫的动态代理实现，基于cglib框架实现的
/** * 爬虫任务代理接口 * * @author liyi * @create 2018-03-17 16:58 **/
public interface CrawlerProxy {
/** * 任务开始 * @param website */
void start(CrawlerWebsiteModelOut website);
}
/** * 爬虫任务类 * * @author liyi * @create 2018-03-17 18:21 **/
public class CrawlerTask implements CrawlerProxy {
@Override
public void start(CrawlerWebsiteModelOut website) {
System.out.println("爬虫任务开始");
}
}
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
import com.lynn.yiyi.http.Request;
import com.lynn.yiyi.http.WebUtils;
import com.lynn.yiyi.model.out.CrawlerWebsiteModelOut;
import java.util.*;
/** * 爬虫定时任务 * * @author liyi * @create 2018-03-17 18:35 **/
public class CrawlerTimerTask extends TimerTask {
private CrawlerWebsiteModelOut website = null;
@Override
public void run() {
String json = WebUtils.executeHttp(Request.options().setMethod(com.lynn.yiyi.http.Method.GET).setUrl(website.getUrl()).build()).getJsonString();
String strs[] = website.getDataField().split(".");
List<String> dataList = new ArrayList<>();
Arrays.stream(strs).forEach(s -> {
dataList.clear();
String data = JSON.parseObject(json, new TypeReference<Map<String, List<Map<String, String>>>>() {}.getType());
dataList.add(data);
});
}
public void setWebsite(CrawlerWebsiteModelOut website) {
this.website = website;
}
public CrawlerWebsiteModelOut getWebsite() {
return website;
}
}
import com.alibaba.fastjson.JSONObject;
import com.lynn.yiyi.http.Request;
import com.lynn.yiyi.http.WebUtils;
import com.lynn.yiyi.model.out.CrawlerWebsiteModelOut;
import com.lynn.yiyi.service.CrawlerService;
import com.lynn.yiyi.utils.SpringUtils;
import net.sf.cglib.proxy.Enhancer;
import net.sf.cglib.proxy.MethodInterceptor;
import net.sf.cglib.proxy.MethodProxy;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/** * 爬虫动态代理类 * * @author liyi * @create 2018-03-17 18:22 **/
public class CrawlerCglibProxy implements MethodInterceptor {
private Map<Long,Timer> timerMap = new HashMap<>();
private Enhancer enhancer = new Enhancer();
Object getProxy(Class<?> clazz){
enhancer.setSuperclass(clazz);
enhancer.setCallback(this);
return enhancer.create();
}
@Override
public Object intercept(Object obj, Method method, Object[] args, MethodProxy proxy) throws Throwable {
Object o = proxy.invokeSuper(obj,args);
if("start".equals(method.getName())){
if(args[0] instanceof CrawlerWebsiteModelOut){
CrawlerWebsiteModelOut website = (CrawlerWebsiteModelOut)args[0];
if(timerMap.get(website.getId()) == null){
Timer timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
String data = WebUtils.executeHttp(Request.options().setMethod(com.lynn.yiyi.http.Method.GET).setUrl(website.getUrl()).build()).getJsonString();
String strs[] = website.getDataField().split("\.");
for (String s : strs) {
JSONObject object = JSONObject.parseObject(data);
data = object.getString(s);
}
//TODO 这里将爬取到的数据写到数据库对应的表中
}
}, 0, website.getInterval());
timerMap.put(website.getId(),timer);
}
}
}
return o;
}
public static <T> T create(Class<T> cls){
CrawlerCglibProxy proxy = new CrawlerCglibProxy();
return (T)proxy.getProxy(cls);
}
}
import java.util.ArrayList;
import java.util.List;
/** * 网站爬虫输出参数 * * @author liyi * @create 2018-03-17 17:04 **/
public class CrawlerWebsiteModelOut extends BaseModelOut {
private String url;
private Long interval;
private Integer websiteType;
private String websiteName;
private String sourceLink;
private String dataField;
private List<CrawlerWebsiteContentModelOut> contentList = new ArrayList<>();
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public Long getInterval() {
return interval;
}
public void setInterval(Long interval) {
this.interval = interval;
}
public Integer getWebsiteType() {
return websiteType;
}
public void setWebsiteType(Integer websiteType) {
this.websiteType = websiteType;
}
public String getWebsiteName() {
return websiteName;
}
public void setWebsiteName(String websiteName) {
this.websiteName = websiteName;
}
public String getSourceLink() {
return sourceLink;
}
public void setSourceLink(String sourceLink) {
this.sourceLink = sourceLink;
}
public String getDataField() {
return dataField;
}
public void setDataField(String dataField) {
this.dataField = dataField;
}
public List<CrawlerWebsiteContentModelOut> getContentList() {
return contentList;
}
public void setContentList(List<CrawlerWebsiteContentModelOut> contentList) {
this.contentList = contentList;
}
}
/** * 爬虫网站内容输出参数 */
public class CrawlerWebsiteContentModelOut extends BaseModelOut{
private Long websiteId;
private String contentName;
private String tableName;
private String columnName;
private String returnField;
private Integer fieldType;
public Long getWebsiteId() {
return websiteId;
}
public void setWebsiteId(Long websiteId) {
this.websiteId = websiteId;
}
public String getContentName() {
return contentName;
}
public void setContentName(String contentName) {
this.contentName = contentName;
}
public String getTableName() {
return tableName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
public String getColumnName() {
return columnName;
}
public void setColumnName(String columnName) {
this.columnName = columnName;
}
public String getReturnField() {
return returnField;
}
public void setReturnField(String returnField) {
this.returnField = returnField;
}
public Integer getFieldType() {
return fieldType;
}
public void setFieldType(Integer fieldType) {
this.fieldType = fieldType;
}
}
import java.util.Date;
/** * 基础输出参数 * * @author liyi * @create 2018-03-17 17:02 **/
public abstract class BaseModelOut{
private Long id;
private Date create;
private Date modified;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public Date getCreate() {
return create;
}
public void setCreate(Date create) {
this.create = create;
}
public Date getModified() {
return modified;
}
public void setModified(Date modified) {
this.modified = modified;
}
}代码语言：javascript复制下面给出测试的main方法
public static void main(String[] args) {
CrawlerProxy proxy = CrawlerFactory.create();
proxy.start(website);//website即当前要爬取的网站对象，可以从数据库中读取
}调用了main方法即可启动定时器，定时从指定website爬取数据。
发布者：全栈程序员栈长，转载请注明出处：https://javaforall.cn/159775.html原文链接：https://javaforall.cn
