这里做得比较暴力,没有分页取出数据解析,O(∩_∩)O哈哈~,居然没有被挂机.不过解析的坑特别多...不过大部分我想要的数据都拿到了.
//解析列表数据var http = require("http"), cheerio = require("cheerio"), mongoose = require('mongoose'), db = mongoose.createConnection('mongodb://127.0.0.1:27017/crawl58');db.on('error', function (error) { console.log('mongodb连接错误: ' + error);});//列表页面数据var mongooseSchema = new mongoose.Schema({ url: {type: String},//抓取地址 type: {type: String},//类型 content: {type: String},//抓取地址 updateTime: {type: Date, default: Date.now},//数据抓取时间 flag: {type: String, default: 0} //用于判断是否抓取过 0表示详情没有抓取过.});// modelvar mongooseModel = db.model('pageList', mongooseSchema);//代理记账//存储数据var parseListSchema = new mongoose.Schema({ url: {type: String},//抓取地址 detailUrl: {type: String},//详情地址 type: {type: String},//类型 title: {type: String},//标题 company: {type: String},//公司名称 contact: {type: String},//联系人 score: {type: String},//评分 phone: {type: String},//电话 updateTime: {type: Date, default: Date.now},//数据解析时间 flag: {type: String, default: 0} //用于判断是否抓取过 0表示详情没有抓取过.});// modelvar parseListModel = db.model('parseList', parseListSchema);//代理记账var pageNo = 0;var data;//保存取出的数据function queryList() { var condition = { url: 'http://cd.58.com/yanzi/pn16/?PGTID=139112794188694845657499716&ClickID=1' } mongooseModel.find(condition, function (error, result) { if (error) { console.log(error); } else { //解析数据 data = result; console.log('开始解析...'); parseList(); } });//.skip(0).limit(100);//分页解析};//解析function parseList() { //解析数据并存入数据库 if (!data[pageNo]) { console.log('解析完成. 页码: ' + pageNo); //更新数据库,修改解析标志位 暂时不处理. return false; } var listItem = data[pageNo]; var listContent = listItem.content; if (!listContent) { pageNo = pageNo + 1; parseList(); return false; } var $ = cheerio.load(listContent); //解析页面 var trElements = $('.small-tbimg>tr'); var docArray = []; trElements.each(function (index, ele) { if ($(ele).find('td.dev').length > 0) { //已经没有这个类型的数据了. return false; } var contact = $(ele).find('div.tdiv .f14').first().text(); if (contact) { contact = contact.replace(':', ''); } var title = $(ele).find('div.tdiv>a').first().text(); var company = $(ele).find('a.u').first().text(); if (!company) { var companyBox = $(ele).find('div.tdiv'); companyBox.find('b,a,span,i').remove(); company = decodeUtf8(companyBox.html()); if (company && company.indexOf('') > 0) { company = company.replace('company', '').replace('%uA0', ''); company = company.split('')[2]; } } var score = $(ele).find('.star00').first().attr('title'); var detailUrl = $(ele).find('div.pjdiv a').first().attr('href'); if (!detailUrl) { detailUrl = $(ele).find('div.tdiv a').first().attr('href'); if (!detailUrl) { detailUrl = $(ele).find('a.t').first().attr('href'); } } else { detailUrl = detailUrl.replace('showtype=yuyue&', ''); } var phone = $(ele).find('.jumpDiv_tel').first().text(); if (phone) { phone = getNumber(phone); } var item = { contact: contact, type: listItem.type, title: title, url: listItem.url, detailUrl: detailUrl, company: company, score: score, phone: phone }; docArray.push(item); }); //存入数据库 parseListModel.create(docArray, function (error) { if (error) { console.log(error); } else { console.log('保存成功 页码: ' + pageNo + ' 条数: ' + docArray.length); pageNo = pageNo + 1; parseList(); } });};//解码utf-8function decodeUtf8(str) { return unescape(str.replace(/&#x/g, '%u').replace(/;/g, ''))};//提取电话号码function getNumber(str) { var reg = /[0-9][0-9]*/g; return str.match(reg).join('-');//带区号的电话号码};//这里为整个解析的开始 -- 特么这么烂的代码自己都看不下去了,唯一看得过去的是,能用 .O(∩_∩)O.//调用...1.取出数据;2 解析数据并存入数据库queryList();