var request=require(‘request’); var cheerio=require(‘cheerio’); var url =‘http://shuju.sxw100.com/search/listUniversity.json?areaCode=-1&categoryCode=-1&administrationCode=&natureCode=1&orderType=&pinYin=&name=&pageNumber=1&pageSize=10&_=1499671138096’;

var i=0;

//读取下一页的信息 抓取6页
for(j=1;j<=6;j++){ var s = url.match(/pageNumber=(\d+)/); //下一页URL var nextUrl = url.replace(/pageNumber=(\d+)/,‘pageNumber=’+j); //调用crawler函数 crawler(nextUrl,function (err,listUniversity2) { if (err) return callback(err); callback(null,listUniversity.concat.(listUniversity2)); }) }

exports.crawler=function(url,callback) { request(url,function (err,res) { if (err) return console.error(err); // json对象 var obj=JSON.parse(res.body.toString()); //获取list列表 var list=obj.list; var $=cheerio.load(list); //定义学校列表 var listUniversity=[]; $(’.academy’).each(function () { var $me=$(this); var title=$me.find(’.academy-name’).find(‘h2’).text(); var location=$me.find(’.academy-name’).find(‘em’).text(); var item={ title:title, location:location, i:i+=1 } listUniversity.push(item); }) console.log(listUniversity) }); }

我参考的是这个爬虫教程。。。。。。。node 刚接触时间不长，求大神修改下代码。。 网络爬虫与数据库操作

var request = require('request');
var cheerio = require('cheerio');
var url = 'http://shuju.sxw100.com/search/listUniversity.json?areaCode=-1&categoryCode=-1&administrationCode=&natureCode=1&orderType=&pinYin=&name=&pageSize=10&_=1499671138096';

(async function() {

    var page = 0,
        result = [];
    while (page++ < 6) {
        result = result.concat(await new Promise((resolve, reject) => {
            console.log(page);
            request(`${url}&pageNumber=${page}`, function(err, res) {
                if (err) {
                    resolve([]);
                    return console.error(err);
                }
                // json对象
                var obj = JSON.parse(res.body.toString());
                //获取list列表
                var list = obj.list.join("");
                var $ = cheerio.load(list);
                //定义学校列表
                var listUniversity = [];
                $('.academy').each(function(i, e) {
                    var $me = $(e);
                    listUniversity.push({
                        title: $me.find('.academy-name').find('h2').text(),
                        location: $me.find('.academy-name').find('em').text(),
                        i: i += 1
                    });

                })
                resolve(listUniversity);
            });

        }))
    };
    return result;
})().then(result => {
    console.dir(result);
});

我运行代码，出现错误，怎么解决？？@godghdai

@Jackzhangpan node版本有点低，不支持async，换下面这种写法试试

var request = require('request');
var cheerio = require('cheerio');
var url = 'http://shuju.sxw100.com/search/listUniversity.json?areaCode=-1&categoryCode=-1&administrationCode=&natureCode=1&orderType=&pinYin=&name=&pageSize=10&_=1499671138096';

Promise.all(((pages) => {
    let range = [];
    while (pages--) range.unshift(pages + 1);
    return range;
})(6).map(page => {
    return new Promise((resolve, reject) => {
        request(`${url}&pageNumber=${page}`, function(err, res) {
            if (err) {
                resolve([]);
                return console.error(err);
            }
            // json对象
            var obj = JSON.parse(res.body.toString());
            //获取list列表
            var list = obj.list.join("");
            var $ = cheerio.load(list);
            //定义学校列表
            var listUniversity = [];
            $('.academy').each(function(i, e) {
                var $me = $(e);
                listUniversity.push({
                    title: $me.find('.academy-name').find('h2').text(),
                    location: $me.find('.academy-name').find('em').text(),
                    i: i += 1
                });

            })
            resolve(listUniversity);
        });
    })

})).then(datas => {
    console.dir(datas.reduce((result, value) => {
        return result = result.concat(value);
    }, []));
});

@godghdai ok了，非常感谢大神帮忙，谢谢