最近中途接手了一个项目,要求使用Node.js是把接收到的post请求处理一下转发到kafka中存放,还没有写消费端。zookeeper是由另外一个人写的,别的系统接入到zookeeper中没有问题,我的node.js程序在Apache Bench测试中发现发送到1W6条的时候就会出现超时报错。现在不知道是zookeeper端还是node.js代码的问题,麻烦大家帮我看看
exports.Kafkatest = function(req,res) { var TOPIC_UPDATE_TIME = getNowTime(new Date()); var topicJSON = ‘{“VERSION”: VERSION,“PRODUCT_LINE”:PRODUCT_LINE,“COMPONENT”:APP,“OPR_TYPE”:OPR_TYPE,“TOPIC_UPDATE_TIME”:TOPIC_UPDATE_TIME,“SRC”:APPSPACE,“PURPOSE”:PURPOSE}’; var args = eval(’(’+topicJSON+’)’); //topic的信息
var post = '';
req.on('data', function(chunk) {
post += chunk;
});
req.on('end',function() {
args["DATA"] = url.parse(post,false).query; //获取post中的信息
args["HEADERS"] = req.headers; //直接使用post的HEADER
const HighLevelProducer = kafka.HighLevelProducer;
const client = new kafka.Client(KAFKA_PORT,0); //KAFKA+PORT = xx.xxx.xxx.xxx:2181
const producer = new HighLevelProducer(client, {
requireAcks:1
});
var playloads = [{ topic: SAMPLE_TOPIC, messages:[JSON.stringify(args)], partition: 0}]; //定义发送发送的信息是存放到哪个topic
producer.on('ready',function(){
producer.send(playloads,function(err) {
if(err) {
console.log('【' + new Date() + '】' + "ERROR!");
console.log(err);
client.close();
producer.close();
}else {
console.log('【' + new Date() + '】' + 'JSON was sent to kafka !!!');
client.close();
producer.close();
}
});
});
producer.on('error',function(err) {
console.log(err);
client.close();
producer.close();
});
res.end();
res.destroy();
});
}
注:
- kafkatest函数是在router中调用的,ab test发送的地址是xx.xxx.xxx.xxx:9090/post
- zookeeper中的sessionTimeout是默认值
- ulimit -n = 65535
- kafka-node的版本是0.5.6