你尚未登录,仅允许查看本站部分内容。请登录使用邀请码注册
thonatos

Gearman & nodejs & python 的任务队列实验 0个回复 专栏 @ Nodejs

thonatos 发布于 3 年前

#Demand

在转码任务完后通过下载阿里云oss视频文件并后台上传视频到xxx网站的接口。

#Solution

  • jobServer
    - Gearman

  • client
    - nodejs
    - php
    - ...

  • worker
    - nodejs
    - c&c++
    - python

#Structure

Gearman: worker/client/server.

----------     ----------     ----------     ----------
| Client |     | Client |     | Client |     | Client |
----------     ----------     ----------     ----------
     \             /              \             /
      \           /                \           /
      --------------               --------------
      | Job Server |               | Job Server |
      --------------               --------------
            |                            |
    ----------------------------------------------
    |              |              |              |
----------     ----------     ----------     ----------
| Worker |     | Worker |     | Worker |     | Worker |
----------     ----------     ----------     ----------

#Code

client.js

var Gearman,
    COUNT = 0, 
    client,
    interval;

Gearman = require('gearman').Gearman;

client = new Gearman("10.211.55.6", 4730);

client.on('WORK_COMPLETE', function (job) {
    console.log('job completed, result:', job.payload.toString());    
    if(COUNT > 5){
        client.close();
    }    
});

client.connect(function () {
    console.log('Gearman Server connected.');
});

interval = setInterval(function() {

    COUNT++;

    if(COUNT > 5){
        clearInterval(interval);        
    }

    client.submitJob('upload', COUNT.toString());    

},1000);

worker.js

var Gearman,
    worker,
    oss;

Gearman = require('gearman').Gearman;    
worker = new Gearman('10.211.55.6', 4730);
oss = require('./oss')();    

worker.on('JOB_ASSIGN', function (job) {    

    // LOG
    console.log('# "' + job.func_name + '" job assigned to this worker with payload: "' + job.payload + '"');

    if(job.func_name === 'upload'){
        oss.upload(job.payload,function(err,result){
            if(err) throw err;            
            worker.sendWorkComplete(job.handle, result);
            return worker.preSleep();            
        });  
    }else{                
        var result = job.payload.toString().split('').reverse().join('');
        worker.sendWorkComplete(job.handle, result);
        return worker.preSleep();         
    }         
});

worker.on('NOOP', function () {
    return worker.grabJob();
});

worker.connect(function () {
    worker.addFunction('upload');
    return worker.preSleep();
});

oss.js

module.exports = function () {

    var obj = {};

    obj.upload = function (fileName,callback) {        
        callback(false,fileName);        
    };

    return obj;   
};

#Result

另外测试了使用nodejs & python 同时作为worker来执行700个任务,发现有python的worker时候nodejs版本基本不会被调用。

#Next

在client之前架设一层中间件,负责查询jobServer的状态,按需(上传带宽占用)增加或者释放ecs,来实现大规模任务队列。

原文:http://blog.thonatos.com/gearman-nodejs-on-ubuntu/

等待第一条回复
登录后回复,如无账号,请使用邀请码注册