Async Hell

Intro

This is not a guide on best practice. This is not something you want to copy. This is me, having trouble (again) with coding.

My issue is that whenever I write any kind of asynchronous code, I find myself hating the end result. It's usually ugly and not easily readable. Either the order of function declarations doesn't make sense, or that the flow feels bloated with unnecessary constructs which seems arbitrary and independent from my intentions.

What I would like to do is to have code which isn't ugly, the flow is clear to anyone, even if they don't quite know what the asynchronous lib I am using does behind the scenes. I like code which is simplistic and minimalistic in nature.

Now anyone can cry about how hard the life of a scripter is (it isn't, really), but without examples these are just empty words.

So lets see if I can show you what I am talking about.

One more note here: I despise short, 3-4 line, out of context examples. I think it's not good, because most of the time you can't test it or try it without doing some more research, which the author obviously was lazy to do. So my code examples will be complete and runnable. This doesn't mean they will make any sense. :)

Synchronous demo

First, some sync code to start up things. It will not be a practical example, so we can focus on the constructs.


void function (){
    'use strict';

    var subject = Object.create(null), sum = 0, root = 0;

    function isInt(n) { return n % 1 === 0; }

    function check(object, level){
        return typeof object[level] !== 'undefined';
    }

    function set(object, level){ return object[level] = Object.create(null) ; }

    function write(object){ object.value = Math.floor(Math.random() * 98) + 1; }

    function findLevels(object){
        var levels = [];
        (function fl(obj, levelIndex){
            var currentLevel=obj['level'+levelIndex];
            if ( currentLevel != null ) {
                levels[levelIndex] = currentLevel ;
                fl(currentLevel, levelIndex+1);
            }
        }(object, 0));
        return levels;
    }

    function makeDivisible(object){
        var levels = findLevels(object);
        levels.forEach(
            function(obj){
                var diff;
                if ( diff = obj.value % levels.length ) {
                    obj.value -= diff;
                }
            }
        );
    }

    function mean(object){
        var levels = findLevels(object),
            mean = levels.reduce(
                function(a, b){ return (a.value || a) + b.value; },
                0
            ) / levels.length;


        levels.forEach(function(obj){
            var val = obj.value;
            obj.value = val*(mean/levels.length);
        });
    }

    if ( ! check(subject, 'level0') ) set(subject, 'level0');
    write(subject.level0);

    if ( ! check(subject.level0, 'level1') ) set(subject.level0, 'level1');
    write(subject.level0.level1);

    if ( ! check(subject.level0.level1, 'level2') ) {
        set(subject.level0.level1, 'level2');
    }
    write(subject.level0.level1.level2);

    if ( ! check(subject.level0.level1.level2, 'level3') ) {
        set(subject.level0.level1.level2, 'level3');
    }
    write(subject.level0.level1.level2.level3);

    makeDivisible(subject);

    mean(subject);

    sum = findLevels(subject).reduce(
            function(a, b){ return (a.value || a) + b.value; }, 0);

    if ( isInt(root = Math.sqrt(sum)) ) {
        console.log('success', JSON.stringify(subject), sum, root);
    } else {
        console.error('failure',JSON.stringify(subject), sum, root);
    }

}();

Of course this is a very silly code, because there is no way those `if`s would return true, but please play along. Another note here is that this little piece of code could be improved a lot if all I wanted to have `subject.level0.level1.level2` with the sum of level values being a square number, but my focus right now is on the `check`, `set`, `write`, `add` and `mean` functions. It's not even what they do what is important but their role, and their dependence on each other. All else is just scenery.

Behold the Pyramids of Doom

This is how I imagine the same script if the three functions are asynchronous and all I am allowed to use are unnamed callbacks:


void function (){
    'use strict';

    var subject = Object.create(null), sum = 0, root = 0;

    function isInt(n) { return n % 1 === 0; }

    function check(object, level, cb){
        function task(){ cb(typeof object[level] !== 'undefined'); }
        setTimeout(task, Math.random()*3000);
    }

    function set(object, level, cb){
        function task(){ object[level] = Object.create(null); cb(); }
        setTimeout(task, Math.random()*3000);
    }

    function write(object, cb){
        function task(){
            cb(object.value = Math.floor(Math.random() * 1000) + 1);
        }
        setTimeout(task, Math.random()*3000);
    }

    function findLevels(object){
        var levels = [];
        (function fl(obj, levelIndex){
            var currentLevel=obj['level'+levelIndex];
            if ( currentLevel != null ) {
                levels[levelIndex] = currentLevel;
                fl(currentLevel, levelIndex+1);
            }
        }(object, 0));
        return levels;
    }

    function makeDivisible(object, cb){
        function task(){
            var levels = findLevels(object);
            levels.forEach(
                function(obj){
                    var diff;
                    if ( diff = obj.value % levels.length ) {
                        obj.value += -1 *diff;
                    }
                }
            );
            cb();
        }
        setTimeout(task, Math.random()*3000);
    }

    function mean(object, cb){
        function task(){
            var levels = findLevels(object),
                mean = levels.reduce(
                    function(a, b){ return (a.value || a) + b.value; },
                    0
                ) / levels.length;


            levels.forEach(function(obj){
                var val = obj.value;
                obj.value = val*(mean/levels.length);
            });
            cb();
        }
        setTimeout(task, Math.random()*3000);
    }

    function result(object){
        sum = findLevels(object).reduce(
                function(a, b){ return (a.value || a) + b.value; }, 0);

        if ( isInt(root = Math.sqrt(sum)) ) {
            console.log('success', JSON.stringify(object), sum, root);
        } else {
            console.error('failure',JSON.stringify(object), sum, root);
        }
    }

    check(subject, 'level0', function(exists){
        if ( ! exists ) {
            set(subject, 'level0', function(){
                write(subject.level0, function(){
                    check(subject.level0, 'level1', function(exists){
                        if ( ! exists ) {
                            set(subject.level0, 'level1', function(){
                                write(subject.level0.level1, function(){
                                    check(subject.level0.level1, 'level2',
                                        function(exists){
                                            if ( ! exists ) {
                                                set(subject.level0.level1,
                                                    'level2',
                                                    function(){
                                                        write(subject.level0.level1.level2,
                                                            function(){
                                                                makeDivisible(subject,
                                                                    function(){
                                                                        mean(subject,
                                                                            function(){
                                                                                result(subject);
                                                                            }
                                                                        );
                                                                    }
                                                                );
                                                            }
                                                        );
                                                    }
                                                );
                                            } else {
                                                write(subject.level0.level1.level2,
                                                    function(){
                                                        makeDivisible(subject,
                                                            function(){
                                                                mean(subject,
                                                                    function(){
                                                                        result(subject);
                                                                    }
                                                                );
                                                            }
                                                        );
                                                    }
                                                );
                                            }
                                        }
                                    );
                                });
                            });
                        } else {
                            write(subject.level0.level1, function(){
                                check(subject.level0.level1, 'level2',
                                    function(exists){
                                        if ( ! exists ) {
                                            set(subject.level0.level1,
                                                'level2',
                                                function(){
                                                    write(subject.level0.level1.level2,
                                                        function(){
                                                            makeDivisible(subject,
                                                                function(){
                                                                    mean(subject,
                                                                        function(){
                                                                            result(subject);
                                                                        }
                                                                    );
                                                                }
                                                            );
                                                        }
                                                    );
                                                }
                                            );
                                        } else {
                                            write(subject.level0.level1.level2,
                                                function(){
                                                    makeDivisible(subject,
                                                        function(){
                                                            mean(subject,
                                                                function(){
                                                                    result(subject);
                                                                }
                                                            );
                                                        }
                                                    );
                                                }
                                            );
                                        }
                                    }
                                );
                            });
                        }
                    });
                });
            });
        } else {
            write(subject.level0, function(){
                check(subject.level0, 'level1', function(exists){
                    if ( ! exists ) {
                        set(subject.level0, 'level1', function(){
                            write(subject.level0.level1, function(){
                                check(subject.level0.level1, 'level2',
                                    function(exists){
                                        if ( ! exists ) {
                                            set(subject.level0.level1,
                                                'level2',
                                                function(){
                                                    write(subject.level0.level1.level2,
                                                        function(){
                                                            makeDivisible(subject,
                                                                function(){
                                                                    mean(subject,
                                                                        function(){
                                                                            result(subject);
                                                                        }
                                                                    );
                                                                }
                                                            );
                                                        }
                                                    );
                                                }
                                            );
                                        } else {
                                            write(subject.level0.level1.level2,
                                                function(){
                                                    makeDivisible(subject,
                                                        function(){
                                                            mean(subject,
                                                                function(){
                                                                    result(subject);
                                                                }
                                                            );
                                                        }
                                                    );
                                                }
                                            );
                                        }
                                    }
                                );
                            });
                        });
                    } else {
                        write(subject.level0.level1, function(){
                            check(subject.level0.level1, 'level2',
                                function(exists){
                                    if ( ! exists ) {
                                        set(subject.level0.level1,
                                            'level2',
                                            function(){
                                                write(subject.level0.level1.level2,
                                                    function(){
                                                        makeDivisible(subject,
                                                            function(){
                                                                mean(subject,
                                                                    function(){
                                                                        result(subject);
                                                                    }
                                                                );
                                                            }
                                                        );
                                                    }
                                                );
                                            }
                                        );
                                    } else {
                                        write(subject.level0.level1.level2,
                                            function(){
                                                makeDivisible(subject,
                                                    function(){
                                                        mean(subject,
                                                            function(){
                                                                result(subject);
                                                            }
                                                        );
                                                    }
                                                );
                                            }
                                        );
                                    }
                                }
                            );
                        });
                    }
                });
            });
        }
    });

}();

Cute, isn't it? Now I dare you to check if that code is doing what it supposed to do without pushing the run button. :D Obviously, this is crazyness and no sane people will ever write such a code. Most of it is going away as soon as I start naming the functions, and use higher order functions to generate callbacks so that I don't have to repeat them over and over again.

Functions of functions


void function (){
    'use strict';

    var subject = Object.create(null), sum = 0, root = 0, levelCount = 3,
        maxCallbackTime = 1000;

    function isInt(n) { return n % 1 === 0; }

    function check(object, level, cb){
        function task(){ cb(typeof object[level] !== 'undefined'); }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function set(object, level, cb){
        function task(){ object[level] = Object.create(null); cb(); }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function write(object, cb){
        function task(){
            cb(object.value = Math.floor(Math.random() * 1000) + 1);
        }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function findLevels(object){
        var levels = [];
        (function fl(obj, levelIndex){
            var currentLevel=obj['level'+levelIndex];
            if ( currentLevel != null ) {
                levels[levelIndex] = currentLevel;
                fl(currentLevel, levelIndex+1);
            }
        }(object, 0));
        return levels;
    }

    function makeDivisible(object, cb){
        function task(){
            var levels = findLevels(object);
            levels.forEach(
                function(obj){
                    var diff;
                    if ( diff = obj.value % levels.length ) {
                        obj.value += -1 *diff;
                    }
                }
            );
            cb();
        }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function mean(object, cb){
        function task(){
            var levels = findLevels(object),
                mean = levels.reduce(
                    function(a, b){ return (a.value || a) + b.value; },
                    0
                )/levels.length/levels.length;


            levels.forEach(function(obj){
                var val = obj.value;
                obj.value = val*mean;
            });
            cb();
        }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function result(object){
        sum = findLevels(object).reduce(
                function(a, b){ return (a.value || a) + b.value; }, 0);

        if ( isInt(root = Math.sqrt(sum)) ) {
            console.log('success', JSON.stringify(object), sum, root);
        } else {
            console.error('failure',JSON.stringify(object), sum, root);
        }
    }

    function createWriter(obj, level, cb){ return function(){
        write(obj[level], cb);
    }; }

    (function run(object, level){
        var currentLevel = 'level'+level;
        if ( level < levelCount ) {
            check(object, currentLevel, function(exists){
                if ( ! exists ) {
                    set(object, currentLevel, createWriter(
                            object,
                            currentLevel,
                            function(){run(object[currentLevel], level+1);}
                        )
                    );
                } else {
                    write(object[currentLevel],function(){
                        run(object[currentLevel],level+1);
                    });
                }
            });
        } else {
            makeDivisible(subject, function(){
                mean(subject, function(){ result(subject); });
            });
        }
    }(subject, 0));
}();

Now please go back to the sync version, read that, then come back here and compare the two. I think it's clear that even though this is a very simple script, it's highly obfuscated compared to the original one. I think this is not because of the use of a recursive function, because that actually simplifies a lot of the code, but because of the order of function calls. Reading this version is not clear, without careful observation of all the code, what is going to happen and in what order.

The code which runs at last is in the middle, because I like to have function declarations before I actually use them. This way you have to read like watching a yo-yo tournament. Of course one might try adding the functions after they are called, but I believe that would be even more confusing, and maybe wrong too.

Caolan's async

I suspect that I am not the only one having a bit trouble with asynchronous code style, because there are zillions of asynchronous libs and approaches. The one which is one of most straightforward ones is caolan's `async` lib.


void function (global){
    'use strict';

    var subject = Object.create(null), sum = 0, root = 0, levelcount = 3,
        maxCallbackTime = 1000, async = global.async || require('async');

    function isInt(n) { return n % 1 === 0; }

    function check(object, level, cb){
        function task(){ cb(typeof object[level] !== 'undefined'); }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function set(object, level, cb){
        function task(){ object[level] = Object.create(null); cb(); }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function write(object, cb){
        function task(){
            object.value = Math.floor(Math.random() * 1000) + 1;
            cb();
        }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function findLevels(object){
        var levels = [];
        (function fl(obj, levelIndex){
            var currentLevel=obj['level'+levelIndex];
            if ( currentLevel != null ) {
                levels[levelIndex] = currentLevel;
                fl(currentLevel, levelIndex+1);
            }
        }(object, 0));
        return levels;
    }

    function makeDivisible(object, cb){
        function task(){
            var levels = findLevels(object);
            levels.forEach(
                function(obj){
                    var diff;
                    if ( diff = obj.value % levels.length ) {
                        obj.value += -1 *diff;
                    }
                }
            );
            cb();
        }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function mean(object, cb){
        function task(){
            var levels = findLevels(object),
                mean = levels.reduce(
                    function(a, b){ return (a.value || a) + b.value; },
                    0
                )/levels.length/levels.length;


            levels.forEach(function(obj){
                var val = obj.value;
                obj.value = val*mean;
            });
            cb();
        }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function result(object){
        sum = findLevels(object).reduce(
                function(a, b){ return (a.value || a) + b.value; }, 0);

        if ( isInt(root = Math.sqrt(sum)) ) {
            console.log('success', JSON.stringify(object), sum, root);
        } else {
            console.error('failure',JSON.stringify(object), sum, root);
        }
    }

    function createWriter(obj, level, cb){ return function(){
        write(obj[level], cb);
    }; }

    async.auto({
        build_levels: function(cb){
            var i, tasks=[];
            for ( i = 0; i < levelcount; i++ ) {
                tasks[i] = function(i){
                    return function(object, callback){
                        var currentLevel = 'level'+i;
                        if ( callback == null ) {
                            callback = object;
                            object = subject;
                        }
                        check(object, currentLevel, function(exists){
                            if ( ! exists ) {
                                set(object, currentLevel, createWriter(
                                        object,
                                        currentLevel,
                                        function(){
                                            callback(null, object[currentLevel]);
                                        }
                                    )
                                );
                            } else {
                                write(object[currentLevel],
                                    function(){
                                        callback(null, object[currentLevel]);
                                    }
                                );
                            }
                        });
                    };
                }(i);
            }
            async.waterfall(tasks, cb);
        },
        makeDivisible : ['build_levels', function(cb){
            makeDivisible(subject, cb);
        }],
        mean : ['makeDivisible', function(cb){
            mean(subject, cb);
        }],
        result : ['mean', function(){
            result(subject);
        }]
    });
}(this);

This async lib is really great. I kinda like it the most all solutions I've seen until now. But when you get to the more complicated flows, you either don't use it, or you get ugly constructs like this. For the most part it is very clear what's happening, but the waterfall tasks construction is crazy. I wish I would never ever have to write code like this again. It feels redundant, and you really have to know the async API to get why it's like this. ( And because this is the first version of this article, I really hope someone more experienced will come along and show me a much better alternative for this).

Promises, deferreds and vows

The latest and greatest of all async approaches are the promises. The web is full of the different implementations, there are even specs for it, and last year even Mr. Douglas "Javascript's foremost authority" Crockford praised them, describing a minimal implementation of them, which he calls VOW.js. After all, promises are not a new concept, they are around since the seventies.

First I wanted to try to use VOW.js but as it turns out, it doesn't follow the A+ spec, and I kinda feel like that's an error with it. (VOW.js now supports most of the spec, there are some edge cases which fail, but nothing important.) So instead I will use medikoo's deferred lib, which also doesn't follow the A+ specs, but it does this with a higher purpose in mind.


void function (global){
    'use strict';

    var subject = Object.create(null), sum = 0, root = 0, levelCount = 3,
        maxCallbackTime = 1000, promisify = global.deferred ? global.deferred.promisify : require('deferred').promisify;

    function isInt(n) { return n % 1 === 0; }

    function check(object, level, cb){
        function task(){
            if ( typeof object[level] === 'undefined' ) {
                cb(new Error('this is really sad, but '+
                        'medikoo doesn\'t believe in functions which would' +
                        'return only true/false'));
            } else {
                cb();
            }
        }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function set(object, level, cb){
        function task(){ object[level] = Object.create(null); cb(); }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function write(object, cb){
        function task(){
            cb(null, object.value = Math.floor(Math.random() * 1000) + 1);
        }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function findLevels(object){
        var levels = [];
        (function fl(obj, levelIndex){
            var currentLevel=obj['level'+levelIndex];
            if ( currentLevel != null ) {
                levels[levelIndex] = currentLevel;
                fl(currentLevel, levelIndex+1);
            }
        }(object, 0));
        return levels;
    }

    function makeDivisible(object, cb){
        function task(){
            var levels = findLevels(object);
            levels.forEach(
                function(obj){
                    var diff;
                    if ( diff = obj.value % levels.length ) {
                        obj.value += -1 *diff;
                    }
                }
            );
            cb();
        }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function mean(object, cb){
        function task(){
            var levels = findLevels(object),
                mean = levels.reduce(
                    function(a, b){ return (a.value || a) + b.value; },
                    0
                )/levels.length/levels.length;


            levels.forEach(function(obj){
                var val = obj.value;
                obj.value = val*mean;
            });
            cb();
        }
        setTimeout(task, Math.random()*maxCallbackTime);
    }

    function result(object, cb){
        sum = findLevels(object).reduce(
                function(a, b){ return (a.value || a) + b.value; }, 0);

        if ( isInt(root = Math.sqrt(sum)) ) {
            console.log('success', JSON.stringify(object), sum, root);
        } else {
            console.error('failure',JSON.stringify(object), sum, root);
        }
        cb();
    }

    var chk = promisify(check);
    var st = promisify(set);
    var wrt = promisify(write);
    var mkdv = promisify(makeDivisible);
    var mn = promisify(mean);
    var rslt = promisify(result);

    chk(subject,'level0')
    (null,function(){return st(subject,'level0')})
    (function(){return wrt(subject.level0)})

    (function(){return chk(subject.level0,'level1')})
    (null,function(){return st(subject.level0,'level1')})
    (function(){return wrt(subject.level0.level1)})

    (function(){return chk(subject.level0.level1,'level2')})
    (null,function(){return st(subject.level0.level1,'level2')})
    (function(){return wrt(subject.level0.level1.level2)})

    (function(){return mkdv(subject)})
    (function(){return mn(subject)})
    (function(){return rslt(subject)})
    .end()

}(this);