Iterate over a task list with the ability to fail and retry and hard fail after n attempts.

Handles both expected Exceptions and Unexpected Exceptions.

def run(tasks):

    queue = collections.deque(tasks)

    while queue:
        task = queue.pop().copy()
        task['retry'] -= 1

        try:
            result = do_work(task)
            save(result)
            task['state'] = 'ok'

        except (Expected1, Expected2, ExpectedN) as exp:
            log.warn(exp)

            if task['retry']:
                task['state'] = 'error'
                queue.appendleft(task)

            else:
                task['state'] = 'failed'

        except Exception as exp:
            task['state'] = 'error'
            log.critical(exp, exc_info=True)

        finally:
            task_queue.save(task)