Initial commit.

.gitignore

+docs/_build
+*.egg-info

docs/Makefile

+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo "  coverage   to run coverage check of the documentation (if enabled)"
+
+.PHONY: clean
+clean:
+	rm -rf $(BUILDDIR)/*
+
+.PHONY: html
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+.PHONY: dirhtml
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+.PHONY: changes
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+.PHONY: linkcheck
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+.PHONY: doctest
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+.PHONY: coverage
+coverage:
+	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+	@echo "Testing of coverage in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/coverage/python.txt."

docs/_static/style.css

+
+div.figure {
+    padding: 0;
+}

docs/_templates/github.html

+<p>
+<br>
+<a href="https://github.com/dw/mitogen/">GitHub Repository</a>
+</p>

docs/_templates/layout.html

+{% extends "!layout.html" %}
+{% set css_files = css_files + ['_static/style.css'] %}

docs/api.rst

+
+API Reference
+*************
+
+
+Package Layout
+==============
+
+
+mitogen Package
+---------------
+
+.. automodule:: mitogen
+
+.. autodata:: mitogen.slave
+.. autodata:: mitogen.context_id
+.. autodata:: mitogen.parent_id
+
+
+mitogen.core
+------------
+
+.. automodule:: mitogen.core
+
+
+mitogen.master
+--------------
+
+.. automodule:: mitogen.master
+
+
+mitogen.fakessh
+---------------
+
+.. automodule:: mitogen.fakessh
+
+.. autofunction:: mitogen.fakessh.run
+
+
+Router Class
+============
+
+.. autoclass:: mitogen.master.Router
+   :members:
+   :inherited-members:
+
+
+Broker Class
+============
+
+.. autoclass:: mitogen.master.Broker
+   :members:
+   :inherited-members:
+
+
+Context Class
+=============
+
+.. autoclass:: mitogen.master.Context
+   :members:
+   :inherited-members:
+
+
+Channel Class
+-------------
+
+.. autoclass:: mitogen.core.Channel
+   :members:
+
+
+Context Class
+-------------
+
+.. autoclass:: mitogen.master.Context
+   :members:
+
+
+Utility Functions
+=================
+
+.. automodule:: mitogen.utils
+  :members:
+
+
+Exceptions
+==========
+
+.. autoclass:: mitogen.core.Error
+.. autoclass:: mitogen.core.CallError
+.. autoclass:: mitogen.core.ChannelError
+.. autoclass:: mitogen.core.StreamError
+.. autoclass:: mitogen.core.TimeoutError

docs/conf.py

+import sys
+sys.path.append('..')
+
+author = u'David Wilson'
+copyright = u'2016, David Wilson'
+exclude_patterns = ['_build']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
+html_show_sourcelink = False
+html_show_sphinx = False
+html_sidebars = {'**': ['globaltoc.html', 'github.html']}
+html_static_path = ['_static']
+html_theme = 'alabaster'
+htmlhelp_basename = 'mitogendoc'
+intersphinx_mapping = {'python': ('https://docs.python.org/2', None)}
+language = None
+master_doc = 'toc'
+project = u'Mitogen'
+pygments_style = 'sphinx'
+release = u'master'
+source_suffix = '.rst'
+templates_path = ['_templates']
+todo_include_todos = False
+version = u'master'

docs/examples.rst

+
+Examples
+========
+
+
+Recursively Nested Bootstrap
+----------------------------
+
+This demonstrates the library's ability to use slave contexts to recursively
+proxy connections to additional slave contexts, with a uniform API to any
+slave, and all features (function calls, import forwarding, stdio forwarding,
+log forwarding) functioning transparently.
+
+This example uses a chain of local contexts for clarity, however SSH and sudo
+contexts work identically.
+
+nested.py:
+
+.. code-block:: python
+
+    import os
+    import mitogen.utils
+
+    @mitogen.utils.run_with_router
+    def main(router):
+        mitogen.utils.log_to_file()
+
+        context = None
+        for x in range(1, 11):
+            print 'Connect local%d via %s' % (x, context)
+            context = router.local(via=context, name='local%d' % x)
+
+        context.call(os.system, 'pstree -s python -s mitogen')
+
+
+Output:
+
+.. code-block:: shell
+
+    $ python nested.py
+    Connect local1 via None
+    Connect local2 via Context(1, 'local1')
+    Connect local3 via Context(2, 'local2')
+    Connect local4 via Context(3, 'local3')
+    Connect local5 via Context(4, 'local4')
+    Connect local6 via Context(5, 'local5')
+    Connect local7 via Context(6, 'local6')
+    Connect local8 via Context(7, 'local7')
+    Connect local9 via Context(8, 'local8')
+    Connect local10 via Context(9, 'local9')
+    18:14:07 I ctx.local10: stdout: -+= 00001 root /sbin/launchd
+    18:14:07 I ctx.local10: stdout:  \-+= 08126 dmw /Applications/iTerm.app/Contents/MacOS/iTerm2
+    18:14:07 I ctx.local10: stdout:    \-+= 10638 dmw /Applications/iTerm.app/Contents/MacOS/iTerm2 --server bash --login
+    18:14:07 I ctx.local10: stdout:      \-+= 10639 dmw bash --login
+    18:14:07 I ctx.local10: stdout:        \-+= 13632 dmw python nested.py
+    18:14:07 I ctx.local10: stdout:          \-+- 13633 dmw mitogen:dmw@Eldil.local:13632
+    18:14:07 I ctx.local10: stdout:            \-+- 13635 dmw mitogen:dmw@Eldil.local:13633
+    18:14:07 I ctx.local10: stdout:              \-+- 13637 dmw mitogen:dmw@Eldil.local:13635
+    18:14:07 I ctx.local10: stdout:                \-+- 13639 dmw mitogen:dmw@Eldil.local:13637
+    18:14:07 I ctx.local10: stdout:                  \-+- 13641 dmw mitogen:dmw@Eldil.local:13639
+    18:14:07 I ctx.local10: stdout:                    \-+- 13643 dmw mitogen:dmw@Eldil.local:13641
+    18:14:07 I ctx.local10: stdout:                      \-+- 13645 dmw mitogen:dmw@Eldil.local:13643
+    18:14:07 I ctx.local10: stdout:                        \-+- 13647 dmw mitogen:dmw@Eldil.local:13645
+    18:14:07 I ctx.local10: stdout:                          \-+- 13649 dmw mitogen:dmw@Eldil.local:13647
+    18:14:07 I ctx.local10: stdout:                            \-+- 13651 dmw mitogen:dmw@Eldil.local:13649
+    18:14:07 I ctx.local10: stdout:                              \-+- 13653 dmw pstree -s python -s mitogen
+    18:14:07 I ctx.local10: stdout:                                \--- 13654 root ps -axwwo user,pid,ppid,pgid,command

docs/getting_started.rst

+
+Getting Started
+===============
+
+xxx

docs/images/billing.graphml

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:java="http://www.yworks.com/xml/yfiles-common/1.0/java" xmlns:sys="http://www.yworks.com/xml/yfiles-common/markup/primitives/2.0" xmlns:x="http://www.yworks.com/xml/yfiles-common/markup/2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:y="http://www.yworks.com/xml/graphml" xmlns:yed="http://www.yworks.com/xml/yed/3" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://www.yworks.com/xml/schema/graphml/1.1/ygraphml.xsd">
+  <!--Created by yEd 3.14.4-->
+  <key attr.name="Description" attr.type="string" for="graph" id="d0"/>
+  <key for="port" id="d1" yfiles.type="portgraphics"/>
+  <key for="port" id="d2" yfiles.type="portgeometry"/>
+  <key for="port" id="d3" yfiles.type="portuserdata"/>
+  <key attr.name="url" attr.type="string" for="node" id="d4"/>
+  <key attr.name="description" attr.type="string" for="node" id="d5"/>
+  <key for="node" id="d6" yfiles.type="nodegraphics"/>
+  <key for="graphml" id="d7" yfiles.type="resources"/>
+  <key attr.name="url" attr.type="string" for="edge" id="d8"/>
+  <key attr.name="description" attr.type="string" for="edge" id="d9"/>
+  <key for="edge" id="d10" yfiles.type="edgegraphics"/>
+  <graph edgedefault="directed" id="G">
+    <data key="d0"/>
+    <node id="n0">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="189.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="44.03125" x="47.984375" y="5.93359375">master<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n1">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="239.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="70.55078125" x="34.724609375" y="5.93359375">ssh:bastion<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n2">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="289.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="32.48828125" x="53.755859375" y="5.93359375">sudo<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n3">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="339.0"/>
+          <y:Fill color="#FFCC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="71.423828125" x="34.2880859375" y="5.93359375">ssh:billing0<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <node id="n4">
+      <data key="d5"/>
+      <data key="d6">
+        <y:ShapeNode>
+          <y:Geometry height="30.0" width="140.0" x="319.0" y="389.0"/>
+          <y:Fill color="#99CC00" transparent="false"/>
+          <y:BorderStyle color="#000000" type="line" width="1.0"/>
+          <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog" fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false" height="18.1328125" modelName="custom" textColor="#000000" visible="true" width="131.740234375" x="4.1298828125" y="5.93359375">run-nightly-billing.py<y:LabelModel>
+              <y:SmartNodeLabelModel distance="4.0"/>
+            </y:LabelModel>
+            <y:ModelParameter>
+              <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0" nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0" upY="-1.0"/>
+            </y:ModelParameter>
+          </y:NodeLabel>
+          <y:Shape type="rectangle"/>
+        </y:ShapeNode>
+      </data>
+    </node>
+    <edge id="e0" source="n0" target="n1">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e1" source="n1" target="n2">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e2" source="n2" target="n3">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+    <edge id="e3" source="n3" target="n4">
+      <data key="d9"/>
+      <data key="d10">
+        <y:PolyLineEdge>
+          <y:Path sx="0.0" sy="0.0" tx="0.0" ty="0.0"/>
+          <y:LineStyle color="#000000" type="line" width="1.0"/>
+          <y:Arrows source="none" target="standard"/>
+          <y:BendStyle smoothed="false"/>
+        </y:PolyLineEdge>
+      </data>
+    </edge>
+  </graph>
+  <data key="d7">
+    <y:Resources/>
+  </data>
+</graphml>

docs/internals.rst

+
+Internal API Reference
+**********************
+
+
+mitogen.core
+============
+
+
+Side Class
+----------
+
+.. autoclass:: mitogen.core.Side
+   :members:
+
+
+Stream Classes
+--------------
+
+.. autoclass:: mitogen.core.BasicStream
+   :members:
+
+.. autoclass:: mitogen.core.Stream
+   :members:
+
+.. autoclass:: mitogen.master.Stream
+   :members:
+
+.. autoclass:: mitogen.ssh.Stream
+   :members:
+
+
+Other Stream Subclasses
+-----------------------
+
+.. autoclass:: mitogen.core.IoLogger
+   :members:
+
+.. autoclass:: mitogen.core.Waker
+   :members:
+
+
+
+ExternalContext Class
+---------------------
+
+.. autoclass:: mitogen.core.ExternalContext
+
+
+mitogen.master
+===============
+
+.. autoclass:: mitogen.master.ProcessMonitor
+
+
+Helper Functions
+----------------
+
+.. autofunction:: mitogen.master.create_child
+.. autofunction:: mitogen.master.get_child_modules
+.. autofunction:: mitogen.master.minimize_source

docs/toc.rst

+
+Table Of Contents
+=================
+
+.. toctree::
+   :maxdepth: 2
+
+   index
+   howitworks
+   examples
+   getting_started
+   api
+   internals

examples/ansible_demo.py

+"""
+Minimal demo of running an Ansible module via mitogen.
+"""
+
+import json
+import logging
+import time
+
+import mitogen
+import mitogen.master
+import mitogen.utils
+
+# Prevent accident import of an Ansible module from hanging on stdin read.
+import ansible.module_utils.basic
+ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
+
+
+class Exit(Exception):
+    """
+    Raised when a module exits with success.
+    """
+    def __init__(self, dct):
+        self.dct = dct
+
+
+class ModuleError(Exception):
+    """
+    Raised when a module voluntarily indicates failure via .fail_json().
+    """
+    def __init__(self, msg, dct):
+        Exception.__init__(self, msg)
+        self.dct = dct
+
+
+def wtf_exit_json(self, **kwargs):
+    """
+    Replace AnsibleModule.exit_json() with something that doesn't try to
+    suicide the process or JSON-encode the dictionary. Instead, cause Exit to
+    be raised, with a `dct` attribute containing the successful result
+    dictionary.
+    """
+    self.add_path_info(kwargs)
+    kwargs.setdefault('changed', False)
+    kwargs.setdefault('invocation', {
+        'module_args': self.params
+    })
+    kwargs = ansible.module_utils.basic.remove_values(kwargs, self.no_log_values)
+    self.do_cleanup_files()
+    raise Exit(kwargs)
+
+
+def wtf_fail_json(self, **kwargs):
+    """
+    Replace AnsibleModule.fail_json() with something that raises ModuleError,
+    which includes a `dct` attribute.
+    """
+    self.add_path_info(kwargs)
+    kwargs.setdefault('failed', True)
+    kwargs.setdefault('invocation', {
+        'module_args': self.params
+    })
+    kwargs = ansible.module_utils.basic.remove_values(kwargs, self.no_log_values)
+    self.do_cleanup_files()
+    raise ModuleError(kwargs.get('msg'), kwargs)
+
+
+def run_module(module, raw_params=None, args=None):
+    """
+    Set up the process environment in preparation for running an Ansible
+    module. The monkey-patches the Ansible libraries in various places to
+    prevent it from trying to kill the process on completion, and to prevent it
+    from reading sys.stdin.
+    """
+    if args is None:
+        args = {}
+    if raw_params is not None:
+        args['_raw_params'] = raw_params
+
+    ansible.module_utils.basic.AnsibleModule.exit_json = wtf_exit_json
+    ansible.module_utils.basic.AnsibleModule.fail_json = wtf_fail_json
+    ansible.module_utils.basic._ANSIBLE_ARGS = json.dumps({
+        'ANSIBLE_MODULE_ARGS': args
+    })
+
+    try:
+        mod = __import__(module, {}, {}, [''])
+        # Ansible modules begin execution on import, because they're crap from
+        # hell. Thus the above __import__ will cause either Exit or
+        # ModuleError to be raised. If we reach the line below, the module did
+        # not execute and must already have been imported for a previous
+        # invocation, so we need to invoke main explicitly.
+        mod.main()
+    except Exit, e:
+        return e.dct
+
+
+def main(router):
+    fmt = '%(asctime)s %(levelname).1s %(name)s: %(message)s'
+    datefmt = '%H:%M:%S'
+    level = logging.DEBUG
+    level = logging.INFO
+    logging.basicConfig(level=level, format=fmt, datefmt=datefmt)
+
+    context = mitogen.master.connect(broker)
+    print context.call(run_module, 'ansible.modules.core.system.setup')
+    for x in xrange(10):
+        print context.call(run_module, 'ansible.modules.core.commands.command', 'hostname')
+
+if __name__ == '__main__' and not mitogen.slave:
+    mitogen.utils.run_with_router(main)

mitogen/__init__.py

+"""
+On the Mitogen master, this is imported from ``mitogen/__init__.py`` as would
+be expected. On the slave, it is built dynamically during startup.
+"""
+
+#: This is ``False`` in slave contexts. It is used in single-file Python
+#: programs to avoid reexecuting the program's :py:func:`main` function in the
+#: slave. For example:
+#:
+#:      .. code-block:: python
+#:
+#:          def do_work():
+#:              os.system('hostname')
+#:
+#:          def main(broker):
+#:              context = mitogen.master.connect(broker)
+#:              context.call(do_work)  # Causes slave to import __main__.
+#:
+#:          if __name__ == '__main__' and mitogen.master:
+#:              import mitogen.utils
+#:              mitogen.utils.run_with_broker(main)
+#:
+master = True
+
+
+#: This is ``0`` in a master, otherwise it is a master-generated ID unique to
+#: the slave context used for message routing.
+context_id = 0
+
+
+#: This is ``None`` in a master, otherwise it is the master-generated ID unique
+#: to the slave's parent context.
+parent_id = None

mitogen/ansible/connection.py

+"""
+Basic Ansible connection plug-in mostly useful for testing functionality,
+due to Ansible's use of the multiprocessing package a lot more work is required
+to share the mitogen SSH connection across tasks.
+
+Enable it by:
+
+    $ cat ansible.cfg
+    [defaults]
+    connection_plugins = plugins/connection
+
+    $ mkdir -p plugins/connection
+    $ cat > plugins/connection/mitogen_conn.py <<-EOF
+    from mitogen.ansible.connection import Connection
+    EOF
+"""
+
+import mitogen.master
+import mitogen.ssh
+import mitogen.utils
+from mitogen.ansible import helpers
+
+import ansible.plugins.connection
+
+
+class Connection(ansible.plugins.connection.ConnectionBase):
+    broker = None
+    context = None
+
+    become_methods = []
+    transport = 'mitogen'
+
+    @property
+    def connected(self):
+        return self.broker is not None
+
+    def _connect(self):
+        if self.connected:
+            return
+        self.broker = mitogen.master.Broker()
+        if self._play_context.remote_addr == 'localhost':
+            self.context = mitogen.master.connect(self.broker)
+        else:
+            self.context = mitogen.ssh.connect(broker,
+                self._play_context.remote_addr)
+
+    def exec_command(self, cmd, in_data=None, sudoable=True):
+        super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
+        if in_data:
+            raise AnsibleError("does not support module pipelining")
+
+        return self.context.call(helpers.exec_command, cmd, in_data)
+
+    def fetch_file(self, in_path, out_path):
+        output = self.context.call(helpers.read_path, in_path)
+        helpers.write_path(out_path, output)
+
+    def put_file(self, in_path, out_path):
+        self.context.call(helpers.write_path, out_path,
+                          helpers.read_path(in_path))
+
+    def close(self):
+        self.broker.shutdown()
+        self.broker.join()

mitogen/ansible/helpers.py

+"""
+Ansible is so poorly layered that attempting to import anything under
+ansible.plugins automatically triggers import of __main__, which causes
+remote execution of the ansible command-line tool. :(
+
+So here we define helpers in some sanely layered package where the entirety of
+Ansible won't be imported.
+"""
+
+import subprocess
+
+
+def exec_command(cmd, in_data=None):
+    proc = subprocess.Popen(cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        stdin=subprocess.PIPE,
+        shell=True)
+    stdout, stderr = proc.communicate(in_data)
+    return proc.returncode, stdout, stderr
+
+
+def read_path(path):
+    return file(path, 'rb').read()
+
+
+def write_path(path, s):
+    open(path, 'wb').write(s)

mitogen/ssh.py

+"""
+Functionality to allow establishing new slave contexts over an SSH connection.
+"""
+
+import commands
+
+import mitogen.master
+
+
+class Stream(mitogen.master.Stream):
+    python_path = 'python'
+
+    #: The path to the SSH binary.
+    ssh_path = 'ssh'
+
+    def construct(self, hostname, username=None, ssh_path=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        self.hostname = hostname
+        self.username = username
+        if ssh_path:
+            self.ssh_path = ssh_path
+
+    def get_boot_command(self):
+        bits = [self.ssh_path]
+        if self.username:
+            bits += ['-l', self.username]
+        bits.append(self.hostname)
+        base = super(Stream, self).get_boot_command()
+        return bits + map(commands.mkarg, base)
+
+    def connect(self):
+        super(Stream, self).connect()
+        self.name = 'ssh.' + self.hostname

mitogen/tcp.py

+"""
+Functionality to allow a slave context to reconnect back to its master using a
+plain TCP connection.
+"""
+
+import socket
+
+import mitogen.core
+
+
+class Listener(mitogen.core.BasicStream):
+    def __init__(self, broker, address=None, backlog=30):
+        self._broker = broker
+        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self._sock.bind(address or ('0.0.0.0', 0))
+        self._sock.listen(backlog)
+        mitogen.core.set_cloexec(self._sock.fileno())
+        self.address = self._sock.getsockname()
+        self.receive_side = mitogen.core.Side(self, self._sock.fileno())
+        broker.start_receive(self)
+
+    def on_receive(self, broker):
+        sock, addr = self._sock.accept()
+        context = Context(self._broker, name=addr)
+        stream = mitogen.core.Stream(context)
+        stream.accept(sock.fileno(), sock.fileno())
+
+
+def listen(broker, address=None, backlog=30):
+    """Listen on `address` for connections from newly spawned contexts."""
+    return Listener(broker, address, backlog)
+
+
+def connect(context):
+    """Connect to a Broker at the address specified in our associated
+    Context."""
+    LOG.debug('%s.connect()', __name__)
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    self.receive_side = mitogen.core.Side(self, sock.fileno())
+    self.transmit_side = mitogen.core.Side(self, sock.fileno())
+    sock.connect(self._context.parent_addr)
+    self.enqueue(0, self._context.name)

mitogen/utils.py

+"""
+A random assortment of utility functions useful on masters and slaves.
+"""
+
+import logging
+import sys
+
+import mitogen
+import mitogen.core
+import mitogen.master
+
+
+LOG = logging.getLogger('mitogen')
+
+
+def disable_site_packages():
+    """Remove all entries mentioning site-packages or Extras from the system
+    path. Used primarily for testing on OS X within a virtualenv, where OS X
+    bundles some ancient version of the 'six' module."""
+    for entry in sys.path[:]:
+        if 'site-packages' in entry or 'Extras' in entry:
+            sys.path.remove(entry)
+
+
+def log_to_tmp():
+    import os
+    log_to_file(path='/tmp/mitogen.%s.log' % (os.getpid(),))
+
+
+def log_to_file(path=None, io=True, level=logging.INFO):
+    """Install a new :py:class:`logging.Handler` writing applications logs to
+    the filesystem. Useful when debugging slave IO problems."""
+    log = logging.getLogger('')
+    if path:
+        fp = open(path, 'w', 1)
+        mitogen.core.set_cloexec(fp.fileno())
+    else:
+        fp = sys.stderr
+
+    log.setLevel(level)
+    if io:
+        logging.getLogger('mitogen.io').setLevel(level)
+
+    fmt = '%(asctime)s %(levelname).1s %(name)s: %(message)s'
+    datefmt = '%H:%M:%S'
+    handler = logging.StreamHandler(fp)
+    handler.formatter = logging.Formatter(fmt, datefmt)
+    log.handlers.insert(0, handler)
+
+
+def run_with_router(func, *args, **kwargs):
+    """Arrange for `func(broker, *args, **kwargs)` to run with a temporary
+    :py:class:`mitogen.master.Router`, ensuring the Router and Broker are
+    correctly shut down during normal or exceptional return."""
+    broker = mitogen.master.Broker()
+    router = mitogen.master.Router(broker)
+    try:
+        return func(router, *args, **kwargs)
+    finally:
+        broker.shutdown()
+        broker.join()
+
+
+def with_router(func):
+    """Decorator version of :py:func:`run_with_broker`. Example:
+
+    .. code-block:: python
+
+        @with_broker
+        def do_stuff(broker, arg):
+            pass
+
+        do_stuff(blah, 123)
+    """
+    def wrapper(*args, **kwargs):
+        return run_with_router(func, *args, **kwargs)
+    wrapper.func_name = func.func_name
+    return wrapper

setup.py

+
+from distutils.core import setup
+
+setup(
+    name = 'mitogen',
+    version = '0.0.0',
+    description = 'Library for writing distributed self-replicating programs. THIS PACKAGE IS INCOMPLETE. IT IS BEING UPLOADED BECAUSE PYPI MAINTAINERS BROKE THE REGISTER COMMAND',
+    author = 'David Wilson',
+    license = 'OpenLDAP BSD',
+    url = 'http://github.com/dw/mitogen/',
+    py_packages = ['Mitogen'],
+    zip_safe = False
+)

tests/data/simple_pkg/a.py

+
+import simple_pkg.b
+
+
+def subtract_one_add_two(n):
+    return simple_pkg.b.subtract_one(n) + 2

tests/data/simple_pkg/b.py

+
+def subtract_one(n):
+    return n - 1