Add support for creating envs from requirements.txt files via pip

conda.recipe/meta.yaml

@@ -19,8 +19,10 @@ source:
 requirements:
   build:
     - python
+    - pip
   run:
     - python
+    - pip
 
 test:
   commands:

conda_env/cli/main_create.py

@@ -80,6 +80,8 @@ def execute(args, parser):
         # FIXME conda code currently requires args to have a name or prefix
         if args.name is None:
             args.name = env.name
+        else:
+            env.name = args.name
 
     except exceptions.SpecNotFound as e:
         common.error_and_exit(str(e), json=args.json)

conda_env/env.py

@@ -1,12 +1,26 @@
 from __future__ import absolute_import, print_function
+
+import errno
+import os
+import subprocess
 from collections import OrderedDict
 from copy import copy
-import os
+from io import open
+from os.path import isdir
+from shutil import rmtree
+
+# Try to import PipSession to support new pips
+try:
+    from pip.download import PipSession
+except ImportError:
+    pass
+from pip.req import parse_requirements
 
 # TODO This should never have to import from conda.cli
-from conda.cli import common
-from conda.cli import main_list
 from conda import install
+from conda.api import get_index
+from conda.cli import common, main_list
+from conda.resolve import NoPackagesFound, Resolve, MatchSpec
 
 from . import compat
 from . import exceptions
@@ -50,19 +64,83 @@ def from_environment(name, prefix, no_builds=False):
 
 
 def from_yaml(yamlstr, **kwargs):
-    """Load and return a ``Environment`` from a given ``yaml string``"""
+    """Load and return an ``Environment`` from a given ``yaml string``"""
     data = yaml.load(yamlstr)
     if kwargs is not None:
         for key, value in kwargs.items():
             data[key] = value
     return Environment(**data)
 
 
+def get_all_pip_dependencies(requirements_path, temp_dir='_delete_when_done'):
+    """
+    Use pip to gather all of the packages that would be installed for the given
+    requirements.txt file.
+    """
+    pip_cmd = ('pip', 'install', '--src', temp_dir, '--download', temp_dir,
+               '--no-use-wheel', '-r', requirements_path)
+    try:
+        os.makedirs(temp_dir)
+    except OSError as exc:
+        # Don't raise exception if directory already exists
+        if not (exc.errno == errno.EEXIST and isdir(temp_dir)):
+            raise
+    process = subprocess.Popen(pip_cmd, stdout=subprocess.PIPE,
+                               universal_newlines=True)
+    stdout_data = process.communicate()[0]
+    reqs = []
+    for line in stdout_data.splitlines():
+        req = ''
+        if line.startswith('Collecting'):
+            req = line.split(' (', 1)[0][11:]
+        elif line.startswith('Obtaining'):
+            req = line.split(' (', 1)[0][10:]
+        if req:
+            if ' from ' in req:
+                req = '-e {}'.format(req.split(' from ', 1)[1])
+            reqs.append(req)
+    # Remove temporary directory
+    rmtree(temp_dir)
+    return reqs
+
+
+def from_requirements_txt(filename, **kwargs):
+    """Load and return an ``Environment`` from a given ``requirements.txt``"""
+    pip_reqs = []
+    dep_list = []
+    r = Resolve(get_index())
+    parsed_reqs = get_all_pip_dependencies(filename)
+    for req in parsed_reqs:
+        if req.startswith('-e'):
+            pip_reqs.append(req)
+        # If it's not an editable package, check if it's available via conda
+        else:
+            try:
+                # If package is available via conda, use that
+                r.get_pkgs(MatchSpec(common.arg2spec(req)))
+                dep_list.append(req)
+            except NoPackagesFound:
+                # Otherwise, just use pip
+                pip_reqs.append(req)
+    # Add pip requirements to environment if there were any left
+    if pip_reqs:
+        dep_list.append('pip')
+        dep_list.append({'pip': pip_reqs})
+    data = {'dependencies': dep_list}
+    if kwargs is not None:
+        for key, value in kwargs.items():
+            data[key] = value
+    return Environment(**data)
+
+
 def from_file(filename):
     if not os.path.exists(filename):
         raise exceptions.EnvironmentFileNotFound(filename)
-    with open(filename, 'rb') as fp:
-        return from_yaml(fp.read(), filename=filename)
+    if filename.endswith('.txt'):
+        return from_requirements_txt(filename)
+    else:
+        with open(filename, 'rb') as fp:
+            return from_yaml(fp.read(), filename=filename)
 
 
 # TODO test explicitly

conda_env/installers/pip.py

@@ -1,10 +1,19 @@
 from __future__ import absolute_import
+
 import subprocess
+from itertools import chain
+from os.path import join
 
+import conda.config as config
 from conda.cli import main_list
 
 
 def install(prefix, specs, args, env):
-    pip_cmd = main_list.pip_args(prefix) + ['install', ] + specs
+    # This mess is necessary to get --editable package sent to subprocess
+    # properly.
+    specs = list(chain(*[s.split() if '-e' in s else [s] for s in specs]))
+    # Directory where pip will store VCS checkouts for editable packages
+    src_dir = join(config.envs_dirs[0], env.name, 'src')
+    pip_cmd = main_list.pip_args(prefix) + ['install', '--src', src_dir] + specs
     process = subprocess.Popen(pip_cmd, universal_newlines=True)
     process.communicate()

tests/support/requirements.txt

@@ -0,0 +1,5 @@
+# Comment to be ignored
+skll
+ # Comment 2
+simplejson
+-e git+https://github.com/fabric/fabric.git#egg=fabric

tests/test_env.py

@@ -43,6 +43,19 @@ def test_with_pip(self):
         self.assert_('foo' in e.dependencies['pip'])
         self.assert_('baz' in e.dependencies['pip'])
 
+    def test_requirements_txt(self):
+        e = env.from_file(utils.support_file('requirements.txt'))
+        # skll could be in either, depending on active channels
+        self.assert_('skll' in e.dependencies['pip'] or
+                     'skll' in e.dependencies['conda'])
+        self.assert_('simplejson' in e.dependencies['pip'])
+        self.assert_('-e git+https://github.com/fabric/fabric.git#egg=fabric' in
+                     e.dependencies['pip'])
+        self.assert_('numpy' in e.dependencies['conda'])
+        self.assert_('# Comment to be ignored' not in e.dependencies['pip'])
+        self.assert_('# Comment 2' not in e.dependencies['pip'])
+        self.assert_(' # Comment 2' not in e.dependencies['pip'])
+
 
 class EnvironmentTestCase(unittest.TestCase):
     def test_has_empty_filename_by_default(self):