Setuptools packaging improvments

fixed-requirements.txt

@@ -0,0 +1 @@
+eventlet>=0.13.0

requirements.txt

@@ -1,3 +1,5 @@
+# 1. Don't include forget to populate spcific requirements.txt. IMORTANT!!!
+# 2. Don't include setuptools into any of requirements.txt (we don't want it bundled).
 apscheduler>=3.0.0rc1
 eventlet>=0.13.0
 flask
@@ -15,19 +17,18 @@ python-dateutil
 python-json-logger
 pyyaml
 requests
-setuptools==11.1
 six==1.9.0
 tooz
-git+https://github.com/StackStorm/[email protected]
-git+https://github.com/StackStorm/fabric.git@stanley-patched
+git+https://github.com/StackStorm/[email protected]#egg=python-mistralclient
+git+https://github.com/StackStorm/fabric.git@stanley-patched#egg=fabric
 passlib>=1.6.2,<1.7
 lockfile>=0.10.2,<0.11
 python-gnupg>=0.3.7,<0.4
 jsonpath-rw>=1.3.0
 # Requirements for linux pack
 # used by file watcher sensor
 pyinotify>=0.9.5,<=0.10
--e git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper
+git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper
 # used by nmap actions
 python-nmap>=0.3.4,<0.4
 semver>=2.1.2

scripts/fixate-requirements.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This script is used to automate generation of requirements.txt
+for st2 components.
+
+The idea behind this script is that that each component has it's own requirements
+in-requirements.txt file (input requirements file). Except this file
+there's also the top-level fixed-requirements.txt which pins production versions
+for the whole st2 stack. During production use (building, packaging, etc)
+requirements.txt is generated from in-requirements.txt where version of packages are
+fixed according to fixed-requirements.txt.
+"""
+
+import argparse
+import os
+import os.path
+import sys
+from distutils.version import StrictVersion
+
+OSCWD = os.path.abspath(os.curdir)
+GET_PIP = '    curl https://bootstrap.pypa.io/get-pip.py | python'
+
+try:
+    import pip
+    from pip.req import parse_requirements
+except ImportError:
+    print 'Download pip:\n', GET_PIP
+    sys.exit(1)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Tool for requirements.txt generation.')
+    parser.add_argument('-s', '--source-requirements', nargs='+',
+                        required=True,
+                        help='Specifiy paths to requirements file(s). '
+                        'In case severasl requirements files are given their content is merged.')
+    parser.add_argument('-f', '--fixed-requirements', required=True,
+                        help='Specifiy path to fixed-requirements.txt file.')
+    parser.add_argument('-o', '--output-file', default='requirements.txt',
+                        help='Specifiy path to the resulting requirements file.')
+    if len(sys.argv) < 2:
+        parser.print_help()
+        sys.exit(1)
+    return vars(parser.parse_args())
+
+
+def check_pip_version():
+    if StrictVersion(pip.__version__) < StrictVersion('6.0.0'):
+        print "Upgrade pip, your version `{}' "\
+              "is outdated:\n".format(pip.__version__), GET_PIP
+        sys.exit(1)
+
+
+def load_requirements(file_path):
+    return tuple((r for r in parse_requirements(file_path, session=False)))
+
+
+def locate_file(path, must_exist=False):
+    if not os.path.isabs(path):
+        path = os.path.join(OSCWD, path)
+    if must_exist and not os.path.isfile(path):
+        print("Error: couldn't locate file `{}'".format(path))
+    return path
+
+
+def merge_source_requirements(sources):
+    """Read requirements source files and merge it's content.
+    """
+    projects = set()
+    merged_requirements = []
+    for infile_path in (locate_file(p, must_exist=True) for p in sources):
+        for req in load_requirements(infile_path):
+            # Requirements lines like "project[version_spec]"
+            if req.req:
+                # Skip already added project name
+                if req.req.project_name in projects:
+                    continue
+                projects.add(req.req.project_name)
+                merged_requirements.append(req)
+
+            # Requirements lines like "vcs+proto://url"
+            elif req.link:
+                merged_requirements.append(req)
+            else:
+                raise RuntimeError('Unexpected requirement {}'.format(req))
+
+    return merged_requirements
+
+
+def write_requirements(sources=None, fixed_requirements=None, output_file=None):
+    """Wrire resulting requirements taking versions from the fixed_requirements.
+    """
+    requirements = merge_source_requirements(sources)
+    fixed = load_requirements(locate_file(fixed_requirements, must_exist=True))
+    fixedreq_hash = {req.req.project_name: req for req in fixed if req.req}
+
+    with open(output_file, 'w') as f:
+        f.write("# Don't edit this file. It's generated automatically!\n")
+        for req in requirements:
+            # we don't have any idea how to process links, so just add them
+            if req.link:
+                rline = str(req.link)
+            else:
+                project = req.req.project_name
+                if project in fixedreq_hash:
+                    rline = str(fixedreq_hash[project])
+                else:
+                    rline = str(req.req)
+            f.write(rline + '\n')
+
+    return
+
+
+if __name__ == '__main__':
+    check_pip_version()
+    args = parse_args()
+    write_requirements(sources=args['source_requirements'],
+                       fixed_requirements=args['fixed_requirements'],
+                       output_file=args['output_file'])

st2_version

@@ -0,0 +1 @@
+0.10dev0

st2api/requirements.txt

@@ -0,0 +1,7 @@
+eventlet>=0.13.0
+jsonschema>=2.3.0
+kombu
+mongoengine>=0.8.7,<0.9
+oslo.config
+pecan==0.7.0
+six==1.9.0

st2api/setup.py

@@ -14,27 +14,43 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-try:
-    from setuptools import setup, find_packages
-except ImportError:
-    from ez_setup import use_setuptools
-    use_setuptools()
-    from setuptools import setup, find_packages
+import re
+import os.path
+from pip.req import parse_requirements
+from setuptools import setup, find_packages
+
+
+def fetch_requirements():
+    links = []
+    reqs = []
+    for req in parse_requirements('requirements.txt', session=False):
+        if req.link:
+            links.append(str(req.link))
+        reqs.append(str(req.req))
+    return (reqs, links)
+
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+version_file = os.path.join(current_dir, '../st2client/st2client/__init__.py')
+with open(version_file, 'r') as f:
+    vmatch = re.search(r'__version__ = [\'\"](.*)[\'\"]$', f.read(), flags=re.MULTILINE)
+
+
+install_reqs, dep_links = fetch_requirements()
+ST2_COMPONENT = os.path.basename(current_dir)
+ST2_VERSION = vmatch.group(1)
+
 
 setup(
-    name='st2api',
-    version='0.4.0',
-    description='',
+    name=ST2_COMPONENT,
+    version=ST2_VERSION,
+    description='{} component'.format(ST2_COMPONENT),
     author='StackStorm',
     author_email='[email protected]',
-    install_requires=[
-        "pecan",
-    ],
-    package_data={
-        'st2api': ['templates/*.html']
-    },
-    test_suite='st2api',
+    install_requires=install_reqs,
+    dependency_links=dep_links,
+    test_suite=ST2_COMPONENT,
     zip_safe=False,
     include_package_data=True,
-    packages=find_packages(exclude=['ez_setup'])
+    packages=find_packages(exclude=['setuptools'])
 )

st2auth/requirements.txt

@@ -0,0 +1,6 @@
+eventlet>=0.13.0
+oslo.config
+passlib>=1.6.2,<1.7
+pecan==0.7.0
+pymongo<3.0
+six==1.9.0

st2client/requirements.txt

@@ -1,6 +1,6 @@
 prettytable
+python-dateutil
 pyyaml
+jsonpath-rw>=1.3.0
 requests
-six
-python-dateutil
-jsonpath-rw
+six==1.9.0

st2common/MANIFEST.in

@@ -0,0 +1,4 @@
+# https://docs.python.org/2/distutils/sourcedist.html#commands
+# Include all files under the source tree by default.
+# Other behaviour can be used though.
+recursive-include st2common *.* *

st2common/Makefile

@@ -32,3 +32,6 @@ deb:
 	tar --transform=s~^~$(COMPONENTS)-$(VER)/~ --exclude=correlation -czf ~/$(COMPONENTS).tar.gz bin st2 logrotate.d $(COMPONENTS) ../contrib ../docs ../tools/ ../requirements.txt packaging/debian
 	pushd ~ && tar -xzf $(COMPONENTS).tar.gz && cd $(COMPONENTS)-$(VER) && cp -Rf packaging/debian ./ && dpkg-buildpackage -us -uc -b && popd
 	cp -f ~/$(COMPONENT)*.deb ~/debbuild/
+
+.PHONY: requirements
+	python ../scripts/fixate-requirements.py -s in-requirements.txt -f ../fixed-requirements.txt

st2common/in-requirements.txt

@@ -0,0 +1,15 @@
+python-dateutil
+eventlet>=0.13.0
+git+https://github.com/StackStorm/fabric.git@stanley-patched#egg=fabric
+jinja2
+jsonschema>=2.3.0
+kombu
+mongoengine>=0.8.7,<0.9
+oslo.config
+paramiko
+pecan==0.7.0
+pyyaml
+requests
+semver>=2.1.2
+six==1.9.0
+tooz

st2common/setup.py

@@ -14,24 +14,45 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-try:
-    from setuptools import setup, find_packages
-except ImportError:
-    from ez_setup import use_setuptools
-    use_setuptools()
-    from setuptools import setup, find_packages
+# Just setup pip before using with a command:
+# curl https://bootstrap.pypa.io/get-pip.py | python
+
+import os.path
+from pip.req import parse_requirements
+from setuptools import setup, find_packages
+
+
+def fetch_requirements():
+    links = []
+    reqs = []
+    for req in parse_requirements('requirements.txt', session=False):
+        if req.link:
+            links.append(str(req.link))
+        reqs.append(str(req.req))
+    return (reqs, links)
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+with open(os.path.join(current_dir, 'st2_version'), 'r') as f:
+    st2_version = f.read().strip()
+
+install_reqs, dep_links = fetch_requirements()
+st2_component = os.path.basename(current_dir)
+
 
 setup(
-    name='st2common',
-    version='0.4.0',
-    description='',
+    name=st2_component,
+    version=st2_version,
+    description='{} component'.format(st2_component),
     author='StackStorm',
     author_email='[email protected]',
-    install_requires=[
-        "pecan",
-    ],
-    test_suite='st2common',
+    install_requires=install_reqs,
+    dependency_links=dep_links,
+    test_suite=st2_component,
     zip_safe=False,
     include_package_data=True,
-    packages=find_packages(exclude=['ez_setup'])
+    packages=find_packages(exclude=['setuptools', 'examples', 'tests']),
+    scripts=[
+        'bin/st2-bootstrap-rmq',
+        'bin/st2-register-content'
+    ]
 )

st2debug/requirements.txt

@@ -0,0 +1,4 @@
+python-gnupg>=0.3.7,<0.4
+requests
+six==1.9.0
+pyyaml

st2reactor/requirement.txt

@@ -0,0 +1,8 @@
+apscheduler>=3.0.0rc1
+python-dateutil
+eventlet>=0.13.0
+jsonpath-rw>=1.3.0
+jsonschema>=2.3.0
+kombu
+oslo.config
+six==1.9.0