Merge pull request #25 from KristofferC/kc/async

Asynchronously install packages

ext/BinaryProvider/.codecov.yml

@@ -0,0 +1 @@
+comment: false

ext/BinaryProvider/.gitignore

@@ -0,0 +1,5 @@
+*.jl.cov
+*.jl.*.cov
+*.jl.mem
+
+global_prefix

ext/BinaryProvider/.travis.yml

@@ -0,0 +1,30 @@
+# Documentation: http://docs.travis-ci.com/user/languages/julia/
+language: julia
+os:
+  - linux
+  - osx
+julia:
+  - 0.6
+  - nightly
+notifications:
+  email: false
+
+matrix:
+  include:
+    # Make sure to override to "wget" at least once
+    - julia: 0.6
+      os: linux
+      env: BINARYPROVIDER_DOWNLOAD_ENGINE="wget"
+
+# Ironic.  He could provide binaries for others but not himself...
+addons:
+  apt:
+    packages:
+      - curl
+      - wget
+      - tar
+      - gzip
+
+after_success:
+  # push coverage results to Codecov
+  - julia -e 'cd(Pkg.dir("BinaryProvider")); Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())'

ext/BinaryProvider/LICENSE.md

@@ -0,0 +1,22 @@
+The BinaryProvider.jl package is licensed under the MIT "Expat" License:
+
+> Copyright (c) 2017: SimonDanisch.
+> 
+> Permission is hereby granted, free of charge, to any person obtaining a copy
+> of this software and associated documentation files (the "Software"), to deal
+> in the Software without restriction, including without limitation the rights
+> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+> copies of the Software, and to permit persons to whom the Software is
+> furnished to do so, subject to the following conditions:
+> 
+> The above copyright notice and this permission notice shall be included in all
+> copies or substantial portions of the Software.
+> 
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+> SOFTWARE.
+> 

ext/BinaryProvider/README.md

@@ -0,0 +1,35 @@
+# BinaryProvider
+
+[![Travis Status](https://travis-ci.org/JuliaPackaging/BinaryProvider.jl.svg?branch=master)](https://travis-ci.org/JuliaPackaging/BinaryProvider.jl)
+
+[![Appveyor Status](https://ci.appveyor.com/api/projects/status/0sbp28iie07c5dn3/branch/master?svg=true)](https://ci.appveyor.com/project/staticfloat/binaryprovider-jl-fu5p5/branch/master)
+
+[![codecov.io](http://codecov.io/github/JuliaPackaging/BinaryProvider.jl/coverage.svg?branch=master)](http://codecov.io/github/JuliaPackaging/BinaryProvider.jl?branch=master)
+
+**staticfloat's third draft**:  This package is intended to work alongside [`BinaryBuilder.jl`](https://github.com/JuliaPackaging/BinaryBuilder.jl); within this is all logic necessary to download and unpack tarballs into `Prefix`es.
+
+## Basic concepts
+
+Packages are installed to a `Prefix`; a folder that acts similar to the `/usr/local` directory on Unix-like systems, containing a `bin` folder for binaries, a `lib` folder for libraries, etc...  `Prefix` objects can have tarballs `install()`'ed within them, `uninstall()`'ed from them, etc...
+
+`BinaryProvider` has the concept of a `Product`, the result of a package installation.  `LibraryProduct` and `ExecutableProduct` are two example `Product` object types that can be used to keep track of the binary objects installed by an `install()` invocation.  `Products` can check to see if they are already satisfied (e.g. whether a file exists, or is executable, or is `dlopen()`'able), allowing for very quick and easy `build.jl` construction.
+
+`BinaryProvider` also contains a platform abstraction layer for common operations like downloading and unpacking tarballs.  The primary method you should be using to interact with these operations is through the `install()` method, however if you need more control, there are more fundamental methods such as `download_verify()`, or `unpack()`, or even the wittingly-named `download_verify_unpack()`.
+
+The method documentation within the `BinaryProvider` module should be considered the primary source of documentation for this package, usage examples are provided in the form of the `LibFoo.jl` mock package [within this repository](test/LibFoo.jl), as well as other packages that use this package for binary installation such as 
+
+## Usage
+
+To download and install a package into a `Prefix`, the basic syntax is:
+```julia
+prefix = Prefix("./deps")
+install(url, tarball_hash; prefix=prefix)
+```
+
+It is recommended to inspect examples for a fuller treatment of installation, the [`LibFoo.jl` package within this repository](test/LibFoo.jl) contains a [`deps/build.jl` file](test/LibFoo.jl/deps/build.jl) that may be instructive.
+
+To actually generate the tarballs that are installed by this package, check out the [`BinaryBuilder.jl` package](https://github.com/JuliaPackaging/BinaryBuilder.jl).
+
+## Miscellanea
+
+* This package contains a `run(::Cmd)` wrapper class named `OutputCollector` that captures the output of shell commands, and in particular, captures the `stdout` and `stderr` streams separately, colorizing, buffering and timestamping appropriately to provide seamless printing of shell output in a consistent and intuitive way.  Critically, it also allows for saving of the captured streams to log files, a very useful feature for [`BinaryBuilder.jl`](https://github.com/JuliaPackaging/BinaryBuilder.jl), which makes extensive use of this class, however all commands run by `BinaryProvider.jl` also use this same mechanism to provide coloring of `stderr`.

ext/BinaryProvider/REQUIRE

@@ -0,0 +1,3 @@
+julia 0.6
+SHA
+Compat 0.27.0

ext/BinaryProvider/appveyor.yml

@@ -0,0 +1,48 @@
+environment:
+  matrix:
+  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe"
+  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
+  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe"
+  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
+
+matrix:
+  allow_failures:
+  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe"
+  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
+
+branches:
+  only:
+    - master
+    - /release-.*/
+
+notifications:
+  - provider: Email
+    on_build_success: false
+    on_build_failure: false
+    on_build_status_changed: false
+
+install:
+  - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12"
+# If there's a newer build queued for the same PR, cancel this one
+  - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
+        https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
+        Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
+        throw "There are newer queued builds for this pull request, failing early." }
+# Download most recent Julia Windows binary
+  - ps: (new-object net.webclient).DownloadFile(
+        $env:JULIA_URL,
+        "C:\projects\julia-binary.exe")
+# Run installer silently, output to C:\projects\julia
+  - C:\projects\julia-binary.exe /S /D=C:\projects\julia
+
+build_script:
+# Need to convert from shallow to complete for Pkg.clone to work
+  - IF EXIST .git\shallow (git fetch --unshallow)
+  - C:\projects\julia\bin\julia -e "versioninfo();
+      Pkg.clone(pwd(), \"BinaryProvider\"); Pkg.build(\"BinaryProvider\")"
+
+test_script:
+  - C:\projects\julia\bin\julia -e "Pkg.test(\"BinaryProvider\", coverage=true)"
+
+after_test:
+  - C:\projects\julia\bin\julia -e "cd(Pkg.dir(\"BinaryProvider\")); Pkg.add(\"Coverage\"); using Coverage; Codecov.submit(process_folder())"

ext/BinaryProvider/src/BinDepsIntegration.jl

@@ -0,0 +1,37 @@
+# This file contains the ingredients to create a PackageManager for BinDeps
+import BinDeps: Binaries, can_use, package_available, bindir, libdir,
+                generate_steps, LibraryDependency, provider, provides
+import Base: show
+
+type BP <: Binaries
+    url::String
+    hash::String
+    prefix::Prefix
+end
+
+show(io::IO, p::BP) = write(io, "BinaryProvider for $(p.url)")
+
+# We are cross-platform baby, and we never say no to a party
+can_use(::Type{BP}) = true
+package_available(p::BP) = true
+libdir(p::BP, dep) = @static if is_windows()
+    joinpath(p.prefix, "bin")
+else
+    joinpath(p.prefix, "lib")
+end
+
+# We provide our own overload of provides() for BP
+macro BP_provides(url, hash, dep, opts...)
+    return quote
+        prefix = Prefix(joinpath(dirname(@__FILE__), "usr"))
+        activate(prefix)
+        return provides(BP, ($url, $hash, prefix), $(esc(dep)), $(opts...))
+    end
+end
+provider(::Type{BP}, data; opts...) = BP(data...)
+
+function generate_steps(dep::LibraryDependency, p::BP, opts)
+    () -> begin
+        install(p.url, p.hash; prefix=p.prefix, verbose=true)
+    end
+end

ext/BinaryProvider/src/BinaryPackage.jl

@@ -0,0 +1,151 @@
+# Package objects provide a _slightly_ higher-level API for dealing with
+# installing/uninstalling tarballs to a Prefix.
+export BinaryPackage, install, uninstall, satisfied
+
+"""
+A `BinaryPackage` collects all the information needed to download and install a
+tarball containing binary objects; it has the `url` to download form, the
+`hash` to verify package integrity, a list of `products` to check for proper
+functioning after installation, and a list of `dependencies` that must also be
+installed before this package can be used.
+
+There exist `install()`, `uninstall()` and `satisfied()` methods for
+`BinaryPackage` objects, similar to the lower-level versions that take direct
+`url` and `hash` arguments.
+"""
+immutable BinaryPackage
+    url::String
+    hash::String
+    platform::Platform
+    products::Vector{Product}
+    dependencies::Vector{BinaryPackage}
+
+    function BinaryPackage(url::AbstractString,
+                           hash::AbstractString,
+                           platform::Platform,
+                           products::Vector{Product}=Product[],
+                           dependencies::Vector{BinaryPackage}=BinaryPackage[])
+        return new(url, hash, platform, products, dependencies)
+    end
+end
+
+function pkg_name(pkg::BinaryPackage)
+    name = basename(pkg.url)
+    if endswith(name, ".tar.gz")
+        return name[1:end-7]
+    end
+    return name
+end
+
+"""
+`install(pkg::BinaryPackage; verbose::Bool = false, kwargs...)`
+
+A thin wrapper over the main `install(url, path; ...)` method.  Installs all
+of `pkg`'s dependencies, then installs `pkg`, but only of it is not already
+satisfied.
+"""
+function install(pkg::BinaryPackage; verbose::Bool = false, kwargs...)
+    name = pkg_name(pkg)
+
+    # If we are already satisfied, don't do nuthin'
+    if satisfied(pkg)
+        if verbose
+            info("Not installing $(name), as it is already satisfied")
+        end
+        return true
+    end
+
+    # Begin by installing all the dependencies if they are not already
+    for dep in pkg.dependencies
+        # TODO: We may want to handle this through `Pkg3` operations
+        install(dep; verbose=verbose, kwargs...)
+    end
+
+    # Finally, install ourselves
+    install(pkg.url, pkg.hash; verbose=verbose, kwargs...)
+
+    # Check to see if we are actually satisfied
+    if !satisfied(pkg; verbose=verbose)
+        warn("$(name) did not satisfy itself after installation!")
+        return false
+    end
+
+    return true
+end
+
+"""
+manifest_path(pkg::BinaryPackage; prefix::Prefix = global_prefix(),
+                                  verbose::Bool = false)
+
+Discovers the manifest path for the given `BinaryPackage` within the given
+`Prefix`.  First attempts to guess from the `url` what the manifest file would
+have been named, if that doesn't work, will search for manifests that contain
+any of the `products` that are within `pkg`.  If neither approach works, throws
+an error.
+"""
+function manifest_path(pkg::BinaryPackage; prefix::Prefix = global_prefix(),
+                                           verbose::Bool = false)
+    name = pkg_name(pkg)
+    # First, see if we can auto-guess the manifest file path:
+    manifest_path = manifest_from_url(pkg.url, prefix=prefix)
+    if isfile(manifest_path)
+        if verbose
+            info("Correctly auto-guessed manifest path $(manifest_path)")
+        end
+        return manifest_path
+    end
+
+    if verbose
+        info("Could not auto-guess manifest path for $(name)")
+    end
+
+    # Otherwise, let's try to guess from our products
+    if isempty(pkg.products)
+        msg = """
+        Cannot find manifest path for package $(name) with unguessable manifest
+        file and no products.
+        """
+        error(replace(strip(msg),"\n", " "))
+    end
+
+    for product in pkg.products
+        product_path = locate(product, platform=pkg.platform)
+        if product_path != nothing
+            try
+                manifest_path = manifest_for_file(product_path; prefix=prefix)
+                relmani = relpath(manifest_path, prefix.path)
+                relprod = relpath(product_path, prefix.path)
+                info("Found $(relmani) for product $(relprod)")
+                return manifest_path
+            end
+        end
+    end
+
+    error("Cannot find manifest path for package $(name)")
+end
+
+
+"""
+uninstall(pkg::BinaryPackage; prefix::Prefix = global_prefix,
+                              verbose::Bool = false)
+
+Uninstall `pkg` from the given `prefix` by automatically determining the
+manifest path created when the package was installed.  Throws an error if
+this `pkg` was not installed in the first place.
+"""
+function uninstall(pkg::BinaryPackage; prefix::Prefix = global_prefix,
+                                       verbose::Bool = false)
+    # Find the manifest path for this pkg, then uninstall it
+    manipath = manifest_path(pkg; prefix=prefix, verbose=verbose)
+    uninstall(manipath; verbose=verbose)
+end
+
+"""
+`satisfied(pkg::BinaryPackage; verbose::Bool = false)`
+
+Returns `true` if all products defined within `pkg` are satisfied.
+"""
+function satisfied(pkg::BinaryPackage; verbose::Bool = false)
+    s = p -> satisfied(p; platform=pkg.platform, verbose=verbose)
+    return all(s(p) for p in pkg.products)
+end

ext/BinaryProvider/src/BinaryProvider.jl

@@ -0,0 +1,37 @@
+module BinaryProvider
+
+# Include our subprocess running funtionality
+include("OutputCollector.jl")
+# External utilities such as downloading/decompressing tarballs
+include("PlatformEngines.jl")
+# Platform naming
+include("PlatformNames.jl")
+# Everything related to file/path management
+include("Prefix.jl")
+# Abstraction of "needing" a file, that would trigger an install
+include("Products.jl")
+# Abstraction of bundled binary package
+include("BinaryPackage.jl")
+
+# BinDeps support, disabled for now because I don't particularly want to force
+# users to install BinDeps to install this package.  That seems counter-productive
+#include("BinDepsIntegration.jl")
+
+
+function __init__()
+    global global_prefix
+
+    # Initialize our global_prefix
+    # global_prefix = Prefix(joinpath(dirname(@__FILE__), "../", "global_prefix"))
+    # activate(global_prefix)
+
+    # Find the right download/compression engines for this platform
+    probe_platform_engines!()
+
+    # If we're on a julia that's too old, then fixup the color mappings
+    # if !haskey(Base.text_colors, :default)
+    #    Base.text_colors[:default] = Base.color_normal
+    # end
+end
+
+end # module