diff --git a/src/Operations.jl b/src/Operations.jl index 4cd8a0fde6..80a5a48d7a 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -759,7 +759,7 @@ function prune_manifest(env::EnvCache) env.manifest = prune_manifest(env.manifest, keep) end -function prune_manifest(manifest::Dict, keep::Vector{UUID}) +function prune_manifest(manifest::Manifest, keep::Vector{UUID}) while !isempty(keep) clean = true for (uuid, entry) in manifest @@ -772,7 +772,8 @@ function prune_manifest(manifest::Dict, keep::Vector{UUID}) end clean && break end - return Dict(uuid => entry for (uuid, entry) in manifest if uuid in keep) + manifest.deps = Dict(uuid => entry for (uuid, entry) in manifest if uuid in keep) + return manifest end @@ -1405,7 +1406,7 @@ function sandbox_preserve(env::EnvCache, target::PackageSpec, test_project::Stri return prune_manifest(env.manifest, keep) end -function abspath!(env::EnvCache, manifest::Dict{UUID,PackageEntry}) +function abspath!(env::EnvCache, manifest::Manifest) for (uuid, entry) in manifest if entry.path !== nothing entry.path = project_rel_path(env, entry.path) @@ -1477,7 +1478,7 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::S allow_reresolve || rethrow() @debug err @warn "Could not use exact versions of packages in manifest, re-resolving" - temp_ctx.env.manifest = Dict(uuid => entry for (uuid, entry) in temp_ctx.env.manifest if isfixed(entry)) + temp_ctx.env.manifest.deps = Dict(uuid => entry for (uuid, entry) in temp_ctx.env.manifest.deps if isfixed(entry)) Pkg.resolve(temp_ctx; io=devnull, skip_writing_project=true) @debug "Using _clean_ dep graph" end diff --git a/src/Types.jl b/src/Types.jl index 512a3a38ff..57cc6c5686 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -245,7 +245,24 @@ Base.:(==)(t1::PackageEntry, t2::PackageEntry) = t1.name == t2.name && t1.uuid == t2.uuid # omits `other` Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.uuid], init=h) # omits `other` -const Manifest = Dict{UUID,PackageEntry} + +Base.@kwdef mutable struct Manifest + julia_version::Union{Nothing,VersionNumber} = Base.VERSION + manifest_format::VersionNumber = v"2.0.0" + deps::Dict{UUID,PackageEntry} = Dict{UUID,PackageEntry}() +end +Base.:(==)(t1::Manifest, t2::Manifest) = all(x -> (getfield(t1, x) == getfield(t2, x))::Bool, fieldnames(Manifest)) +Base.hash(m::Manifest, h::UInt) = foldr(hash, [getfield(m, x) for x in fieldnames(Manifest)], init=h) +Base.getindex(m::Manifest, i_or_key) = getindex(m.deps, i_or_key) +Base.get(m::Manifest, key, default) = get(m.deps, key, default) +Base.setindex!(m::Manifest, i_or_key, value) = setindex!(m.deps, i_or_key, value) +Base.iterate(m::Manifest) = iterate(m.deps) +Base.iterate(m::Manifest, i::Int) = iterate(m.deps, i) +Base.length(m::Manifest) = length(m.deps) +Base.empty!(m::Manifest) = empty!(m.deps) +Base.values(m::Manifest) = values(m.deps) +Base.keys(m::Manifest) = keys(m.deps) +Base.haskey(m::Manifest, key) = haskey(m.deps, key) function Base.show(io::IO, pkg::PackageEntry) f = [] diff --git a/src/manifest.jl b/src/manifest.jl index 951ea9dca6..602ced90fe 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -105,19 +105,19 @@ function normalize_deps(name, uuid, deps::Vector{String}, manifest::Dict{String, return final end -function validate_manifest(stage1::Dict{String,Vector{Stage1}}) +function validate_manifest(julia_version::Union{Nothing,VersionNumber}, manifest_format::VersionNumber, stage1::Dict{String,Vector{Stage1}}) # expand vector format deps for (name, infos) in stage1, info in infos info.entry.deps = normalize_deps(name, info.uuid, info.deps, stage1) end # invariant: all dependencies are now normalized to Dict{String,UUID} - manifest = Dict{UUID, PackageEntry}() + deps = Dict{UUID, PackageEntry}() for (name, infos) in stage1, info in infos - manifest[info.uuid] = info.entry + deps[info.uuid] = info.entry end # now just verify the graph structure - for (entry_uuid, entry) in manifest, (name, uuid) in entry.deps - dep_entry = get(manifest, uuid, nothing) + for (entry_uuid, entry) in deps, (name, uuid) in entry.deps + dep_entry = get(deps, uuid, nothing) if dep_entry === nothing pkgerror("`$(entry.name)=$(entry_uuid)` depends on `$name=$uuid`, ", "but no such entry exists in the manifest.") @@ -127,38 +127,42 @@ function validate_manifest(stage1::Dict{String,Vector{Stage1}}) "but entry with UUID `$uuid` has name `$(dep_entry.name)`.") end end - return manifest + return Manifest(; julia_version, manifest_format, deps) end function Manifest(raw::Dict)::Manifest + julia_version = isnothing(raw["julia_version"]) ? nothing : VersionNumber(raw["julia_version"]) + manifest_format = VersionNumber(raw["manifest_format"]) stage1 = Dict{String,Vector{Stage1}}() - for (name, infos) in raw, info in infos - entry = PackageEntry() - entry.name = name - uuid = nothing - deps = nothing - try - entry.pinned = read_pinned(get(info, "pinned", nothing)) - uuid = read_field("uuid", nothing, info, safe_uuid)::UUID - entry.version = read_field("version", nothing, info, safe_version) - entry.path = read_field("path", nothing, info, safe_path) - entry.repo.source = read_field("repo-url", nothing, info, identity) - entry.repo.rev = read_field("repo-rev", nothing, info, identity) - entry.repo.subdir = read_field("repo-subdir", nothing, info, identity) - entry.tree_hash = read_field("git-tree-sha1", nothing, info, safe_SHA1) - entry.uuid = uuid - deps = read_deps(get(info::Dict, "deps", nothing)) - catch - # TODO: Should probably not unconditionally log something - @error "Could not parse entry for `$name`" - rethrow() + if haskey(raw, "deps") # deps field doesn't exist if there are no deps + for (name, infos) in raw["deps"], info in infos + entry = PackageEntry() + entry.name = name + uuid = nothing + deps = nothing + try + entry.pinned = read_pinned(get(info, "pinned", nothing)) + uuid = read_field("uuid", nothing, info, safe_uuid)::UUID + entry.version = read_field("version", nothing, info, safe_version) + entry.path = read_field("path", nothing, info, safe_path) + entry.repo.source = read_field("repo-url", nothing, info, identity) + entry.repo.rev = read_field("repo-rev", nothing, info, identity) + entry.repo.subdir = read_field("repo-subdir", nothing, info, identity) + entry.tree_hash = read_field("git-tree-sha1", nothing, info, safe_SHA1) + entry.uuid = uuid + deps = read_deps(get(info::Dict, "deps", nothing)) + catch + # TODO: Should probably not unconditionally log something + @error "Could not parse entry for `$name`" + rethrow() + end + entry.other = info::Union{Dict,Nothing} + stage1[name] = push!(get(stage1, name, Stage1[]), Stage1(uuid, entry, deps)) end - entry.other = info::Union{Dict,Nothing} - stage1[name] = push!(get(stage1, name, Stage1[]), Stage1(uuid, entry, deps)) + # by this point, all the fields of the `PackageEntry`s have been type casted + # but we have *not* verified the _graph_ structure of the manifest end - # by this point, all the fields of the `PackageEntry`s have been type casted - # but we have *not* verified the _graph_ structure of the manifest - return validate_manifest(stage1) + return validate_manifest(julia_version, manifest_format, stage1) end function read_manifest(f_or_io::Union{String, IO}) @@ -174,9 +178,23 @@ function read_manifest(f_or_io::Union{String, IO}) end rethrow() end + if !isempty(raw) && Base.is_v1_format_manifest(raw) + raw = convert_flat_format_manifest(raw) + end return Manifest(raw) end +function convert_flat_format_manifest(old_raw_manifest::Dict) + new_raw_manifest = Dict{String,Any}() + new_raw_manifest["deps"] = Dict{String,Vector{Any}}() + for (key, value) in old_raw_manifest + new_raw_manifest["deps"][key] = value + end + new_raw_manifest["julia_version"] = nothing + new_raw_manifest["manifest_format"] = "1" + return new_raw_manifest +end + ########### # WRITING # ########### @@ -194,7 +212,16 @@ function destructure(manifest::Manifest)::Dict unique_name[entry.name] = !haskey(unique_name, entry.name) end - raw = Dict{String,Any}() + # maintain the format of the manifest when writing + if manifest.manifest_format.major == 1 + raw = Dict{String,Vector{Dict{String,Any}}}() + elseif manifest.manifest_format.major == 2 + raw = Dict{String,Any}() + raw["julia_version"] = manifest.julia_version + raw["manifest_format"] = manifest.manifest_format + raw["deps"] = Dict{String,Vector{Dict{String,Any}}}() + end + for (uuid, entry) in manifest new_entry = something(entry.other, Dict{String,Any}()) new_entry["uuid"] = string(uuid) @@ -225,7 +252,11 @@ function destructure(manifest::Manifest)::Dict end end end - push!(get!(raw, entry.name, Dict{String,Any}[]), new_entry) + if manifest.manifest_format.major == 1 + push!(get!(raw, entry.name, Dict{String,Any}[]), new_entry) + elseif manifest.manifest_format.major == 2 + push!(get!(raw["deps"], entry.name, Dict{String,Any}[]), new_entry) + end end return raw end @@ -234,13 +265,20 @@ function write_manifest(env::EnvCache) mkpath(dirname(env.manifest_file)) write_manifest(env.manifest, env.manifest_file) end -write_manifest(manifest::Manifest, manifest_file::AbstractString) = - write_manifest(destructure(manifest), manifest_file) +function write_manifest(manifest::Manifest, manifest_file::AbstractString) + if manifest.manifest_format.major == 1 + @warn """The active manifest file has an old format that is being maintained. + To update to the new format: + 1. Delete the manifest file at `$(manifest_file)` + 2. Run `import Pkg; Pkg.resolve()`""" maxlog = 1 + end + return write_manifest(destructure(manifest), manifest_file) +end function write_manifest(io::IO, manifest::Dict) print(io, "# This file is machine-generated - editing it directly is not advised\n\n") TOML.print(io, manifest, sorted=true) do x - x isa UUID || x isa SHA1 || x isa VersionNumber || pkgerror("unhandled type `$(typeof(x))`") - return string(x) + (typeof(x) in [String, Nothing, UUID, SHA1, VersionNumber]) && return string(x) + error("unhandled type `$(typeof(x))`") end return nothing end diff --git a/test/new.jl b/test/new.jl index 42875e2824..b70b19e4b3 100644 --- a/test/new.jl +++ b/test/new.jl @@ -2490,7 +2490,7 @@ tree_hash(root::AbstractString; kwargs...) = bytes2hex(@inferred Pkg.GitTools.tr chmod(joinpath(dir, "FooGit", "foo"), 0o644) write(joinpath(dir, "FooGit", ".git", "foo"), "foo") chmod(joinpath(dir, "FooGit", ".git", "foo"), 0o644) - @test tree_hash(joinpath(dir, "Foo")) == + @test tree_hash(joinpath(dir, "Foo")) == tree_hash(joinpath(dir, "FooGit")) == "2f42e2c1c1afd4ef8c66a2aaba5d5e1baddcab33" end @@ -2658,9 +2658,9 @@ end function get_manifest_block(name) manifest_path = joinpath(dirname(Base.active_project()), "Manifest.toml") @test isfile(manifest_path) - manifest = TOML.parsefile(manifest_path) - @test haskey(manifest, name) - return only(manifest[name]) + deps = Base.get_deps(TOML.parsefile(manifest_path)) + @test haskey(deps, name) + return only(deps[name]) end isolate(loaded_depot=true) do diff --git a/test/test_packages/BuildProjectFixedDeps/deps/build.jl b/test/test_packages/BuildProjectFixedDeps/deps/build.jl index f592e8f7c8..d63fe55dcc 100644 --- a/test/test_packages/BuildProjectFixedDeps/deps/build.jl +++ b/test/test_packages/BuildProjectFixedDeps/deps/build.jl @@ -3,7 +3,7 @@ build_artifact = joinpath(@__DIR__, "artifact") isfile(build_artifact) && rm(build_artifact) project = TOML.parsefile(Base.active_project()) @assert get(project["deps"], "JSON", nothing) === nothing -manifest = TOML.parsefile(joinpath(dirname(Base.active_project()), "Manifest.toml")) +manifest = Base.get_deps(TOML.parsefile(joinpath(dirname(Base.active_project()), "Manifest.toml"))) json = manifest["JSON"][1] @assert json["uuid"] == "682c06a0-de6a-54ab-a142-c8b1cf79cde6" @assert json["version"] == "0.19.0"