implement manifest format with deps field

src/Operations.jl

@@ -759,7 +759,7 @@ function prune_manifest(env::EnvCache)
     env.manifest = prune_manifest(env.manifest, keep)
 end
 
-function prune_manifest(manifest::Dict, keep::Vector{UUID})
+function prune_manifest(manifest::Manifest, keep::Vector{UUID})
     while !isempty(keep)
         clean = true
         for (uuid, entry) in manifest
@@ -772,7 +772,8 @@ function prune_manifest(manifest::Dict, keep::Vector{UUID})
         end
         clean && break
     end
-    return Dict(uuid => entry for (uuid, entry) in manifest if uuid in keep)
+    manifest.deps = Dict(uuid => entry for (uuid, entry) in manifest if uuid in keep)
+    return manifest
 end
 
 
@@ -1405,7 +1406,7 @@ function sandbox_preserve(env::EnvCache, target::PackageSpec, test_project::Stri
     return prune_manifest(env.manifest, keep)
 end
 
-function abspath!(env::EnvCache, manifest::Dict{UUID,PackageEntry})
+function abspath!(env::EnvCache, manifest::Manifest)
     for (uuid, entry) in manifest
         if entry.path !== nothing
             entry.path = project_rel_path(env, entry.path)
@@ -1477,7 +1478,7 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::S
                 allow_reresolve || rethrow()
                 @debug err
                 @warn "Could not use exact versions of packages in manifest, re-resolving"
-                temp_ctx.env.manifest = Dict(uuid => entry for (uuid, entry) in temp_ctx.env.manifest if isfixed(entry))
+                temp_ctx.env.manifest.deps = Dict(uuid => entry for (uuid, entry) in temp_ctx.env.manifest.deps if isfixed(entry))
                 Pkg.resolve(temp_ctx; io=devnull, skip_writing_project=true)
                 @debug "Using _clean_ dep graph"
             end

src/Types.jl

@@ -245,7 +245,24 @@ Base.:(==)(t1::PackageEntry, t2::PackageEntry) = t1.name == t2.name &&
     t1.uuid == t2.uuid
     # omits `other`
 Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.uuid], init=h)  # omits `other`
-const Manifest = Dict{UUID,PackageEntry}
+
+Base.@kwdef mutable struct Manifest
+    julia_version::Union{Nothing,VersionNumber} = Base.VERSION
+    manifest_format::VersionNumber = v"2.0.0"
+    deps::Dict{UUID,PackageEntry} = Dict{UUID,PackageEntry}()
+end
+Base.:(==)(t1::Manifest, t2::Manifest) = all(x -> (getfield(t1, x) == getfield(t2, x))::Bool, fieldnames(Manifest))
+Base.hash(m::Manifest, h::UInt) = foldr(hash, [getfield(m, x) for x in fieldnames(Manifest)], init=h)
+Base.getindex(m::Manifest, i_or_key) = getindex(m.deps, i_or_key)
+Base.get(m::Manifest, key, default) = get(m.deps, key, default)
+Base.setindex!(m::Manifest, i_or_key, value) = setindex!(m.deps, i_or_key, value)
+Base.iterate(m::Manifest) = iterate(m.deps)
+Base.iterate(m::Manifest, i::Int) = iterate(m.deps, i)
+Base.length(m::Manifest) = length(m.deps)
+Base.empty!(m::Manifest) = empty!(m.deps)
+Base.values(m::Manifest) = values(m.deps)
+Base.keys(m::Manifest) = keys(m.deps)
+Base.haskey(m::Manifest, key) = haskey(m.deps, key)
 
 function Base.show(io::IO, pkg::PackageEntry)
     f = []

src/manifest.jl

@@ -105,19 +105,19 @@ function normalize_deps(name, uuid, deps::Vector{String}, manifest::Dict{String,
     return final
 end
 
-function validate_manifest(stage1::Dict{String,Vector{Stage1}})
+function validate_manifest(julia_version::Union{Nothing,VersionNumber}, manifest_format::VersionNumber, stage1::Dict{String,Vector{Stage1}})
     # expand vector format deps
     for (name, infos) in stage1, info in infos
         info.entry.deps = normalize_deps(name, info.uuid, info.deps, stage1)
     end
     # invariant: all dependencies are now normalized to Dict{String,UUID}
-    manifest = Dict{UUID, PackageEntry}()
+    deps = Dict{UUID, PackageEntry}()
     for (name, infos) in stage1, info in infos
-        manifest[info.uuid] = info.entry
+        deps[info.uuid] = info.entry
     end
     # now just verify the graph structure
-    for (entry_uuid, entry) in manifest, (name, uuid) in entry.deps
-        dep_entry = get(manifest, uuid, nothing)
+    for (entry_uuid, entry) in deps, (name, uuid) in entry.deps
+        dep_entry = get(deps, uuid, nothing)
         if dep_entry === nothing
             pkgerror("`$(entry.name)=$(entry_uuid)` depends on `$name=$uuid`, ",
                      "but no such entry exists in the manifest.")
@@ -127,38 +127,42 @@ function validate_manifest(stage1::Dict{String,Vector{Stage1}})
                      "but entry with UUID `$uuid` has name `$(dep_entry.name)`.")
         end
     end
-    return manifest
+    return Manifest(; julia_version, manifest_format, deps)
 end
 
 function Manifest(raw::Dict)::Manifest
+    julia_version = isnothing(raw["julia_version"]) ? nothing : VersionNumber(raw["julia_version"])
+    manifest_format = VersionNumber(raw["manifest_format"])
     stage1 = Dict{String,Vector{Stage1}}()
-    for (name, infos) in raw, info in infos
-        entry = PackageEntry()
-        entry.name = name
-        uuid = nothing
-        deps = nothing
-        try
-            entry.pinned      = read_pinned(get(info, "pinned", nothing))
-            uuid              = read_field("uuid",          nothing, info, safe_uuid)::UUID
-            entry.version     = read_field("version",       nothing, info, safe_version)
-            entry.path        = read_field("path",          nothing, info, safe_path)
-            entry.repo.source = read_field("repo-url",      nothing, info, identity)
-            entry.repo.rev    = read_field("repo-rev",      nothing, info, identity)
-            entry.repo.subdir = read_field("repo-subdir",   nothing, info, identity)
-            entry.tree_hash   = read_field("git-tree-sha1", nothing, info, safe_SHA1)
-            entry.uuid        = uuid
-            deps = read_deps(get(info::Dict, "deps", nothing))
-        catch
-            # TODO: Should probably not unconditionally log something
-            @error "Could not parse entry for `$name`"
-            rethrow()
+    if haskey(raw, "deps") # deps field doesn't exist if there are no deps
+        for (name, infos) in raw["deps"], info in infos
+            entry = PackageEntry()
+            entry.name = name
+            uuid = nothing
+            deps = nothing
+            try
+                entry.pinned      = read_pinned(get(info, "pinned", nothing))
+                uuid              = read_field("uuid",          nothing, info, safe_uuid)::UUID
+                entry.version     = read_field("version",       nothing, info, safe_version)
+                entry.path        = read_field("path",          nothing, info, safe_path)
+                entry.repo.source = read_field("repo-url",      nothing, info, identity)
+                entry.repo.rev    = read_field("repo-rev",      nothing, info, identity)
+                entry.repo.subdir = read_field("repo-subdir",   nothing, info, identity)
+                entry.tree_hash   = read_field("git-tree-sha1", nothing, info, safe_SHA1)
+                entry.uuid        = uuid
+                deps = read_deps(get(info::Dict, "deps", nothing))
+            catch
+                # TODO: Should probably not unconditionally log something
+                @error "Could not parse entry for `$name`"
+                rethrow()
+            end
+            entry.other = info::Union{Dict,Nothing}
+            stage1[name] = push!(get(stage1, name, Stage1[]), Stage1(uuid, entry, deps))
         end
-        entry.other = info::Union{Dict,Nothing}
-        stage1[name] = push!(get(stage1, name, Stage1[]), Stage1(uuid, entry, deps))
+        # by this point, all the fields of the `PackageEntry`s have been type casted
+        # but we have *not* verified the _graph_ structure of the manifest
     end
-    # by this point, all the fields of the `PackageEntry`s have been type casted
-    # but we have *not* verified the _graph_ structure of the manifest
-    return validate_manifest(stage1)
+    return validate_manifest(julia_version, manifest_format, stage1)
 end
 
 function read_manifest(f_or_io::Union{String, IO})
@@ -174,9 +178,23 @@ function read_manifest(f_or_io::Union{String, IO})
         end
         rethrow()
     end
+    if !isempty(raw) && Base.is_v1_format_manifest(raw)
+        raw = convert_flat_format_manifest(raw)
+    end
     return Manifest(raw)
 end
 
+function convert_flat_format_manifest(old_raw_manifest::Dict)
+    new_raw_manifest = Dict{String,Any}()
+    new_raw_manifest["deps"] = Dict{String,Vector{Any}}()
+    for (key, value) in old_raw_manifest
+        new_raw_manifest["deps"][key] = value
+    end
+    new_raw_manifest["julia_version"] = nothing
+    new_raw_manifest["manifest_format"] = "1"
+    return new_raw_manifest
+end
+
 ###########
 # WRITING #
 ###########
@@ -194,7 +212,16 @@ function destructure(manifest::Manifest)::Dict
         unique_name[entry.name] = !haskey(unique_name, entry.name)
     end
 
-    raw = Dict{String,Any}()
+    # maintain the format of the manifest when writing
+    if manifest.manifest_format.major == 1
+        raw = Dict{String,Vector{Dict{String,Any}}}()
+    elseif manifest.manifest_format.major == 2
+        raw = Dict{String,Any}()
+        raw["julia_version"] = manifest.julia_version
+        raw["manifest_format"] = manifest.manifest_format
+        raw["deps"] = Dict{String,Vector{Dict{String,Any}}}()
+    end
+
     for (uuid, entry) in manifest
         new_entry = something(entry.other, Dict{String,Any}())
         new_entry["uuid"] = string(uuid)
@@ -225,7 +252,11 @@ function destructure(manifest::Manifest)::Dict
                 end
             end
         end
-        push!(get!(raw, entry.name, Dict{String,Any}[]), new_entry)
+        if manifest.manifest_format.major == 1
+            push!(get!(raw, entry.name, Dict{String,Any}[]), new_entry)
+        elseif manifest.manifest_format.major == 2
+            push!(get!(raw["deps"], entry.name, Dict{String,Any}[]), new_entry)
+        end
     end
     return raw
 end
@@ -234,13 +265,20 @@ function write_manifest(env::EnvCache)
     mkpath(dirname(env.manifest_file))
     write_manifest(env.manifest, env.manifest_file)
 end
-write_manifest(manifest::Manifest, manifest_file::AbstractString) =
-    write_manifest(destructure(manifest), manifest_file)
+function write_manifest(manifest::Manifest, manifest_file::AbstractString)
+    if manifest.manifest_format.major == 1
+        @warn """The active manifest file has an old format that is being maintained.
+            To update to the new format:
+                1. Delete the manifest file at `$(manifest_file)`
+                2. Run `import Pkg; Pkg.resolve()`""" maxlog = 1
+    end
+    return write_manifest(destructure(manifest), manifest_file)
+end
 function write_manifest(io::IO, manifest::Dict)
     print(io, "# This file is machine-generated - editing it directly is not advised\n\n")
     TOML.print(io, manifest, sorted=true) do x
-        x isa UUID || x isa SHA1 || x isa VersionNumber || pkgerror("unhandled type `$(typeof(x))`")
-        return string(x)
+        (typeof(x) in [String, Nothing, UUID, SHA1, VersionNumber]) && return string(x)
+        error("unhandled type `$(typeof(x))`")
     end
     return nothing
 end

test/new.jl

@@ -2490,7 +2490,7 @@ tree_hash(root::AbstractString; kwargs...) = bytes2hex(@inferred Pkg.GitTools.tr
         chmod(joinpath(dir, "FooGit", "foo"), 0o644)
         write(joinpath(dir, "FooGit", ".git", "foo"), "foo")
         chmod(joinpath(dir, "FooGit", ".git", "foo"), 0o644)
-        @test tree_hash(joinpath(dir, "Foo")) == 
+        @test tree_hash(joinpath(dir, "Foo")) ==
               tree_hash(joinpath(dir, "FooGit")) ==
               "2f42e2c1c1afd4ef8c66a2aaba5d5e1baddcab33"
     end
@@ -2658,9 +2658,9 @@ end
     function get_manifest_block(name)
         manifest_path = joinpath(dirname(Base.active_project()), "Manifest.toml")
         @test isfile(manifest_path)
-        manifest = TOML.parsefile(manifest_path)
-        @test haskey(manifest, name)
-        return only(manifest[name])
+        deps = Base.get_deps(TOML.parsefile(manifest_path))
+        @test haskey(deps, name)
+        return only(deps[name])
     end
 
     isolate(loaded_depot=true) do

test/test_packages/BuildProjectFixedDeps/deps/build.jl

@@ -3,7 +3,7 @@ build_artifact = joinpath(@__DIR__, "artifact")
 isfile(build_artifact) && rm(build_artifact)
 project = TOML.parsefile(Base.active_project())
 @assert get(project["deps"], "JSON", nothing) === nothing
-manifest = TOML.parsefile(joinpath(dirname(Base.active_project()), "Manifest.toml"))
+manifest = Base.get_deps(TOML.parsefile(joinpath(dirname(Base.active_project()), "Manifest.toml")))
 json = manifest["JSON"][1]
 @assert json["uuid"] == "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 @assert json["version"] == "0.19.0"