zfs: new zfs_builder to create ZFS images

This patch makes some incremental improvements to the process
of building ZFS images. However, the main driving force here is
to support building and running ZFS images with the kernel built
with most symbols hidden.

The key missing part to support the above is adding libstdc++.so
to the image so that it can be loaded as needed by cpiod.so and mkfs.so.
However, this would make the loader.elf even larger than the non-hidden
version of it. So instead this patch adds new artifact - zfs_builder.elf
- which is intended to be used by upload_manifest.py only to build ZFS-images.

In essence this patch modifies the main makefile to build new zfs_builder.elf
with bootfs populated with all artifacts necessary to build and load ZFS filesystem
(cpiod.so, mkfs.so, etc). At the same time we drop support of building
kernel.elf which became obsolete in favor of standard loader.elf which
now is even leaner - it does not carry the bootfs footprint with ZFS
building tools. On top of that we optimise the build time of ZFS image
by making it run zfs_builder.elf in QEMU kernel direct mode and VGA
console off that cuts boot time to around 20ms.

At the end of the day we come close to satisfying #1068 - "Building a
full OSv image without running it" - we use new OSv-based build tool zfs_builder.elf
to build final ZFS image - usr.img.

Another benefit of this approach is that we can use some "old"
pre-existing version of zfs_builder.elf (possibly retrieved from github)
to build ZFS image that includes new version of kernel (loader.elf).

Refs #1068
Refs #1186

Signed-off-by: Waldemar Kozaczuk <[email protected]>

Makefile

@@ -144,10 +144,13 @@ endif
 quiet = $(if $V, $1, @echo " $2"; $1)
 very-quiet = $(if $V, $1, @$1)
 
-all: $(out)/loader.img links $(out)/kernel.elf
+all: $(out)/loader.img links $(out)/zfs_builder-stripped.elf
 ifeq ($(arch),x64)
 all: $(out)/vmlinuz.bin
 endif
+ifeq ($(arch),aarch64)
+all: $(out)/zfs_builder.img
+endif
 .PHONY: all
 
 links:
@@ -532,6 +535,12 @@ $(out)/loader.img: $(out)/preboot.bin $(out)/loader-stripped.elf
 	$(call quiet, scripts/imgedit.py setsize_aarch64 "-f raw $@" $(image_size), IMGEDIT $@)
 	$(call quiet, scripts/imgedit.py setargs "-f raw $@" $(cmdline), IMGEDIT $@)
 
+$(out)/zfs_builder.img: $(out)/preboot.bin $(out)/zfs_builder-stripped.elf
+	$(call quiet, dd if=$(out)/preboot.bin of=$@ > /dev/null 2>&1, DD $@ preboot.bin)
+	$(call quiet, dd if=$(out)/zfs_builder-stripped.elf of=$@ conv=notrunc obs=4096 seek=16 > /dev/null 2>&1, DD $@ zfs_builder-stripped.elf)
+	$(call quiet, scripts/imgedit.py setsize_aarch64 "-f raw $@" $(image_size), IMGEDIT $@)
+	$(call quiet, scripts/imgedit.py setargs "-f raw $@" $(cmdline), IMGEDIT $@)
+
 endif # aarch64
 
 $(out)/bsd/sys/crypto/rijndael/rijndael-api-fst.o: COMMON+=-fno-strict-aliasing
@@ -2068,7 +2077,7 @@ $(loader_options_dep): stage1
 ifeq ($(conf_hide_symbols),1)
 version_script_file:=$(out)/version_script
 #Detect which version script to be used and copy to $(out)/version_script
-#so that loader.elf/kernel.elf is rebuilt accordingly if version script has changed
+#so that loader.elf/zfs_builder.elf is rebuilt accordingly if version script has changed
 ifdef conf_version_script
 ifeq (,$(wildcard $(conf_version_script)))
     $(error Missing version script: $(conf_version_script))
@@ -2121,13 +2130,14 @@ $(out)/loader.elf: $(stage1_targets) arch/$(arch)/loader.ld $(out)/bootfs.o $(lo
 	@scripts/libosv.py $(out)/osv.syms $(out)/libosv.ld `scripts/osv-version.sh` | $(CC) -c -o $(out)/osv.o -x assembler -
 	$(call quiet, $(CC) $(out)/osv.o -nostdlib -shared -o $(out)/libosv.so -T $(out)/libosv.ld, LIBOSV.SO)
 
-$(out)/kernel.elf: $(stage1_targets) arch/$(arch)/loader.ld $(out)/empty_bootfs.o $(loader_options_dep) $(version_script_file)
+$(out)/zfs_builder.elf: $(stage1_targets) arch/$(arch)/loader.ld $(out)/zfs_builder_bootfs.o $(loader_options_dep) $(version_script_file)
 	$(call quiet, $(LD) -o $@ $(def_symbols) \
 		-Bdynamic --export-dynamic --eh-frame-hdr --enable-new-dtags -L$(out)/arch/$(arch) \
             $(patsubst %version_script,--version-script=%version_script,$(patsubst %.ld,-T %.ld,$^)) \
 	    $(linker_archives_options) $(conf_linker_extra_options), \
-		LINK kernel.elf)
-	$(call quiet, $(STRIP) $(out)/kernel.elf -o $(out)/kernel-stripped.elf, STRIP kernel.elf -> kernel-stripped.elf )
+		LINK zfs_builder.elf)
+$(out)/zfs_builder-stripped.elf:  $(out)/zfs_builder.elf
+	$(call quiet, $(STRIP) $(out)/zfs_builder.elf -o $(out)/zfs_builder-stripped.elf, STRIP zfs_builder.elf -> zfs_builder-stripped.elf )
 
 $(out)/bsd/%.o: COMMON += -DSMP -D'__FBSDID(__str__)=extern int __bogus__'
 
@@ -2163,9 +2173,8 @@ libgcc_s_dir := ../../$(aarch64_gccbase)/lib64
 endif
 
 $(out)/bootfs.bin: scripts/mkbootfs.py $(bootfs_manifest) $(bootfs_manifest_dep) $(tools:%=$(out)/%) \
-		$(out)/zpool.so $(out)/zfs.so $(out)/libenviron.so $(out)/libvdso.so
-	$(call quiet, olddir=`pwd`; cd $(out); "$$olddir"/scripts/mkbootfs.py -o bootfs.bin -d bootfs.bin.d -m "$$olddir"/$(bootfs_manifest) \
-		-D libgcc_s_dir=$(libgcc_s_dir), MKBOOTFS $@)
+		$(out)/libenviron.so $(out)/libvdso.so $(out)/libsolaris.so
+	$(call quiet, olddir=`pwd`; cd $(out); "$$olddir"/scripts/mkbootfs.py -o bootfs.bin -d bootfs.bin.d -m "$$olddir"/$(bootfs_manifest), MKBOOTFS $@)
 
 $(out)/bootfs.o: $(out)/bootfs.bin
 $(out)/bootfs.o: ASFLAGS += -I$(out)
@@ -2184,7 +2193,17 @@ else
 endif
 endif
 
-$(out)/empty_bootfs.o: ASFLAGS += -I$(out)
+$(shell mkdir -p $(out) && cp zfs_builder_bootfs.manifest.skel $(out)/zfs_builder_bootfs.manifest)
+ifeq ($(conf_hide_symbols),1)
+$(shell echo "/usr/lib/libstdc++.so.6: $$(readlink -f $(libstd_dir))/libstdc++.so" >> $(out)/zfs_builder_bootfs.manifest)
+endif
+$(out)/zfs_builder_bootfs.bin: scripts/mkbootfs.py $(zfs_builder_bootfs_manifest) $(tools:%=$(out)/%) \
+		$(out)/zpool.so $(out)/zfs.so $(out)/libenviron.so $(out)/libvdso.so $(out)/libsolaris.so
+	$(call quiet, olddir=`pwd`; cd $(out); "$$olddir"/scripts/mkbootfs.py -o zfs_builder_bootfs.bin -d zfs_builder_bootfs.bin.d -m zfs_builder_bootfs.manifest \
+		-D libgcc_s_dir=$(libgcc_s_dir), MKBOOTFS $@)
+
+$(out)/zfs_builder_bootfs.o: $(out)/zfs_builder_bootfs.bin
+$(out)/zfs_builder_bootfs.o: ASFLAGS += -I$(out)
 
 $(out)/tools/mkfs/mkfs.so: $(out)/tools/mkfs/mkfs.o $(out)/libzfs.so
 	$(makedir)

bootfs.manifest.skel

@@ -1,10 +1,2 @@
 [manifest]
-/libvdso.so: libvdso.so
-/libuutil.so: libuutil.so
-/zpool.so: zpool.so
-/libzfs.so: libzfs.so
 /libsolaris.so: libsolaris.so
-/zfs.so: zfs.so
-/tools/mkfs.so: tools/mkfs/mkfs.so
-/tools/cpiod.so: tools/cpiod/cpiod.so
-/usr/lib/libgcc_s.so.1: %(libgcc_s_dir)s/libgcc_s.so.1

modules/zfs-tools/usr.manifest

@@ -0,0 +1,5 @@
+[manifest]
+/zpool.so: zpool.so
+/libzfs.so: libzfs.so
+/libuutil.so: libuutil.so
+/libsolaris.so: libsolaris.so

scripts/build

@@ -301,9 +301,6 @@ if [[ ${vars[create_disk]} == "true" ]]; then
 	bare="$SRC"/scripts/disk.bin
 	raw_disk=disk
 	qcow2_disk=disk
-	if [[ "$arch" == 'x64' ]]; then
-		upload_kernel_mode="-k"
-	fi
 else
 	partition_offset=$kernel_end
 	bare=loader.img
@@ -316,7 +313,7 @@ create_zfs_disk() {
 	"$SRC"/scripts/imgedit.py setpartition "-f raw ${raw_disk}.raw" 2 $partition_offset $partition_size
 	qemu-img convert -f raw -O qcow2 $raw_disk.raw $qcow2_disk.img
 	qemu-img resize $qcow2_disk.img ${image_size}b >/dev/null 2>&1
-	"$SRC"/scripts/upload_manifest.py --arch=$arch -o $qcow2_disk.img -m usr.manifest -D libgcc_s_dir="$libgcc_s_dir" $upload_kernel_mode
+	"$SRC"/scripts/upload_manifest.py --arch=$arch -o $qcow2_disk.img -m usr.manifest -D libgcc_s_dir="$libgcc_s_dir"
 }
 
 create_rofs_disk() {

scripts/firecracker.py

@@ -370,7 +370,7 @@ def main(options):
     parser.add_argument("-i", "--image", action="store", default=None, metavar="CMD",
                         help="path to disk image file. defaults to ../build/last/usr.img")
     parser.add_argument("-k", "--kernel", action="store", default=None, metavar="CMD",
-                        help="path to kernel loader file. defaults to ../build/last/kernel.elf")
+                        help="path to kernel loader file. defaults to ../build/last/loader-stripped.elf")
     parser.add_argument("-n", "--networking", action="store_true",
                         help="needs root to setup tap networking first time")
     parser.add_argument("-b", "--bridge", action="store", default=None,
@@ -390,7 +390,7 @@ def main(options):
         default_kernel_file_name = "loader.img"
         default_image_file_name = "disk.img"
     else:
-        default_kernel_file_name = "kernel.elf"
+        default_kernel_file_name = "loader-stripped.elf"
         default_image_file_name = "usr.img"
     cmdargs.kernel_path = os.path.abspath(cmdargs.kernel or os.path.join(osv_base, "build/%s/%s" % (cmdargs.opt_path, default_kernel_file_name)))
     cmdargs.image_path = os.path.abspath(cmdargs.image or os.path.join(osv_base, "build/%s/%s" % (cmdargs.opt_path, default_image_file_name)))

scripts/run.py

@@ -594,7 +594,7 @@ def main(options):
     parser.add_argument("-k", "--kernel", action="store_true",
                         help="Run OSv in QEMU kernel mode as PVH.")
     parser.add_argument("--kernel-path", action="store",
-                        help="path to kernel.elf. defaults to build/$mode/kernel.elf")
+                        help="path to loader-stripped.elf. defaults to build/$mode/loader-stripped.elf")
     parser.add_argument("--virtio", action="store", choices=["legacy","transitional","modern"], default="transitional",
                         help="specify virtio version: legacy, transitional or modern")
     parser.add_argument("--arch", action="store", choices=["x86_64","aarch64"], default=host_arch,
@@ -618,7 +618,7 @@ def main(options):
         default_kernel_file_name = "loader.img"
         default_image_file_name = "disk.img"
     else:
-        default_kernel_file_name = "kernel.elf"
+        default_kernel_file_name = "loader-stripped.elf"
         default_image_file_name = "usr.img"
     cmdargs.kernel_file = os.path.abspath(cmdargs.kernel_path or os.path.join(osv_base, "build/%s/%s" % (cmdargs.opt_path, default_kernel_file_name)))
     cmdargs.image_file = os.path.abspath(cmdargs.image or os.path.join(osv_base, "build/%s/%s" % (cmdargs.opt_path, default_image_file_name)))

scripts/upload_manifest.py

@@ -136,10 +136,6 @@ def main():
                         metavar='VAR=DATA',
                         action='callback',
                         callback=add_var),
-            make_option('-k',
-                        dest='kernel',
-                        action='store_true',
-                        help='run OSv in direct kernel mode'),
             make_option('--arch',
                         dest='arch',
                         default=host_arch,
@@ -157,14 +153,18 @@ def main():
 
     image_path = os.path.abspath(options.output)
     upload_port = find_free_port()
-    if options.kernel:
-        kernel_mode_flag = '-k --kernel-path build/release/loader-stripped.elf'
-    else:
-        kernel_mode_flag = ''
     arch = options.arch
     if arch == 'x64':
         arch = 'x86_64'
-    osv = subprocess.Popen('cd ../..; scripts/run.py %s --arch=%s --vnc none -m 512 -c1 -i "%s" --block-device-cache unsafe -s -e "--norandom --nomount --noinit /tools/mkfs.so; /tools/cpiod.so --prefix /zfs/zfs/; /zfs.so set compression=off osv" --forward tcp:127.0.0.1:%s-:10000' % (kernel_mode_flag,arch,image_path,upload_port), shell=True, stdout=subprocess.PIPE)
+
+    if arch == 'aarch64':
+        console = ''
+        zfs_builder_name = 'zfs_builder.img'
+    else:
+        console = '--console=serial'
+        zfs_builder_name = 'zfs_builder-stripped.elf'
+
+    osv = subprocess.Popen('cd ../..; scripts/run.py -k --kernel-path build/release/%s --arch=%s --vnc none -m 512 -c1 -i "%s" --block-device-cache unsafe -s -e "%s --norandom --nomount --noinit /tools/mkfs.so; /tools/cpiod.so --prefix /zfs/zfs/; /zfs.so set compression=off osv" --forward tcp:127.0.0.1:%s-:10000' % (zfs_builder_name,arch,image_path,console,upload_port), shell=True, stdout=subprocess.PIPE)
 
     upload(osv, manifest, depends, upload_port)
 

usr.manifest.skel

@@ -1,13 +1,6 @@
 [manifest]
 /libenviron.so: libenviron.so
 /libvdso.so: libvdso.so
-/zpool.so: zpool.so
-/libzfs.so: libzfs.so
-/libuutil.so: libuutil.so
-/zfs.so: zfs.so
-/libsolaris.so: libsolaris.so
-/tools/mkfs.so: tools/mkfs/mkfs.so
-/tools/cpiod.so: tools/cpiod/cpiod.so
 /tools/mount-fs.so: tools/mount/mount-fs.so
 /tools/umount.so: tools/mount/umount.so
 /usr/lib/libgcc_s.so.1: %(libgcc_s_dir)s/libgcc_s.so.1

empty_bootfs.S → zfs_builder_bootfs.S

@@ -2,4 +2,5 @@
 .global bootfs_start
 .hidden bootfs_start
 bootfs_start:
+.incbin "zfs_builder_bootfs.bin"
 .popsection

zfs_builder_bootfs.manifest.skel

@@ -0,0 +1,10 @@
+[manifest]
+/libvdso.so: libvdso.so
+/libuutil.so: libuutil.so
+/zpool.so: zpool.so
+/libzfs.so: libzfs.so
+/libsolaris.so: libsolaris.so
+/zfs.so: zfs.so
+/tools/mkfs.so: tools/mkfs/mkfs.so
+/tools/cpiod.so: tools/cpiod/cpiod.so
+/usr/lib/libgcc_s.so.1: %(libgcc_s_dir)s/libgcc_s.so.1