Signed-off-by: BassMatt <[email protected]>

Main scripts in scripts/ folder updated to use Python3

I went through the scripts detailed in scripts/README and updated them to use Python3. I used the Python "Future" module to provide suggestions, then manually went through and applied the changes. The "Future" module gives suggestions to allow for cross-compatibility between Python2/3, but since it was expressed that only Python3 needed to be supported, I left all that out.

The issue is detailed here:
#1056
Message-Id: <[email protected]>

scripts/export_manifest.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 import optparse, os, shutil
 from manifest_common import add_var, expand, unsymlink, read_manifest, defines, strip_file
@@ -8,7 +8,7 @@
 # support for links in OSv, e.g., /etc/mnttab: ->/proc/mounts.
 def export_package(manifest, dest):
     abs_dest = os.path.abspath(dest)
-    print "[INFO] exporting into directory %s" % abs_dest
+    print("[INFO] exporting into directory %s" % abs_dest)
 
     # Remove and create the base directory where we are going to put all package files.
     if os.path.exists(abs_dest):
@@ -39,7 +39,7 @@ def export_package(manifest, dest):
                 os.makedirs(target_dir)
 
             os.symlink(link_source, name)
-            print "[INFO] added link %s -> %s" % (name, link_source)
+            print("[INFO] added link %s -> %s" % (name, link_source))
 
         else:
             # If it is a symlink, then resolve it add to the list of host symlinks to be created later
@@ -58,23 +58,23 @@ def export_package(manifest, dest):
                 hostname = strip_file(hostname)
 
                 shutil.copy(hostname, name)
-                print "[INFO] exported %s" % name
+                print("[INFO] exported %s" % name)
             elif os.path.isdir(hostname):
                 # If hostname is a dir, it is only a request to create the folder on guest. Nothing to copy.
                 if not os.path.exists(name):
                     os.makedirs(name)
-                print "[INFO] created dir %s" % name
+                print("[INFO] created dir %s" % name)
 
             else:
                 # Inform the user that the rule cannot be applied. For example, this happens for links in OSv.
-                print "[ERR] unable to export %s" % hostname
+                print("[ERR] unable to export %s" % hostname)
 
     for link_source, name in host_symlinks:
         target_dir = os.path.dirname(name)
         if not os.path.exists(target_dir):
             os.makedirs(target_dir)
         os.symlink(link_source, name)
-        print "[INFO] added link %s -> %s" % (name, link_source)
+        print("[INFO] added link %s -> %s" % (name, link_source))
 
 def main():
     make_option = optparse.make_option

scripts/firecracker.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # pip install requests-unixsocket
 import sys
@@ -182,7 +182,7 @@ def find_firecracker(dirname):
     if not os.path.exists(firecracker_path):
         url_base = 'https://github.com/firecracker-microvm/firecracker/releases/download'
         download_url = '%s/%s/firecracker-%s' % (url_base, firecracker_version, firecracker_version)
-        answer = raw_input("Firecracker executable has not been found under %s. "
+        answer = input("Firecracker executable has not been found under %s. "
                            "Would you like to download it from %s and place it under %s? [y|n]" %
                            (firecracker_path, download_url, firecracker_path))
         if answer.capitalize() != 'Y':

scripts/gen-rofs-img.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 #
 # Copyright (c) 2015 Carnegie Mellon University.
@@ -218,13 +218,13 @@ def write_dir(fp, manifest, dirpath, parent_dir):
                 inode.data_offset = symlinks_count
                 inode.count = 1
                 next_symlink(val[2:],manifest)
-                print 'Link %s to %s' % (dirpath + '/' + entry, val[2:])
+                print('Link %s to %s' % (dirpath + '/' + entry, val[2:]))
             else: #file
                 inode.mode = REG_MODE
                 global block
                 inode.data_offset = block
                 inode.count = write_file(fp, val)
-                print 'Adding %s' % (dirpath + '/' + entry)
+                print('Adding %s' % (dirpath + '/' + entry))
 
     # This needs to be added so that later we can walk the tree
     # when fining symlinks
@@ -264,15 +264,15 @@ def write_fs(fp, manifest):
     return (block_no, bytes_written)
 
 def gen_image(out, manifest):
-    print 'Writing image'
+    print('Writing image')
     fp = open(out, 'wb')
 
     # write the initial superblock
     write_initial_superblock(fp)
 
     system_structure_block, bytes_written = write_fs(fp, manifest)
     structure_info_last_block_bytes = bytes_written % OSV_BLOCK_SIZE
-    structure_info_blocks_count = bytes_written / OSV_BLOCK_SIZE + (1 if structure_info_last_block_bytes > 0 else 0)
+    structure_info_blocks_count = bytes_written // OSV_BLOCK_SIZE + (1 if structure_info_last_block_bytes > 0 else 0)
 
     pad(fp,OSV_BLOCK_SIZE - structure_info_last_block_bytes)
 
@@ -290,10 +290,10 @@ def gen_image(out, manifest):
     sb.symlinks_count = len(symlinks)
     sb.inodes_count = len(inodes)
 
-    print 'First block: %d, blocks count: %d' % (sb.structure_info_first_block, sb.structure_info_blocks_count)
-    print 'Directory entries count %d' % sb.directory_entries_count
-    print 'Symlinks count %d' % sb.symlinks_count
-    print 'Inodes count %d' % sb.inodes_count
+    print('First block: %d, blocks count: %d' % (sb.structure_info_first_block, sb.structure_info_blocks_count))
+    print('Directory entries count %d' % sb.directory_entries_count)
+    print('Symlinks count %d' % sb.symlinks_count)
+    print('Inodes count %d' % sb.inodes_count)
 
     fp.seek(0)
     fp.write(sb)

scripts/loader.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 
 import gdb
 import re
@@ -37,8 +37,8 @@ def phys_cast(addr, type):
 
 def values(_dict):
     if hasattr(_dict, 'viewvalues'):
-        return _dict.viewvalues()
-    return _dict.values()
+        return _dict.values()
+    return list(_dict.values())
 
 def read_vector(v):
     impl = v['_M_impl']
@@ -426,19 +426,19 @@ def invoke(self, arg, from_tty):
 
         print ("\n:: ARC SIZES ::")
         print ("\tCurrent size:    %d (%d MB)" %
-               (arc_size, arc_size / 1024 / 1024))
+               (arc_size, arc_size // 1024 // 1024))
         print ("\tTarget size:     %d (%d MB)" %
-               (arc_target_size, arc_target_size / 1024 / 1024))
+               (arc_target_size, arc_target_size // 1024 // 1024))
         print ("\tMin target size: %d (%d MB)" %
-               (arc_min_size, arc_min_size / 1024 / 1024))
+               (arc_min_size, arc_min_size // 1024 // 1024))
         print ("\tMax target size: %d (%d MB)" %
-               (arc_max_size, arc_max_size / 1024 / 1024))
+               (arc_max_size, arc_max_size // 1024 // 1024))
 
         print ("\n:: ARC SIZE BREAKDOWN ::")
         print ("\tMost recently used cache size:   %d (%d MB) (%.2f%%)" %
-               (arc_mru_size, arc_mru_size / 1024 / 1024, arc_mru_perc))
+               (arc_mru_size, arc_mru_size // 1024 // 1024, arc_mru_perc))
         print ("\tMost frequently used cache size: %d (%d MB) (%.2f%%)" %
-               (arc_mfu_size, arc_mfu_size / 1024 / 1024, arc_mfu_perc))
+               (arc_mfu_size, arc_mfu_size // 1024 // 1024, arc_mfu_perc))
 
         # Cache efficiency
         arc_hits = get_stat_by_name(arc_stats_struct, arc_stats_cast, 'arcstat_hits')
@@ -618,7 +618,7 @@ def invoke(self, arg, from_tty):
             end = ulong(vma['_range']['_end'])
             flags = flagstr(ulong(vma['_flags']))
             perm = permstr(ulong(vma['_perm']))
-            size = '{:<16}'.format('[%s kB]' % (ulong(end - start)/1024))
+            size = '{:<16}'.format('[%s kB]' % (ulong(end - start)//1024))
 
             if 'F' in flags:
                 file_vma = vma.cast(gdb.lookup_type('mmu::file_vma').pointer())
@@ -648,7 +648,7 @@ def invoke(self, arg, from_tty):
                 if start <= addr and end > addr:
                     flags = flagstr(ulong(vma['_flags']))
                     perm = permstr(ulong(vma['_perm']))
-                    size = '{:<16}'.format('[%s kB]' % (ulong(end - start)/1024))
+                    size = '{:<16}'.format('[%s kB]' % (ulong(end - start)//1024))
                     print('0x%016x -> vma 0x%016x' % (addr, vma_addr))
                     print('0x%016x 0x%016x %s flags=%s perm=%s' % (start, end, size, flags, perm))
                     break
@@ -671,7 +671,7 @@ def ulong(x):
 def to_int(gdb_value):
     if hasattr(globals()['__builtins__'], 'long'):
         # For GDB with python2
-        return long(gdb_value)
+        return int(gdb_value)
     return int(gdb_value)
 
 class osv_syms(gdb.Command):
@@ -685,9 +685,9 @@ def invoke(self, arg, from_tty):
             obj_path = obj['_pathname']['_M_dataplus']['_M_p'].string()
             path = translate(obj_path)
             if not path:
-                print('ERROR: Unable to locate object file for:', obj_path, hex(base))
+                print(('ERROR: Unable to locate object file for:', obj_path, hex(base)))
             else:
-                print(path, hex(base))
+                print((path, hex(base)))
                 load_elf(path, base)
 
 class osv_load_elf(gdb.Command):
@@ -751,7 +751,7 @@ def get_base_class_offset(gdb_type, base_class_name):
     name_pattern = re.escape(base_class_name) + "(<.*>)?$"
     for field in gdb_type.fields():
         if field.is_base_class and re.match(name_pattern, field.name):
-            return field.bitpos / 8
+            return field.bitpos // 8
 
 def derived_from(type, base_class):
     return len([x for x in type.fields()
@@ -808,11 +808,8 @@ def __iter__(self):
             yield node_ptr.cast(self.node_type.pointer()).dereference()
             hook = hook['next_']
 
-    def __nonzero__(self):
-        return self.root['next_'] != self.root.address
-
     def __bool__(self):
-        return self.__nonzero__()
+        return self.root['next_'] != self.root.address
 
 class vmstate(object):
     def __init__(self):
@@ -832,7 +829,7 @@ def load_cpu_list(self):
         self.cpu_list = cpu_list
 
     def load_thread_list(self):
-        threads = map(gdb.Value.dereference, unordered_map(gdb.lookup_global_symbol('sched::thread_map').value()))
+        threads = list(map(gdb.Value.dereference, unordered_map(gdb.lookup_global_symbol('sched::thread_map').value())))
         self.thread_list = sorted(threads, key=lambda x: int(x["_id"]))
 
     def cpu_from_thread(self, thread):
@@ -896,7 +893,7 @@ def show_thread_timers(t):
         gdb.write('  timers:')
         for timer in timer_list:
             expired = '*' if timer['_state'] == timer_state_expired else ''
-            expiration = int(timer['_time']['__d']['__r']) / 1.0e9
+            expiration = int(timer['_time']['__d']['__r']) // 1.0e9
             gdb.write(' %11.9f%s' % (expiration, expired))
         gdb.write('\n')
 
@@ -911,7 +908,7 @@ def __init__(self, frame, file_name, line, func_name):
         self.frame = frame
         self.file_name = file_name
         self.line = line
-        self.func_name = func_name
+        self.__name__ = func_name
 
 def traverse_resolved_frames(frame):
     while frame:
@@ -989,14 +986,14 @@ def invoke(self, arg, for_tty):
                 function_whitelist = [sched_thread_join]
 
                 def is_interesting(resolved_frame):
-                    is_whitelisted = resolved_frame.func_name in function_whitelist
+                    is_whitelisted = resolved_frame.__name__ in function_whitelist
                     is_blacklisted = os.path.basename(resolved_frame.file_name) in file_blacklist
                     return is_whitelisted or not is_blacklisted
 
                 fr = find_or_give_last(is_interesting, traverse_resolved_frames(newest_frame))
 
                 if fr:
-                    location = '%s at %s:%s' % (fr.func_name, strip_dotdot(fr.file_name), fr.line)
+                    location = '%s at %s:%s' % (fr.__name__, strip_dotdot(fr.file_name), fr.line)
                 else:
                     location = '??'
 
@@ -1009,7 +1006,7 @@ def is_interesting(resolved_frame):
                            )
                           )
 
-                if fr and fr.func_name == sched_thread_join:
+                if fr and fr.__name__ == sched_thread_join:
                     gdb.write("\tjoining on %s\n" % fr.frame.read_var("this"))
 
                 show_thread_timers(t)
@@ -1214,7 +1211,7 @@ def one_cpu_trace(cpu):
             unpacker.align_up(8)
             yield Trace(tp, Thread(thread, thread_name), time, cpu, data, backtrace=backtrace)
 
-    iters = map(lambda cpu: one_cpu_trace(cpu), values(state.cpu_list))
+    iters = [one_cpu_trace(cpu) for cpu in values(state.cpu_list)]
     return heapq.merge(*iters)
 
 def save_traces_to_file(filename):
@@ -1281,7 +1278,7 @@ def show_leak():
     gdb.flush()
     allocs = []
     for i in range(size_allocations):
-        newpercent = '%2d%%' % round(100.0*i/(size_allocations-1))
+        newpercent = '%2d%%' % round(100.0*i//(size_allocations-1))
         if newpercent != percent:
             percent = newpercent
             gdb.write('\b\b\b%s' % newpercent)
@@ -1343,10 +1340,10 @@ def show_leak():
                    allocations=cur_n,
                    minsize=cur_min_size,
                    maxsize=cur_max_size,
-                   avgsize=cur_total_size/cur_n,
+                   avgsize=cur_total_size//cur_n,
                    minbirth=cur_first_seq,
                    maxbirth=cur_last_seq,
-                   avgbirth=cur_total_seq/cur_n,
+                   avgbirth=cur_total_seq//cur_n,
                    callchain=callchain)
         records.append(r)
         cur_n = 0
@@ -1538,7 +1535,7 @@ def invoke(self, args, from_tty):
             gdb.write('%s\n'%e)
             return
         percpu_addr = percpu.address
-        for cpu in vmstate().cpu_list.values():
+        for cpu in list(vmstate().cpu_list.values()):
             gdb.write("CPU %d:\n" % cpu.id)
             base = cpu.obj['percpu_base']
             addr = base+to_int(percpu_addr)

scripts/metadata.py

@@ -1,7 +1,7 @@
-import SimpleHTTPServer as http
-import SocketServer
 import subprocess
 import os
+import http.server as http
+import socketserver
 
 METADATA_IP = '169.254.169.254'
 port = 80
@@ -32,7 +32,7 @@ def start_server(path):
     try:
         os.chdir(path)
         handler = http.SimpleHTTPRequestHandler
-        server = SocketServer.TCPServer(("", port), handler, False)
+        server = socketserver.TCPServer(("", port), handler, False)
         server.allow_reuse_address = True
         server.server_bind()
         server.server_activate()

scripts/mkbootfs.py

@@ -1,10 +1,7 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 import os, struct, optparse, io
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
+import configparser
 from manifest_common import add_var, expand, unsymlink, read_manifest, defines, strip_file
 
 def main():

scripts/module.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 import re
 import os
@@ -226,7 +226,7 @@ def build(args):
         else:
             print(prefix)
 
-    for module, run_config_name in modules_to_run.items():
+    for module, run_config_name in list(modules_to_run.items()):
         run_config = resolve.get_run_config(module, run_config_name)
         if run_config:
             run_list.append(run_config)

scripts/setup.py

@@ -1,8 +1,8 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 
 # set up a development environment for OSv.  Run as root.
 
-import sys, platform, argparse
+import sys, distro, argparse
 import subprocess
 
 standard_ec2_packages = ['python-pip', 'wget']
@@ -319,11 +319,11 @@ class LinuxMint_19(object):
                     help="install packages required by testing tools")
 cmdargs = parser.parse_args()
 
-(name, version, id) = platform.linux_distribution()
+(name, version, id) = distro.linux_distribution()
 
 for distro in distros:
     if type(distro.name) == type([]):
-        dname = filter(lambda n: name.startswith(n), distro.name)
+        dname = [n for n in distro.name if name.startswith(n)]
         if len(dname):
             distro.name = dname[0]
         else:
@@ -349,5 +349,5 @@ class LinuxMint_19(object):
         print ('Your distribution %s version %s is not supported by this script' % (name, version))
         sys.exit(1)
 
-print 'Your distribution is not supported by this script.'
+print('Your distribution is not supported by this script.')
 sys.exit(2)