progress tracked in docs/progress/10-31.md

benchmark/ansible/group_vars/all.yml

@@ -3,10 +3,10 @@ ansible_ssh_public_key_file: ~/.ssh/id_rsa.pub
 ansible_ssh_user: centos
 client:
   dir: /
-  host: 10.52.100.3
-  n_parallel: 4
-  opts: mds_namespace=alpha
-  type: ceph
+  host: 10.52.100.9
+  n_parallel: 1
+  opts: vers=4
+  type: nfs
 disks:
 - delete_on_termination: true
   mode: READ_WRITE
@@ -43,7 +43,7 @@ options:
     version: 4.0
   iozone:
     docker_image: dchampion24/iozone:nfs
-    opts: -Rac -r 4k -s 32m
+    opts: -Rac -r 4k -s 512m
   nfsv4:
     mountpoint: /data
     nfsd_image: heliumdatacommons/nfsd
@@ -70,6 +70,6 @@ project: helium-dc
 running: true
 supernet: 10.52.0.0/16
 zone:
-- name: us-east1-b
+- end: 1
+  name: us-east1-b
   start: 1
-  end: 1

benchmark/ansible/playbooks/cephfs/main.yml

@@ -51,6 +51,7 @@
         image: "{{ options.cephfs.daemon_image }}"
         network_mode: host
         command: mon
+        restart_policy: always
         volumes:
           - /opt/ceph/etc:/etc/ceph
           - /opt/ceph/var/lib:/var/lib/ceph/
@@ -63,6 +64,7 @@
         name: ceph-mgr
         image: "{{ options.cephfs.daemon_image }}"
         network_mode: host
+        restart_policy: always
         command: mgr
         volumes:
           - /opt/ceph/etc:/etc/ceph
@@ -112,6 +114,7 @@
         name: ceph-osd-{{ item }}
         image: "{{ options.cephfs.daemon_image }}"
         network_mode: host
+        restart_policy: always
         pid_mode: host
         privileged: true
         command: osd
@@ -132,6 +135,7 @@
         name: cephfs-{{ options.cephfs.name }}
         image: "{{ options.cephfs.daemon_image }}"
         network_mode: host
+        restart_policy: always
         privileged: true
         command: mds
         volumes:

benchmark/ansible/playbooks/ganesha/main.yml

@@ -35,6 +35,7 @@
         image: "{{ options.ganesha.docker_image }}"
         network_mode: host
         privileged: true
+        restart_policy: always
         pull: true
         volumes:
           - "{{ options.ganesha.mountpoint }}:/data/nfs"

benchmark/ansible/playbooks/nfsv4/main.yml

@@ -51,6 +51,7 @@
         image: "{{ options.nfsv4.nfsd_image }}"
         network_mode: host
         privileged: true
+        restart_policy: always
         volumes:
           - "{{ options.nfsv4.mountpoint }}:/data"
         env:

benchmark/ansible/playbooks/rbd/main.yml

@@ -45,6 +45,7 @@
         name: ceph-mon
         image: "{{ options.rbd.daemon_image }}"
         network_mode: host
+        restart_policy: always
         command: mon
         volumes:
           - /opt/ceph/etc:/etc/ceph
@@ -58,6 +59,7 @@
         name: ceph-mgr
         image: "{{ options.rbd.daemon_image }}"
         network_mode: host
+        restart_policy: always
         command: mgr
         volumes:
           - /opt/ceph/etc:/etc/ceph
@@ -111,6 +113,7 @@
         pid_mode: host
         privileged: true
         command: osd
+        restart_policy: always
         volumes:
           - /opt/ceph/etc:/etc/ceph
           - /opt/ceph/var/lib:/var/lib/ceph/
@@ -187,6 +190,7 @@
         image: "{{ options.rbd.nfsd_image }}"
         network_mode: host
         privileged: true
+        restart_policy: always
         volumes:
           - "{{ options.rbd.mountpoint }}:/data"
         env:

benchmark/ansible/playbooks/rbdganesha/main.yml

@@ -45,6 +45,7 @@
         name: ceph-mon
         image: "{{ options.rbdganesha.daemon_image }}"
         network_mode: host
+        restart_policy: always
         command: mon
         volumes:
           - /opt/ceph/etc:/etc/ceph
@@ -56,6 +57,7 @@
     - name: Run Ceph Manager
       docker_container:
         name: ceph-mgr
+        restart_policy: always
         image: "{{ options.rbdganesha.daemon_image }}"
         network_mode: host
         command: mgr
@@ -94,6 +96,7 @@
         name: ceph-zap-device
         image: "{{ options.rbdganesha.daemon_image }}"
         command: zap_device
+        restart_policy: always
         privileged: true
         auto_remove: true
         volumes:
@@ -110,6 +113,7 @@
         network_mode: host
         pid_mode: host
         privileged: true
+        restart_policy: always
         command: osd
         volumes:
           - /opt/ceph/etc:/etc/ceph
@@ -171,6 +175,7 @@
         name: nfs-ganesha
         image: "{{ options.rbdganesha.ganesha_image }}"
         network_mode: host
+        restart_policy: always
         privileged: true
         pull: true
         volumes:

benchmark/ansible/runner.py

@@ -12,11 +12,11 @@
 
 clients = {
   # 'rbd': dict(host='10.52.100.16', type='nfs', dir='/', opts='vers=4'),
-  'cephfs': dict(host='10.52.100.3', type='ceph', dir='/', opts='mds_namespace=alpha'),
+  'cephfs': dict(host='10.52.100.11', type='ceph', dir='/', opts='mds_namespace=alpha'),
   # 'nfsv4': dict(host='10.52.100.22', type='nfs', dir='/', opts='vers=4'),
   # 'ganesha': dict(host='10.52.100.23', type='nfs', dir='/', opts='vers=4'),
-  'gfganesha': dict(host='10.52.100.11', type='nfs', dir='/data', opts='vers=4'),
-  'rbdganesha': dict(host='10.52.100.8', type='nfs', dir='/', opts='vers=4'),
+  'gfganesha': dict(host='10.52.100.3', type='nfs', dir='/data', opts='vers=4'),
+  'rbdganesha': dict(host='10.52.100.9', type='nfs', dir='/', opts='vers=4'),
 }
 
 
@@ -51,8 +51,8 @@ def run_experiment():
         start = False
       if start:
         sys.stdout.write(l)
+        sys.stdout.flush()
       if not l and p.poll() is not None:
-        start = False
         break
   except KeyboardInterrupt as e:
     p.kill()
@@ -65,6 +65,7 @@ def collect_output(name):
   output_dir = [d for d in os.listdir(base_dir) if len(d) == 36]
   if len(output_dir) == 0:
     sys.stderr.write('Output directory for %s is not found'%name)
+    sys.stderr.flush()
     sys.exit(1)
   os.makedirs('%s/%s/%s'%(base_dir, exp_id, name), exist_ok=True)
   print('Move %s/%s to %s/%s/%s'%(base_dir, output_dir[0], base_dir, exp_id, name))
@@ -90,17 +91,18 @@ def test_scalability():
     dict(name='us-east1-b', start=1, end=1)
   ]
   try:
-    n_parallel = 1
-    while n_parallel <= 16:
-      for _ in range(n_repeat):
+    for i in range(n_repeat):
+      n_parallel = 1
+      while n_parallel <= 16:
+        print('Parallel #: %d, repeat #: %d'%(n_parallel, i))
         clis = shuffle_clients()
         while clis:
           name, params = clis.pop()
           print('Testing %s ...'%name)
           update_client_config(n_parallel=n_parallel, zone=zone, **params)
           run_experiment()
           collect_output(name)
-      n_parallel *= 2
+        n_parallel *= 2
   except KeyboardInterrupt:
     sys.stdout.write('Stopped\n')
 

benchmark/tools/iozone/entrypoint.sh

@@ -37,20 +37,20 @@ if [ -n "${FS_TYPE}" ];then
   fi
 fi
 
-rm -rf ${MOUNTPOINT}/${SUB_DIR} && mkdir -p ${MOUNTPOINT}/${SUB_DIR}
+mkdir -p ${MOUNTPOINT}/${SUB_DIR}
 cd ${MOUNTPOINT}/${SUB_DIR}
 
-exec "${@}" | parse_output.py
-
-
-function umount_fs() {
-  umount -f ${MOUNTPOINT} \
-  && echo "${MOUNTPOINT} is umounted successfully" \
-  || echo "Failed to unmount ${MOUNTPOINT}"
-}
-
-
-if [ -n "${FS_TYPE}" ];then
-  umount_fs
-  umount_fs
-fi
+${@} > /data/stats-${CLIENT_ID}.txt
+python3 /bin/parse_output.py
+
+#function umount_fs() {
+#  umount -f ${MOUNTPOINT} \
+#  && echo "${MOUNTPOINT} is umounted successfully" \
+#  || echo "Failed to unmount ${MOUNTPOINT}"
+#}
+#
+#
+#if [ -n "${FS_TYPE}" ];then
+#  umount_fs
+#  umount_fs
+#fi

benchmark/tools/iozone/parse_output.py

@@ -1,47 +1,31 @@
 #!/usr/bin/env python3
 
 import os
-import sys
 import json
-import time
 
 
 field_names = ('file_size', 'reclen', 'write', 'rewrite', 'read', 'reread', 'random_read',
                'random_write', 'bkwd_read', 'record_rewrite', 'stride_read', 'fwrite', 'frewrite',
                'fread', 'freread')
 
 
-def read_stdin():
-  while True:
-    yield sys.stdin.readline().strip()
-
-
-def parse_line(l):
-  res = {field_names[i]: int(f) for i, f in enumerate(l.split())}
-  sys.stdout.write(json.dumps(res) + '\n')
-  sys.stdout.flush()
-  return res
-
-
-def filter_input():
+def parse_output():
   start = False
   output = []
-  for l in read_stdin():
-    if 'reclen' in l:
-      start = True
-      continue
-    if not start: continue
-    if len(l.split()) < 14: break
-    output += [parse_line(l)]
-  return output
-
-
-def write(output):
-  fn = '/data/stats-%s.json'%os.environ['CLIENT_ID']
-  with open(fn, 'w') as f:
-    json.dump(output, f, indent=2)
+  output_fn = '/data/stats-%s.txt'%os.environ['CLIENT_ID']
+  with open(output_fn) as f:
+    for l in f.readlines():
+      if 'reclen' in l:
+        start = True
+        continue
+      if not start: continue
+      if len(l.split()) < 14: break
+      output += [{field_names[i]: int(f) for i, f in enumerate(l.split())}]
+  if len(output) > 0:
+    json.dump(output, open('/data/stats-%s.json' % os.environ['CLIENT_ID'], 'w'), indent=2)
+  os.remove(output_fn)
 
 
 if __name__ == '__main__':
-  write(filter_input())
+  parse_output()