[benchmark] used pre-built image for CephFS to accelerate the deploym…

…ent; fixed issues with multiple Ceph MONs

benchmark/ansible/group_vars/all.yml

@@ -11,22 +11,24 @@ disks:
 - delete_on_termination: true
   mode: READ_WRITE
   name: boot
-  size: 30
+  size: 10
   type: pd-ssd
 - delete_on_termination: true
   mode: READ_WRITE
   name: ext-0
   size: 100
-  type: pd-ssd
-image: ubuntu-1804
-machine_type: n1-standard-4
+  type: pd-standard
+
+image: ceph-ubuntu-1804 #ubuntu-1804
+custom_image: true
+machine_type: n1-standard-2
 max_retries: 5
 mode: server
 options:
   cephfs:
     daemon_image: ceph/daemon:latest-luminous
-    mds_group_size: 1
-    mon_group_size: 1
+    mds_group_size: 0
+    mon_group_size: 3
     name: alpha
   ganesha:
     docker_image: janeczku/nfs-ganesha
@@ -66,10 +68,17 @@ options:
     num_pgs: 8
     unwanted_rbd_features: exclusive-lock object-map fast-diff deep-flatten
 preemptible: false
-project: helium-dc
-running: true
+project: helium-skunkworks
+running: false
 supernet: 10.52.0.0/16
 zone:
-- end: 1
-  name: us-east1-b
+- name: us-east1-b
+  start: 1
+  end: 3
+- name: us-east4-c
+  start: 1
+  end: 2
+- name: us-west1-a
   start: 1
+  end: 2
+

benchmark/ansible/playbooks/cephfs/main.yml

@@ -7,6 +7,7 @@
         name: ceph-base
         update_cache: true
         autoremove: true
+      when: not custom_image
 
     - name: Load Ceph kernel module
       modprobe:
@@ -19,8 +20,10 @@
         state: directory
         recurse: true
       with_items:
-      - /opt/ceph/etc
-      - /opt/ceph/var/lib/bootstrap-osd
+        - /opt/ceph/etc
+        - /opt/ceph/var/lib/bootstrap-osd
+        - /opt/ceph/var/lib/bootstrap-rbd
+      when: not custom_image
 
 - hosts: localhost
   gather_facts: false
@@ -31,18 +34,34 @@
         groupname: ceph-mon
       with_items: "{{ groups[fs][:options.cephfs.mon_group_size] }}"
 
-    - name: Assign host(s) to the CephFS group
+    - name: Assign host(s) to the Ceph MDS group
       add_host:
         hostname: "{{ item }}"
         groupname: ceph-mds
       with_items: "{{ groups[fs][:options.cephfs.mds_group_size] }}"
 
+- hosts: cephfs
+  tasks:
+    - name: Set Ceph Monitor hostnames
+      set_fact:
+        ceph_mon_hostnames: >
+          {{ ceph_mon_hostnames | default([]) }}
+          + [ '{{ hostvars[item].ansible_hostname }}' ]
+      with_items: "{{ groups['ceph-mon'] }}"
+
+    - name: Set Ceph Monitor IP addresses
+      set_fact:
+        ceph_mon_ips: >
+          {{ ceph_mon_ips | default([]) }}
+          + [ '{{ hostvars[item].ansible_default_ipv4.address }}' ]
+      with_items: "{{ groups['ceph-mon'] }}"
+
 - hosts: ceph-mon
   become: true
   tasks:
     - name: Create Ceph configuration file
       template:
-        src: ceph.conf.mon.j2
+        src: ceph.conf.j2
         dest: /opt/ceph/etc/ceph.conf
 
     - name: Run Ceph Monitor
@@ -58,6 +77,7 @@
         env:
           MON_IP: "{{ ansible_default_ipv4.address }}"
           CEPH_PUBLIC_NETWORK: "{{ supernet }}"
+          CEPH_CLUSTER_NETWORK: "{{ supernet }}"
 
     - name: Run Ceph Manager
       docker_container:
@@ -70,22 +90,38 @@
           - /opt/ceph/etc:/etc/ceph
           - /opt/ceph/var/lib:/var/lib/ceph/
 
+    - name: Run Ceph REST API
+      docker_container:
+        name: ceph-restapi
+        image: "{{ options.cephfs.daemon_image }}"
+        network_mode: host
+        restart_policy: always
+        command: restapi
+        volumes:
+          - /opt/ceph/etc:/etc/ceph
+          - /opt/ceph/var/lib:/var/lib/ceph/
+
 - hosts: cephfs
   become: true
   tasks:
     - name: Create Ceph configuration file
       template:
-        src: ceph.conf.osd.j2
+        src: ceph.conf.j2
         dest: "{{ item }}"
         force: false
       with_items:
         - /opt/ceph/etc/ceph.conf
         - /etc/ceph/ceph.conf
 
-    - name: Create an empty keyring file as placeholder
+    - name: Create empty keyring files as placeholder
       file:
-        path: /opt/ceph/var/lib/bootstrap-osd/ceph.keyring
+        path: "{{ item }}"
         state: touch
+      with_items:
+        - /opt/ceph/var/lib/bootstrap-osd/ceph.keyring
+        - /opt/ceph/etc/ceph.client.admin.keyring
+      when: not custom_image
+
 
     - name: Find attached block devices
       shell: >
@@ -126,6 +162,26 @@
           OSD_DEVICE: /dev/{{ item }}
       with_items: "{{ block_devices.stdout_lines }}"
 
+    - name: Run Ceph REST API
+      docker_container:
+        name: ceph-restapi
+        image: "{{ options.cephfs.daemon_image }}"
+        network_mode: host
+        restart_policy: always
+        command: restapi
+        volumes:
+          - /opt/ceph/etc:/etc/ceph
+          - /opt/ceph/var/lib:/var/lib/ceph/
+
+    - name: Run CephFS API
+      docker_container:
+        name: cephfs-api
+        image: dchampion24/cephfs-api
+        network_mode: host
+        restart_policy: always
+        volumes:
+          - /var/run/docker.sock:/var/run/docker.sock
+
 - hosts: ceph-mds
   gather_facts: false
   become: true

benchmark/ansible/playbooks/cephfs/templates/ceph.conf.mon.j2 → benchmark/ansible/playbooks/cephfs/templates/ceph.conf.j2

@@ -1,7 +1,7 @@
 [global]
 fsid = ab9314b3-5b09-4fb7-b457-e90b3a05a777
-mon initial members = {{ ansible_hostname }}
-mon host = {{ ansible_default_ipv4.address }}
+mon initial members = {{ ceph_mon_hostnames | join(', ') }}
+mon host = {{ ceph_mon_ips | join(', ') }}
 public network = {{ supernet }}
 cluster network = {{ supernet }}
 osd journal size = 100

benchmark/ansible/roles/install_docker/tasks/main.yml

@@ -9,35 +9,35 @@
       - python-pip
     update_cache: true
     autoremove: true
-  when: not running
+  when: not running and not custom_image
 
 - name: Install required Python dependencies
   pip:
     name: docker
     state: latest
-  when: not running
+  when: not running and not custom_image
 
 - name: Add Docker CE official GPG key
   apt_key:
     url: https://download.docker.com/linux/ubuntu/gpg
     state: present
-  when: not running
+  when: not running and not custom_image
 
 - name: Add Docker CE repository
   apt_repository:
     repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable
     state: present
-  when: not running
+  when: not running and not custom_image
 
 - name: Install Docker CE
   apt:
     name: docker-ce
     update_cache: true
-  when: not running
+  when: not running and not custom_image
 
 - name: Add sudo user to "docker" group
   user:
     name: "{{ ansible_ssh_user }}"
     groups: docker
     append: true
-  when: not running
+  when: not running and not custom_image

benchmark/ansible/setup.yml

@@ -9,7 +9,7 @@
   tasks:
     - name: Install Python
       raw: apt-get update && apt-get install -y python
-      when: not running
+      when: not running and not custom_image
       register: _install_python
       until: not _install_python.failed
       retries: 3