From a6edef2fa52f5b7771cc5751cde1a190d96ffa50 Mon Sep 17 00:00:00 2001 From: Joe LeVeque Date: Mon, 13 Nov 2017 14:16:19 -0800 Subject: [PATCH] [docker-teamd]: Manage teamd and teamsyncd processes with supervisor (#1137) --- dockers/docker-teamd/Dockerfile.j2 | 7 +- dockers/docker-teamd/docker-teamd-init.sh | 43 +++++++++++++ dockers/docker-teamd/docker-teamd-start.sh | 9 +++ .../docker-teamd.supervisord.conf.j2 | 52 +++++++++++++++ dockers/docker-teamd/start.sh | 35 ---------- dockers/docker-teamd/teamd.sh | 27 ++------ .../docker-dhcp-relay.supervisord.conf | 9 ++- .../docker-teamd.supervisord.conf | 64 +++++++++++++++++++ .../tests/sample_output/wait_for_intf.sh | 28 ++++++++ src/sonic-config-engine/tests/test_j2files.py | 30 ++++++++- 10 files changed, 240 insertions(+), 64 deletions(-) create mode 100755 dockers/docker-teamd/docker-teamd-init.sh create mode 100755 dockers/docker-teamd/docker-teamd-start.sh create mode 100644 dockers/docker-teamd/docker-teamd.supervisord.conf.j2 delete mode 100755 dockers/docker-teamd/start.sh rename dockers/docker-teamd/supervisord.conf => src/sonic-config-engine/tests/sample_output/docker-dhcp-relay.supervisord.conf (58%) create mode 100644 src/sonic-config-engine/tests/sample_output/docker-teamd.supervisord.conf create mode 100644 src/sonic-config-engine/tests/sample_output/wait_for_intf.sh diff --git a/dockers/docker-teamd/Dockerfile.j2 b/dockers/docker-teamd/Dockerfile.j2 index f604433a6c3f..9fded5183c3c 100644 --- a/dockers/docker-teamd/Dockerfile.j2 +++ b/dockers/docker-teamd/Dockerfile.j2 @@ -22,11 +22,10 @@ RUN dpkg -i \ debs/{{ deb }}{{' '}} {%- endfor %} -COPY ["start.sh", "teamd.sh", "/usr/bin/"] -COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] -COPY ["teamd.j2", "/usr/share/sonic/templates/"] +COPY ["docker-teamd-init.sh", "docker-teamd-start.sh", "teamd.sh", "/usr/bin/"] +COPY ["docker-teamd.supervisord.conf.j2", "teamd.j2", "/usr/share/sonic/templates/"] RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y RUN rm -rf /debs -ENTRYPOINT ["/usr/bin/supervisord"] +ENTRYPOINT ["/usr/bin/docker-teamd-init.sh"] diff --git a/dockers/docker-teamd/docker-teamd-init.sh b/dockers/docker-teamd/docker-teamd-init.sh new file mode 100755 index 000000000000..a06ca4e8f890 --- /dev/null +++ b/dockers/docker-teamd/docker-teamd-init.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +TEAMD_CONF_PATH="/etc/teamd" + +rm -rf $TEAMD_CONF_PATH +mkdir -p $TEAMD_CONF_PATH + +SONIC_ASIC_TYPE=$(sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type) +MAC_ADDRESS=$(ip link show eth0 | grep ether | awk '{print $2}') + +# Align last byte +if [ "$SONIC_ASIC_TYPE" == "mellanox" -o "$SONIC_ASIC_TYPE" == "centec" ]; then + last_byte=$(python -c "print '$MAC_ADDRESS'[-2:]") + aligned_last_byte=$(python -c "print format(int(int('$last_byte', 16) & 0b11000000), '02x')") # put mask and take away the 0x prefix + MAC_ADDRESS=$(python -c "print '$MAC_ADDRESS'[:-2] + '$aligned_last_byte'") # put aligned byte into the end of MAC +fi + +for pc in `sonic-cfggen -d -v "PORTCHANNEL.keys() | join(' ') if PORTCHANNEL"`; do + sonic-cfggen -d -a '{"pc":"'$pc'","hwaddr":"'$MAC_ADDRESS'"}' -t /usr/share/sonic/templates/teamd.j2 > $TEAMD_CONF_PATH/$pc.conf + # bring down all member ports before starting teamd + for member in $(sonic-cfggen -d -v "PORTCHANNEL['$pc']['members'] | join(' ')" ); do + if [ -L /sys/class/net/$member ]; then + ip link set $member down + fi + done +done + +# Create a Python dictionary where the key is the Jinja2 variable name +# "lags" and the value is a list of dctionaries containing the name of +# the LAG and the path of the LAG config file. Then output this in +# JSON format, as we will pass it to sonic-cfggen as additional data +# below for generating the supervisord config file. +# Example output: {"lags": [{"name": "PortChannel1", "file": "/etc/teamd/PortChannel1.conf"}, {"name": "PortChannel2", "file": "/etc/teamd/PortChannel2.conf"}]} +LAG_INFO_DICT=$(python -c "import json,os,sys; lags_dict = {}; lags_dict['lags'] = [{'name': os.path.basename(file).split('.')[0], 'file': os.path.join('${TEAMD_CONF_PATH}', file)} for file in sorted(os.listdir('${TEAMD_CONF_PATH}'))]; sys.stdout.write(json.dumps(lags_dict))") + +# Generate supervisord config file +mkdir -p /etc/supervisor/conf.d/ +sonic-cfggen -d -a "${LAG_INFO_DICT}" -t /usr/share/sonic/templates/docker-teamd.supervisord.conf.j2 > /etc/supervisor/conf.d/docker-teamd.supervisord.conf + +# The Docker container should start this script as PID 1, so now that we +# have generated the proper supervisord configuration, we exec supervisord +# so that it runs as PID 1 for the duration of the container's lifetime +exec /usr/bin/supervisord diff --git a/dockers/docker-teamd/docker-teamd-start.sh b/dockers/docker-teamd/docker-teamd-start.sh new file mode 100755 index 000000000000..95b167d22b40 --- /dev/null +++ b/dockers/docker-teamd/docker-teamd-start.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +rm -f /var/run/rsyslogd.pid + +supervisorctl start rsyslogd + +supervisorctl start teamd:* + +supervisorctl start teamsyncd diff --git a/dockers/docker-teamd/docker-teamd.supervisord.conf.j2 b/dockers/docker-teamd/docker-teamd.supervisord.conf.j2 new file mode 100644 index 000000000000..abd3203b3477 --- /dev/null +++ b/dockers/docker-teamd/docker-teamd.supervisord.conf.j2 @@ -0,0 +1,52 @@ +[supervisord] +logfile_maxbytes=1MB +logfile_backups=2 +nodaemon=true + +[program:docker-teamd-start.sh] +command=/usr/bin/docker-teamd-start.sh +priority=1 +autostart=true +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:rsyslogd] +command=/usr/sbin/rsyslogd -n +priority=2 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +{# If there are LAGs... #} +{% if lags -%} +[group:teamd] +programs= +{%- set add_preceding_comma = { 'flag': False } -%} +{%- for lag in lags -%} +{%- if add_preceding_comma.flag %},{% endif -%} +{%- set _dummy = add_preceding_comma.update({'flag': True}) -%} +teamd-{{ lag['name'] }} +{%- endfor %} + +{# Create a program entry for each teamd instance #} +{% for lag in lags %} + +[program:teamd-{{ lag['name'] }}] +command=/usr/bin/teamd.sh {{ lag['file'] }} +priority=3 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog +{% endfor %} + +[program:teamsyncd] +command=/usr/bin/teamsyncd +priority=4 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog +{% endif %} diff --git a/dockers/docker-teamd/start.sh b/dockers/docker-teamd/start.sh deleted file mode 100755 index 6e80f6eb9a83..000000000000 --- a/dockers/docker-teamd/start.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -TEAMD_CONF_PATH=/etc/teamd - -rm -rf $TEAMD_CONF_PATH -mkdir -p $TEAMD_CONF_PATH - -SONIC_ASIC_TYPE=$(sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type) -MAC_ADDRESS=$(ip link show eth0 | grep ether | awk '{print $2}') - -# Align last byte -if [ "$SONIC_ASIC_TYPE" == "mellanox" -o "$SONIC_ASIC_TYPE" == "centec" ]; then - last_byte=$(python -c "print '$MAC_ADDRESS'[-2:]") - aligned_last_byte=$(python -c "print format(int(int('$last_byte', 16) & 0b11000000), '02x')") # put mask and take away the 0x prefix - MAC_ADDRESS=$(python -c "print '$MAC_ADDRESS'[:-2] + '$aligned_last_byte'") # put aligned byte into the end of MAC -fi - -for pc in `sonic-cfggen -d -v "PORTCHANNEL.keys() | join(' ') if PORTCHANNEL"`; do - sonic-cfggen -d -a '{"pc":"'$pc'","hwaddr":"'$MAC_ADDRESS'"}' -t /usr/share/sonic/templates/teamd.j2 > $TEAMD_CONF_PATH/$pc.conf - # bring down all member ports before starting teamd - for member in $(sonic-cfggen -d -v "PORTCHANNEL['$pc']['members'] | join(' ')" ); do - if [ -L /sys/class/net/$member ]; then - ip link set $member down - fi - done -done - -mkdir -p /var/sonic -echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status - -rm -f /var/run/rsyslogd.pid - -supervisorctl start rsyslogd - -supervisorctl start teamd diff --git a/dockers/docker-teamd/teamd.sh b/dockers/docker-teamd/teamd.sh index 2b6d9fb53970..7192a2bdeca2 100755 --- a/dockers/docker-teamd/teamd.sh +++ b/dockers/docker-teamd/teamd.sh @@ -1,28 +1,15 @@ #!/usr/bin/env bash -TEAMD_CONF_PATH=/etc/teamd - -function start_app { - rm -f /var/run/teamd/* - if [ "$(ls -A $TEAMD_CONF_PATH)" ]; then - for f in $TEAMD_CONF_PATH/*; do - teamd -f $f -d - done - fi - teamsyncd & -} +TEAMD_CONF_FILE=$1 function clean_up { - if [ "$(ls -A $TEAMD_CONF_PATH)" ]; then - for f in $TEAMD_CONF_PATH/*; do - teamd -f $f -k - done - fi - pkill -9 teamsyncd - exit + teamd -f $TEAMD_CONF_FILE -k + exit $? } trap clean_up SIGTERM SIGKILL -start_app -read +teamd -f $TEAMD_CONF_FILE & +TEAMD_PID=$! +wait $TEAMD_PID +exit $? diff --git a/dockers/docker-teamd/supervisord.conf b/src/sonic-config-engine/tests/sample_output/docker-dhcp-relay.supervisord.conf similarity index 58% rename from dockers/docker-teamd/supervisord.conf rename to src/sonic-config-engine/tests/sample_output/docker-dhcp-relay.supervisord.conf index 48ef0ca9f29e..f955546cdc09 100644 --- a/dockers/docker-teamd/supervisord.conf +++ b/src/sonic-config-engine/tests/sample_output/docker-dhcp-relay.supervisord.conf @@ -19,10 +19,15 @@ autorestart=false stdout_logfile=syslog stderr_logfile=syslog -[program:teamd] -command=/usr/bin/teamd.sh +[group:isc-dhcp-relay] +programs=isc-dhcp-relay-Vlan1000 + +[program:isc-dhcp-relay-Vlan1000] +command=/usr/sbin/dhcrelay -d -m discard -a %%h:%%p %%P --name-alias-map-file /tmp/port-name-alias-map.txt -i Vlan1000 -i PortChannel02 -i PortChannel03 -i PortChannel04 -i PortChannel01 192.0.0.1 192.0.0.2 priority=3 autostart=false autorestart=false stdout_logfile=syslog stderr_logfile=syslog + + diff --git a/src/sonic-config-engine/tests/sample_output/docker-teamd.supervisord.conf b/src/sonic-config-engine/tests/sample_output/docker-teamd.supervisord.conf new file mode 100644 index 000000000000..83feaf05b8c8 --- /dev/null +++ b/src/sonic-config-engine/tests/sample_output/docker-teamd.supervisord.conf @@ -0,0 +1,64 @@ +[supervisord] +logfile_maxbytes=1MB +logfile_backups=2 +nodaemon=true + +[program:docker-teamd-start.sh] +command=/usr/bin/docker-teamd-start.sh +priority=1 +autostart=true +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:rsyslogd] +command=/usr/sbin/rsyslogd -n +priority=2 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[group:teamd] +programs=teamd-PortChannel01,teamd-PortChannel02,teamd-PortChannel03,teamd-PortChannel04 + +[program:teamd-PortChannel01] +command=/usr/bin/teamd.sh /sonic/src/sonic-config-engine/tests/sample_output/t0_sample_output/PortChannel01.conf +priority=3 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:teamd-PortChannel02] +command=/usr/bin/teamd.sh /sonic/src/sonic-config-engine/tests/sample_output/t0_sample_output/PortChannel02.conf +priority=3 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:teamd-PortChannel03] +command=/usr/bin/teamd.sh /sonic/src/sonic-config-engine/tests/sample_output/t0_sample_output/PortChannel03.conf +priority=3 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:teamd-PortChannel04] +command=/usr/bin/teamd.sh /sonic/src/sonic-config-engine/tests/sample_output/t0_sample_output/PortChannel04.conf +priority=3 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:teamsyncd] +command=/usr/bin/teamsyncd +priority=4 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + diff --git a/src/sonic-config-engine/tests/sample_output/wait_for_intf.sh b/src/sonic-config-engine/tests/sample_output/wait_for_intf.sh new file mode 100644 index 000000000000..26c453d487af --- /dev/null +++ b/src/sonic-config-engine/tests/sample_output/wait_for_intf.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +function wait_until_iface_exists +{ + IFACE=$1 + + echo "Waiting for interface ${IFACE}..." + + # Wait for the interface to come up (i.e., 'ip link show' returns 0) + until ip link show $IFACE > /dev/null 2>&1; do + sleep 1 + done + + echo "Interface ${IFACE} is created" +} + + +# Wait for all interfaces to come up before starting the DHCP relay +wait_until_iface_exists Vlan1000 +wait_until_iface_exists PortChannel04 +wait_until_iface_exists PortChannel02 +wait_until_iface_exists PortChannel03 +wait_until_iface_exists PortChannel03 +wait_until_iface_exists PortChannel01 +wait_until_iface_exists PortChannel02 +wait_until_iface_exists PortChannel04 +wait_until_iface_exists PortChannel01 + diff --git a/src/sonic-config-engine/tests/test_j2files.py b/src/sonic-config-engine/tests/test_j2files.py index d1cc0d2a3296..fb450ba77ae2 100644 --- a/src/sonic-config-engine/tests/test_j2files.py +++ b/src/sonic-config-engine/tests/test_j2files.py @@ -33,6 +33,19 @@ def test_alias_map(self): data = json.loads(output) self.assertEqual(data["Ethernet4"], "fortyGigE0/4") + def test_dhcp_relay(self): + # Test generation of wait_for_intf.sh + template_path = os.path.join(self.test_dir, '..', '..', '..', 'dockers', 'docker-dhcp-relay', 'wait_for_intf.sh.j2') + argument = '-m ' + self.t0_minigraph + ' -p ' + self.t0_port_config + ' -t ' + template_path + ' > ' + self.output_file + self.run_script(argument) + self.assertTrue(filecmp.cmp(os.path.join(self.test_dir, 'sample_output', 'wait_for_intf.sh'), self.output_file)) + + # Test generation of docker-dhcp-relay.supervisord.conf + template_path = os.path.join(self.test_dir, '..', '..', '..', 'dockers', 'docker-dhcp-relay', 'docker-dhcp-relay.supervisord.conf.j2') + argument = '-m ' + self.t0_minigraph + ' -p ' + self.t0_port_config + ' -t ' + template_path + ' > ' + self.output_file + self.run_script(argument) + self.assertTrue(filecmp.cmp(os.path.join(self.test_dir, 'sample_output', 'docker-dhcp-relay.supervisord.conf'), self.output_file)) + def test_lldp(self): lldpd_conf_template = os.path.join(self.test_dir, '..', '..', '..', 'dockers', 'docker-lldp-sv2', 'lldpd.conf.j2') argument = '-m ' + self.t0_minigraph + ' -p ' + self.t0_port_config + ' -t ' + lldpd_conf_template + ' > ' + self.output_file @@ -49,7 +62,7 @@ def test_render_teamd(self, pc, minigraph, sample_output): # Test T0 minigraph argument = '-m ' + self.t0_minigraph + ' -p ' + self.t0_port_config + ' -v "PORTCHANNEL.keys() | join(\' \') if PORTCHANNEL"' - output = self.run_script(argument) # Mock the output via config.sh in docker-teamd + output = self.run_script(argument) # Mock the output via docker-teamd-init.sh in docker-teamd pc_list = output.split() for i in range(1, 5): @@ -58,9 +71,9 @@ def test_render_teamd(self, pc, minigraph, sample_output): sample_output = os.path.join(self.test_dir, 'sample_output', 't0_sample_output', pc_name + '.conf') test_render_teamd(self, pc_name, self.t0_minigraph, sample_output) - # Test port channel test minigraph + # Test port channel test minigraph (for testing proper 'min_ports' attribute generation) argument = '-m ' + self.pc_minigraph + ' -p ' + self.t0_port_config + ' -v "PORTCHANNEL.keys() | join(\' \') if PORTCHANNEL"' - output = self.run_script(argument) # Mock the output via config.sh in docker-teamd + output = self.run_script(argument) # Mock the output via docker-teamd-init.sh in docker-teamd pc_list = output.split() pc_name = 'PortChannel01' @@ -68,6 +81,17 @@ def test_render_teamd(self, pc, minigraph, sample_output): sample_output = os.path.join(self.test_dir, 'sample_output', 'pc_sample_output', pc_name + '.conf') test_render_teamd(self, pc_name, self.pc_minigraph, sample_output) + # Test generation of docker-teamd.supervisord.conf + template_path = os.path.join(self.test_dir, '..', '..', '..', 'dockers', 'docker-teamd', 'docker-teamd.supervisord.conf.j2') + teamd_conf_dir = os.path.join(self.test_dir, 'sample_output', 't0_sample_output') + + lags_dict = {} + lags_dict['lags'] = [{'name': os.path.basename(file).split('.')[0], 'file': os.path.join(teamd_conf_dir, file)} for file in sorted(os.listdir(teamd_conf_dir))] + argument = '-m ' + self.t0_minigraph + ' -p ' + self.t0_port_config + ' -a \'' + json.dumps(lags_dict) + '\' -t ' + template_path + ' > ' + self.output_file + + self.run_script(argument) + self.assertTrue(filecmp.cmp(os.path.join(self.test_dir, 'sample_output', 'docker-teamd.supervisord.conf'), self.output_file)) + def test_ipinip(self): ipinip_file = os.path.join(self.test_dir, '..', '..', '..', 'dockers', 'docker-orchagent', 'ipinip.json.j2') argument = '-m ' + self.t0_minigraph + ' -p ' + self.t0_port_config + ' -t ' + ipinip_file + ' > ' + self.output_file