Fix python3 IAM role name encoded format, add optional override for stunnel log

.circleci/config.yml

@@ -132,7 +132,13 @@ workflows:
           image: ubuntu:18.04
       - build-deb-package:
           name: debian9
-          image: debian:9
+          image: debian:stretch
+      - build-deb-package:
+          name: debian10
+          image: debian:buster
+      - build-deb-package:
+          name: debian11
+          image: debian:bullseye
       - build-rpm-package:
           name: centos7
           image: centos:centos7
@@ -144,4 +150,16 @@ workflows:
           image: amazonlinux:2
       - build-rpm-package:
           name: amazon-linux
-          image: amazonlinux:1
\ No newline at end of file
+          image: amazonlinux:1
+      - build-rpm-package:
+          name: fedora30
+          image: fedora:30
+      - build-rpm-package:
+          name: fedora31
+          image: fedora:31
+      - build-rpm-package:
+          name: fedora32
+          image: fedora:32
+      - build-rpm-package:
+          name: fedora33
+          image: fedora:33
\ No newline at end of file

build-deb.sh

@@ -12,6 +12,10 @@ set -ex
 BASE_DIR=$(pwd)
 BUILD_ROOT=${BASE_DIR}/build/debbuild
 VERSION=1.25
+RELEASE=2
+DEB_SYSTEM_RELEASE_PATH=/etc/os-release
+UBUNTU18_REGEX="Ubuntu 18"
+DEBIAN11_REGEX="Debian GNU/Linux bullseye"
 
 echo 'Cleaning deb build workspace'
 rm -rf ${BUILD_ROOT}
@@ -26,6 +30,15 @@ mkdir -p ${BUILD_ROOT}/usr/bin
 mkdir -p ${BUILD_ROOT}/var/log/amazon/efs
 mkdir -p ${BUILD_ROOT}/usr/share/man/man8
 
+if [ -f $DEB_SYSTEM_RELEASE_PATH ] && echo "$(grep PRETTY_NAME $DEB_SYSTEM_RELEASE_PATH)" \
+          | grep -e "$UBUNTU18_REGEX" -e "$DEBIAN11_REGEX"; then
+    echo 'Correcting python executable'
+    sed -i -e 's/python|python2/python3/' dist/amazon-efs-utils.control
+    # Replace the first line in .py to "#!/usr/bin/env python3" no matter what it was before
+    sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python3/' src/watchdog/__init__.py
+    sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python3/' src/mount_efs/__init__.py
+fi
+
 echo 'Copying application files'
 install -p -m 644 dist/amazon-efs-mount-watchdog.conf ${BUILD_ROOT}/etc/init
 install -p -m 644 dist/amazon-efs-mount-watchdog.service ${BUILD_ROOT}/etc/systemd/system
@@ -61,7 +74,7 @@ tar czf data.tar.gz etc sbin usr var --owner=0 --group=0
 cd ${BASE_DIR}
 
 echo 'Building deb'
-DEB=${BUILD_ROOT}/amazon-efs-utils-${VERSION}-1.deb
+DEB=${BUILD_ROOT}/amazon-efs-utils-${VERSION}-${RELEASE}.deb
 ar r ${DEB} ${BUILD_ROOT}/debian-binary
 ar r ${DEB} ${BUILD_ROOT}/control.tar.gz
 ar r ${DEB} ${BUILD_ROOT}/data.tar.gz

config.ini

@@ -8,4 +8,4 @@
 
 [global]
 version=1.25
-release=1
+release=2

dist/amazon-efs-utils.spec

@@ -9,10 +9,16 @@
 %if 0%{?amzn1}
 %global python_requires system-python
 
+%else
+
+%if 0%{?fedora} || 0%{?el8}
+%global python_requires python3
 %else
 %global python_requires python2
 %endif
 
+%endif
+
 %if 0%{?amzn1} || 0%{?rhel} == 6
 %global with_systemd 0
 %else
@@ -21,7 +27,7 @@
 
 Name      : amazon-efs-utils
 Version   : 1.25
-Release   : 1%{?dist}
+Release   : 2%{?dist}
 Summary   : This package provides utilities for simplifying the use of EFS file systems
 
 Group     : Amazon/Tools
@@ -120,6 +126,10 @@ fi
 %clean
 
 %changelog
+* Tue May 05 2020 Yuan Gao <ygaochn@amazon.com> - 1.25-2
+- Fix the issue that IAM role name format is not correctly encoded in python3
+- Add optional override for stunnel debug log output location
+
 * Mon Apr 20 2020 Yuan Gao <ygaochn@amazon.com> - 1.25-1
 - Create self-signed certificate for tls-only mount
 

dist/efs-utils.conf

@@ -19,6 +19,8 @@ dns_name_suffix = amazonaws.com
 #The region of the file system when mounting from on-premises or cross region.
 #region = us-east-1
 stunnel_debug_enabled = false
+#Uncomment the below option to save all stunnel logs for a file system to the same file
+#stunnel_logs_file = /var/log/amazon/efs/{fs_id}.stunnel.log
 stunnel_cafile = /etc/amazon/efs/efs-utils.crt
 
 # Validate the certificate hostname on mount. This option is not supported by certain stunnel versions.

mangle-shebangs.sh

 #!/bin/bash
 
-SYSTEM_RELEASE_PATH=/etc/system-release
+RPM_SYSTEM_RELEASE_PATH=/etc/system-release
 RHEL8_REGEX="Red Hat Enterprise Linux release 8"
 FEDORA_REGEX="Fedora release"
 CENTOS8_REGEX="CentOS Linux release 8"
 
 # RHEL8, Fedora30+ and CentOS8 treat shebangs of the form "#!/usr/bin/env python" as errors
-if [ -f $SYSTEM_RELEASE_PATH ] && [[ "$(cat $SYSTEM_RELEASE_PATH)" =~ $RHEL8_REGEX|$FEDORA_REGEX|$CENTOS8_REGEX ]]; then
-    # Replace the first line in .py to "#!/usr/bin/env python2" no
-    # matter what it was before
-    sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python2/' src/watchdog/__init__.py
-    sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python2/' src/mount_efs/__init__.py
-fi
+if [ -f $RPM_SYSTEM_RELEASE_PATH ] && [[ "$(cat $RPM_SYSTEM_RELEASE_PATH)" =~ $RHEL8_REGEX|$FEDORA_REGEX|$CENTOS8_REGEX ]]; then
+    echo 'Correcting python executable'
+    # Replace the first line in .py to "#!/usr/bin/env python3" no matter what it was before
+    sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python3/' src/watchdog/__init__.py
+    sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python3/' src/mount_efs/__init__.py
+fi
\ No newline at end of file

src/mount_efs/__init__.py

@@ -34,7 +34,6 @@ import base64
 import errno
 import hashlib
 import hmac
-import itertools
 import json
 import logging
 import os
@@ -424,27 +423,18 @@ def credentials_file_helper(file_path, awsprofile):
     return credentials
 
 
-def get_correct_default_case_combination(aws_credentials_configs):
-    default_str = 'default'
-    for perm in map(''.join, itertools.product(*zip(default_str.upper(), default_str.lower()))):
-        try:
-            access_key = aws_credentials_configs.get(perm, 'aws_access_key_id')
-            if access_key is not None:
-                return perm
-        except (NoSectionError, NoOptionError):
-            continue
-
-    return None
-
-
 def get_aws_profile(options, use_iam):
     awsprofile = options.get('awsprofile')
     if not awsprofile and use_iam:
         for file_path in [AWS_CREDENTIALS_FILE, AWS_CONFIG_FILE]:
             aws_credentials_configs = read_config(file_path)
-            awsprofile = get_correct_default_case_combination(aws_credentials_configs)
-            if awsprofile:
-                break
+            # check if aws access key id is found under [default] section in current file and return 'default' if so
+            try:
+                access_key = aws_credentials_configs.get('default', 'aws_access_key_id')
+                if access_key is not None:
+                    return 'default'
+            except (NoSectionError, NoOptionError):
+                continue
 
     return awsprofile
 
@@ -641,7 +631,11 @@ def write_stunnel_config_file(config, state_file_dir, fs_id, mountpoint, tls_por
     global_config = dict(STUNNEL_GLOBAL_CONFIG)
     if config.getboolean(CONFIG_SECTION, 'stunnel_debug_enabled'):
         global_config['debug'] = 'debug'
-        global_config['output'] = os.path.join(log_dir, '%s.stunnel.log' % mount_filename)
+
+        if config.has_option(CONFIG_SECTION, 'stunnel_logs_file'):
+            global_config['output'] = config.get(CONFIG_SECTION, 'stunnel_logs_file').replace('{fs_id}', fs_id)
+        else:
+            global_config['output'] = os.path.join(log_dir, '%s.stunnel.log' % mount_filename)
 
     efs_config = dict(STUNNEL_EFS_CONFIG)
     efs_config['accept'] = efs_config['accept'] % tls_port

test/global_test/test_global_version_match.py

@@ -16,6 +16,7 @@ except ImportError:
 FILE_LIST = ['src/watchdog/__init__.py', 'src/mount_efs/__init__.py', 'dist/amazon-efs-utils.spec',
              'dist/amazon-efs-utils.control', 'build-deb.sh']
 RPM_FILE = 'dist/amazon-efs-utils.spec'
+DEB_FILE = 'build-deb.sh'
 
 GLOBAL_CONFIG = 'config.ini'
 
@@ -32,9 +33,12 @@ def test_global_version_match():
 def test_global_release_match():
     global_release = get_global_value('release')
 
-    release_in_file = get_release_for_file(RPM_FILE)
-    assert release_in_file == global_release, \
-        'release in {} is {}, does not match global release {}'.format(RPM_FILE, release_in_file, global_release)
+    release_in_rpm_file = get_release_for_file(RPM_FILE)
+    release_in_deb_file = get_release_for_file(DEB_FILE)
+    assert release_in_rpm_file == global_release, \
+        'release in {} is {}, does not match global release {}'.format(RPM_FILE, release_in_rpm_file, global_release)
+    assert release_in_deb_file == global_release, \
+        'release in {} is {}, does not match global release {}'.format(RPM_FILE, release_in_deb_file, global_release)
 
 
 def test_changelog_version_match():
@@ -83,6 +87,8 @@ def get_release_for_file(file_path):
     for line in lines:
         if line.startswith('Release'):
             return line.split(':')[1].strip().split('%')[0]
+        elif line.startswith('RELEASE'):
+            return line.split('=')[1].strip()
     return None
 
 

test/mount_efs_test/test_write_stunnel_config_file.py

@@ -24,11 +24,12 @@ MOUNT_POINT = '/mnt'
 PORT = 12345
 VERIFY_LEVEL = 2
 OCSP_ENABLED = False
+STUNNEL_LOGS_FILE = '/var/log/amazon/efs/%s.stunnel.log' % FS_ID
 
 
 def _get_config(mocker, stunnel_debug_enabled=False, stunnel_check_cert_hostname_supported=True,
                 stunnel_check_cert_validity_supported=True, stunnel_check_cert_hostname=None,
-                stunnel_check_cert_validity=False):
+                stunnel_check_cert_validity=False, stunnel_logs_file=None):
 
     mocker.patch('mount_efs.get_version_specific_stunnel_options',
                  return_value=(stunnel_check_cert_hostname_supported, stunnel_check_cert_validity_supported, ))
@@ -47,6 +48,11 @@ def _get_config(mocker, stunnel_debug_enabled=False, stunnel_check_cert_hostname
     config.set(mount_efs.CONFIG_SECTION, 'stunnel_debug_enabled', str(stunnel_debug_enabled))
     config.set(mount_efs.CONFIG_SECTION, 'stunnel_check_cert_hostname', str(stunnel_check_cert_hostname))
     config.set(mount_efs.CONFIG_SECTION, 'stunnel_check_cert_validity', str(stunnel_check_cert_validity))
+
+    # This option is only written if stunnel debug logs are enabled and a log file is specified
+    if stunnel_debug_enabled and stunnel_logs_file:
+        config.set(mount_efs.CONFIG_SECTION, 'stunnel_logs_file', str(stunnel_logs_file))
+
     return config
 
 
@@ -180,6 +186,23 @@ def test_write_stunnel_config_with_debug(mocker, tmpdir):
     _validate_config(config_file, expected_global_config, _get_expected_efs_config())
 
 
+def test_write_stunnel_config_with_debug_and_logs_file(mocker, tmpdir):
+    ca_mocker = mocker.patch('mount_efs.add_stunnel_ca_options')
+    state_file_dir = str(tmpdir)
+    config_file = mount_efs.write_stunnel_config_file(_get_config(mocker, stunnel_debug_enabled=True,
+                                                                  stunnel_logs_file=STUNNEL_LOGS_FILE),
+                                                      state_file_dir, FS_ID,
+                                                      MOUNT_POINT, PORT, DNS_NAME, VERIFY_LEVEL, OCSP_ENABLED,
+                                                      _get_mount_options())
+    utils.assert_called_once(ca_mocker)
+
+    expected_global_config = dict(mount_efs.STUNNEL_GLOBAL_CONFIG)
+    expected_global_config['debug'] = 'debug'
+    expected_global_config['output'] = STUNNEL_LOGS_FILE
+
+    _validate_config(config_file, expected_global_config, _get_expected_efs_config())
+
+
 def test_write_stunnel_config_check_cert_hostname_supported_flag_not_set(mocker, tmpdir):
     _test_check_cert_hostname(mocker, tmpdir, stunnel_check_cert_hostname_supported=True, stunnel_check_cert_hostname=None,
                               expected_check_cert_hostname_config_value=True)