Commit 7b1f2d0f authored by Yuan Gao's avatar Yuan Gao
Browse files

Fix python3 IAM role name encoded format, add optional override for stunnel log

parent 4fe2b3c7
...@@ -132,7 +132,13 @@ workflows: ...@@ -132,7 +132,13 @@ workflows:
image: ubuntu:18.04 image: ubuntu:18.04
- build-deb-package: - build-deb-package:
name: debian9 name: debian9
image: debian:9 image: debian:stretch
- build-deb-package:
name: debian10
image: debian:buster
- build-deb-package:
name: debian11
image: debian:bullseye
- build-rpm-package: - build-rpm-package:
name: centos7 name: centos7
image: centos:centos7 image: centos:centos7
...@@ -144,4 +150,16 @@ workflows: ...@@ -144,4 +150,16 @@ workflows:
image: amazonlinux:2 image: amazonlinux:2
- build-rpm-package: - build-rpm-package:
name: amazon-linux name: amazon-linux
image: amazonlinux:1 image: amazonlinux:1
\ No newline at end of file - build-rpm-package:
name: fedora30
image: fedora:30
- build-rpm-package:
name: fedora31
image: fedora:31
- build-rpm-package:
name: fedora32
image: fedora:32
- build-rpm-package:
name: fedora33
image: fedora:33
\ No newline at end of file
...@@ -12,6 +12,10 @@ set -ex ...@@ -12,6 +12,10 @@ set -ex
BASE_DIR=$(pwd) BASE_DIR=$(pwd)
BUILD_ROOT=${BASE_DIR}/build/debbuild BUILD_ROOT=${BASE_DIR}/build/debbuild
VERSION=1.25 VERSION=1.25
RELEASE=2
DEB_SYSTEM_RELEASE_PATH=/etc/os-release
UBUNTU18_REGEX="Ubuntu 18"
DEBIAN11_REGEX="Debian GNU/Linux bullseye"
echo 'Cleaning deb build workspace' echo 'Cleaning deb build workspace'
rm -rf ${BUILD_ROOT} rm -rf ${BUILD_ROOT}
...@@ -26,6 +30,15 @@ mkdir -p ${BUILD_ROOT}/usr/bin ...@@ -26,6 +30,15 @@ mkdir -p ${BUILD_ROOT}/usr/bin
mkdir -p ${BUILD_ROOT}/var/log/amazon/efs mkdir -p ${BUILD_ROOT}/var/log/amazon/efs
mkdir -p ${BUILD_ROOT}/usr/share/man/man8 mkdir -p ${BUILD_ROOT}/usr/share/man/man8
if [ -f $DEB_SYSTEM_RELEASE_PATH ] && echo "$(grep PRETTY_NAME $DEB_SYSTEM_RELEASE_PATH)" \
| grep -e "$UBUNTU18_REGEX" -e "$DEBIAN11_REGEX"; then
echo 'Correcting python executable'
sed -i -e 's/python|python2/python3/' dist/amazon-efs-utils.control
# Replace the first line in .py to "#!/usr/bin/env python3" no matter what it was before
sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python3/' src/watchdog/__init__.py
sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python3/' src/mount_efs/__init__.py
fi
echo 'Copying application files' echo 'Copying application files'
install -p -m 644 dist/amazon-efs-mount-watchdog.conf ${BUILD_ROOT}/etc/init install -p -m 644 dist/amazon-efs-mount-watchdog.conf ${BUILD_ROOT}/etc/init
install -p -m 644 dist/amazon-efs-mount-watchdog.service ${BUILD_ROOT}/etc/systemd/system install -p -m 644 dist/amazon-efs-mount-watchdog.service ${BUILD_ROOT}/etc/systemd/system
...@@ -61,7 +74,7 @@ tar czf data.tar.gz etc sbin usr var --owner=0 --group=0 ...@@ -61,7 +74,7 @@ tar czf data.tar.gz etc sbin usr var --owner=0 --group=0
cd ${BASE_DIR} cd ${BASE_DIR}
echo 'Building deb' echo 'Building deb'
DEB=${BUILD_ROOT}/amazon-efs-utils-${VERSION}-1.deb DEB=${BUILD_ROOT}/amazon-efs-utils-${VERSION}-${RELEASE}.deb
ar r ${DEB} ${BUILD_ROOT}/debian-binary ar r ${DEB} ${BUILD_ROOT}/debian-binary
ar r ${DEB} ${BUILD_ROOT}/control.tar.gz ar r ${DEB} ${BUILD_ROOT}/control.tar.gz
ar r ${DEB} ${BUILD_ROOT}/data.tar.gz ar r ${DEB} ${BUILD_ROOT}/data.tar.gz
......
...@@ -8,4 +8,4 @@ ...@@ -8,4 +8,4 @@
[global] [global]
version=1.25 version=1.25
release=1 release=2
...@@ -9,10 +9,16 @@ ...@@ -9,10 +9,16 @@
%if 0%{?amzn1} %if 0%{?amzn1}
%global python_requires system-python %global python_requires system-python
%else
%if 0%{?fedora} || 0%{?el8}
%global python_requires python3
%else %else
%global python_requires python2 %global python_requires python2
%endif %endif
%endif
%if 0%{?amzn1} || 0%{?rhel} == 6 %if 0%{?amzn1} || 0%{?rhel} == 6
%global with_systemd 0 %global with_systemd 0
%else %else
...@@ -21,7 +27,7 @@ ...@@ -21,7 +27,7 @@
Name : amazon-efs-utils Name : amazon-efs-utils
Version : 1.25 Version : 1.25
Release : 1%{?dist} Release : 2%{?dist}
Summary : This package provides utilities for simplifying the use of EFS file systems Summary : This package provides utilities for simplifying the use of EFS file systems
Group : Amazon/Tools Group : Amazon/Tools
...@@ -120,6 +126,10 @@ fi ...@@ -120,6 +126,10 @@ fi
%clean %clean
%changelog %changelog
* Tue May 05 2020 Yuan Gao <ygaochn@amazon.com> - 1.25-2
- Fix the issue that IAM role name format is not correctly encoded in python3
- Add optional override for stunnel debug log output location
* Mon Apr 20 2020 Yuan Gao <ygaochn@amazon.com> - 1.25-1 * Mon Apr 20 2020 Yuan Gao <ygaochn@amazon.com> - 1.25-1
- Create self-signed certificate for tls-only mount - Create self-signed certificate for tls-only mount
......
...@@ -19,6 +19,8 @@ dns_name_suffix = amazonaws.com ...@@ -19,6 +19,8 @@ dns_name_suffix = amazonaws.com
#The region of the file system when mounting from on-premises or cross region. #The region of the file system when mounting from on-premises or cross region.
#region = us-east-1 #region = us-east-1
stunnel_debug_enabled = false stunnel_debug_enabled = false
#Uncomment the below option to save all stunnel logs for a file system to the same file
#stunnel_logs_file = /var/log/amazon/efs/{fs_id}.stunnel.log
stunnel_cafile = /etc/amazon/efs/efs-utils.crt stunnel_cafile = /etc/amazon/efs/efs-utils.crt
# Validate the certificate hostname on mount. This option is not supported by certain stunnel versions. # Validate the certificate hostname on mount. This option is not supported by certain stunnel versions.
......
#!/bin/bash #!/bin/bash
SYSTEM_RELEASE_PATH=/etc/system-release RPM_SYSTEM_RELEASE_PATH=/etc/system-release
RHEL8_REGEX="Red Hat Enterprise Linux release 8" RHEL8_REGEX="Red Hat Enterprise Linux release 8"
FEDORA_REGEX="Fedora release" FEDORA_REGEX="Fedora release"
CENTOS8_REGEX="CentOS Linux release 8" CENTOS8_REGEX="CentOS Linux release 8"
# RHEL8, Fedora30+ and CentOS8 treat shebangs of the form "#!/usr/bin/env python" as errors # RHEL8, Fedora30+ and CentOS8 treat shebangs of the form "#!/usr/bin/env python" as errors
if [ -f $SYSTEM_RELEASE_PATH ] && [[ "$(cat $SYSTEM_RELEASE_PATH)" =~ $RHEL8_REGEX|$FEDORA_REGEX|$CENTOS8_REGEX ]]; then if [ -f $RPM_SYSTEM_RELEASE_PATH ] && [[ "$(cat $RPM_SYSTEM_RELEASE_PATH)" =~ $RHEL8_REGEX|$FEDORA_REGEX|$CENTOS8_REGEX ]]; then
# Replace the first line in .py to "#!/usr/bin/env python2" no echo 'Correcting python executable'
# matter what it was before # Replace the first line in .py to "#!/usr/bin/env python3" no matter what it was before
sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python2/' src/watchdog/__init__.py sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python3/' src/watchdog/__init__.py
sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python2/' src/mount_efs/__init__.py sed -i -e '1 s/^.*$/\#!\/usr\/bin\/env python3/' src/mount_efs/__init__.py
fi fi
\ No newline at end of file
...@@ -34,7 +34,6 @@ import base64 ...@@ -34,7 +34,6 @@ import base64
import errno import errno
import hashlib import hashlib
import hmac import hmac
import itertools
import json import json
import logging import logging
import os import os
...@@ -424,27 +423,18 @@ def credentials_file_helper(file_path, awsprofile): ...@@ -424,27 +423,18 @@ def credentials_file_helper(file_path, awsprofile):
return credentials return credentials
def get_correct_default_case_combination(aws_credentials_configs):
default_str = 'default'
for perm in map(''.join, itertools.product(*zip(default_str.upper(), default_str.lower()))):
try:
access_key = aws_credentials_configs.get(perm, 'aws_access_key_id')
if access_key is not None:
return perm
except (NoSectionError, NoOptionError):
continue
return None
def get_aws_profile(options, use_iam): def get_aws_profile(options, use_iam):
awsprofile = options.get('awsprofile') awsprofile = options.get('awsprofile')
if not awsprofile and use_iam: if not awsprofile and use_iam:
for file_path in [AWS_CREDENTIALS_FILE, AWS_CONFIG_FILE]: for file_path in [AWS_CREDENTIALS_FILE, AWS_CONFIG_FILE]:
aws_credentials_configs = read_config(file_path) aws_credentials_configs = read_config(file_path)
awsprofile = get_correct_default_case_combination(aws_credentials_configs) # check if aws access key id is found under [default] section in current file and return 'default' if so
if awsprofile: try:
break access_key = aws_credentials_configs.get('default', 'aws_access_key_id')
if access_key is not None:
return 'default'
except (NoSectionError, NoOptionError):
continue
return awsprofile return awsprofile
...@@ -641,7 +631,11 @@ def write_stunnel_config_file(config, state_file_dir, fs_id, mountpoint, tls_por ...@@ -641,7 +631,11 @@ def write_stunnel_config_file(config, state_file_dir, fs_id, mountpoint, tls_por
global_config = dict(STUNNEL_GLOBAL_CONFIG) global_config = dict(STUNNEL_GLOBAL_CONFIG)
if config.getboolean(CONFIG_SECTION, 'stunnel_debug_enabled'): if config.getboolean(CONFIG_SECTION, 'stunnel_debug_enabled'):
global_config['debug'] = 'debug' global_config['debug'] = 'debug'
global_config['output'] = os.path.join(log_dir, '%s.stunnel.log' % mount_filename)
if config.has_option(CONFIG_SECTION, 'stunnel_logs_file'):
global_config['output'] = config.get(CONFIG_SECTION, 'stunnel_logs_file').replace('{fs_id}', fs_id)
else:
global_config['output'] = os.path.join(log_dir, '%s.stunnel.log' % mount_filename)
efs_config = dict(STUNNEL_EFS_CONFIG) efs_config = dict(STUNNEL_EFS_CONFIG)
efs_config['accept'] = efs_config['accept'] % tls_port efs_config['accept'] = efs_config['accept'] % tls_port
......
...@@ -16,6 +16,7 @@ except ImportError: ...@@ -16,6 +16,7 @@ except ImportError:
FILE_LIST = ['src/watchdog/__init__.py', 'src/mount_efs/__init__.py', 'dist/amazon-efs-utils.spec', FILE_LIST = ['src/watchdog/__init__.py', 'src/mount_efs/__init__.py', 'dist/amazon-efs-utils.spec',
'dist/amazon-efs-utils.control', 'build-deb.sh'] 'dist/amazon-efs-utils.control', 'build-deb.sh']
RPM_FILE = 'dist/amazon-efs-utils.spec' RPM_FILE = 'dist/amazon-efs-utils.spec'
DEB_FILE = 'build-deb.sh'
GLOBAL_CONFIG = 'config.ini' GLOBAL_CONFIG = 'config.ini'
...@@ -32,9 +33,12 @@ def test_global_version_match(): ...@@ -32,9 +33,12 @@ def test_global_version_match():
def test_global_release_match(): def test_global_release_match():
global_release = get_global_value('release') global_release = get_global_value('release')
release_in_file = get_release_for_file(RPM_FILE) release_in_rpm_file = get_release_for_file(RPM_FILE)
assert release_in_file == global_release, \ release_in_deb_file = get_release_for_file(DEB_FILE)
'release in {} is {}, does not match global release {}'.format(RPM_FILE, release_in_file, global_release) assert release_in_rpm_file == global_release, \
'release in {} is {}, does not match global release {}'.format(RPM_FILE, release_in_rpm_file, global_release)
assert release_in_deb_file == global_release, \
'release in {} is {}, does not match global release {}'.format(RPM_FILE, release_in_deb_file, global_release)
def test_changelog_version_match(): def test_changelog_version_match():
...@@ -83,6 +87,8 @@ def get_release_for_file(file_path): ...@@ -83,6 +87,8 @@ def get_release_for_file(file_path):
for line in lines: for line in lines:
if line.startswith('Release'): if line.startswith('Release'):
return line.split(':')[1].strip().split('%')[0] return line.split(':')[1].strip().split('%')[0]
elif line.startswith('RELEASE'):
return line.split('=')[1].strip()
return None return None
......
...@@ -24,11 +24,12 @@ MOUNT_POINT = '/mnt' ...@@ -24,11 +24,12 @@ MOUNT_POINT = '/mnt'
PORT = 12345 PORT = 12345
VERIFY_LEVEL = 2 VERIFY_LEVEL = 2
OCSP_ENABLED = False OCSP_ENABLED = False
STUNNEL_LOGS_FILE = '/var/log/amazon/efs/%s.stunnel.log' % FS_ID
def _get_config(mocker, stunnel_debug_enabled=False, stunnel_check_cert_hostname_supported=True, def _get_config(mocker, stunnel_debug_enabled=False, stunnel_check_cert_hostname_supported=True,
stunnel_check_cert_validity_supported=True, stunnel_check_cert_hostname=None, stunnel_check_cert_validity_supported=True, stunnel_check_cert_hostname=None,
stunnel_check_cert_validity=False): stunnel_check_cert_validity=False, stunnel_logs_file=None):
mocker.patch('mount_efs.get_version_specific_stunnel_options', mocker.patch('mount_efs.get_version_specific_stunnel_options',
return_value=(stunnel_check_cert_hostname_supported, stunnel_check_cert_validity_supported, )) return_value=(stunnel_check_cert_hostname_supported, stunnel_check_cert_validity_supported, ))
...@@ -47,6 +48,11 @@ def _get_config(mocker, stunnel_debug_enabled=False, stunnel_check_cert_hostname ...@@ -47,6 +48,11 @@ def _get_config(mocker, stunnel_debug_enabled=False, stunnel_check_cert_hostname
config.set(mount_efs.CONFIG_SECTION, 'stunnel_debug_enabled', str(stunnel_debug_enabled)) config.set(mount_efs.CONFIG_SECTION, 'stunnel_debug_enabled', str(stunnel_debug_enabled))
config.set(mount_efs.CONFIG_SECTION, 'stunnel_check_cert_hostname', str(stunnel_check_cert_hostname)) config.set(mount_efs.CONFIG_SECTION, 'stunnel_check_cert_hostname', str(stunnel_check_cert_hostname))
config.set(mount_efs.CONFIG_SECTION, 'stunnel_check_cert_validity', str(stunnel_check_cert_validity)) config.set(mount_efs.CONFIG_SECTION, 'stunnel_check_cert_validity', str(stunnel_check_cert_validity))
# This option is only written if stunnel debug logs are enabled and a log file is specified
if stunnel_debug_enabled and stunnel_logs_file:
config.set(mount_efs.CONFIG_SECTION, 'stunnel_logs_file', str(stunnel_logs_file))
return config return config
...@@ -180,6 +186,23 @@ def test_write_stunnel_config_with_debug(mocker, tmpdir): ...@@ -180,6 +186,23 @@ def test_write_stunnel_config_with_debug(mocker, tmpdir):
_validate_config(config_file, expected_global_config, _get_expected_efs_config()) _validate_config(config_file, expected_global_config, _get_expected_efs_config())
def test_write_stunnel_config_with_debug_and_logs_file(mocker, tmpdir):
ca_mocker = mocker.patch('mount_efs.add_stunnel_ca_options')
state_file_dir = str(tmpdir)
config_file = mount_efs.write_stunnel_config_file(_get_config(mocker, stunnel_debug_enabled=True,
stunnel_logs_file=STUNNEL_LOGS_FILE),
state_file_dir, FS_ID,
MOUNT_POINT, PORT, DNS_NAME, VERIFY_LEVEL, OCSP_ENABLED,
_get_mount_options())
utils.assert_called_once(ca_mocker)
expected_global_config = dict(mount_efs.STUNNEL_GLOBAL_CONFIG)
expected_global_config['debug'] = 'debug'
expected_global_config['output'] = STUNNEL_LOGS_FILE
_validate_config(config_file, expected_global_config, _get_expected_efs_config())
def test_write_stunnel_config_check_cert_hostname_supported_flag_not_set(mocker, tmpdir): def test_write_stunnel_config_check_cert_hostname_supported_flag_not_set(mocker, tmpdir):
_test_check_cert_hostname(mocker, tmpdir, stunnel_check_cert_hostname_supported=True, stunnel_check_cert_hostname=None, _test_check_cert_hostname(mocker, tmpdir, stunnel_check_cert_hostname_supported=True, stunnel_check_cert_hostname=None,
expected_check_cert_hostname_config_value=True) expected_check_cert_hostname_config_value=True)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment