Commit 973d4269 authored by Yuan Gao's avatar Yuan Gao
Browse files

Support mounting to specific AZ mount target

* We can use the new option az, e.g. `sudo mount -t efs -o az=${AZ_NAME} ${FS_ID} ${MOUNT_POINT}` to mount to specific AZ mount target
parent 578ed405
......@@ -171,6 +171,12 @@ To mount file system within a given network namespace, run:
$ sudo mount -t efs -o netns=netns-path file-system-id efs-mount-point/
```
To mount file system to the mount target in specific availability zone (e.g. us-east-1a), run:
```
$ sudo mount -t efs -o az=az-name file-system-id efs-mount-point/
```
To mount over TLS, simply add the `tls` option:
```
......
......@@ -31,7 +31,7 @@
%endif
Name : amazon-efs-utils
Version : 1.29.1
Version : 1.30.1
Release : 1%{platform}
Summary : This package provides utilities for simplifying the use of EFS file systems
......@@ -131,6 +131,10 @@ fi
%clean
%changelog
* Mon Mar 22 2021 Yuan Gao <ygaochn@amazon.com> - 1.30.1
- Support new option: az, enable mount file system to specific availability zone mount target
- Merge PR #84 on Github. Fix to use regional AWS STS endpoints instead of the global endpoint to reduce latency
* Mon Jan 25 2021 Yuan Gao <ygaochn@amazon.com> - 1.29.1
- Update the python dependency to python3
- Support SLES and OpenSUSE
......
......@@ -11,7 +11,7 @@ set -ex
BASE_DIR=$(pwd)
BUILD_ROOT=${BASE_DIR}/build/debbuild
VERSION=1.29.1
VERSION=1.30.1
RELEASE=1
DEB_SYSTEM_RELEASE_PATH=/etc/os-release
......
......@@ -7,5 +7,5 @@
#
[global]
version=1.29.1
version=1.30.1
release=1
Package: amazon-efs-utils
Architecture: all
Version: 1.29.1
Version: 1.30.1
Section: utils
Depends: python3, nfs-common, stunnel4 (>= 4.56), openssl (>= 1.0.2), util-linux
Priority: optional
......
......@@ -14,7 +14,7 @@ logging_file_count = 10
state_file_dir_mode = 750
[mount]
dns_name_format = {fs_id}.efs.{region}.{dns_name_suffix}
dns_name_format = {az}.{fs_id}.efs.{region}.{dns_name_suffix}
dns_name_suffix = amazonaws.com
#The region of the file system when mounting from on-premises or cross region.
#region = us-east-1
......
......@@ -99,7 +99,19 @@ Use the named profile used to lookup IAM credentials in the AWS CLI credentials
(~/.aws/credentials) or AWS CLI config file (~/.aws/config). If "awsprofile" is not \
specified, the "default" profile is used\&.
.TP
mountport
\fBawscredsuri\fR
Use the relative uri to lookup IAM credentials from ecs task metadata endpoint\&.
.TP
\fBcafile\fR
Use the cafile as the stunnel certificate authority file.\&.
.TP
\fBnetns\fR
Mount the EFS file system to the specified network namespace\&.
.TP
\fBaz\fR
Mount the EFS file system to the specified availability zone mount target\&.
.TP
\fBmountport\fR
Use the port 2049 to bypass portmapper daemon on EC2 Mac instances running macOS Big Sur\&.
.if n \{\
.RE
......@@ -115,6 +127,10 @@ Mount an EFS file system with file system ID "fs-abcd1234" at mount point \
"/mnt/efs" without encryption of data in transit in given network namespace \
'/proc/1/net/ns'
.TP
sudo mount -t efs -o az=us-east-1a fs-abcd1234 /mnt/efs
Mount an EFS file system with file system ID "fs-abcd1234" at mount point \
"/mnt/efs" to the mount target in availability zone us-east-1a
.TP
sudo mount -t efs fs-abcd1234:/child /mnt/efs
Mount a non-root directory of an EFS file system with file system ID \
"fs-abcd1234" at mount point "/mnt/efs" without encryption of data in transit\&.
......
......@@ -78,7 +78,7 @@ except ImportError:
BOTOCORE_PRESENT = False
VERSION = '1.29.1'
VERSION = '1.30.1'
SERVICE = 'elasticfilesystem'
CLONE_NEWNET = 0x40000000
......@@ -180,6 +180,7 @@ EFS_ONLY_OPTIONS = [
'accesspoint',
'awscredsuri',
'awsprofile',
'az',
'cafile',
'iam',
'netns',
......@@ -819,7 +820,10 @@ def write_stunnel_config_file(config, state_file_dir, fs_id, mountpoint, tls_por
if config.getboolean(CONFIG_SECTION, 'stunnel_check_cert_hostname'):
if check_host_supported:
efs_config['checkHost'] = dns_name
# Stunnel checkHost option checks if the specified DNS host name or wildcard matches any of the provider in peer
# certificate's CN fields, after introducing the AZ field in dns name, the host name in the stunnel config file
# is not valid, remove the az info there
efs_config['checkHost'] = dns_name[dns_name.index(fs_id):]
else:
fatal_error(tls_controls_message % 'stunnel_check_cert_hostname')
......@@ -1453,7 +1457,7 @@ def bootstrap_logging(config, log_dir=LOG_DIR):
logging.error('Malformed logging level "%s", setting logging level to %s', raw_level, level)
def get_dns_name(config, fs_id):
def get_dns_name(config, fs_id, options):
def _validate_replacement_field_count(format_str, expected_ct):
if format_str.count('{') != expected_ct or format_str.count('}') != expected_ct:
raise ValueError('DNS name format has an incorrect number of replacement fields')
......@@ -1467,6 +1471,14 @@ def get_dns_name(config, fs_id):
expected_replacement_field_ct = 1
if '{az}' in dns_name_format:
az = options.get('az')
if az:
expected_replacement_field_ct += 1
format_args['az'] = az
else:
dns_name_format = dns_name_format.replace('{az}.', '')
if '{region}' in dns_name_format:
expected_replacement_field_ct += 1
format_args['region'] = get_target_region(config)
......@@ -2150,7 +2162,7 @@ def main():
init_system = get_init_system()
check_network_status(fs_id, init_system)
dns_name = get_dns_name(config, fs_id)
dns_name = get_dns_name(config, fs_id, options)
if 'tls' in options:
mount_tls(config, init_system, dns_name, path, fs_id, mountpoint, options)
......
......@@ -48,7 +48,7 @@ except ImportError:
from urllib import urlencode
VERSION = '1.29.1'
VERSION = '1.30.1'
SERVICE = 'elasticfilesystem'
CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'
......
......@@ -15,8 +15,15 @@ from mock import MagicMock
from .. import utils
try:
from configparser import NoOptionError
except ImportError:
import ConfigParser
from ConfigParser import NoOptionError
FS_ID = 'fs-deadbeef'
DEFAULT_REGION = 'us-east-1'
DEFAULT_AZ = 'us-east-1a'
SPECIAL_REGION_DNS_DICT = {
"cn-north-1": "amazonaws.com.cn",
"cn-northwest-1": "amazonaws.com.cn",
......@@ -24,6 +31,8 @@ SPECIAL_REGION_DNS_DICT = {
"us-isob-east-1": "sc2s.sgov.gov"
}
SPECIAL_REGIONS = ["cn-north-1", "cn-northwest-1", "us-iso-east-1", "us-isob-east-1"]
DEFAULT_NFS_OPTIONS = {}
OPTIONS_WITH_AZ = {'az': DEFAULT_AZ}
@pytest.fixture(autouse=True)
......@@ -32,7 +41,7 @@ def setup(mocker):
mocker.patch('socket.gethostbyname')
def _get_mock_config(dns_name_format='{fs_id}.efs.{region}.{dns_name_suffix}', dns_name_suffix='amazonaws.com',
def _get_mock_config(dns_name_format='{az}.{fs_id}.efs.{region}.{dns_name_suffix}', dns_name_suffix='amazonaws.com',
config_section='mount'):
def config_get_side_effect(section, field):
if section == mount_efs.CONFIG_SECTION and field == 'dns_name_format':
......@@ -51,25 +60,41 @@ def _get_mock_config(dns_name_format='{fs_id}.efs.{region}.{dns_name_suffix}', d
def test_get_dns_name(mocker):
config = _get_mock_config()
dns_name = mount_efs.get_dns_name(config, FS_ID)
dns_name = mount_efs.get_dns_name(config, FS_ID, DEFAULT_NFS_OPTIONS)
assert '%s.efs.%s.amazonaws.com' % (FS_ID, DEFAULT_REGION) == dns_name
def test_get_dns_name_with_az_in_options(mocker):
config = _get_mock_config('{az}.{fs_id}.efs.{region}.amazonaws.com')
dns_name = mount_efs.get_dns_name(config, FS_ID, OPTIONS_WITH_AZ)
assert '%s.%s.efs.%s.amazonaws.com' % (DEFAULT_AZ, FS_ID, DEFAULT_REGION) == dns_name
def test_get_dns_name_without_az_in_options(mocker):
config = _get_mock_config('{az}.{fs_id}.efs.{region}.amazonaws.com')
dns_name = mount_efs.get_dns_name(config, FS_ID, DEFAULT_NFS_OPTIONS)
assert '%s.efs.%s.amazonaws.com' % (FS_ID, DEFAULT_REGION) == dns_name
def test_get_dns_name_suffix_hardcoded(mocker):
config = _get_mock_config('{fs_id}.elastic-file-system.{region}.amazonaws.com')
config = _get_mock_config('{az}.{fs_id}.efs.{region}.amazonaws.com')
dns_name = mount_efs.get_dns_name(config, FS_ID)
dns_name = mount_efs.get_dns_name(config, FS_ID, DEFAULT_NFS_OPTIONS)
assert '%s.elastic-file-system.%s.amazonaws.com' % (FS_ID, DEFAULT_REGION) == dns_name
assert '%s.efs.%s.amazonaws.com' % (FS_ID, DEFAULT_REGION) == dns_name
def test_get_dns_name_region_hardcoded(mocker):
get_target_region_mock = mocker.patch('mount_efs.get_target_region')
config = _get_mock_config('{fs_id}.efs.%s.{dns_name_suffix}' % DEFAULT_REGION)
config = _get_mock_config('{az}.{fs_id}.efs.%s.{dns_name_suffix}' % DEFAULT_REGION)
dns_name = mount_efs.get_dns_name(config, FS_ID)
dns_name = mount_efs.get_dns_name(config, FS_ID, DEFAULT_NFS_OPTIONS)
utils.assert_not_called(get_target_region_mock)
......@@ -79,20 +104,20 @@ def test_get_dns_name_region_hardcoded(mocker):
def test_get_dns_name_region_and_suffix_hardcoded(mocker):
get_target_region_mock = mocker.patch('mount_efs.get_target_region')
config = _get_mock_config('{fs_id}.elastic-file-system.us-west-2.amazonaws.com')
config = _get_mock_config('{az}.{fs_id}.efs.us-west-2.amazonaws.com')
dns_name = mount_efs.get_dns_name(config, FS_ID)
dns_name = mount_efs.get_dns_name(config, FS_ID, DEFAULT_NFS_OPTIONS)
utils.assert_not_called(get_target_region_mock)
assert '%s.elastic-file-system.us-west-2.amazonaws.com' % FS_ID == dns_name
assert '%s.efs.us-west-2.amazonaws.com' % FS_ID == dns_name
def test_get_dns_name_bad_format_wrong_specifiers(mocker):
config = _get_mock_config('{foo}.efs.{bar}')
with pytest.raises(ValueError) as ex:
mount_efs.get_dns_name(config, FS_ID)
mount_efs.get_dns_name(config, FS_ID, DEFAULT_NFS_OPTIONS)
assert 'must include' in str(ex.value)
......@@ -101,7 +126,7 @@ def test_get_dns_name_bad_format_too_many_specifiers_1(mocker):
config = _get_mock_config('{fs_id}.efs.{foo}')
with pytest.raises(ValueError) as ex:
mount_efs.get_dns_name(config, FS_ID)
mount_efs.get_dns_name(config, FS_ID, DEFAULT_NFS_OPTIONS)
assert 'incorrect number' in str(ex.value)
......@@ -110,7 +135,7 @@ def test_get_dns_name_bad_format_too_many_specifiers_2(mocker):
config = _get_mock_config('{fs_id}.efs.{region}.{foo}')
with pytest.raises(ValueError) as ex:
mount_efs.get_dns_name(config, FS_ID)
mount_efs.get_dns_name(config, FS_ID, DEFAULT_NFS_OPTIONS)
assert 'incorrect number' in str(ex.value)
......@@ -121,7 +146,7 @@ def test_get_dns_name_unresolvable(mocker, capsys):
mocker.patch('socket.gethostbyname', side_effect=socket.gaierror)
with pytest.raises(SystemExit) as ex:
mount_efs.get_dns_name(config, FS_ID)
mount_efs.get_dns_name(config, FS_ID, DEFAULT_NFS_OPTIONS)
assert 0 != ex.value.code
......@@ -139,7 +164,7 @@ def test_get_dns_name_special_region(mocker):
config = _get_mock_config(dns_name_suffix=special_dns_name_suffix, config_section=config_section)
config.has_section.return_value = True
dns_name = mount_efs.get_dns_name(config, FS_ID)
dns_name = mount_efs.get_dns_name(config, FS_ID, DEFAULT_NFS_OPTIONS)
assert '%s.efs.%s.%s' % (FS_ID, special_region, special_dns_name_suffix) == dns_name
......@@ -153,9 +178,8 @@ def test_get_dns_name_region_in_suffix(mocker):
config = _get_mock_config('{fs_id}.efs.{dns_name_suffix}', dns_name_suffix=dns_name_suffix)
dns_name = mount_efs.get_dns_name(config, FS_ID)
dns_name = mount_efs.get_dns_name(config, FS_ID, DEFAULT_NFS_OPTIONS)
utils.assert_not_called(get_target_region_mock)
assert '%s.efs.%s.%s' % (FS_ID, special_region, special_dns_name_suffix) == dns_name
......@@ -79,7 +79,7 @@ def get_config(dns_name_format, region=None):
def get_target_region_helper():
config = get_config('{fs_id}.efs.{region}.{dns_name_suffix}')
config = get_config('{az}.{fs_id}.efs.{region}.{dns_name_suffix}')
return mount_efs.get_target_region(config)
......@@ -184,7 +184,7 @@ def test_get_target_region_missing_region(mocker, capsys):
Get target region from configuration file
"""
def test_get_target_region_from_config_variable(mocker):
config = get_config('{fs_id}.efs.us-east-2.{dns_name_suffix}', TARGET_REGION)
config = get_config('{az}.{fs_id}.efs.us-east-2.{dns_name_suffix}', TARGET_REGION)
assert TARGET_REGION == mount_efs.get_target_region(config)
......@@ -195,11 +195,11 @@ def _test_get_target_region_from_dns_format(mocker, config):
def test_get_target_region_from_legacy_dns_name_format(mocker):
config = get_config('{fs_id}.efs.us-east-1.amazonaws.com')
config = get_config('{az}.{fs_id}.efs.us-east-1.amazonaws.com')
_test_get_target_region_from_dns_format(mocker, config)
def test_get_target_region_from_suffixed_dns_name_format(mocker):
config = get_config('{fs_id}.efs.us-east-1.{dns_name_suffix}')
config = get_config('{az}.{fs_id}.efs.us-east-1.{dns_name_suffix}')
config.set(mount_efs.CONFIG_SECTION, 'dns_name_suffix', DNS_NAME_SUFFIX)
_test_get_target_region_from_dns_format(mocker, config)
\ No newline at end of file
......@@ -20,6 +20,7 @@ except ImportError:
FS_ID = 'fs-deadbeef'
DNS_NAME = 'fs-deadbeef.com'
DNS_NAME_WITH_AZ = 'us-east-1a.fs-deadbeef.com'
MOUNT_POINT = '/mnt'
PORT = 12345
VERIFY_LEVEL = 2
......@@ -106,7 +107,7 @@ def _get_expected_efs_config(port=PORT, dns_name=DNS_NAME, verify=mount_efs.DEFA
expected_efs_config['verify'] = str(verify)
if check_cert_hostname:
expected_efs_config['checkHost'] = dns_name
expected_efs_config['checkHost'] = dns_name[dns_name.index(FS_ID):]
if check_cert_validity and ocsp_override:
expected_efs_config['OCSPaia'] = 'yes'
......@@ -157,6 +158,17 @@ def _test_write_stunnel_config_file(mocker, tmpdir):
_validate_config(config_file, mount_efs.STUNNEL_GLOBAL_CONFIG, _get_expected_efs_config())
def _test_write_stunnel_config_file_with_az_as_dns_name(mocker, tmpdir):
ca_mocker = mocker.patch('mount_efs.add_stunnel_ca_options')
state_file_dir = str(tmpdir)
config_file = mount_efs.write_stunnel_config_file(_get_config(mocker), state_file_dir, FS_ID, MOUNT_POINT, PORT,
DNS_NAME_WITH_AZ, VERIFY_LEVEL, OCSP_ENABLED, _get_mount_options())
utils.assert_called_once(ca_mocker)
_validate_config(config_file, mount_efs.STUNNEL_GLOBAL_CONFIG, _get_expected_efs_config(dns_name=DNS_NAME_WITH_AZ))
def _test_disable_libwrap(mocker, tmpdir, system_release='unknown', disable_libwrap=True):
mocker.patch('mount_efs.add_stunnel_ca_options')
ver_mocker = mocker.patch('mount_efs.get_system_release_version', return_value=system_release)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment