Commit cfd6a546 authored by Yuan Gao's avatar Yuan Gao
Browse files

Added region localization, Integrate repository with CircleCI to enable...

Added region localization, Integrate repository with CircleCI to enable package build visualization, Unit tests bug fixes for python3.5
parent 4e7ffcd4
version: 2.1
executors:
python:
parameters:
image:
type: string
docker:
- image: << parameters.image >>
commands:
runtest:
steps:
- checkout
- restore_cache:
key: efs-utils-{{ .Branch }}-{{ checksum "requirements.txt" }}
- run:
name: Install Python dependencies in a virtual env
command: |
python -m pip install --user virtualenv
python -m virtualenv ~/.env/efs-utils
source ~/.env/efs-utils/bin/activate
pip install -r requirements.txt
- save_cache:
key: efs-utils-{{ .Branch }}-{{ checksum "requirements.txt" }}
paths:
- ~/.env/efs-utils
- run:
name: Run all tests
command: |
source ~/.env/efs-utils/bin/activate
make test
jobs:
test:
parameters:
image:
type: string
executor:
name: python
image: << parameters.image >>
steps:
- runtest
workflows:
workflow:
jobs:
- test:
name: python38
image: circleci/python:3.8.1
- test:
name: python37
image: circleci/python:3.7.3
- test:
name: python36
image: circleci/python:3.6.9
- test:
name: python35
image: circleci/python:3.5.9
- test:
name: python34
image: circleci/python:3.4.9
- test:
name: python27
image: circleci/python:2.7.13
\ No newline at end of file
......@@ -11,7 +11,7 @@ set -ex
BASE_DIR=$(pwd)
BUILD_ROOT=${BASE_DIR}/build/debbuild
VERSION=1.18
VERSION=1.19
echo 'Cleaning deb build workspace'
rm -rf ${BUILD_ROOT}
......
......@@ -7,4 +7,4 @@
#
[global]
version=1.18
version=1.19
Package: amazon-efs-utils
Architecture: all
Version: 1.18
Version: 1.19
Section: utils
Depends: python|python2, nfs-common, stunnel4 (>= 4.56)
Priority: optional
......
......@@ -20,7 +20,7 @@
%endif
Name : amazon-efs-utils
Version : 1.18
Version : 1.19
Release : 1%{?dist}
Summary : This package provides utilities for simplifying the use of EFS file systems
......
......@@ -29,6 +29,18 @@ stunnel_check_cert_validity = false
port_range_lower_bound = 20049
port_range_upper_bound = 20449
[mount.cn-north-1]
dns_name_suffix = amazonaws.com.cn
[mount.cn-northwest-1]
dns_name_suffix = amazonaws.com.cn
[mount.us-iso-east-1]
dns_name_suffix = c2s.ic.gov
[mount.us-isob-east-1]
dns_name_suffix = sc2s.sgov.gov
[mount-watchdog]
enabled = true
poll_interval_sec = 1
......
......@@ -68,7 +68,7 @@ except ImportError:
from urllib.error import URLError
from urllib.request import urlopen
VERSION = '1.18'
VERSION = '1.19'
SERVICE = 'elasticfilesystem'
CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'
......
......@@ -46,7 +46,7 @@ except ImportError:
from urllib.error import URLError
from urllib.request import urlopen
VERSION = '1.18'
VERSION = '1.19'
SERVICE = 'elasticfilesystem'
CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'
......
File added
#
# Copyright 2017-2018 Amazon.com, Inc. and its affiliates. All Rights Reserved.
#
# Licensed under the MIT License. See the LICENSE accompanying this file
# for the specific language governing permissions and limitations under
# the License.
#
......@@ -12,6 +12,8 @@ import pytest
from mock import MagicMock
from .. import utils
FS_ID = 'fs-deadbeef'
......@@ -25,7 +27,7 @@ def test_non_systemd(mocker):
mount_efs.check_network_status(FS_ID, 'init')
call_mock.assert_not_called()
utils.assert_not_called(call_mock)
def test_systemd_network_up(mocker):
......@@ -33,7 +35,7 @@ def test_systemd_network_up(mocker):
mount_efs.check_network_status(FS_ID, 'systemd')
call_mock.assert_called_once()
utils.assert_called_once(call_mock)
def test_systemd_network_down(mocker):
......@@ -42,5 +44,5 @@ def test_systemd_network_down(mocker):
with pytest.raises(SystemExit) as ex:
mount_efs.check_network_status(FS_ID, 'systemd')
call_mock.assert_called_once()
utils.assert_called_once(call_mock)
assert 0 == ex.value.code
......@@ -13,10 +13,17 @@ import pytest
from mock import MagicMock
from .. import utils
FS_ID = 'fs-deadbeef'
DEFAULT_REGION = 'us-east-1'
SPECIAL_DNS_NAME_SUFFIX = 'amazonaws.com.cn'
SPECIAL_REGION = 'cn-north-1'
SPECIAL_REGION_DNS_DICT = {
"cn-north-1": "amazonaws.com.cn",
"cn-northwest-1": "amazonaws.com.cn",
"us-iso-east-1": "c2s.ic.gov",
"us-isob-east-1": "sc2s.sgov.gov"
}
SPECIAL_REGIONS = ["cn-north-1", "cn-northwest-1", "us-iso-east-1", "us-isob-east-1"]
@pytest.fixture(autouse=True)
......@@ -64,7 +71,7 @@ def test_get_dns_name_region_hardcoded(mocker):
dns_name = mount_efs.get_dns_name(config, FS_ID)
get_region_mock.assert_not_called()
utils.assert_not_called(get_region_mock)
assert '%s.efs.%s.amazonaws.com' % (FS_ID, DEFAULT_REGION) == dns_name
......@@ -76,7 +83,7 @@ def test_get_dns_name_region_and_suffix_hardcoded(mocker):
dns_name = mount_efs.get_dns_name(config, FS_ID)
get_region_mock.assert_not_called()
utils.assert_not_called(get_region_mock)
assert '%s.elastic-file-system.us-west-2.amazonaws.com' % FS_ID == dns_name
......@@ -123,25 +130,32 @@ def test_get_dns_name_unresolvable(mocker, capsys):
def test_get_dns_name_special_region(mocker):
mocker.patch('mount_efs.get_region', return_value=SPECIAL_REGION)
for special_region in SPECIAL_REGIONS:
mocker.patch('mount_efs.get_region', return_value=special_region)
config_section = 'mount.%s' % SPECIAL_REGION
config = _get_mock_config(dns_name_suffix=SPECIAL_DNS_NAME_SUFFIX, config_section=config_section)
config.has_section.return_value = True
config_section = 'mount.%s' % special_region
special_dns_name_suffix = SPECIAL_REGION_DNS_DICT[special_region]
dns_name = mount_efs.get_dns_name(config, FS_ID)
config = _get_mock_config(dns_name_suffix=special_dns_name_suffix, config_section=config_section)
config.has_section.return_value = True
dns_name = mount_efs.get_dns_name(config, FS_ID)
assert '%s.efs.%s.%s' % (FS_ID, SPECIAL_REGION, SPECIAL_DNS_NAME_SUFFIX) == dns_name
assert '%s.efs.%s.%s' % (FS_ID, special_region, special_dns_name_suffix) == dns_name
def test_get_dns_name_region_in_suffix(mocker):
get_region_mock = mocker.patch('mount_efs.get_region')
dns_name_suffix = '%s.%s' % (SPECIAL_REGION, SPECIAL_DNS_NAME_SUFFIX)
config = _get_mock_config('{fs_id}.efs.{dns_name_suffix}', dns_name_suffix=dns_name_suffix)
for special_region in SPECIAL_REGIONS:
special_dns_name_suffix = SPECIAL_REGION_DNS_DICT[special_region]
dns_name_suffix = '%s.%s' % (special_region, special_dns_name_suffix)
dns_name = mount_efs.get_dns_name(config, FS_ID)
config = _get_mock_config('{fs_id}.efs.{dns_name_suffix}', dns_name_suffix=dns_name_suffix)
dns_name = mount_efs.get_dns_name(config, FS_ID)
utils.assert_not_called(get_region_mock)
get_region_mock.assert_not_called()
assert '%s.efs.%s.%s' % (FS_ID, special_region, special_dns_name_suffix) == dns_name
assert '%s.efs.%s.%s' % (FS_ID, SPECIAL_REGION, SPECIAL_DNS_NAME_SUFFIX) == dns_name
......@@ -14,6 +14,9 @@ from contextlib import contextmanager
from mock import patch
from .. import utils
AP_ID = 'fsap-0123456789abcdef0'
BAD_AP_ID_INCORRECT_START = 'bad-fsap-0123456789abc'
BAD_AP_ID_TOO_SHORT = 'fsap-0123456789abcdef'
......@@ -61,15 +64,15 @@ def _test_main(mocker, tls=False, root=True, ap_id=None, iam=False, awsprofile=N
mount_efs.main()
bootstrap_logging_mock.assert_called_once()
get_dns_mock.assert_called_once()
parse_arguments_mock.assert_called_once()
mount_mock.assert_called_once()
utils.assert_called_once(bootstrap_logging_mock)
utils.assert_called_once(get_dns_mock)
utils.assert_called_once(parse_arguments_mock)
utils.assert_called_once(mount_mock)
if tls:
bootstrap_tls_mock.assert_called_once()
utils.assert_called_once(bootstrap_tls_mock)
else:
bootstrap_tls_mock.assert_not_called()
utils.assert_not_called(bootstrap_tls_mock)
def _test_main_assert_error(mocker, capsys, expected_err, **kwargs):
......
......@@ -11,6 +11,9 @@ import pytest
import mount_efs
from .. import utils
CORRECT_DEVICE_DESCRIPTORS_FS_ID = [
('fs-deadbeef', ('fs-deadbeef', '/')),
('fs-deadbeef:/', ('fs-deadbeef', '/')),
......@@ -36,8 +39,8 @@ def test_match_device_correct_descriptors_cname_dns_primary(mocker):
return_value=('fs-deadbeef.efs.us-east-1.amazonaws.com', [], None))
for device, (fs_id, path) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS:
assert (fs_id, path) == mount_efs.match_device(None, device)
get_dns_name_mock.assert_called()
gethostbyname_ex_mock.assert_called()
utils.assert_called(get_dns_name_mock)
utils.assert_called(gethostbyname_ex_mock)
def test_match_device_correct_descriptors_cname_dns_secondary(mocker):
......@@ -46,8 +49,8 @@ def test_match_device_correct_descriptors_cname_dns_secondary(mocker):
return_value=(None, ['fs-deadbeef.efs.us-east-1.amazonaws.com'], None))
for device, (fs_id, path) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS:
assert (fs_id, path) == mount_efs.match_device(None, device)
get_dns_name_mock.assert_called()
gethostbyname_ex_mock.assert_called()
utils.assert_called(get_dns_name_mock)
utils.assert_called(gethostbyname_ex_mock)
def test_match_device_correct_descriptors_cname_dns_tertiary(mocker):
......@@ -56,8 +59,8 @@ def test_match_device_correct_descriptors_cname_dns_tertiary(mocker):
return_value=(None, [None, 'fs-deadbeef.efs.us-east-1.amazonaws.com'], None))
for device, (fs_id, path) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS:
assert (fs_id, path) == mount_efs.match_device(None, device)
get_dns_name_mock.assert_called()
gethostbyname_ex_mock.assert_called()
utils.assert_called(get_dns_name_mock)
utils.assert_called(gethostbyname_ex_mock)
def test_match_device_correct_descriptors_cname_dns_amongst_invalid(mocker):
......@@ -70,8 +73,8 @@ def test_match_device_correct_descriptors_cname_dns_amongst_invalid(mocker):
)
for device, (fs_id, path) in CORRECT_DEVICE_DESCRIPTORS_CNAME_DNS:
assert (fs_id, path) == mount_efs.match_device(None, device)
get_dns_name_mock.assert_called()
gethostbyname_ex_mock.assert_called()
utils.assert_called(get_dns_name_mock)
utils.assert_called(gethostbyname_ex_mock)
def test_match_device_unresolvable_domain(mocker, capsys):
......@@ -93,7 +96,7 @@ def test_match_device_no_hostnames(mocker, capsys):
assert 0 != ex.value.code
out, err = capsys.readouterr()
assert 'did not resolve to an EFS mount target' in err
gethostbyname_ex_mock.assert_called()
utils.assert_called(gethostbyname_ex_mock)
def test_match_device_no_hostnames2(mocker, capsys):
......@@ -105,7 +108,7 @@ def test_match_device_no_hostnames2(mocker, capsys):
assert 0 != ex.value.code
out, err = capsys.readouterr()
assert 'did not resolve to an EFS mount target' in err
gethostbyname_ex_mock.assert_called()
utils.assert_called(gethostbyname_ex_mock)
def test_match_device_resolve_to_invalid_efs_dns_name(mocker, capsys):
......@@ -117,7 +120,7 @@ def test_match_device_resolve_to_invalid_efs_dns_name(mocker, capsys):
assert 0 != ex.value.code
out, err = capsys.readouterr()
assert 'did not resolve to a valid DNS name' in err
gethostbyname_ex_mock.assert_called()
utils.assert_called(gethostbyname_ex_mock)
def test_match_device_resolve_to_unexpected_efs_dns_name(mocker, capsys):
......@@ -130,5 +133,5 @@ def test_match_device_resolve_to_unexpected_efs_dns_name(mocker, capsys):
assert 0 != ex.value.code
out, err = capsys.readouterr()
assert 'did not resolve to a valid DNS name' in err
get_dns_name_mock.assert_called()
gethostbyname_ex_mock.assert_called()
utils.assert_called(get_dns_name_mock)
utils.assert_called(gethostbyname_ex_mock)
......@@ -10,6 +10,8 @@ import mount_efs
from mock import MagicMock
from .. import utils
FS_ID = 'fs-deadbeef'
......@@ -31,10 +33,10 @@ def test_systemd_system(mocker):
mount_efs.start_watchdog('systemd')
call_mock.assert_called_once()
utils.assert_called_once(call_mock)
assert 'systemctl' in call_mock.call_args[0][0]
assert 'is-active' in call_mock.call_args[0][0]
popen_mock.assert_called_once()
utils.assert_called_once(popen_mock)
assert 'systemctl' in popen_mock.call_args[0][0]
assert 'start' in popen_mock.call_args[0][0]
......@@ -44,4 +46,4 @@ def test_unknown_system(mocker):
mount_efs.start_watchdog('unknown')
popen_mock.assert_not_called()
utils.assert_not_called(popen_mock)
......@@ -11,6 +11,8 @@ import os
import pytest
from .. import utils
try:
import ConfigParser
except ImportError:
......@@ -118,7 +120,7 @@ def _test_check_cert_hostname(mocker, tmpdir, stunnel_check_cert_hostname_suppor
stunnel_check_cert_hostname=stunnel_check_cert_hostname),
str(tmpdir), FS_ID, MOUNT_POINT, PORT, DNS_NAME, VERIFY_LEVEL, OCSP_ENABLED, _get_mount_options())
ca_mocker.assert_called_once()
utils.assert_called_once(ca_mocker)
_validate_config(config_file, mount_efs.STUNNEL_GLOBAL_CONFIG,
_get_expected_efs_config(check_cert_hostname=expected_check_cert_hostname_config_value))
......@@ -132,7 +134,7 @@ def _test_check_cert_validity(mocker, tmpdir, stunnel_check_cert_validity_suppor
_get_config(mocker, stunnel_check_cert_validity_supported=stunnel_check_cert_validity_supported),
str(tmpdir), FS_ID, MOUNT_POINT, PORT, DNS_NAME, VERIFY_LEVEL, stunnel_check_cert_validity, _get_mount_options())
ca_mocker.assert_called_once()
utils.assert_called_once(ca_mocker)
_validate_config(config_file, mount_efs.STUNNEL_GLOBAL_CONFIG,
_get_expected_efs_config(check_cert_validity=expected_check_cert_validity_config_value))
......@@ -144,7 +146,7 @@ def _test_write_stunnel_config_file(mocker, tmpdir):
config_file = mount_efs.write_stunnel_config_file(_get_config(mocker), state_file_dir, FS_ID, MOUNT_POINT, PORT, DNS_NAME,
VERIFY_LEVEL, OCSP_ENABLED, _get_mount_options())
ca_mocker.assert_called_once()
utils.assert_called_once(ca_mocker)
_validate_config(config_file, mount_efs.STUNNEL_GLOBAL_CONFIG, _get_expected_efs_config())
......@@ -156,7 +158,7 @@ def _test_disable_libwrap(mocker, tmpdir, system_release='unknown', disable_libw
config_file = mount_efs.write_stunnel_config_file(_get_config(mocker), str(tmpdir), FS_ID, MOUNT_POINT, PORT, DNS_NAME,
VERIFY_LEVEL, OCSP_ENABLED, _get_mount_options())
ver_mocker.assert_called_once()
utils.assert_called_once(ver_mocker)
_validate_config(config_file, mount_efs.STUNNEL_GLOBAL_CONFIG, _get_expected_efs_config(disable_libwrap=disable_libwrap))
......@@ -167,7 +169,7 @@ def test_write_stunnel_config_with_debug(mocker, tmpdir):
config_file = mount_efs.write_stunnel_config_file(_get_config(mocker, stunnel_debug_enabled=True), state_file_dir, FS_ID,
MOUNT_POINT, PORT, DNS_NAME, VERIFY_LEVEL, OCSP_ENABLED,
_get_mount_options())
ca_mocker.assert_called_once()
utils.assert_called_once(ca_mocker)
expected_global_config = dict(mount_efs.STUNNEL_GLOBAL_CONFIG)
expected_global_config['debug'] = 'debug'
......@@ -254,7 +256,7 @@ def test_write_stunnel_config_with_verify_level(mocker, tmpdir):
verify = 0
config_file = mount_efs.write_stunnel_config_file(_get_config(mocker, stunnel_check_cert_validity=True), str(tmpdir), FS_ID,
MOUNT_POINT, PORT, DNS_NAME, verify, OCSP_ENABLED, _get_mount_options())
ca_mocker.assert_not_called()
utils.assert_not_called(ca_mocker)
_validate_config(config_file, mount_efs.STUNNEL_GLOBAL_CONFIG,
_get_expected_efs_config(check_cert_validity=False, verify=verify))
......
#
# Copyright 2017-2018 Amazon.com, Inc. and its affiliates. All Rights Reserved.
#
# Licensed under the MIT License. See the LICENSE accompanying this file
# for the specific language governing permissions and limitations under
# the License.
#
def assert_called_once(mock):
assert mock.call_count == 1, 'Expected mock to have been called once. Called {} times.'.format(mock.call_count)
def assert_not_called(mock):
assert mock.call_count == 0, 'Expected mock to have been not called. Called {} times.'.format(mock.call_count)
def assert_called(mock):
assert mock.call_count != 0, 'Expected mock to have been called. While the mock is not called.'
\ No newline at end of file
......@@ -10,6 +10,8 @@ import watchdog
from mock import MagicMock
from .. import utils
def test_child_procs_empty():
watchdog.check_child_procs([])
......@@ -27,6 +29,6 @@ def test_child_procs():
assert 1 == len(children)
assert dead_proc not in children
dead_proc.poll.assert_called_once()
utils.assert_called_once(dead_proc.poll)
assert live_proc in children
live_proc.poll.assert_called_once()
utils.assert_called_once(live_proc.poll)
......@@ -11,6 +11,8 @@ import watchdog
import json
import tempfile
from .. import utils
from datetime import datetime
try:
import ConfigParser
......@@ -72,8 +74,8 @@ def test_no_state_files(mocker):
watchdog.check_efs_mounts(_get_config(), [], GRACE_PERIOD)
clean_up_mock.assert_not_called()
restart_tls_mock.assert_not_called()
utils.assert_not_called(clean_up_mock)
utils.assert_not_called(restart_tls_mock)
def test_malformed_state_file(mocker, tmpdir):
......@@ -83,8 +85,8 @@ def test_malformed_state_file(mocker, tmpdir):
watchdog.check_efs_mounts(_get_config(), [], GRACE_PERIOD, state_file_dir)
clean_up_mock.assert_not_called()
restart_tls_mock.assert_not_called()
utils.assert_not_called(clean_up_mock)
utils.assert_not_called(restart_tls_mock)
def test_no_mount_for_state_file(mocker, tmpdir):
......@@ -96,8 +98,8 @@ def test_no_mount_for_state_file(mocker, tmpdir):
watchdog.check_efs_mounts(_get_config(), [], GRACE_PERIOD, state_file_dir)
clean_up_mock.assert_not_called()
restart_tls_mock.assert_not_called()
utils.assert_not_called(clean_up_mock)
utils.assert_not_called(restart_tls_mock)
def test_no_mount_for_state_file_out_of_grace_period(mocker, tmpdir):
......@@ -110,9 +112,9 @@ def test_no_mount_for_state_file_out_of_grace_period(mocker, tmpdir):
watchdog.check_efs_mounts(_get_config(), [], GRACE_PERIOD, state_file_dir)
clean_up_mock.assert_called_once()
restart_tls_mock.assert_not_called()
check_certificate_call.assert_not_called()
utils.assert_called_once(clean_up_mock)
utils.assert_not_called(restart_tls_mock)
utils.assert_not_called(check_certificate_call)
def test_no_mount_for_state_file_in_grace_period(mocker, tmpdir):
......@@ -125,8 +127,8 @@ def test_no_mount_for_state_file_in_grace_period(mocker, tmpdir):
watchdog.check_efs_mounts(_get_config(), [], GRACE_PERIOD, state_file_dir)
clean_up_mock.assert_not_called()
restart_tls_mock.assert_not_called()
utils.assert_not_called(clean_up_mock)
utils.assert_not_called(restart_tls_mock)
def test_tls_not_running(mocker, tmpdir):
......@@ -138,8 +140,8 @@ def test_tls_not_running(mocker, tmpdir):
watchdog.check_efs_mounts(_get_config(), [], GRACE_PERIOD, state_file_dir)
clean_up_mock.assert_not_called()
restart_tls_mock.assert_called_once()
utils.assert_not_called(clean_up_mock)
utils.assert_called_once(restart_tls_mock)
def test_ap_mount_with_extra_mount(mocker, tmpdir):
......@@ -153,6 +155,6 @@ def test_ap_mount_with_extra_mount(mocker, tmpdir):
watchdog.check_efs_mounts(_get_config(), [], GRACE_PERIOD, state_file_dir)
clean_up_mock.assert_not_called()
restart_tls_mock.assert_not_called()
check_certificate_call.assert_called_once()
utils.assert_not_called(clean_up_mock)
utils.assert_not_called(restart_tls_mock)
utils.assert_called_once(check_certificate_call)
......@@ -11,6 +11,8 @@ import json
import os
import tempfile
from .. import utils
PID = 99999999999999999
......@@ -48,7 +50,7 @@ def test_clean_up_on_first_try(mocker, tmpdir):
watchdog.clean_up_mount_state(state_dir, state_file, PID, is_running=True)
killpg_mock.assert_called_once()
utils.assert_called_once(killpg_mock)
assert not os.path.exists(abs_state_file)
......@@ -70,7 +72,7 @@ def _test_clean_up_files(mocker, tmpdir, files_should_exist):
watchdog.clean_up_mount_state(state_dir, state_file, PID, is_running=True)