Commit 60e865f5 authored by Yuan Gao's avatar Yuan Gao
Browse files

Improvements to metadata retrieval and IAM authentication

* Support retrieving metadata using IMDSv2
* Allow specification of interval for certificate renewal in the config file
* Fix retrieval of instance metadata on ECS
parent 153b3462
......@@ -15,6 +15,7 @@ The `efs-utils` package has been verified against the following Linux distributi
| RHEL 8 | `rpm`| `systemd` |
| Debian 9 | `deb` | `systemd` |
| Ubuntu 16.04 | `deb` | `systemd` |
| Ubuntu 18.04 | `deb` | `systemd` |
## Prerequisites
......@@ -37,17 +38,12 @@ $ sudo yum -y install amazon-efs-utils
Other distributions require building the package from source and installing it.
- Clone this repository:
- To build and install an RPM:
```
$ sudo yum -y install git rpm-build
$ git clone https://github.com/aws/efs-utils
$ cd efs-utils
```
- To build and install an RPM:
```
$ sudo yum -y install rpm-build
$ make rpm
$ sudo yum -y install build/amazon-efs-utils*rpm
```
......@@ -56,7 +52,9 @@ $ sudo yum -y install build/amazon-efs-utils*rpm
```
$ sudo apt-get update
$ sudo apt-get -y install binutils
$ sudo apt-get -y install git binutils
$ git clone https://github.com/aws/efs-utils
$ cd efs-utils
$ ./build-deb.sh
$ sudo apt-get -y install ./build/amazon-efs-utils*deb
```
......
......@@ -11,7 +11,7 @@ set -ex
BASE_DIR=$(pwd)
BUILD_ROOT=${BASE_DIR}/build/debbuild
VERSION=1.21
VERSION=1.22
echo 'Cleaning deb build workspace'
rm -rf ${BUILD_ROOT}
......
......@@ -7,4 +7,4 @@
#
[global]
version=1.21
version=1.22
Package: amazon-efs-utils
Architecture: all
Version: 1.21
Version: 1.22
Section: utils
Depends: python|python2, nfs-common, stunnel4 (>= 4.56)
Priority: optional
......
......@@ -20,7 +20,7 @@
%endif
Name : amazon-efs-utils
Version : 1.21
Version : 1.22
Release : 1%{?dist}
Summary : This package provides utilities for simplifying the use of EFS file systems
......
......@@ -45,3 +45,6 @@ dns_name_suffix = sc2s.sgov.gov
enabled = true
poll_interval_sec = 1
unmount_grace_period_sec = 30
# Set client auth/access point certificate renewal rate. Minimum value is 1 minute.
tls_cert_renewal_interval_min = 60
\ No newline at end of file
......@@ -63,12 +63,13 @@ except ImportError:
from urllib import quote_plus
try:
from urllib2 import urlopen, URLError
from urllib2 import URLError, HTTPError, build_opener, urlopen, Request, HTTPHandler
except ImportError:
from urllib.error import URLError
from urllib.request import urlopen
from urllib.request import urlopen, Request
from urllib.error import URLError, HTTPError
VERSION = '1.21'
VERSION = '1.22'
SERVICE = 'elasticfilesystem'
CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'
......@@ -136,7 +137,8 @@ FS_ID_RE = re.compile('^(?P<fs_id>fs-[0-9a-f]+)$')
EFS_FQDN_RE = re.compile(r'^(?P<fs_id>fs-[0-9a-f]+)\.efs\.(?P<region>[a-z0-9-]+)\.(?P<dns_name_suffix>[a-z0-9.]+)$')
AP_ID_RE = re.compile('^fsap-[0-9a-f]{17}$')
ECS_TASK_METADATA_API = '169.254.170.2'
ECS_TASK_METADATA_API = 'http://169.254.170.2'
INSTANCE_METADATA_TOKEN_URL = 'http://169.254.169.254/latest/api/token'
INSTANCE_METADATA_SERVICE_URL = 'http://169.254.169.254/latest/dynamic/instance-identity/document/'
INSTANCE_IAM_URL = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
SECURITY_CREDS_ECS_URI_HELP_URL = 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html'
......@@ -205,19 +207,16 @@ def get_region():
fatal_error('Error retrieving region', message)
try:
resource = urlopen(INSTANCE_METADATA_SERVICE_URL, timeout=1)
if resource.getcode() != 200:
_fatal_error('Unable to reach instance metadata service at %s: status=%d'
% (INSTANCE_METADATA_SERVICE_URL, resource.getcode()))
data = resource.read()
if type(data) is str:
instance_identity = json.loads(data)
else:
instance_identity = json.loads(data.decode(resource.headers.get_content_charset() or 'us-ascii'))
token = get_aws_ec2_metadata_token()
headers = {}
if token:
headers = {'X-aws-ec2-metadata-token': token}
instance_identity = get_aws_ec2_metadata(headers)
return instance_identity['region']
except HTTPError as e:
_fatal_error('Unable to reach instance metadata service at %s: status=%d'
% (INSTANCE_METADATA_SERVICE_URL, e.code))
except URLError as e:
_fatal_error('Unable to reach the instance metadata service at %s. If this is an on-premises instance, replace '
'"{region}" in the "dns_name_format" option in %s with the region of the EFS file system you are mounting.\n'
......@@ -229,6 +228,32 @@ def get_region():
_fatal_error('Region not present in %s: %s' % (instance_identity, e))
def get_aws_ec2_metadata_token():
try:
opener = build_opener(HTTPHandler)
request = Request(INSTANCE_METADATA_TOKEN_URL)
request.add_header('X-aws-ec2-metadata-token-ttl-seconds', 21600)
request.get_method = lambda: 'PUT'
res = opener.open(request)
return res.read()
except NameError:
headers = {'X-aws-ec2-metadata-token-ttl-seconds': 21600}
req = Request(INSTANCE_METADATA_TOKEN_URL, headers=headers, method='PUT')
res = urlopen(req)
return res.read()
def get_aws_ec2_metadata(headers):
request = Request(INSTANCE_METADATA_SERVICE_URL, headers=headers)
response = urlopen(request, timeout=1)
data = response.read()
if type(data) is str:
instance_identity = json.loads(data)
else:
instance_identity = json.loads(data.decode(response.headers.get_content_charset() or 'us-ascii'))
return instance_identity
def get_region_helper(config):
dns_name_format = config.get(CONFIG_SECTION, 'dns_name_format')
if '{region}' in dns_name_format:
......
......@@ -45,7 +45,7 @@ except ImportError:
from urllib.error import URLError
from urllib.request import urlopen
VERSION = '1.21'
VERSION = '1.22'
SERVICE = 'elasticfilesystem'
CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'
......@@ -57,7 +57,7 @@ LOG_FILE = 'mount-watchdog.log'
STATE_FILE_DIR = '/var/run/efs'
PRIVATE_KEY_FILE = '/etc/amazon/efs/privateKey.pem'
REFRESH_SELF_SIGNED_CERT_INTERVAL_SEC = 60 * 60
DEFAULT_REFRESH_SELF_SIGNED_CERT_INTERVAL_MIN = 60
NOT_BEFORE_MINS = 15
NOT_AFTER_HOURS = 3
DATE_ONLY_FORMAT = '%Y%m%d'
......@@ -116,7 +116,7 @@ REQUEST_PAYLOAD = ''
AP_ID_RE = re.compile('^fsap-[0-9a-f]{17}$')
ECS_TASK_METADATA_API = '169.254.170.2'
ECS_TASK_METADATA_API = 'http://169.254.170.2'
INSTANCE_IAM_URL = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
SECURITY_CREDS_ECS_URI_HELP_URL = 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html'
SECURITY_CREDS_IAM_ROLE_HELP_URL = 'https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html'
......@@ -512,8 +512,9 @@ def read_config(config_file=CONFIG_FILE):
def check_certificate(config, state, state_file_dir, state_file, base_path=STATE_FILE_DIR):
certificate_creation_time = datetime.strptime(state['certificateCreationTime'], CERT_DATETIME_FORMAT)
certificate_exists = os.path.isfile(state['certificate'])
certificate_renewal_interval_secs = get_certificate_renewal_interval_mins(config) * 60
# creation instead of NOT_BEFORE datetime is used for refresh of cert because NOT_BEFORE derives from creation datetime
should_refresh_cert = (get_utc_now() - certificate_creation_time).total_seconds() > REFRESH_SELF_SIGNED_CERT_INTERVAL_SEC
should_refresh_cert = (get_utc_now() - certificate_creation_time).total_seconds() > certificate_renewal_interval_secs
if certificate_exists and not should_refresh_cert:
return
......@@ -906,6 +907,26 @@ def calculate_signature(string_to_sign, date, secret_access_key, region):
return _sign(signing_key, string_to_sign).hexdigest()
def get_certificate_renewal_interval_mins(config):
interval = DEFAULT_REFRESH_SELF_SIGNED_CERT_INTERVAL_MIN
try:
mins_from_config = config.get(CONFIG_SECTION, 'tls_cert_renewal_interval_min')
try:
if int(mins_from_config) > 0:
interval = int(mins_from_config)
else:
logging.warning('tls_cert_renewal_interval_min value in config file "%s" is lower than 1 minute. Defaulting '
'to %d minutes.', CONFIG_FILE, DEFAULT_REFRESH_SELF_SIGNED_CERT_INTERVAL_MIN)
except ValueError:
logging.warning('Bad tls_cert_renewal_interval_min value, "%s", in config file "%s". Defaulting to %d minutes.',
mins_from_config, CONFIG_FILE, DEFAULT_REFRESH_SELF_SIGNED_CERT_INTERVAL_MIN)
except NoOptionError:
logging.warning('No tls_cert_renewal_interval_min value in config file "%s". Defaulting to %d minutes.', CONFIG_FILE,
DEFAULT_REFRESH_SELF_SIGNED_CERT_INTERVAL_MIN)
return interval
def get_credential_scope(date, region):
return '/'.join([date.strftime(DATE_ONLY_FORMAT), region, SERVICE, AWS4_REQUEST])
......@@ -932,10 +953,12 @@ def main():
child_procs = []
if config.getboolean(CONFIG_SECTION, 'enabled'):
logging.info('amazon-efs-mount-watchdog, version %s, is enabled and started', VERSION)
poll_interval_sec = config.getint(CONFIG_SECTION, 'poll_interval_sec')
unmount_grace_period_sec = config.getint(CONFIG_SECTION, 'unmount_grace_period_sec')
while True:
config = read_config()
check_efs_mounts(config, child_procs, unmount_grace_period_sec)
check_child_procs(child_procs)
......
......@@ -12,9 +12,9 @@ import json
import pytest
try:
from urllib2 import URLError
from urllib2 import URLError, HTTPError
except ImportError:
from urllib.error import URLError
from urllib.error import URLError, HTTPError
INSTANCE_DATA = {
'devpayProductCodes': None,
......@@ -57,13 +57,20 @@ class MockUrlLibResponse(object):
return self.data
def test_get_region(mocker):
def test_get_region_with_token(mocker):
mocker.patch('mount_efs.get_aws_ec2_metadata_token', return_value='ABCDEFG==')
mocker.patch('mount_efs.urlopen', return_value=MockUrlLibResponse())
assert 'us-east-1' == mount_efs.get_region()
def test_get_region_without_token(mocker):
mocker.patch('mount_efs.get_aws_ec2_metadata_token', return_value=None)
mocker.patch('mount_efs.urlopen', return_value=MockUrlLibResponse())
assert 'us-east-1' == mount_efs.get_region()
def test_get_region_py3_no_charset(mocker):
mocker.patch('mount_efs.get_aws_ec2_metadata_token', return_value=None)
mocker.patch('mount_efs.urlopen', return_value=MockUrlLibResponse(data=bytearray(INSTANCE_DOCUMENT, 'us-ascii')))
assert 'us-east-1' == mount_efs.get_region()
......@@ -71,6 +78,7 @@ def test_get_region_py3_no_charset(mocker):
def test_get_region_py3_utf8_charset(mocker):
charset = 'utf-8'
mocker.patch('mount_efs.get_aws_ec2_metadata_token', return_value=None)
mocker.patch('mount_efs.urlopen', return_value=MockUrlLibResponse(data=bytearray(INSTANCE_DOCUMENT, charset)),
headers=MockHeaders(content_charset=charset))
......@@ -78,6 +86,7 @@ def test_get_region_py3_utf8_charset(mocker):
def _test_get_region_error(mocker, capsys, response=None, error=None):
mocker.patch('mount_efs.get_aws_ec2_metadata_token', return_value=None)
if (response and error) or (not response and not error):
raise ValueError('Invalid arguments')
elif response:
......@@ -95,7 +104,7 @@ def _test_get_region_error(mocker, capsys, response=None, error=None):
def test_get_region_bad_response(mocker, capsys):
_test_get_region_error(mocker, capsys, response=MockUrlLibResponse(code=400))
_test_get_region_error(mocker, capsys, error=HTTPError('url', 400, 'Bad Request Error', None, None))
def test_get_region_error_response(mocker, capsys):
......
......@@ -58,12 +58,14 @@ def setup(mocker):
mocker.patch('watchdog.get_aws_security_credentials', return_value=CREDENTIALS)
def _get_config(dns_name_format='{fs_id}.efs.{region}.amazonaws.com'):
def _get_config(dns_name_format='{fs_id}.efs.{region}.amazonaws.com', certificate_renewal_interval=60):
def config_get_side_effect(section, field):
if field == 'state_file_dir_mode':
return '0755'
elif field == 'dns_name_format':
return dns_name_format
elif field == 'tls_cert_renewal_interval_min':
return certificate_renewal_interval
else:
raise ValueError('Unexpected arguments')
......@@ -142,6 +144,48 @@ def _create_ca_conf_helper(mocker, tmpdir, current_time, iam=True, ap=True):
return tls_dict, full_config_body
def _test_refresh_certificate_helper(mocker, tmpdir, caplog, minutes_back, renewal_interval=60, with_iam=True, with_ap=True):
mocker.patch('watchdog.get_utc_now', return_value=FIXED_DT)
config = _get_config(certificate_renewal_interval=renewal_interval)
pk_path = _get_mock_private_key_path(mocker, tmpdir)
minutes_back = (FIXED_DT - timedelta(minutes=minutes_back)).strftime(DT_PATTERN)
tls_dict = watchdog.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
if not with_iam and with_ap:
state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, minutes_back, ap_id=AP_ID)
elif with_iam and not with_ap:
state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, minutes_back, security_credentials=CREDENTIALS,
credentials_source=CREDENTIALS_SOURCE)
else:
state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, minutes_back, security_credentials=CREDENTIALS,
credentials_source=CREDENTIALS_SOURCE, ap_id=AP_ID)
watchdog.check_certificate(config, state, str(tmpdir), STATE_FILE, base_path=str(tmpdir))
with open(os.path.join(str(tmpdir), STATE_FILE), 'r') as state_json:
state = json.load(state_json)
if not with_iam and with_ap:
assert state['accessPoint'] == AP_ID
assert not state.get('awsCredentialsMethod')
assert not os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
elif with_iam and not with_ap:
assert 'accessPoint' not in state
assert state['awsCredentialsMethod'] == CREDENTIALS_SOURCE
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
else:
assert state['accessPoint'] == AP_ID
assert state['awsCredentialsMethod'] == CREDENTIALS_SOURCE
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
assert datetime.strptime(state['certificateCreationTime'], DT_PATTERN) > datetime.strptime(minutes_back, DT_PATTERN)
assert os.path.exists(pk_path)
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'request.csr'))
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
return caplog
def test_do_not_refresh_self_signed_certificate(mocker, tmpdir):
mocker.patch('watchdog.get_utc_now', return_value=FIXED_DT)
config = _get_config()
......@@ -237,72 +281,36 @@ def test_recreate_missing_self_signed_certificate(mocker, tmpdir):
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
def test_refresh_self_signed_certificate_without_iam_with_ap_id(mocker, tmpdir):
mocker.patch('watchdog.get_utc_now', return_value=FIXED_DT)
config = _get_config()
pk_path = _get_mock_private_key_path(mocker, tmpdir)
four_hours_back = (FIXED_DT - timedelta(hours=4)).strftime(DT_PATTERN)
tls_dict = watchdog.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, four_hours_back, ap_id=AP_ID)
def test_refresh_self_signed_certificate_without_iam_with_ap_id(mocker, caplog, tmpdir):
_test_refresh_certificate_helper(mocker, tmpdir, caplog, 240, with_iam=False)
watchdog.check_certificate(config, state, str(tmpdir), STATE_FILE, base_path=str(tmpdir))
with open(os.path.join(str(tmpdir), STATE_FILE), 'r') as state_json:
state = json.load(state_json)
def test_refresh_self_signed_certificate_with_iam_without_ap_id(mocker, caplog, tmpdir):
_test_refresh_certificate_helper(mocker, tmpdir, caplog, 240, with_ap=False)
assert datetime.strptime(state['certificateCreationTime'], DT_PATTERN) > datetime.strptime(four_hours_back, DT_PATTERN)
assert state['accessPoint'] == AP_ID
assert not state.get('awsCredentialsMethod')
assert os.path.exists(pk_path)
assert not os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'request.csr'))
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
def test_refresh_self_signed_certificate_with_iam_with_ap_id(mocker, caplog, tmpdir):
_test_refresh_certificate_helper(mocker, tmpdir, caplog, 240)
def test_refresh_self_signed_certificate_with_iam_without_ap_id(mocker, tmpdir):
mocker.patch('watchdog.get_utc_now', return_value=FIXED_DT)
config = _get_config()
pk_path = _get_mock_private_key_path(mocker, tmpdir)
four_hours_back = (FIXED_DT - timedelta(hours=4)).strftime(DT_PATTERN)
tls_dict = watchdog.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, four_hours_back, security_credentials=CREDENTIALS,
credentials_source=CREDENTIALS_SOURCE)
watchdog.check_certificate(config, state, str(tmpdir), STATE_FILE, base_path=str(tmpdir))
def test_refresh_self_signed_certificate_custom_renewal_interval(mocker, caplog, tmpdir):
_test_refresh_certificate_helper(mocker, tmpdir, caplog, 45, renewal_interval=30)
with open(os.path.join(str(tmpdir), STATE_FILE), 'r') as state_json:
state = json.load(state_json)
assert datetime.strptime(state['certificateCreationTime'], DT_PATTERN) > datetime.strptime(four_hours_back, DT_PATTERN)
assert 'accessPoint' not in state
assert state['awsCredentialsMethod'] == CREDENTIALS_SOURCE
assert os.path.exists(pk_path)
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'request.csr'))
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
def test_refresh_self_signed_certificate_invalid_refresh_interval(mocker, caplog, tmpdir):
caplog.set_level(logging.WARNING)
caplog = _test_refresh_certificate_helper(mocker, tmpdir, caplog, 240, renewal_interval='not_an_int')
assert 'Bad tls_cert_renewal_interval_min value, "not_an_int", in config file "/etc/amazon/efs/efs-utils.conf". Defaulting' \
' to 60 minutes.' in caplog.text
def test_refresh_self_signed_certificate_with_iam_with_ap_id(mocker, tmpdir):
mocker.patch('watchdog.get_utc_now', return_value=FIXED_DT)
config = _get_config()
pk_path = _get_mock_private_key_path(mocker, tmpdir)
four_hours_back = (FIXED_DT - timedelta(hours=4)).strftime(DT_PATTERN)
tls_dict = watchdog.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, four_hours_back, security_credentials=CREDENTIALS,
credentials_source=CREDENTIALS_SOURCE, ap_id=AP_ID)
watchdog.check_certificate(config, state, str(tmpdir), STATE_FILE, base_path=str(tmpdir))
def test_refresh_self_signed_certificate_too_low_refresh_interval(mocker, caplog, tmpdir):
caplog.set_level(logging.WARNING)
caplog = _test_refresh_certificate_helper(mocker, tmpdir, caplog, 240, renewal_interval=0)
with open(os.path.join(str(tmpdir), STATE_FILE), 'r') as state_json:
state = json.load(state_json)
assert datetime.strptime(state['certificateCreationTime'], DT_PATTERN) > datetime.strptime(four_hours_back, DT_PATTERN)
assert state['accessPoint'] == AP_ID
assert state['awsCredentialsMethod'] == CREDENTIALS_SOURCE
assert os.path.exists(pk_path)
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'request.csr'))
assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
assert 'tls_cert_renewal_interval_min value in config file "/etc/amazon/efs/efs-utils.conf" is lower than 1 minute. ' \
'Defaulting to 60 minutes.' in caplog.text
def test_refresh_self_signed_certificate_send_sighup(mocker, tmpdir, caplog):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment