Improvements to metadata retrieval and IAM authentication

* Support retrieving metadata using IMDSv2
* Allow specification of interval for certificate renewal in the config file
* Fix retrieval of instance metadata on ECS

README.md

@@ -15,6 +15,7 @@ The `efs-utils` package has been verified against the following Linux distributi
 | RHEL 8 | `rpm`| `systemd` |
 | Debian 9 | `deb` | `systemd` |
 | Ubuntu 16.04 | `deb` | `systemd` |
+| Ubuntu 18.04 | `deb` | `systemd` |
 
 ## Prerequisites
 
@@ -37,17 +38,12 @@ $ sudo yum -y install amazon-efs-utils
 
 Other distributions require building the package from source and installing it.
 
-- Clone this repository:
+- To build and install an RPM:
 
 ```
+$ sudo yum -y install git rpm-build
 $ git clone https://github.com/aws/efs-utils
 $ cd efs-utils
-```
-
-- To build and install an RPM:
-
-```
-$ sudo yum -y install rpm-build
 $ make rpm
 $ sudo yum -y install build/amazon-efs-utils*rpm
 ```
@@ -56,7 +52,9 @@ $ sudo yum -y install build/amazon-efs-utils*rpm
 
 ```
 $ sudo apt-get update
-$ sudo apt-get -y install binutils
+$ sudo apt-get -y install git binutils
+$ git clone https://github.com/aws/efs-utils
+$ cd efs-utils
 $ ./build-deb.sh
 $ sudo apt-get -y install ./build/amazon-efs-utils*deb
 ```

build-deb.sh

@@ -11,7 +11,7 @@ set -ex
 
 BASE_DIR=$(pwd)
 BUILD_ROOT=${BASE_DIR}/build/debbuild
-VERSION=1.21
+VERSION=1.22
 
 echo 'Cleaning deb build workspace'
 rm -rf ${BUILD_ROOT}

config.ini

@@ -7,4 +7,4 @@
 #
 
 [global]
-version=1.21
+version=1.22

dist/amazon-efs-utils.control

 Package: amazon-efs-utils
 Architecture: all
-Version: 1.21
+Version: 1.22
 Section: utils
 Depends: python|python2, nfs-common, stunnel4 (>= 4.56)
 Priority: optional

dist/amazon-efs-utils.spec

@@ -20,7 +20,7 @@
 %endif
 
 Name      : amazon-efs-utils
-Version   : 1.21
+Version   : 1.22
 Release   : 1%{?dist}
 Summary   : This package provides utilities for simplifying the use of EFS file systems
 

dist/efs-utils.conf

@@ -45,3 +45,6 @@ dns_name_suffix = sc2s.sgov.gov
 enabled = true
 poll_interval_sec = 1
 unmount_grace_period_sec = 30
+
+# Set client auth/access point certificate renewal rate. Minimum value is 1 minute.
+tls_cert_renewal_interval_min = 60
\ No newline at end of file

src/mount_efs/__init__.py

@@ -63,12 +63,13 @@ except ImportError:
     from urllib import quote_plus
 
 try:
-    from urllib2 import urlopen, URLError
+    from urllib2 import URLError, HTTPError, build_opener, urlopen, Request, HTTPHandler
 except ImportError:
-    from urllib.error import URLError
-    from urllib.request import urlopen
+    from urllib.request import urlopen, Request
+    from urllib.error import URLError, HTTPError
 
-VERSION = '1.21'
+
+VERSION = '1.22'
 SERVICE = 'elasticfilesystem'
 
 CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'
@@ -136,7 +137,8 @@ FS_ID_RE = re.compile('^(?P<fs_id>fs-[0-9a-f]+)$')
 EFS_FQDN_RE = re.compile(r'^(?P<fs_id>fs-[0-9a-f]+)\.efs\.(?P<region>[a-z0-9-]+)\.(?P<dns_name_suffix>[a-z0-9.]+)$')
 AP_ID_RE = re.compile('^fsap-[0-9a-f]{17}$')
 
-ECS_TASK_METADATA_API = '169.254.170.2'
+ECS_TASK_METADATA_API = 'http://169.254.170.2'
+INSTANCE_METADATA_TOKEN_URL = 'http://169.254.169.254/latest/api/token'
 INSTANCE_METADATA_SERVICE_URL = 'http://169.254.169.254/latest/dynamic/instance-identity/document/'
 INSTANCE_IAM_URL = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
 SECURITY_CREDS_ECS_URI_HELP_URL = 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html'
@@ -205,19 +207,16 @@ def get_region():
         fatal_error('Error retrieving region', message)
 
     try:
-        resource = urlopen(INSTANCE_METADATA_SERVICE_URL, timeout=1)
-
-        if resource.getcode() != 200:
-            _fatal_error('Unable to reach instance metadata service at %s: status=%d'
-                         % (INSTANCE_METADATA_SERVICE_URL, resource.getcode()))
-
-        data = resource.read()
-        if type(data) is str:
-            instance_identity = json.loads(data)
-        else:
-            instance_identity = json.loads(data.decode(resource.headers.get_content_charset() or 'us-ascii'))
+        token = get_aws_ec2_metadata_token()
+        headers = {}
+        if token:
+            headers = {'X-aws-ec2-metadata-token': token}
+        instance_identity = get_aws_ec2_metadata(headers)
 
         return instance_identity['region']
+    except HTTPError as e:
+        _fatal_error('Unable to reach instance metadata service at %s: status=%d'
+                     % (INSTANCE_METADATA_SERVICE_URL, e.code))
     except URLError as e:
         _fatal_error('Unable to reach the instance metadata service at %s. If this is an on-premises instance, replace '
                      '"{region}" in the "dns_name_format" option in %s with the region of the EFS file system you are mounting.\n'
@@ -229,6 +228,32 @@ def get_region():
         _fatal_error('Region not present in %s: %s' % (instance_identity, e))
 
 
+def get_aws_ec2_metadata_token():
+    try:
+        opener = build_opener(HTTPHandler)
+        request = Request(INSTANCE_METADATA_TOKEN_URL)
+        request.add_header('X-aws-ec2-metadata-token-ttl-seconds', 21600)
+        request.get_method = lambda: 'PUT'
+        res = opener.open(request)
+        return res.read()
+    except NameError:
+        headers = {'X-aws-ec2-metadata-token-ttl-seconds': 21600}
+        req = Request(INSTANCE_METADATA_TOKEN_URL, headers=headers, method='PUT')
+        res = urlopen(req)
+        return res.read()
+
+
+def get_aws_ec2_metadata(headers):
+    request = Request(INSTANCE_METADATA_SERVICE_URL, headers=headers)
+    response = urlopen(request, timeout=1)
+    data = response.read()
+    if type(data) is str:
+        instance_identity = json.loads(data)
+    else:
+        instance_identity = json.loads(data.decode(response.headers.get_content_charset() or 'us-ascii'))
+    return instance_identity
+
+
 def get_region_helper(config):
     dns_name_format = config.get(CONFIG_SECTION, 'dns_name_format')
     if '{region}' in dns_name_format:

src/watchdog/__init__.py

@@ -45,7 +45,7 @@ except ImportError:
     from urllib.error import URLError
     from urllib.request import urlopen
 
-VERSION = '1.21'
+VERSION = '1.22'
 SERVICE = 'elasticfilesystem'
 
 CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'
@@ -57,7 +57,7 @@ LOG_FILE = 'mount-watchdog.log'
 STATE_FILE_DIR = '/var/run/efs'
 
 PRIVATE_KEY_FILE = '/etc/amazon/efs/privateKey.pem'
-REFRESH_SELF_SIGNED_CERT_INTERVAL_SEC = 60 * 60
+DEFAULT_REFRESH_SELF_SIGNED_CERT_INTERVAL_MIN = 60
 NOT_BEFORE_MINS = 15
 NOT_AFTER_HOURS = 3
 DATE_ONLY_FORMAT = '%Y%m%d'
@@ -116,7 +116,7 @@ REQUEST_PAYLOAD = ''
 
 AP_ID_RE = re.compile('^fsap-[0-9a-f]{17}$')
 
-ECS_TASK_METADATA_API = '169.254.170.2'
+ECS_TASK_METADATA_API = 'http://169.254.170.2'
 INSTANCE_IAM_URL = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
 SECURITY_CREDS_ECS_URI_HELP_URL = 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html'
 SECURITY_CREDS_IAM_ROLE_HELP_URL = 'https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html'
@@ -512,8 +512,9 @@ def read_config(config_file=CONFIG_FILE):
 def check_certificate(config, state, state_file_dir, state_file, base_path=STATE_FILE_DIR):
     certificate_creation_time = datetime.strptime(state['certificateCreationTime'], CERT_DATETIME_FORMAT)
     certificate_exists = os.path.isfile(state['certificate'])
+    certificate_renewal_interval_secs = get_certificate_renewal_interval_mins(config) * 60
     # creation instead of NOT_BEFORE datetime is used for refresh of cert because NOT_BEFORE derives from creation datetime
-    should_refresh_cert = (get_utc_now() - certificate_creation_time).total_seconds() > REFRESH_SELF_SIGNED_CERT_INTERVAL_SEC
+    should_refresh_cert = (get_utc_now() - certificate_creation_time).total_seconds() > certificate_renewal_interval_secs
 
     if certificate_exists and not should_refresh_cert:
         return
@@ -906,6 +907,26 @@ def calculate_signature(string_to_sign, date, secret_access_key, region):
     return _sign(signing_key, string_to_sign).hexdigest()
 
 
+def get_certificate_renewal_interval_mins(config):
+    interval = DEFAULT_REFRESH_SELF_SIGNED_CERT_INTERVAL_MIN
+    try:
+        mins_from_config = config.get(CONFIG_SECTION, 'tls_cert_renewal_interval_min')
+        try:
+            if int(mins_from_config) > 0:
+                interval = int(mins_from_config)
+            else:
+                logging.warning('tls_cert_renewal_interval_min value in config file "%s" is lower than 1 minute. Defaulting '
+                                'to %d minutes.', CONFIG_FILE, DEFAULT_REFRESH_SELF_SIGNED_CERT_INTERVAL_MIN)
+        except ValueError:
+            logging.warning('Bad tls_cert_renewal_interval_min value, "%s", in config file "%s". Defaulting to %d minutes.',
+                            mins_from_config, CONFIG_FILE, DEFAULT_REFRESH_SELF_SIGNED_CERT_INTERVAL_MIN)
+    except NoOptionError:
+        logging.warning('No tls_cert_renewal_interval_min value in config file "%s". Defaulting to %d minutes.', CONFIG_FILE,
+                        DEFAULT_REFRESH_SELF_SIGNED_CERT_INTERVAL_MIN)
+
+    return interval
+
+
 def get_credential_scope(date, region):
     return '/'.join([date.strftime(DATE_ONLY_FORMAT), region, SERVICE, AWS4_REQUEST])
 
@@ -932,10 +953,12 @@ def main():
     child_procs = []
 
     if config.getboolean(CONFIG_SECTION, 'enabled'):
+        logging.info('amazon-efs-mount-watchdog, version %s, is enabled and started', VERSION)
         poll_interval_sec = config.getint(CONFIG_SECTION, 'poll_interval_sec')
         unmount_grace_period_sec = config.getint(CONFIG_SECTION, 'unmount_grace_period_sec')
 
         while True:
+            config = read_config()
             check_efs_mounts(config, child_procs, unmount_grace_period_sec)
             check_child_procs(child_procs)
 

test/mount_efs_test/test_get_region.py

@@ -12,9 +12,9 @@ import json
 import pytest
 
 try:
-    from urllib2 import URLError
+    from urllib2 import URLError, HTTPError
 except ImportError:
-    from urllib.error import URLError
+    from urllib.error import URLError, HTTPError
 
 INSTANCE_DATA = {
   'devpayProductCodes': None,
@@ -57,13 +57,20 @@ class MockUrlLibResponse(object):
         return self.data
 
 
-def test_get_region(mocker):
+def test_get_region_with_token(mocker):
+    mocker.patch('mount_efs.get_aws_ec2_metadata_token', return_value='ABCDEFG==')
     mocker.patch('mount_efs.urlopen', return_value=MockUrlLibResponse())
+    assert 'us-east-1' == mount_efs.get_region()
+
 
+def test_get_region_without_token(mocker):
+    mocker.patch('mount_efs.get_aws_ec2_metadata_token', return_value=None)
+    mocker.patch('mount_efs.urlopen', return_value=MockUrlLibResponse())
     assert 'us-east-1' == mount_efs.get_region()
 
 
 def test_get_region_py3_no_charset(mocker):
+    mocker.patch('mount_efs.get_aws_ec2_metadata_token', return_value=None)
     mocker.patch('mount_efs.urlopen', return_value=MockUrlLibResponse(data=bytearray(INSTANCE_DOCUMENT, 'us-ascii')))
 
     assert 'us-east-1' == mount_efs.get_region()
@@ -71,6 +78,7 @@ def test_get_region_py3_no_charset(mocker):
 
 def test_get_region_py3_utf8_charset(mocker):
     charset = 'utf-8'
+    mocker.patch('mount_efs.get_aws_ec2_metadata_token', return_value=None)
     mocker.patch('mount_efs.urlopen', return_value=MockUrlLibResponse(data=bytearray(INSTANCE_DOCUMENT, charset)),
                  headers=MockHeaders(content_charset=charset))
 
@@ -78,6 +86,7 @@ def test_get_region_py3_utf8_charset(mocker):
 
 
 def _test_get_region_error(mocker, capsys, response=None, error=None):
+    mocker.patch('mount_efs.get_aws_ec2_metadata_token', return_value=None)
     if (response and error) or (not response and not error):
         raise ValueError('Invalid arguments')
     elif response:
@@ -95,7 +104,7 @@ def _test_get_region_error(mocker, capsys, response=None, error=None):
 
 
 def test_get_region_bad_response(mocker, capsys):
-    _test_get_region_error(mocker, capsys, response=MockUrlLibResponse(code=400))
+    _test_get_region_error(mocker, capsys, error=HTTPError('url', 400, 'Bad Request Error', None, None))
 
 
 def test_get_region_error_response(mocker, capsys):

test/watchdog_test/test_refresh_self_signed_certificate.py

@@ -58,12 +58,14 @@ def setup(mocker):
     mocker.patch('watchdog.get_aws_security_credentials', return_value=CREDENTIALS)
 
 
-def _get_config(dns_name_format='{fs_id}.efs.{region}.amazonaws.com'):
+def _get_config(dns_name_format='{fs_id}.efs.{region}.amazonaws.com', certificate_renewal_interval=60):
     def config_get_side_effect(section, field):
         if field == 'state_file_dir_mode':
             return '0755'
         elif field == 'dns_name_format':
             return dns_name_format
+        elif field == 'tls_cert_renewal_interval_min':
+            return certificate_renewal_interval
         else:
             raise ValueError('Unexpected arguments')
 
@@ -142,6 +144,48 @@ def _create_ca_conf_helper(mocker, tmpdir, current_time, iam=True, ap=True):
     return tls_dict, full_config_body
 
 
+def _test_refresh_certificate_helper(mocker, tmpdir, caplog, minutes_back, renewal_interval=60, with_iam=True, with_ap=True):
+    mocker.patch('watchdog.get_utc_now', return_value=FIXED_DT)
+    config = _get_config(certificate_renewal_interval=renewal_interval)
+    pk_path = _get_mock_private_key_path(mocker, tmpdir)
+    minutes_back = (FIXED_DT - timedelta(minutes=minutes_back)).strftime(DT_PATTERN)
+    tls_dict = watchdog.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
+
+    if not with_iam and with_ap:
+        state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, minutes_back, ap_id=AP_ID)
+    elif with_iam and not with_ap:
+        state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, minutes_back, security_credentials=CREDENTIALS,
+                                              credentials_source=CREDENTIALS_SOURCE)
+    else:
+        state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, minutes_back, security_credentials=CREDENTIALS,
+                                              credentials_source=CREDENTIALS_SOURCE, ap_id=AP_ID)
+
+    watchdog.check_certificate(config, state, str(tmpdir), STATE_FILE, base_path=str(tmpdir))
+
+    with open(os.path.join(str(tmpdir), STATE_FILE), 'r') as state_json:
+        state = json.load(state_json)
+
+    if not with_iam and with_ap:
+        assert state['accessPoint'] == AP_ID
+        assert not state.get('awsCredentialsMethod')
+        assert not os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
+    elif with_iam and not with_ap:
+        assert 'accessPoint' not in state
+        assert state['awsCredentialsMethod'] == CREDENTIALS_SOURCE
+        assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
+    else:
+        assert state['accessPoint'] == AP_ID
+        assert state['awsCredentialsMethod'] == CREDENTIALS_SOURCE
+        assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
+
+    assert datetime.strptime(state['certificateCreationTime'], DT_PATTERN) > datetime.strptime(minutes_back, DT_PATTERN)
+    assert os.path.exists(pk_path)
+    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'request.csr'))
+    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
+
+    return caplog
+
+
 def test_do_not_refresh_self_signed_certificate(mocker, tmpdir):
     mocker.patch('watchdog.get_utc_now', return_value=FIXED_DT)
     config = _get_config()
@@ -237,72 +281,36 @@ def test_recreate_missing_self_signed_certificate(mocker, tmpdir):
     assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
 
 
-def test_refresh_self_signed_certificate_without_iam_with_ap_id(mocker, tmpdir):
-    mocker.patch('watchdog.get_utc_now', return_value=FIXED_DT)
-    config = _get_config()
-    pk_path = _get_mock_private_key_path(mocker, tmpdir)
-    four_hours_back = (FIXED_DT - timedelta(hours=4)).strftime(DT_PATTERN)
-    tls_dict = watchdog.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
-    state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, four_hours_back, ap_id=AP_ID)
-
-    watchdog.check_certificate(config, state, str(tmpdir), STATE_FILE, base_path=str(tmpdir))
+def test_refresh_self_signed_certificate_without_iam_with_ap_id(mocker, caplog, tmpdir):
+    _test_refresh_certificate_helper(mocker, tmpdir, caplog, 240, with_iam=False)
 
-    with open(os.path.join(str(tmpdir), STATE_FILE), 'r') as state_json:
-        state = json.load(state_json)
 
-    assert datetime.strptime(state['certificateCreationTime'], DT_PATTERN) > datetime.strptime(four_hours_back, DT_PATTERN)
-    assert state['accessPoint'] == AP_ID
-    assert not state.get('awsCredentialsMethod')
-    assert os.path.exists(pk_path)
-    assert not os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
-    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'request.csr'))
-    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
+def test_refresh_self_signed_certificate_with_iam_without_ap_id(mocker, caplog, tmpdir):
+    _test_refresh_certificate_helper(mocker, tmpdir, caplog, 240, with_ap=False)
 
 
-def test_refresh_self_signed_certificate_with_iam_without_ap_id(mocker, tmpdir):
-    mocker.patch('watchdog.get_utc_now', return_value=FIXED_DT)
-    config = _get_config()
-    pk_path = _get_mock_private_key_path(mocker, tmpdir)
-    four_hours_back = (FIXED_DT - timedelta(hours=4)).strftime(DT_PATTERN)
-    tls_dict = watchdog.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
-    state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, four_hours_back, security_credentials=CREDENTIALS,
-                                          credentials_source=CREDENTIALS_SOURCE)
+def test_refresh_self_signed_certificate_with_iam_with_ap_id(mocker, caplog, tmpdir):
+    _test_refresh_certificate_helper(mocker, tmpdir, caplog, 240)
 
-    watchdog.check_certificate(config, state, str(tmpdir), STATE_FILE, base_path=str(tmpdir))
 
-    with open(os.path.join(str(tmpdir), STATE_FILE), 'r') as state_json:
-        state = json.load(state_json)
+def test_refresh_self_signed_certificate_custom_renewal_interval(mocker, caplog, tmpdir):
+    _test_refresh_certificate_helper(mocker, tmpdir, caplog, 45, renewal_interval=30)
 
-    assert datetime.strptime(state['certificateCreationTime'], DT_PATTERN) > datetime.strptime(four_hours_back, DT_PATTERN)
-    assert 'accessPoint' not in state
-    assert state['awsCredentialsMethod'] == CREDENTIALS_SOURCE
-    assert os.path.exists(pk_path)
-    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
-    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'request.csr'))
-    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
 
+def test_refresh_self_signed_certificate_invalid_refresh_interval(mocker, caplog, tmpdir):
+    caplog.set_level(logging.WARNING)
+    caplog = _test_refresh_certificate_helper(mocker, tmpdir, caplog, 240, renewal_interval='not_an_int')
 
-def test_refresh_self_signed_certificate_with_iam_with_ap_id(mocker, tmpdir):
-    mocker.patch('watchdog.get_utc_now', return_value=FIXED_DT)
-    config = _get_config()
-    pk_path = _get_mock_private_key_path(mocker, tmpdir)
-    four_hours_back = (FIXED_DT - timedelta(hours=4)).strftime(DT_PATTERN)
-    tls_dict = watchdog.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
-    state = _create_certificate_and_state(tls_dict, str(tmpdir), pk_path, four_hours_back, security_credentials=CREDENTIALS,
-                                          credentials_source=CREDENTIALS_SOURCE, ap_id=AP_ID)
+    assert 'Bad tls_cert_renewal_interval_min value, "not_an_int", in config file "/etc/amazon/efs/efs-utils.conf". Defaulting' \
+           ' to 60 minutes.' in caplog.text
 
-    watchdog.check_certificate(config, state, str(tmpdir), STATE_FILE, base_path=str(tmpdir))
 
-    with open(os.path.join(str(tmpdir), STATE_FILE), 'r') as state_json:
-        state = json.load(state_json)
+def test_refresh_self_signed_certificate_too_low_refresh_interval(mocker, caplog, tmpdir):
+    caplog.set_level(logging.WARNING)
+    caplog = _test_refresh_certificate_helper(mocker, tmpdir, caplog, 240, renewal_interval=0)
 
-    assert datetime.strptime(state['certificateCreationTime'], DT_PATTERN) > datetime.strptime(four_hours_back, DT_PATTERN)
-    assert state['accessPoint'] == AP_ID
-    assert state['awsCredentialsMethod'] == CREDENTIALS_SOURCE
-    assert os.path.exists(pk_path)
-    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
-    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'request.csr'))
-    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
+    assert 'tls_cert_renewal_interval_min value in config file "/etc/amazon/efs/efs-utils.conf" is lower than 1 minute. ' \
+           'Defaulting to 60 minutes.' in caplog.text