Create self-signed certificate for tls-only mount

.circleci/config.yml

@@ -144,4 +144,4 @@ workflows:
           image: amazonlinux:2
       - build-rpm-package:
           name: amazon-linux
-          image: amazonlinux:1
+          image: amazonlinux:1
\ No newline at end of file

build-deb.sh

@@ -11,7 +11,7 @@ set -ex
 
 BASE_DIR=$(pwd)
 BUILD_ROOT=${BASE_DIR}/build/debbuild
-VERSION=1.24
+VERSION=1.25
 
 echo 'Cleaning deb build workspace'
 rm -rf ${BUILD_ROOT}

config.ini

@@ -7,5 +7,5 @@
 #
 
 [global]
-version=1.24
-release=4
+version=1.25
+release=1

dist/amazon-efs-utils.control

 Package: amazon-efs-utils
 Architecture: all
-Version: 1.24
+Version: 1.25
 Section: utils
 Depends: python|python2, nfs-common, stunnel4 (>= 4.56), openssl (>= 1.0.2), util-linux
 Priority: optional

dist/amazon-efs-utils.spec

@@ -20,8 +20,8 @@
 %endif
 
 Name      : amazon-efs-utils
-Version   : 1.24
-Release   : 4%{?dist}
+Version   : 1.25
+Release   : 1%{?dist}
 Summary   : This package provides utilities for simplifying the use of EFS file systems
 
 Group     : Amazon/Tools
@@ -120,6 +120,9 @@ fi
 %clean
 
 %changelog
+* Mon Apr 20 2020 Yuan Gao <ygaochn@amazon.com> - 1.25-1
+- Create self-signed certificate for tls-only mount
+
 * Tue Apr 7 2020 Yuan Gao <ygaochn@amazon.com> - 1.24-4
 - Fix the malformed certificate info
 

src/mount_efs/__init__.py

@@ -69,7 +69,7 @@ except ImportError:
     from urllib.error import URLError, HTTPError
 
 
-VERSION = '1.24'
+VERSION = '1.25'
 SERVICE = 'elasticfilesystem'
 
 CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'
@@ -810,8 +810,10 @@ def bootstrap_tls(config, init_system, dns_name, fs_id, mountpoint, options, sta
     use_iam = 'iam' in options
     ap_id = options.get('accesspoint')
     cert_details = {}
+    security_credentials = None
+    client_info = get_client_info(config)
 
-    if use_iam or ap_id:
+    if use_iam:
         aws_creds_uri = options.get('awscredsuri')
         if aws_creds_uri:
             kwargs = {'aws_creds_uri': aws_creds_uri}
@@ -820,23 +822,24 @@ def bootstrap_tls(config, init_system, dns_name, fs_id, mountpoint, options, sta
 
         security_credentials, credentials_source = get_aws_security_credentials(use_iam, **kwargs)
 
-        # additional symbol appended to avoid naming collisions
-        cert_details['mountStateDir'] = get_mount_specific_filename(fs_id, mountpoint, tls_port) + '+'
-        # common name for certificate signing request is max 64 characters
-        cert_details['commonName'] = socket.gethostname()[0:64]
-        cert_details['region'] = get_target_region(config)
-        cert_details['certificateCreationTime'] = create_certificate(config, cert_details['mountStateDir'],
-                                                                     cert_details['commonName'], cert_details['region'], fs_id,
-                                                                     security_credentials, ap_id=ap_id, base_path=state_file_dir)
-        cert_details['certificate'] = os.path.join(state_file_dir, cert_details['mountStateDir'], 'certificate.pem')
-        cert_details['privateKey'] = get_private_key_path()
-        cert_details['fsId'] = fs_id
-
         if credentials_source:
             cert_details['awsCredentialsMethod'] = credentials_source
 
-        if ap_id:
-            cert_details['accessPoint'] = ap_id
+    if ap_id:
+        cert_details['accessPoint'] = ap_id
+
+    # additional symbol appended to avoid naming collisions
+    cert_details['mountStateDir'] = get_mount_specific_filename(fs_id, mountpoint, tls_port) + '+'
+    # common name for certificate signing request is max 64 characters
+    cert_details['commonName'] = socket.gethostname()[0:64]
+    cert_details['region'] = get_target_region(config)
+    cert_details['certificateCreationTime'] = create_certificate(config, cert_details['mountStateDir'],
+                                                                 cert_details['commonName'], cert_details['region'], fs_id,
+                                                                 security_credentials, ap_id, client_info,
+                                                                 base_path=state_file_dir)
+    cert_details['certificate'] = os.path.join(state_file_dir, cert_details['mountStateDir'], 'certificate.pem')
+    cert_details['privateKey'] = get_private_key_path()
+    cert_details['fsId'] = fs_id
 
     start_watchdog(init_system)
 
@@ -976,7 +979,8 @@ def get_client_info(config):
     return client_info
 
 
-def create_certificate(config, mount_name, common_name, region, fs_id, security_credentials, ap_id, base_path=STATE_FILE_DIR):
+def create_certificate(config, mount_name, common_name, region, fs_id, security_credentials, ap_id, client_info,
+                       base_path=STATE_FILE_DIR):
     current_time = get_utc_now()
     tls_paths = tls_paths_dictionary(mount_name, base_path)
 
@@ -993,7 +997,6 @@ def create_certificate(config, mount_name, common_name, region, fs_id, security_
         public_key = os.path.join(tls_paths['mount_dir'], 'publicKey.pem')
         create_public_key(private_key, public_key)
 
-    client_info = get_client_info(config)
     create_ca_conf(certificate_config, common_name, tls_paths['mount_dir'], private_key, current_time, region, fs_id,
                    security_credentials, ap_id, client_info)
     create_certificate_signing_request(certificate_config, private_key, certificate_signing_request)
@@ -1123,14 +1126,18 @@ def create_public_key(private_key, public_key):
 
 def subprocess_call(cmd, error_message):
     """Helper method to run shell openssl command and to handle response error messages"""
-    process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
-    (output, err) = process.communicate()
-    rc = process.poll()
-    if rc != 0:
-        logging.error('Command %s failed, rc=%s, stdout="%s", stderr="%s"' % (cmd, rc, output, err))
-        fatal_error(error_message, error_message)
-    else:
-        return output, err
+    retry_times = 3
+    for retry in range(retry_times):
+        process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
+        (output, err) = process.communicate()
+        rc = process.poll()
+        if rc != 0:
+            logging.error('Command %s failed, rc=%s, stdout="%s", stderr="%s"' % (cmd, rc, output, err), exc_info=True)
+            process.kill()
+        else:
+            return output, err
+    error_message = '%s, error is: %s' % (error_message, err)
+    fatal_error(error_message, error_message)
 
 
 def ca_dirs_check(config, database_dir, certs_dir):

src/watchdog/__init__.py

@@ -45,7 +45,7 @@ except ImportError:
     from urllib.error import URLError
     from urllib.request import urlopen
 
-VERSION = '1.24'
+VERSION = '1.25'
 SERVICE = 'elasticfilesystem'
 
 CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'

test/mount_efs_test/test_bootstrap_tls.py

@@ -85,6 +85,8 @@ def test_bootstrap_tls_state_file_nonexistent_dir(mocker, tmpdir):
             return '0755'
         elif section == mount_efs.CONFIG_SECTION and field == 'dns_name_format':
             return '{fs_id}.efs.{region}.amazonaws.com'
+        elif section == mount_efs.CLIENT_INFO_SECTION and field == 'source':
+            return CLIENT_SOURCE
         else:
             raise ValueError('Unexpected arguments')
 
@@ -99,38 +101,6 @@ def test_bootstrap_tls_state_file_nonexistent_dir(mocker, tmpdir):
     assert os.path.exists(state_file_dir)
 
 
-def test_bootstrap_tls_no_cert_creation(mocker, tmpdir):
-    setup_mocks_without_popen(mocker)
-    mocker.patch('mount_efs.get_mount_specific_filename', return_value=DNS_NAME)
-    state_file_dir = str(tmpdir)
-    tls_dict = mount_efs.tls_paths_dictionary(DNS_NAME, state_file_dir)
-
-    pk_path = os.path.join(str(tmpdir), 'privateKey.pem')
-    mocker.patch('mount_efs.get_private_key_path', return_value=pk_path)
-
-    def config_get_side_effect(section, field):
-        if section == mount_efs.CONFIG_SECTION and field == 'state_file_dir_mode':
-            return '0755'
-        elif section == mount_efs.CONFIG_SECTION and field == 'dns_name_format':
-            return '{fs_id}.efs.{region}.amazonaws.com'
-        else:
-            raise ValueError('Unexpected arguments')
-
-    MOCK_CONFIG.get.side_effect = config_get_side_effect
-
-    mocker.patch('mount_efs._stunnel_bin', return_value='/usr/bin/stunnel')
-    try:
-        with mount_efs.bootstrap_tls(MOCK_CONFIG, INIT_SYSTEM, DNS_NAME, FS_ID, MOUNT_POINT, {}, state_file_dir):
-            pass
-    except OSError as e:
-        assert '[Errno 2] No such file or directory' in str(e)
-
-    assert not os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
-    assert not os.path.exists(os.path.join(tls_dict['mount_dir'], 'request.csr'))
-    assert not os.path.exists(os.path.join(tls_dict['mount_dir'], 'config.conf'))
-    assert not os.path.exists(pk_path)
-
-
 def test_bootstrap_tls_cert_created(mocker, tmpdir):
     setup_mocks_without_popen(mocker)
     mocker.patch('mount_efs.get_mount_specific_filename', return_value=DNS_NAME)

test/mount_efs_test/test_create_self_signed_cert.py

@@ -112,7 +112,7 @@ def test_certificate_without_iam_with_ap_id(mocker, tmpdir):
     pk_path = _get_mock_private_key_path(mocker, tmpdir)
     tls_dict = mount_efs.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
     tmp_config_path = os.path.join(str(tmpdir), MOUNT_NAME, 'tmpConfig')
-    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, None, AP_ID, base_path=str(tmpdir))
+    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, None, AP_ID, CLIENT_INFO, base_path=str(tmpdir))
     with open(os.path.join(tls_dict['mount_dir'], 'config.conf')) as f:
         conf_body = f.read()
         assert conf_body == mount_efs.create_ca_conf(tmp_config_path, COMMON_NAME, tls_dict['mount_dir'], pk_path, FIXED_DT, REGION,
@@ -128,7 +128,7 @@ def test_certificate_with_iam_with_ap_id(mocker, tmpdir):
     pk_path = _get_mock_private_key_path(mocker, tmpdir)
     tls_dict = mount_efs.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
     tmp_config_path = os.path.join(str(tmpdir), MOUNT_NAME, 'tmpConfig')
-    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, CREDENTIALS, AP_ID, base_path=str(tmpdir))
+    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, CREDENTIALS, AP_ID, CLIENT_INFO, base_path=str(tmpdir))
     with open(os.path.join(tls_dict['mount_dir'], 'config.conf')) as f:
         conf_body = f.read()
         assert conf_body == mount_efs.create_ca_conf(tmp_config_path, COMMON_NAME, tls_dict['mount_dir'], pk_path, FIXED_DT, REGION,
@@ -144,7 +144,7 @@ def _test_certificate_with_iam_with_ap_with_invalid_client_source_config(mocker,
     pk_path = _get_mock_private_key_path(mocker, tmpdir)
     tls_dict = mount_efs.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
     tmp_config_path = os.path.join(str(tmpdir), MOUNT_NAME, 'tmpConfig')
-    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, CREDENTIALS, AP_ID,
+    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, CREDENTIALS, AP_ID, None,
                                  base_path=str(tmpdir))
     with open(os.path.join(tls_dict['mount_dir'], 'config.conf')) as f:
         conf_body = f.read()
@@ -173,7 +173,7 @@ def test_certificate_with_iam_without_ap_id(mocker, tmpdir):
     pk_path = _get_mock_private_key_path(mocker, tmpdir)
     tls_dict = mount_efs.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
     tmp_config_path = os.path.join(str(tmpdir), MOUNT_NAME, 'tmpConfig')
-    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, CREDENTIALS, None, base_path=str(tmpdir))
+    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, CREDENTIALS, None, CLIENT_INFO, base_path=str(tmpdir))
     with open(os.path.join(tls_dict['mount_dir'], 'config.conf')) as f:
         conf_body = f.read()
         assert conf_body == mount_efs.create_ca_conf(tmp_config_path, COMMON_NAME, tls_dict['mount_dir'], pk_path, FIXED_DT, REGION,
@@ -184,6 +184,38 @@ def test_certificate_with_iam_without_ap_id(mocker, tmpdir):
     assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
 
 
+def test_certificate_without_iam_without_ap_id_without_client_source(mocker, tmpdir):
+    config = _get_config()
+    pk_path = _get_mock_private_key_path(mocker, tmpdir)
+    tls_dict = mount_efs.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
+    tmp_config_path = os.path.join(str(tmpdir), MOUNT_NAME, 'tmpConfig')
+    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, None, None, None, base_path=str(tmpdir))
+    with open(os.path.join(tls_dict['mount_dir'], 'config.conf')) as f:
+        conf_body = f.read()
+        assert conf_body == mount_efs.create_ca_conf(tmp_config_path, COMMON_NAME, tls_dict['mount_dir'], pk_path, FIXED_DT,
+                                                     REGION, FS_ID, None, None, None)
+    assert os.path.exists(pk_path)
+    assert not os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
+    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'request.csr'))
+    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
+
+
+def test_certificate_without_iam_without_ap_id_with_client_source(mocker, tmpdir):
+    config = _get_mock_config()
+    pk_path = _get_mock_private_key_path(mocker, tmpdir)
+    tls_dict = mount_efs.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
+    tmp_config_path = os.path.join(str(tmpdir), MOUNT_NAME, 'tmpConfig')
+    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, None, None, CLIENT_INFO, base_path=str(tmpdir))
+    with open(os.path.join(tls_dict['mount_dir'], 'config.conf')) as f:
+        conf_body = f.read()
+        assert conf_body == mount_efs.create_ca_conf(tmp_config_path, COMMON_NAME, tls_dict['mount_dir'], pk_path, FIXED_DT,
+                                                     REGION, FS_ID, None, None, CLIENT_INFO)
+    assert os.path.exists(pk_path)
+    assert not os.path.exists(os.path.join(tls_dict['mount_dir'], 'publicKey.pem'))
+    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'request.csr'))
+    assert os.path.exists(os.path.join(tls_dict['mount_dir'], 'certificate.pem'))
+
+
 def test_create_ca_supporting_dirs(tmpdir):
     config = _get_config()
     tls_dict = mount_efs.tls_paths_dictionary(MOUNT_NAME, str(tmpdir))
@@ -341,4 +373,37 @@ def test_create_ca_conf_with_accesspoint_no_iam(tmpdir):
     matching_config_body = mount_efs.CA_CONFIG_BODY % (tls_dict['mount_dir'], tls_dict['private_key'], COMMON_NAME,
                                                        ca_extension_body, efs_client_auth_body, efs_client_info_body)
 
+    assert full_config_body == matching_config_body
+
+
+def test_create_ca_conf_no_ap_no_iam_no_client_source(tmpdir):
+    current_time = mount_efs.get_utc_now()
+    tls_dict, full_config_body = _create_ca_conf_helper(tmpdir, current_time, iam=False, ap=False, client_info=False)
+
+    ca_extension_body = ('[ v3_ca ]\n'
+                         'subjectKeyIdentifier = hash\n'
+                         '1.3.6.1.4.1.4843.7.3 = ASN1:UTF8String:%s'
+                         ) % (FS_ID)
+    efs_client_auth_body = ''
+    efs_client_info_body = ''
+    matching_config_body = mount_efs.CA_CONFIG_BODY % (tls_dict['mount_dir'], tls_dict['private_key'], COMMON_NAME,
+                                                       ca_extension_body, efs_client_auth_body, efs_client_info_body)
+
+    assert full_config_body == matching_config_body
+
+
+def test_create_ca_conf_no_ap_no_iam_with_client_source(tmpdir):
+    current_time = mount_efs.get_utc_now()
+    tls_dict, full_config_body = _create_ca_conf_helper(tmpdir, current_time, iam=False, ap=False, client_info=True)
+
+    ca_extension_body = ('[ v3_ca ]\n'
+                         'subjectKeyIdentifier = hash\n'
+                         '1.3.6.1.4.1.4843.7.3 = ASN1:UTF8String:%s\n' 
+                        '1.3.6.1.4.1.4843.7.4 = ASN1:SEQUENCE:efs_client_info'
+                         ) % (FS_ID)
+    efs_client_auth_body = ''
+    efs_client_info_body = mount_efs.efs_client_info_builder(CLIENT_INFO)
+    matching_config_body = mount_efs.CA_CONFIG_BODY % (tls_dict['mount_dir'], tls_dict['private_key'], COMMON_NAME,
+                                                       ca_extension_body, efs_client_auth_body, efs_client_info_body)
+
     assert full_config_body == matching_config_body
\ No newline at end of file

test/watchdog_test/test_refresh_self_signed_certificate.py

@@ -87,10 +87,10 @@ def _get_mock_private_key_path(mocker, tmpdir):
 
 
 def _create_certificate_and_state(tls_dict, temp_dir, pk_path, timestamp, security_credentials=None,
-                                  credentials_source=None, ap_id=None, remove_cert=False):
+                                  credentials_source=None, ap_id=None, remove_cert=False, client_info=None):
     config = _get_config()
     good_ap_id = AP_ID if ap_id else None
-    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, security_credentials, good_ap_id,
+    mount_efs.create_certificate(config, MOUNT_NAME, COMMON_NAME, REGION, FS_ID, security_credentials, good_ap_id, client_info,
                                  base_path=str(temp_dir))
 
     assert os.path.exists(pk_path)