Commit 4e7ffcd4 authored by stvhu's avatar stvhu
Browse files

Support IAM authentication and access points.

parent 0fd7b833
......@@ -87,27 +87,39 @@ To mount with the recommended default options, simply run:
$ sudo mount -t efs file-system-id efs-mount-point/
```
To mount automatically with recommended options, add an `/etc/fstab` entry like:
To mount over TLS, simply add the `tls` option:
```
file-system-id efs-mount-point efs _netdev 0 0
$ sudo mount -t efs -o tls file-system-id efs-mount-point/
```
To mount over TLS, simply add the `tls` option:
To authenticate with EFS using the system’s IAM identity, add the `iam` option. This option requires the `tls` option.
```
$ sudo mount -t efs -o tls file-system-id efs-mount-point/
$ sudo mount -t efs -o tls,iam file-system-id efs-mount-point/
```
To mount using an access point, use the `accesspoint=` option. This option requires the `tls` option.
```
$ sudo mount -t efs -o tls,accesspoint=access-point-id file-system-id efs-mount-point/
```
To mount your file system automatically with any of the options above, you can add entries to `/efs/fstab` like:
```
file-system-id efs-mount-point efs _netdev,tls,iam,accesspoint=access-point-id 0 0
```
To mount over TLS automatically, add an `/etc/fstab` entry like:
For more information on mounting with the mount helper, see the manual page:
```
file-system-id efs-mount-point efs _netdev,tls 0 0
man mount.efs
```
For more information on mounting with the mount helper, see the [documentation](https://docs.aws.amazon.com/efs/latest/ug/using-amazon-efs-utils.html).
or refer to the [documentation](https://docs.aws.amazon.com/efs/latest/ug/using-amazon-efs-utils.html).
#### amazon-efs-mount-watchdog
### amazon-efs-mount-watchdog
`efs-utils` contains a watchdog process to monitor the health of TLS mounts. This process is managed by either `upstart` or `systemd` depending on your Linux distribution, and is started automatically the first time an EFS file system is mounted over TLS.
......
......@@ -11,7 +11,7 @@ set -ex
BASE_DIR=$(pwd)
BUILD_ROOT=${BASE_DIR}/build/debbuild
VERSION=1.17
VERSION=1.18
echo 'Cleaning deb build workspace'
rm -rf ${BUILD_ROOT}
......
#
# Copyright 2017-2018 Amazon.com, Inc. and its affiliates. All Rights Reserved.
#
# Licensed under the MIT License. See the LICENSE accompanying this file
# for the specific language governing permissions and limitations under
# the License.
#
[global]
version=1.18
Package: amazon-efs-utils
Architecture: all
Version: 1.17
Version: 1.18
Section: utils
Depends: python|python2, nfs-common, stunnel4 (>= 4.56)
Priority: optional
......
......@@ -20,7 +20,7 @@
%endif
Name : amazon-efs-utils
Version : 1.17
Version : 1.18
Release : 1%{?dist}
Summary : This package provides utilities for simplifying the use of EFS file systems
......
......@@ -10,12 +10,14 @@
logging_level = INFO
logging_max_bytes = 1048576
logging_file_count = 10
# mode for /var/run/efs in octal
# mode for /var/run/efs and subdirectories in octal
state_file_dir_mode = 750
[mount]
dns_name_format = {fs_id}.efs.{region}.amazonaws.com
dns_name_format = {fs_id}.efs.{region}.{dns_name_suffix}
dns_name_suffix = amazonaws.com
stunnel_debug_enabled = false
stunnel_cafile = /etc/amazon/efs/efs-utils.crt
# Validate the certificate hostname on mount. This option is not supported by certain stunnel versions.
stunnel_check_cert_hostname = true
......
......@@ -81,6 +81,22 @@ more information, see \fBstunnel(8)\fR\&.
Selects whether to perform OCSP validation on TLS certificates\&, \
overriding /etc/amazon/efs/efs-utils.conf. \
For more information, see \fBstunnel(8)\fR\&.
.TP
\fBiam\fR
Use the system's IAM identity to authenticate with EFS. The mount helper will try \
to retrieve the required IAM credentials from the following locations: the EC2 instance \
profile, the AWS CLI credentials file (~/.aws/credentials), and the AWS CLI config \
file (~/.aws/config). The first location that has credentials will be used. \
This option requires the \fBtls\fR option\&.
.TP
\fBaccesspoint\fR
Mount the EFS file system using the specified access point. This option requires the \
\fBtls\fR option\&.
.TP
\fBawsprofile\fR
Use the named profile used to lookup IAM credentials in the AWS CLI credentials file \
(~/.aws/credentials) or AWS CLI config file (~/.aws/config). If "awsprofile" is not \
specified, the "default" profile is used\&.
.if n \{\
.RE
.\}
......@@ -121,6 +137,16 @@ resolve to a fully-qualified EFS DNS name such as \
"fs\-abcd1234\&.efs\&.us-east-1\&.amazonaws\&.com" \
\(em at mount point "/mnt/efs" using encryption \
of data in transit\&.
.TP
sudo mount -t efs -o tls,iam fs-abcd1234 /mnt/efs
Mount an EFS file system with file system ID "fs-abcd1234" at mount point "/mnt/efs" \
with encryption of data in transit. The mount helper will authenticate with EFS using \
the system's IAM identity\&.
.TP
sudo mount -t efs -o tls,accesspoint=fsap-12345678 fs-abcd1234 /mnt/efs
Mount an EFS file system with file system ID "fs-abcd1234" at mount point "/mnt/efs" \
with encryption of data in transit. The file system is mounted using the access point \
"fsap-12345678"\&.
.SH "FILES"
.TP
\fI/sbin/mount.efs\fR
......
......@@ -30,29 +30,37 @@
#
# The script will add recommended mount options, if not provided in fstab.
import base64
import errno
import hashlib
import hmac
import itertools
import json
import logging
import os
import pwd
import random
import re
import socket
import subprocess
import sys
import threading
import time
from contextlib import contextmanager
from datetime import datetime, timedelta
from logging.handlers import RotatingFileHandler
try:
from ConfigParser import NoOptionError
except Exception:
from configparser import NoOptionError
import ConfigParser
from ConfigParser import NoOptionError, NoSectionError
except ImportError:
from configparser import ConfigParser, NoOptionError, NoSectionError
try:
import ConfigParser
from urllib.parse import quote_plus
except ImportError:
from configparser import ConfigParser
from urllib import quote_plus
try:
from urllib2 import urlopen, URLError
......@@ -60,7 +68,8 @@ except ImportError:
from urllib.error import URLError
from urllib.request import urlopen
VERSION = '1.17'
VERSION = '1.18'
SERVICE = 'elasticfilesystem'
CONFIG_FILE = '/etc/amazon/efs/efs-utils.conf'
CONFIG_SECTION = 'mount'
......@@ -70,25 +79,85 @@ LOG_FILE = 'mount.log'
STATE_FILE_DIR = '/var/run/efs'
PRIVATE_KEY_FILE = '/etc/amazon/efs/privateKey.pem'
DATE_ONLY_FORMAT = '%Y%m%d'
SIGV4_DATETIME_FORMAT = '%Y%m%dT%H%M%SZ'
CERT_DATETIME_FORMAT = '%y%m%d%H%M%SZ'
CA_CONFIG_BODY = """dir = %s
RANDFILE = $dir/database/.rand
[ ca ]
default_ca = local_ca
[ local_ca ]
database = $dir/database/index.txt
serial = $dir/database/serial
private_key = %s
cert = $dir/certificate.pem
new_certs_dir = $dir/certs
default_md = sha256
preserve = no
policy = efsPolicy
x509_extensions = v3_ca
[ efsPolicy ]
CN = supplied
[ req ]
prompt = no
distinguished_name = req_distinguished_name
[ req_distinguished_name ]
CN = %s
%s
%s
"""
# SigV4 Auth
ALGORITHM = 'AWS4-HMAC-SHA256'
AWS4_REQUEST = 'aws4_request'
HTTP_REQUEST_METHOD = 'GET'
CANONICAL_URI = '/'
CANONICAL_HEADERS_DICT = {
'host': '%s'
}
CANONICAL_HEADERS = '\n'.join(['%s:%s' % (k, v) for k, v in sorted(CANONICAL_HEADERS_DICT.items())])
SIGNED_HEADERS = ';'.join(CANONICAL_HEADERS_DICT.keys())
REQUEST_PAYLOAD = ''
FS_ID_RE = re.compile('^(?P<fs_id>fs-[0-9a-f]+)$')
EFS_FQDN_RE = re.compile(r'^(?P<fs_id>fs-[0-9a-f]+)\.efs\.(?P<region>[a-z0-9-]+)\.amazonaws.com$')
AP_ID_RE = re.compile('^fsap-[0-9a-f]{17}$')
INSTANCE_METADATA_SERVICE_URL = 'http://169.254.169.254/latest/dynamic/instance-identity/document/'
INSTANCE_IAM_URL = 'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
SECURITY_CREDS_ECS_URI_HELP_URL = 'https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html'
SECURITY_CREDS_IAM_ROLE_HELP_URL = 'https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html'
DEFAULT_STUNNEL_VERIFY_LEVEL = 2
DEFAULT_STUNNEL_CAFILE = '/etc/amazon/efs/efs-utils.crt'
NOT_BEFORE_MINS = 15
NOT_AFTER_HOURS = 3
EFS_ONLY_OPTIONS = [
'iam',
'noocsp',
'ocsp',
'accesspoint',
'awsprofile',
'cafile',
'tls',
'tlsport',
'verify',
'ocsp',
'noocsp'
'verify'
]
UNSUPPORTED_OPTIONS = [
'cafile',
'capath',
'capath'
]
STUNNEL_GLOBAL_CONFIG = {
......@@ -156,6 +225,143 @@ def get_region():
_fatal_error('Region not present in %s: %s' % (instance_identity, e))
def get_region_helper(config):
dns_name_format = config.get(CONFIG_SECTION, 'dns_name_format')
if '{region}' in dns_name_format:
return get_region()
else:
return dns_name_format.split('.')[-3]
def get_aws_security_credentials(awsprofile=None):
"""
Lookup AWS security credentials (access key ID and secret access key). Adapted credentials provider chain from:
https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html and
https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html
"""
aws_credentials_file = os.path.expanduser(os.path.join('~' + pwd.getpwuid(os.getuid()).pw_name, '.aws', 'credentials'))
aws_config_file = os.path.expanduser(os.path.join('~' + pwd.getpwuid(os.getuid()).pw_name, '.aws', 'config'))
# attempt to lookup AWS access key ID and secret access key in AWS credentials file (~/.aws/credentials)
if os.path.exists(aws_credentials_file):
credentials = credentials_file_helper(aws_credentials_file, awsprofile=awsprofile)
if credentials['AccessKeyId']:
return credentials
# in AWS configs file (~/.aws/config)
if os.path.exists(aws_config_file):
credentials = credentials_file_helper(aws_config_file, awsprofile=awsprofile)
if credentials['AccessKeyId']:
return credentials
keys = ['AccessKeyId', 'SecretAccessKey', 'Token']
# through ECS security credentials uri found in AWS_CONTAINER_CREDENTIALS_RELATIVE_URI environment variable
ecs_uri_env = 'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI'
if ecs_uri_env in os.environ:
ecs_uri = '169.254.170.2' + os.environ[ecs_uri_env]
ecs_unsuccessful_resp = 'Unsuccessful retrieval of AWS security credentials at %s.' % ecs_uri
ecs_url_error_msg = 'Unable to reach %s to retrieve AWS security credentials. See %s for more info.', ecs_uri, \
SECURITY_CREDS_ECS_URI_HELP_URL
ecs_security_dict = url_request_helper(ecs_uri, ecs_unsuccessful_resp, ecs_url_error_msg)
if ecs_security_dict and all(k in ecs_security_dict for k in keys):
return ecs_security_dict
# through IAM role name security credentials lookup uri (after lookup for IAM role name attached to instance)
iam_role_unsuccessful_resp = 'Unsuccessful retrieval of IAM role name at %s.' % INSTANCE_IAM_URL
iam_role_url_error_msg = 'Unable to reach %s to retrieve IAM role name. See %s for more info.', INSTANCE_IAM_URL, \
SECURITY_CREDS_IAM_ROLE_HELP_URL
iam_role_name = url_request_helper(INSTANCE_IAM_URL, iam_role_unsuccessful_resp, iam_role_url_error_msg)
if iam_role_name:
security_creds_lookup_url = INSTANCE_IAM_URL + str(iam_role_name)
unsuccessful_resp = 'Unsuccessful retrieval of AWS security credentials at %s.' % security_creds_lookup_url
url_error_msg = 'Unable to reach %s to retrieve AWS security credentials. See %s for more info.',\
security_creds_lookup_url, SECURITY_CREDS_IAM_ROLE_HELP_URL
iam_security_dict = url_request_helper(security_creds_lookup_url, unsuccessful_resp, url_error_msg)
if iam_security_dict and all(k in iam_security_dict for k in keys):
return iam_security_dict
error_msg = 'AWS Access Key ID and Secret Access Key are not found in AWS credentials file (%s), config file (%s), ' \
'from ECS credentials relative uri, or from the instance security credentials service' % \
(aws_credentials_file, aws_config_file)
fatal_error(error_msg, error_msg)
def credentials_file_helper(file_path, awsprofile=None):
aws_credentials_configs = read_config(file_path)
credentials = {'AccessKeyId': None, 'SecretAccessKey': None, 'Token': None}
profile = awsprofile or get_correct_default_case_combination(aws_credentials_configs)
try:
access_key = aws_credentials_configs.get(profile, 'aws_access_key_id')
secret_key = aws_credentials_configs.get(profile, 'aws_secret_access_key')
session_token = aws_credentials_configs.get(profile, 'aws_session_token')
credentials['AccessKeyId'] = access_key
credentials['SecretAccessKey'] = secret_key
credentials['Token'] = session_token
except NoOptionError as e:
if 'aws_access_key_id' in str(e) or 'aws_secret_access_key' in str(e):
log_message = 'aws_access_key_id or aws_secret_access_key not found in %s under named profile [%s]' % \
(file_path, profile)
if awsprofile:
fatal_error(log_message)
else:
logging.debug(log_message)
return credentials
if 'aws_session_token' in str(e):
logging.debug('aws_session_token not found in %s', file_path)
credentials['AccessKeyId'] = aws_credentials_configs.get(profile, 'aws_access_key_id')
credentials['SecretAccessKey'] = aws_credentials_configs.get(profile, 'aws_secret_access_key')
except NoSectionError:
log_message = 'No [%s] section found in config file %s' % (profile, file_path)
if awsprofile:
fatal_error(log_message)
else:
logging.debug(log_message)
return credentials
def get_correct_default_case_combination(aws_credentials_configs):
default_str = 'default'
for perm in map(''.join, itertools.product(*zip(default_str.upper(), default_str.lower()))):
try:
access_key = aws_credentials_configs.get(perm, 'aws_access_key_id')
if access_key is not None:
return perm
except (NoSectionError, NoOptionError):
continue
return default_str
def url_request_helper(url, unsuccessful_resp, url_error_msg):
try:
request_resp = urlopen(url, timeout=1)
if request_resp.getcode() != 200:
logging.debug(unsuccessful_resp + ' %s: ResponseCode=%d', url, request_resp.getcode())
return None
resp_body = request_resp.read()
try:
if type(resp_body) is str:
resp_dict = json.loads(resp_body)
else:
resp_dict = json.loads(resp_body.decode(request_resp.headers.get_content_charset() or 'us-ascii'))
return resp_dict
except ValueError as e:
logging.debug('Error parsing json: %s, returning raw response body: %s' % (e, str(resp_body)))
return resp_body
except URLError as e:
logging.debug('%s %s', url_error_msg, e)
return None
def parse_options(options):
opts = {}
for o in options.split(','):
......@@ -181,10 +387,7 @@ def get_tls_port_range(config):
def choose_tls_port(config, options):
if 'tlsport' in options:
try:
ports_to_try = [int(options['tlsport'])]
except ValueError:
fatal_error('tlsport option [%s] is not an integer' % options['tlsport'])
else:
lower_bound, upper_bound = get_tls_port_range(config)
......@@ -215,9 +418,7 @@ def choose_tls_port(config, options):
def is_ocsp_enabled(config, options):
if 'ocsp' in options and 'noocsp' in options:
fatal_error('The "ocsp" and "noocsp" options are mutually exclusive')
elif 'ocsp' in options:
if 'ocsp' in options:
return True
elif 'noocsp' in options:
return False
......@@ -245,10 +446,20 @@ def serialize_stunnel_config(config, header=None):
return lines
def add_stunnel_ca_options(efs_config, stunnel_cafile=DEFAULT_STUNNEL_CAFILE):
def add_stunnel_ca_options(efs_config, config, options):
if 'cafile' in options:
stunnel_cafile = options['cafile']
else:
try:
stunnel_cafile = config.get(CONFIG_SECTION, 'stunnel_cafile')
except NoOptionError:
logging.debug('No CA file configured, using default CA file %s', DEFAULT_STUNNEL_CAFILE)
stunnel_cafile = DEFAULT_STUNNEL_CAFILE
if not os.path.exists(stunnel_cafile):
fatal_error('Failed to find the EFS certificate authority file for verification',
'Failed to find the EFS CAfile "%s"' % stunnel_cafile)
fatal_error('Failed to find certificate authority file for verification',
'Failed to find CAfile "%s"' % stunnel_cafile)
efs_config['CAfile'] = stunnel_cafile
......@@ -290,7 +501,7 @@ def get_system_release_version():
def write_stunnel_config_file(config, state_file_dir, fs_id, mountpoint, tls_port, dns_name, verify_level, ocsp_enabled,
log_dir=LOG_DIR):
options, log_dir=LOG_DIR, cert_details=None):
"""
Serializes stunnel configuration to a file. Unfortunately this does not conform to Python's config file format, so we have to
hand-serialize it.
......@@ -308,7 +519,11 @@ def write_stunnel_config_file(config, state_file_dir, fs_id, mountpoint, tls_por
efs_config['connect'] = efs_config['connect'] % dns_name
efs_config['verify'] = verify_level
if verify_level > 0:
add_stunnel_ca_options(efs_config)
add_stunnel_ca_options(efs_config, config, options)
if cert_details:
efs_config['cert'] = cert_details['certificate']
efs_config['key'] = cert_details['privateKey']
check_host_supported, ocsp_aia_supported = get_version_specific_stunnel_options(config)
......@@ -343,7 +558,7 @@ def write_stunnel_config_file(config, state_file_dir, fs_id, mountpoint, tls_por
return stunnel_config_file
def write_tls_tunnel_state_file(fs_id, mountpoint, tls_port, tunnel_pid, command, files, state_file_dir):
def write_tls_tunnel_state_file(fs_id, mountpoint, tls_port, tunnel_pid, command, files, state_file_dir, cert_details=None):
"""
Return the name of the temporary file containing TLS tunnel state, prefixed with a '~'. This file needs to be renamed to a
non-temporary version following a successful mount.
......@@ -356,6 +571,9 @@ def write_tls_tunnel_state_file(fs_id, mountpoint, tls_port, tunnel_pid, command
'files': files,
}
if cert_details:
state.update(cert_details)
with open(os.path.join(state_file_dir, state_file), 'w') as f:
json.dump(state, f)
......@@ -438,7 +656,7 @@ def start_watchdog(init_system):
logging.warning(error_message)
def create_state_file_dir(config, state_file_dir):
def create_required_directory(config, directory):
mode = 0o750
try:
mode_str = config.get(CONFIG_SECTION, 'state_file_dir_mode')
......@@ -450,31 +668,54 @@ def create_state_file_dir(config, state_file_dir):
pass
try:
os.makedirs(state_file_dir, mode)
os.makedirs(directory, mode)
except OSError as e:
if errno.EEXIST != e.errno or not os.path.isdir(state_file_dir):
if errno.EEXIST != e.errno or not os.path.isdir(directory):
raise
@contextmanager
def bootstrap_tls(config, init_system, dns_name, fs_id, mountpoint, options, state_file_dir=STATE_FILE_DIR):
start_watchdog(init_system)
if not os.path.exists(state_file_dir):
create_state_file_dir(config, state_file_dir)
def bootstrap_tls(config, init_system, dns_name, fs_id, ap_id, mountpoint, options, state_file_dir=STATE_FILE_DIR):
tls_port = choose_tls_port(config, options)
# override the tlsport option so that we can later override the port the NFS client uses to connect to stunnel.
# if the user has specified tlsport=X at the command line this will just re-set tlsport to X.
options['tlsport'] = tls_port
use_iam = 'iam' in options
awsprofile = options.get('awsprofile')
cert_details = {}
if use_iam or ap_id:
# additional symbol appended to avoid naming collisions
cert_details['mountStateDir'] = get_mount_specific_filename(fs_id, mountpoint, tls_port) + '+'
# common name for certificate signing request is max 64 characters
cert_details['commonName'] = socket.gethostname()[0:64]
cert_details['region'] = get_region_helper(config)
cert_details['certificateCreationTime'] = create_certificate(config, cert_details['mountStateDir'],
cert_details['commonName'], cert_details['region'], fs_id,
use_iam, ap_id=ap_id, awsprofile=awsprofile,
base_path=state_file_dir)
cert_details['certificate'] = os.path.join(state_file_dir, cert_details['mountStateDir'], 'certificate.pem')
cert_details['privateKey'] = get_private_key_path()
cert_details['fsId'] = fs_id
cert_details['useIam'] = use_iam
if awsprofile:
cert_details['awsprofile'] = awsprofile
if ap_id:
cert_details['accessPoint'] = ap_id
start_watchdog(init_system)
if not os.path.exists(state_file_dir):
create_required_directory(config, state_file_dir)
verify_level = int(options.get('verify', DEFAULT_STUNNEL_VERIFY_LEVEL))
ocsp_enabled = is_ocsp_enabled(config, options)
stunnel_config_file = write_stunnel_config_file(config, state_file_dir, fs_id, mountpoint, tls_port, dns_name, verify_level,
ocsp_enabled)
ocsp_enabled, options, cert_details=cert_details)
tunnel_args = ['stunnel', stunnel_config_file]
# launch the tunnel in a process group so if it has any child processes, they can be killed easily by the mount watchdog
......@@ -484,7 +725,7 @@ def bootstrap_tls(config, init_system, dns_name, fs_id, mountpoint, options, sta
logging.info('Started TLS tunnel, pid: %d', tunnel_proc.pid)
temp_tls_state_file = write_tls_tunnel_state_file(fs_id, mountpoint, tls_port, tunnel_proc.pid, tunnel_args,
[stunnel_config_file], state_file_dir)
[stunnel_config_file], state_file_dir, cert_details=cert_details)
try:
yield tunnel_proc
......@@ -510,8 +751,6 @@ def get_nfs_mount_options(options):
options['noresvport'] = None
if 'tls' in options:
if 'port' in options:
fatal_error('The "port" and "tls" options are mutually exclusive')
options['port'] = options['tlsport']
def to_nfs_option(k, v):
......@@ -525,6 +764,7 @@ def get_nfs_mount_options(options):
def mount_nfs(dns_name, path, mountpoint, options):
if 'tls' in options:
mount_path = '127.0.0.1:%s' % path
else:
......@@ -587,6 +827,184 @@ def parse_arguments(config, args=None):
return fs_id, path, mountpoint, options
def create_certificate(config, mount_name, common_name, region, fs_id, use_iam, ap_id=None, awsprofile=None,
base_path=STATE_FILE_DIR):
current_time = get_utc_now()
tls_paths = tls_paths_dictionary(mount_name, base_path)
<