Lambdaでファイルのダウンロード速度をEFSとS3で比較してみた

cat /dev/urandom | base64 | head -c $((1024*1024*1024)) > ~/rand_1g.txt

def download_file_from_s3(bucket, key):
    output_path = f'/tmp/{key}'

    start_time = time.time()
    s3_client.download_file(bucket, key, output_path)
    total_time = time.time() - start_time

    os.remove(output_path)

    return total_time

def download_file_from_efs(dir, file):
    input_path = f'{dir}/{file}'
    output_path = f'/tmp/{file}'

    start_time = time.time()
    shutil.copy(input_path, output_path)
    total_time = time.time() - start_time

    os.remove(output_path)

    return total_time

for i in $(seq 1000); do
  FILENAME=$(cat /dev/urandom | base64 | tr '/' '_' | head -c 16).txt
  cat /dev/urandom | base64 | head -c $(((RANDOM % 10 + 1) * 10 * 1024)) > ${FILENAME}
done

def download_directory_from_s3(bucket_name, directory_prefix):
    directory_prefix = directory_prefix.rstrip('/') + '/'
    output_dir_path = '/tmp/small'
    os.makedirs(output_dir_path, exist_ok=True)

    start_time = time.time()
    with concurrent.futures.ThreadPoolExecutor(max_workers=30) as executor:
        paginator = s3_client.get_paginator('list_objects_v2')
        for page in paginator.paginate(Bucket=bucket_name, Prefix=directory_prefix):
            for obj in page.get('Contents', []):
                file_key = obj['Key']
                output_path = os.path.join(output_dir_path, file_key[len(directory_prefix):])
                executor.submit(s3_client.download_file, bucket_name, file_key, output_path)
    total_time = time.time() - start_time

    shutil.rmtree(output_dir_path)

    return total_time

def download_directory_from_efs(directory_path):
    input_path = directory_path.rstrip('/') + '/'
    output_dir_path = '/tmp/small'

    start_time = time.time()
    shutil.copytree(input_path, output_dir_path)
    total_time = time.time() - start_time

    shutil.rmtree(output_dir_path)

    return total_time