#                           PUBLIC DOMAIN NOTICE
#              National Center for Biotechnology Information
#  
# This software is a "United States Government Work" under the
# terms of the United States Copyright Act.  It was written as part of
# the authors' official duties as United States Government employees and
# thus cannot be copyrighted.  This software is freely available
# to the public for use.  The National Library of Medicine and the U.S.
# Government have not placed any restriction on its use or reproduction.
#   
# Although all reasonable efforts have been taken to ensure the accuracy
# and reliability of the software and data, the NLM and the U.S.
# Government do not and cannot warrant the performance or results that
# may be obtained by using this software or data.  The NLM and the U.S.
# Government disclaim all warranties, express or implied, including
# warranties of performance, merchantability or fitness for any particular
# purpose.
#   
# Please cite NCBI in any work or product based on this material.

"""
Application tests for elastic-blast

Author: Greg Boratyn boratyng@ncbi.nlm.nih.gov
"""

import subprocess
import os
import tempfile
import configparser
import signal
import time
import io
import re
import logging
import sys
from typing import List
from tempfile import TemporaryDirectory, NamedTemporaryFile
from unittest import mock
from unittest.mock import patch, MagicMock
import contextlib
from pathlib import Path
import shlex
from botocore.exceptions import ClientError
from elastic_blast import constants
from elastic_blast.util import safe_exec, ElbSupportedPrograms
from elastic_blast.base import InstanceProperties, QuerySplittingResults
from tests.utils import gke_mock, aws_credentials, NOT_WRITABLE_BUCKET
from tests.utils import MockedEC2ClientBase
# TODO: refactor bin/elastic-blast to a sub-module inside the elastic_blast module
from .elastic_blast_app import main

import pytest


TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
SUBCOMMANDS = ['submit', 'status', 'delete']
INI_NO_BLASTDB = os.path.join(TEST_DATA_DIR, 'blastdb-notfound.ini')
INI_TOO_MANY_K8S_JOBS = os.path.join(TEST_DATA_DIR, 'too-many-k8s-jobs.ini')
INI_INCOMPLETE_MEM_LIMIT_OPTIMAL_MACHINE_TYPE_AWS = os.path.join(TEST_DATA_DIR, 'incomplete-mem-limit-optimal-aws-machine-type.ini')
INI_NO_NUM_CPUS_OPTIMAL_MACHINE_TYPE_AWS = os.path.join(TEST_DATA_DIR, 'no-num-cpus-optimal-aws-machine-type.ini')
INI_INVALID_MACHINE_TYPE_AWS = os.path.join(TEST_DATA_DIR, 'invalid-machine-type-aws.ini')
INI_INVALID_MACHINE_TYPE_GCP = os.path.join(TEST_DATA_DIR, 'invalid-machine-type-gcp.ini')
INI_INVALID_CHARACTER = os.path.join(TEST_DATA_DIR, 'invalid-dollar-sign-char.ini')
INI_INVALID_MEM_LIMIT = os.path.join(TEST_DATA_DIR, 'invalid-mem-req.ini')
INI_INVALID_CPU_CONFIGURATION = os.path.join(TEST_DATA_DIR, 'invalid-cpu-req-gcp.ini')
INI_BLAST_OPT_NO_CLOSING_QUOTE = os.path.join(TEST_DATA_DIR, 'invalid-blast-opt-no-closing-quote.ini')
INI_VALID = os.path.join(TEST_DATA_DIR, 'good_conf.ini')
ELB_EXENAME = 'elastic-blast'


def run_elastic_blast(cmd: List[str], env=None) -> subprocess.CompletedProcess:
    """Run Elastic-BLAST application with given command line parameters.

    Arguments:
        cmd: A list of command line parameters

    Returns:
        subprocess.CompletedProcess object"""
    effective_env = dict(os.environ)
    if env:
        for key, value in env.items():
            effective_env[key] = str(value)
    p = subprocess.run([ELB_EXENAME] + cmd, check=False,
                       stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                       env=effective_env)
    return p


## Mocked tests

class SetupObjects:
    """A class that holds mocking objects for easier use in tests"""
    def __init__(self, caplog, gke_mock):
        # builtin pytest fixture that captures logs generated by code in unit tests: https://docs.pytest.org/en/6.2.x/logging.html
        self.caplog = caplog
        self.gke_mock = gke_mock


def remove_logfile():
    """Remove logfile generated by an elastic-blast run during a test"""
    #FIXME: Path.unlink(missing_ok=True) supported from python3.8
    logfile = Path(constants.ELB_DFLT_LOGFILE)
    if logfile.exists():
        logfile.unlink()


@pytest.fixture
def app_mocks(caplog, aws_credentials, gke_mock, mocker):
    """A fixture that mocks AWS credentials and simulates gcloud, gsutil, and
    kubectl calls via gke_mock. There is quite extensive cloud simulation for GCP.
    Currently AWS simulation is very limited, with most ElasticBlastAws methods
    mocked as no-ops. app_mocks already depends on gke_mock, so each test in in
    this file should only depend on app_mocks."""
    # capture only error level logs
    caplog.set_level(logging.ERROR)
    gke_mock.options = ['no-cluster']

    #TODO: These function should be mocked at the level of cloud API calls
    mocker.patch('tests.app.elastic_blast_app.clean_up', new=MagicMock(return_value=[]))
    mocker.patch('elastic_blast.commands.submit.check_resource_quotas', new=MagicMock(return_value=None))
    mocker.patch('elastic_blast.elasticblast.copy_to_bucket', new=MagicMock(return_value=None))
    mocker.patch(target='elastic_blast.elb_config.aws_get_machine_properties', new=MagicMock(return_value=InstanceProperties(32, 120)))
    mocker.patch('elastic_blast.elasticblast_factory.ElasticBlastAws', new=MagicMock(return_value=MagicMock()))
    mocker.patch(target='elastic_blast.tuner.aws_get_machine_properties', new=MagicMock(return_value=InstanceProperties(32, 120)))
    mocker.patch('elastic_blast.commands.submit.harvest_query_splitting_results', new=MagicMock(return_value=QuerySplittingResults(query_length=5, query_batches=['batch_0.fa'])))
    mocker.patch('elastic_blast.commands.submit.get_blastdb_size', new=MagicMock(return_value=1.0))
    mocker.patch('elastic_blast.gcp.get_blastdb_info', new=MagicMock(return_value=('gs://test-bucket/testdb', 'gs://test-bucket/testdb.tar.gz', 'testdb')))

    yield SetupObjects(caplog = caplog, gke_mock = gke_mock)

    remove_logfile()


def test_no_cmd_params(app_mocks):
    """Test that running elastic-blast with no parameters produces an error
    message, appropriate exit code, and no python traceback information"""
    # mock command line parameters
    cmd = 'elastic-blast'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    # no stacktrace on stderr
    assert 'Traceback' not in stderr.getvalue()

    # all error messages go into logs
    msg = app_mocks.caplog.text
    assert returncode == constants.INPUT_ERROR
    assert 'error' in msg or 'ERROR' in msg
    assert 'Elastic-BLAST task was not specified' in msg
    for subcommand in SUBCOMMANDS:
        assert subcommand in msg


def test_no_cfg(app_mocks):
    """Test that running elastic-blast with no config parameters produces an error
    message, appropriate exit code, and no python traceback information"""
    for subcommand in SUBCOMMANDS:
        # mock command line parameters
        cmd = f'elastic-blast {subcommand}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()

        # no stacktrace on stderr
        assert 'Traceback' not in stderr.getvalue()

        # all error messages go into logs
        msg = app_mocks.caplog.text
        assert returncode == constants.INPUT_ERROR
        assert 'ERROR' in msg
        assert 'Missing results' in msg


def test_cfg_file_not_found(app_mocks):
    """Test that missing cfg file is found by argparser"""
    FILENAME = 'some-non-existant-file'
    cmd = f'elastic-blast submit --cfg {FILENAME}'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            # argparse raises SystemExit on problems with command line parameters
            # with exit code in SystemExit.code
            with pytest.raises(SystemExit) as err:
                main()
    assert err.value.code == constants.INPUT_ERROR
    msg = stderr.getvalue()
    assert 'Traceback' not in msg
    assert 'error' in msg
    assert f'{FILENAME} was not found' in msg


def test_unicode_args(app_mocks):
    """Test that arguments with Unicode letters are rejected"""
    # NB: in following line the argument --cfg has long dash instead of normal
    # which can't be processed by elastic-blast
    with patch.object(sys, 'argv', ['elastic-blast', 'submit', '–-cfg', 'some-file']):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert returncode == constants.INPUT_ERROR
    assert 'ERROR' in msg
    assert 'Command line has Unicode letters in argument' in msg
    assert "can't be processed" in msg


@pytest.fixture
def client_query_split():
    """Use local query splitting. The fixture sets environment variable
    that invokes local query spliting and resets it after the test."""
    ELB_USE_CLIENT_SPLIT = 'ELB_USE_CLIENT_SPLIT'
    old_query_split = None
    if ELB_USE_CLIENT_SPLIT in os.environ:
        old_query_split = os.environ[ELB_USE_CLIENT_SPLIT]

    if not old_query_split:
        os.environ[ELB_USE_CLIENT_SPLIT] = '1'

    yield

    if old_query_split is not None:
        os.environ[ELB_USE_CLIENT_SPLIT] = old_query_split
    else:
        del os.environ[ELB_USE_CLIENT_SPLIT]

@mock.patch.dict(os.environ, {'ELB_DISABLE_JOB_SUBMISSION_ON_THE_CLOUD': '1'})
def test_too_many_k8s_jobs_client_split(app_mocks, client_query_split):
    """Test that providing a configuration that generates k8s jobs that exceed the limit produces 
    a sensible error message and exit code.
    """
    # FASTA string with 20000 sequences, 800 bases each
    LONG_FASTA = '\n'.join(['>test-query', 'ACTG' * 200] * 20000)

    # simulate a query file in GS
    app_mocks.gke_mock.cloud.storage['gs://test-bucket/long-query.fa'] = LONG_FASTA

    cmd = f'elastic-blast submit --cfg {INI_TOO_MANY_K8S_JOBS}'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    print(stderr.getvalue())
    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'ERROR' in msg
    assert 'Please increase the batch-len' in msg


def test_bad_num_nodes(app_mocks):
    """Test that providing negative number of nodes produces a correct error
    message and exit code"""
    cmd = f'elastic-blast submit --num-nodes -1'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            # argparse raises SystemExit on problems with command line parameters
            # with exit code in SystemExit.code
            with pytest.raises(SystemExit) as err:
                main()

    assert err.value.code == constants.INPUT_ERROR

    msg = stderr.getvalue()
    assert 'Traceback' not in msg
    assert 'error' in msg
    assert '-1' in msg
    assert 'not a positive integer' in msg


def test_invalid_machine_type_gcp(app_mocks):
    """Test that providing an invalid machine type produces a correct error
    message and exit code"""
    cmd = f'elastic-blast submit --cfg {INI_INVALID_MACHINE_TYPE_GCP}'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'ERROR' in msg
    assert 'Invalid machine type' in msg


def test_invalid_machine_type_aws(app_mocks, mocker):
    """Test that providing an invalid machine type produces a correct error
    message and exit code"""

    # Mock EC2 client so that any machine type name is incorrect
    class MockedEC2Client(MockedEC2ClientBase):
        """Mocked boto3 EC2 client that reports any machine type as incorrect"""
        def describe_instance_types(self, InstanceTypes):
            """Always raise ClientError"""
            raise ClientError

    def mocked_client(cli, config=None):
        """Mocked boto3.client() that returns a MockedEC2Client object"""
        if cli == 'ec2':
            return MockedEC2Client()
        else:
            return app_mocks.gke_mock.mocked_client(cli, config)

    mocker.patch('boto3.client', side_effect=mocked_client)
    mocker.patch('botocore.exceptions.ClientError.__init__', new=MagicMock(return_value=None))
    # basic_mocks mock aws_traits.get_machine_properties, we are bringing back
    # the orignal functionality
    from elastic_blast.aws_traits import get_machine_properties
    mocker.patch(target='elastic_blast.elb_config.aws_get_machine_properties', side_effect=get_machine_properties)

    cmd = f'elastic-blast submit --cfg {INI_INVALID_MACHINE_TYPE_AWS}'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'ERROR' in msg
    assert 'Invalid AWS machine type' in msg


def test_invalid_optimal_machine_type_aws_incomplete_mem_limit(app_mocks):
    """Test that providing a machine type 'optimal' with incomplete
    configuration produces a correct error message and exit code"""
    cmd = f'elastic-blast submit --cfg {INI_INCOMPLETE_MEM_LIMIT_OPTIMAL_MACHINE_TYPE_AWS}'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'ERROR' in msg
    assert 'requires configuring blast.mem-limit' in msg


def test_optimal_machine_type_aws_no_num_cpus(app_mocks):
    """Test that providing a machine type 'optimal' without num-cpus works fine"""
    cmd = f'elastic-blast submit --cfg {INI_NO_NUM_CPUS_OPTIMAL_MACHINE_TYPE_AWS}'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    assert returncode == 0
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Traceback' not in msg
    assert 'ERROR' not in msg


def test_invalid_mem_limit(app_mocks):
    """Test that providing an invalid memory limit configuration produces a correct error
    message and exit code"""
    cmd = f'elastic-blast submit --cfg {INI_INVALID_MEM_LIMIT}'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'ERROR' in msg
    assert ' has an invalid value:' in msg


def test_invalid_character(app_mocks):
    """Test that providing an invalid character in the configuration produces a correct error
    message and exit code"""
    cmd = f'elastic-blast submit --cfg {INI_INVALID_CHARACTER}'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'ERROR' in msg
    assert ' do not support variable substitution' in msg


def test_invalid_cpu_config(app_mocks):
    """Test that providing an invalid CPU configuration produces a correct error
    message and exit code"""
    cmd = f'elastic-blast submit --cfg {INI_INVALID_CPU_CONFIGURATION} --num-cpus 32'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'ERROR' in msg
    assert ' does not leave any CPU' in msg


#def test_suboptimal_cpu_config(app_mocks):
#    """Test that providing an invalid CPU configuration produces a correct error
#    message and exit code"""
#    cmd = f'elastic-blast submit --cfg {INI_INVALID_CPU_CONFIGURATION}'
#    with patch.object(sys, 'argv', shlex.split(cmd)):
#        with contextlib.redirect_stderr(io.StringIO()) as stderr:
#            returncode = main()
#
#    assert returncode == constants.INPUT_ERROR
#    assert 'Traceback' not in stderr.getvalue()
#
#    msg = app_mocks.caplog.text
#    assert 'ERROR' in msg
#    assert ' does not optimally use ' in msg


def test_invalid_blast_option_no_closing_quote(app_mocks):
    """Test that providing an invalid memory limit configuration produces a correct error
    message and exit code"""
    cmd = f'elastic-blast submit --dry-run --cfg {INI_BLAST_OPT_NO_CLOSING_QUOTE}'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'ERROR' in msg
    assert 'Incorrect BLAST options: No closing quotation' in msg


def test_non_existent_option(app_mocks):
    """Test that providing a non-existent option produces a correct error
    message and exit code"""
    opt = '--option-that-does-not-exist'
    for subcommand in SUBCOMMANDS:
        cmd = f'elastic-blast {subcommand} {opt}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                with pytest.raises(SystemExit) as err:
                    main()

        assert err.value.code == constants.INPUT_ERROR
        msg = stderr.getvalue()
        assert 'Traceback' not in msg
        assert 'error' in msg
        assert opt in msg
        assert 'unrecognized argument' in msg


def test_wrong_input_query(app_mocks):
    cmd = f'elastic-blast submit --query invalid-file --db testdb --cfg {os.path.join(TEST_DATA_DIR, "good_conf.ini")}'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Query input invalid-file is not readable or does not exist' in msg


def test_wrong_results_bucket(app_mocks):
    """Test that not writable results bucket is properly reported"""
    conf = f"""[cloud-provider]
gcp-project = test-project
gcp-region = us-east4
gcp-zone = us-east4-b

[blast]
program = blastn
results = gs://{NOT_WRITABLE_BUCKET}
queries = gs://test-bucket/test-query.fa
db = testdb
"""

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --query {os.path.join(TEST_DATA_DIR, "query.fa")} --db testdb --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()

    assert returncode == constants.PERMISSIONS_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Cannot write into bucket' in msg


def test_no_results_bucket(app_mocks):
    """Test that non-existent results bucket is properly reported"""
    cmd = f'elastic-blast submit --query {os.path.join(TEST_DATA_DIR, "query.fa")} --db testdb --cfg {os.path.join(TEST_DATA_DIR, "bad_bucket_conf.ini")}'
    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    assert returncode == constants.PERMISSIONS_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Cannot write into bucket' in msg



def test_gcp_config_errors(app_mocks):
    """Test that errors in GCP config are reported"""
    cfg = configparser.ConfigParser()
    cfg[constants.CFG_CLOUD_PROVIDER] = {constants.CFG_CP_GCP_PROJECT: 'GCP-project',
                                         constants.CFG_CP_GCP_REGION: 'region # with comment',
                                         constants.CFG_CP_GCP_ZONE: 'us-east$-a'}
    cfg[constants.CFG_BLAST] = { constants.CFG_BLAST_RESULTS : 'gs://elasticblast-tomcat',
                                 constants.CFG_BLAST_PROGRAM: 'blastn',
                                 constants.CFG_BLAST_DB: 'some-db',
                                 constants.CFG_BLAST_QUERY: 'test-queries.fa',
                                 constants.CFG_BLAST_RESULTS: 'gs://elasticblast-tomcat'}


    for subcommand in SUBCOMMANDS:
        with tempfile.NamedTemporaryFile('w') as cfg_file:
            cfg.write(cfg_file)
            cfg_file.flush()
            cmd = f'elastic-blast {subcommand} --cfg {cfg_file.name}'
            with patch.object(sys, 'argv', shlex.split(cmd)):
                with contextlib.redirect_stderr(io.StringIO()) as stderr:
                    returncode = main()

        assert returncode == constants.INPUT_ERROR
        assert 'Traceback' not in stderr.getvalue()

        msg = app_mocks.caplog.text
        assert '"gcp-project" has an invalid value' in msg
        assert '"gcp-region" has an invalid value' in msg
        assert '"gcp-zone" has an invalid value' in msg


def test_blastdb_error(app_mocks):
    cmd = f'elastic-blast submit --cfg {INI_NO_BLASTDB}'

    with patch.object(sys, 'argv', shlex.split(cmd)):
        with contextlib.redirect_stderr(io.StringIO()) as stderr:
            returncode = main()

    assert returncode == constants.BLASTDB_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'BLAST database' in msg
    assert 'not found' in msg


# This test requires running elastic-blast in subprocess and cannot be mocked
@pytest.mark.skipif(os.getenv('TEAMCITY_VERSION') is not None, reason='This test is flaky in TC')
def test_interrupt_error():
    p = subprocess.Popen([ELB_EXENAME, 'submit',
        '--query', os.path.join(TEST_DATA_DIR, 'query.fa'),
        '--cfg', os.path.join(TEST_DATA_DIR, 'good_conf.ini'),
        '--dry-run'],
        stderr=subprocess.PIPE)
    # it takes arund 0.3 seconds to establish the process
    # and 3.5 seconds to run to completion with dry-run,
    # so 1 second wait is well within these safety margins
    time.sleep(2)
    p.send_signal(signal.SIGINT)
    p.wait()
    print(p.stderr.read().decode())
    assert p.returncode == constants.INTERRUPT_ERROR
    msg = p.stderr.read().decode()
    assert 'Traceback' not in msg


# FIXME: set PATH in a fixture, make sure that real gcloud is never called
def test_dependency_error():
    """Test that missing or not-executable gcloud or kubectl are handled
    properly"""
    p = safe_exec('which elastic-blast')
    exepath = p.stdout.decode()
    newpath = os.path.dirname(exepath)
    orig_exe_path = exepath
    # Check gcloud missing
    p = subprocess.run([ELB_EXENAME, 'submit', '--cfg', INI_VALID, '--dry-run'], env={'PATH': newpath}, stderr=subprocess.PIPE)
    assert p.returncode == constants.DEPENDENCY_ERROR
    msg = p.stderr.decode()
    print(msg)
    assert 'Traceback' not in msg
    assert "Required pre-requisite 'gcloud' doesn't work" in msg
    # Eliminate gcloud, check kubectl missing
    p = safe_exec('which gcloud')
    exepath = p.stdout.decode()
    newpath += ':' + os.path.dirname(exepath)
    p = subprocess.run([ELB_EXENAME, 'submit', '--cfg', INI_VALID, '--dry-run'], env={'PATH': newpath}, stderr=subprocess.PIPE)
    assert p.returncode == constants.DEPENDENCY_ERROR
    msg = p.stderr.decode()
    print(msg)
    assert 'Traceback' not in msg
    assert "Required pre-requisite 'kubectl' doesn't work" in msg
    from tempfile import TemporaryDirectory
    # Provide non-executable gcloud file
    with TemporaryDirectory() as d:
        safe_exec(f'touch {d}/gcloud')
        newpath = os.path.dirname(orig_exe_path) + ':' + d
        p = subprocess.run([ELB_EXENAME, 'submit', '--cfg', INI_VALID, '--dry-run'], env={'PATH': newpath}, stderr=subprocess.PIPE)
        assert p.returncode == constants.DEPENDENCY_ERROR
        msg = p.stderr.decode()
        print(msg)
        assert 'Traceback' not in msg
        assert "Required pre-requisite 'gcloud' doesn't work" in msg


# Disabled until EB-726 is fixed
#def test_cleanup_gsutil_error():
#    p = subprocess.run([ELB_EXENAME, 'delete',
#        '--cfg', os.path.join(TEST_DATA_DIR, 'cleanup-error.ini')],
#        stderr=subprocess.PIPE)
#    assert p.returncode == 0
#    msg = p.stderr.decode()
#    print(msg)
#    assert 'Traceback' not in msg
#    assert 'WARNING' in msg
#    assert 'could not be removed' in msg


# FIXME: set PATH in a fixture, make sure that real gcloud is never called
@pytest.mark.skip(reason="gcloud compute regions list does not work")
def test_cluster_error():
    p = safe_exec('which gcloud')
    gcloud_exepath = p.stdout.decode().strip()
    p = safe_exec('which gsutil')
    gsutil_exepath = p.stdout.decode().strip()
    spy_file = os.path.join(os.getcwd(), 'spy_file.txt')
    json_output = '[{"name":"us-east4"},{"name":"us-east1"}]'

    gcloud = f"""\
#!/bin/bash
echo "gcloud $@" >>{spy_file}
if [ "${1} ${2} ${3}" == "container clusters list" ]; then
echo STOPPING
elif [ "${1} ${2} ${3}" == "compute regions list" ]; then
echo {json_output}
else
TMP=`mktemp`
res=`{gcloud_exepath} $@` > $TMP
exit_code=$?
cat $TMP >>{spy_file}
echo "Exit code $exit_code" >>{spy_file} 
cat $TMP
exit $exit_code
fi"""
    gsutil = f"""\
#!/bin/bash
echo "gsutil $@" >>{spy_file}
if [ "${1} ${2}" == "-q stat" ]; then
   if [[ ${3} == *"nt-nucl-metadata.json" ]] ; then
   exit 0
else
exit 1
fi
else
TMP=`mktemp`
{gsutil_exepath} $@ > $TMP
exit_code=$?
cat $TMP >>{spy_file}
echo "Exit code $exit_code" >>{spy_file} 
cat $TMP
exit $exit_code
fi"""
    env = dict(os.environ)
    with TemporaryDirectory() as d:
        env['PATH'] = d + ':' + env['PATH']
        gcloud_fname = os.path.join(d, 'gcloud')
        with open(gcloud_fname, 'wt') as f:
            f.write(gcloud)
        gsutil_fname = os.path.join(d, 'gsutil')
        with open(gsutil_fname, 'wt') as f:
            f.write(gsutil)
        import stat
        os.chmod(gcloud_fname, stat.S_IRWXU)
        os.chmod(gsutil_fname, stat.S_IRWXU)

        fn_config = os.path.join(TEST_DATA_DIR, 'cluster-error.ini')

        # elastic-blast delete --cfg tests/app/data/cluster-error.ini --logfile stderr
        p = subprocess.run([ELB_EXENAME, 'delete',
            '--cfg', fn_config,
            '--logfile', 'stderr'],
            env=env, stderr=subprocess.PIPE)
        msg = p.stderr.decode()
        print(msg)
        assert p.returncode == constants.CLUSTER_ERROR
        assert 'Traceback' not in msg
        assert 'ERROR' in msg
        assert 'is already being deleted' in msg

        # elastic-blast submit --cfg tests/app/data/cluster-error.ini --logfile stderr
        p = subprocess.run([ELB_EXENAME, 'submit',
            '--cfg', fn_config,
            '--logfile', 'stderr'],
            env=env, stderr=subprocess.PIPE)
        msg = p.stderr.decode()
        print(msg)
        assert p.returncode == constants.CLUSTER_ERROR
        assert 'Traceback' not in msg
        assert 'Previous instance of cluster' in msg
        assert 'is still STOPPING' in msg
        assert 'delete the previous ElasticBLAST search' in msg

        # elastic-blast status --cfg tests/app/data/cluster-error.ini --loglevel DEBUG --logfile stderr
        p = subprocess.run([ELB_EXENAME, 'status',
            '--cfg', fn_config,
            '--logfile', 'stderr'],
            env=env, stderr=subprocess.PIPE)
        msg = p.stderr.decode()
        print(msg)
        assert p.returncode == constants.CLUSTER_ERROR
        assert 'Traceback' not in msg
    run_elastic_blast(['delete', '--cfg', fn_config])


# Failing tests for not implemented error codes
# TODO: Update the call and modify running condition when implemented

@pytest.mark.skip(reason="Not yet implemented return code")
def test_blast_timeout_error():
    p = run_elastic_blast(f'submit --cfg timeout.ini'.split())
    run_elastic_blast(f'delete --cfg timeout.ini'.split())
    assert p.returncode == constants.TIMEOUT_ERROR
    msg = p.stderr.decode()
    print(msg)
    assert 'Traceback' not in msg


def test_memory_limit_too_large(app_mocks):
    """Test that selecting memory limit that exceeds memory available on a
    selected instance type results in the appropriate error code and message."""

    conf = """[cloud-provider]
aws-region = us-east-1

[cluster]
machine-type = m5.large

[blast]
program = blastp
db = testdb
queries = some-queries
mem-limit = 900G
results = s3://some-bucket
"""

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()
            assert returncode == constants.INPUT_ERROR
            assert re.search(r'Memory limit [\w"]* exceeds memory available on the selected machine type', stderr.getvalue())


def test_misplaced_config_parameter(app_mocks):
    """Test that using a config parameter in a wrong section results in an
    appropriate error code and message"""

    # num-cpus should be in [cluster]
    conf = """[cloud-provider]
aws-region = us-east-1

[cluster]
machine-type = m5.large

[blast]
num-cpus = 8
program = blastp
db = testdb
queries = some-queries
mem-limit = 900G
results = s3://some-bucket
"""

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()
            assert returncode == constants.INPUT_ERROR
            assert re.search(r'Unrecognized configuration parameter [\w"-]* in section [\w"-]*', stderr.getvalue())


DB_METADATA = """{
  "dbname": "swissprot",
  "version": "1.1",
  "dbtype": "Protein",
  "description": "Non-redundant UniProtKB/SwissProt sequences",
  "number-of-letters": 180911227,
  "number-of-sequences": 477327,
  "files": [
    "gs://blast-db/2021-09-28-01-05-02/swissprot.ppi",
    "gs://blast-db/2021-09-28-01-05-02/swissprot.pos",
    "gs://blast-db/2021-09-28-01-05-02/swissprot.pog",
    "gs://blast-db/2021-09-28-01-05-02/swissprot.phr",
    "gs://blast-db/2021-09-28-01-05-02/swissprot.ppd",
    "gs://blast-db/2021-09-28-01-05-02/swissprot.psq",
    "gs://blast-db/2021-09-28-01-05-02/swissprot.pto",
    "gs://blast-db/2021-09-28-01-05-02/swissprot.pin",
    "gs://blast-db/2021-09-28-01-05-02/swissprot.pot",
    "gs://blast-db/2021-09-28-01-05-02/swissprot.ptf",
    "gs://blast-db/2021-09-28-01-05-02/swissprot.pdb"
  ],
  "last-updated": "2021-09-19T00:00:00",
  "bytes-total": 353839003,
  "bytes-to-cache": 999185207299,
  "number-of-volumes": 1
}
"""

def test_database_too_large(app_mocks):
    """Test that a database too large to fit in an instance RAM will be reported"""
    DB = 's3://bucket/testdb'
    app_mocks.gke_mock.cloud.storage[f'{DB}-prot-metadata.json'] = DB_METADATA

    conf = f"""[cloud-provider]
aws-region = us-east-1

[cluster]
machine-type = m5.large

[blast]
program = blastp
db = {DB}
queries = some-queries
results = s3://some-bucket
"""

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()
            assert returncode == constants.INPUT_ERROR
            assert re.search(r'BLAST database [\w/:"-]* memory requirements exceed', stderr.getvalue())


def test_missing_ncbi_database_metadata(app_mocks):
    """Test that a missing NCBI database metadata file results in an error"""

    conf = f"""[cloud-provider]
aws-region = us-east-1

[blast]
program = blastp
db = db-no-metadata
queries = s3://test-bucket/test-query.fa
results = s3://test-results
"""

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()

    assert returncode == constants.BLASTDB_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Traceback' not in msg
    assert 'Metadata for BLAST database' in msg


def test_missing_user_database_metadata(app_mocks):
    """Test that a missing user database metadata file is fine"""

    conf = f"""[cloud-provider]
aws-region = us-east-1

[blast]
program = blastp
db = s3://some-db
queries = s3://test-bucket/test-query.fa
results = s3://test-results
"""

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()

    assert returncode == 0
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Traceback' not in msg
    assert 'ERROR' not in msg


def test_incorrect_db_mol_type(app_mocks):
    """Test that incorrect BLAST database molecule type is reported"""

    DB = 'prot-db'
    DB_MOL_TYPE = 'Protein'
    PROGRAM = 'blastn'

    DB_METADATA = '{' + f"""
      "dbname": "{DB}",
      "version": "1.1",
      "dbtype": "{DB_MOL_TYPE}",
      "description": "Some large database",
      "number-of-letters": 180911227,
      "number-of-sequences": 477327,
      "files": [
        "gs://blast-db/2021-09-28-01-05-02/largedb.ppi",
        "gs://blast-db/2021-09-28-01-05-02/largedb.pos",
        "gs://blast-db/2021-09-28-01-05-02/largedb.pog",
        "gs://blast-db/2021-09-28-01-05-02/largedb.phr",
        "gs://blast-db/2021-09-28-01-05-02/largedb.ppd",
        "gs://blast-db/2021-09-28-01-05-02/largedb.psq",
        "gs://blast-db/2021-09-28-01-05-02/largedb.pto",
        "gs://blast-db/2021-09-28-01-05-02/largedb.pin",
        "gs://blast-db/2021-09-28-01-05-02/largedb.pot",
        "gs://blast-db/2021-09-28-01-05-02/largedb.ptf",
        "gs://blast-db/2021-09-28-01-05-02/largedb.pdb"
      ],
      "last-updated": "2021-09-19T00:00:00",
      "bytes-total": 353839003,
      "bytes-to-cache": 185207299,
      "number-of-volumes": 1
""" + '}'

    app_mocks.gke_mock.cloud.storage[f's3://ncbi-blast-databases/000/{DB}-{DB_MOL_TYPE.lower()[:4]}-metadata.json'] = DB_METADATA
    assert constants.MolType[DB_MOL_TYPE.upper()] != ElbSupportedPrograms().get_db_mol_type(PROGRAM)

    conf = f"""[cloud-provider]
aws-region = us-east-1

[blast]
program = {PROGRAM}
db = {DB}
queries = s3://test-bucket/test-query.fa
results = s3://test-results
"""

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()

    assert returncode == constants.BLASTDB_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Traceback' not in msg
    assert 'database molecular type' in msg


def test_gcp_insufficient_ssd_quota(app_mocks, mocker):
    """Test that an insufficient GCP SSD quota is properly reported"""

    conf = f"""[cloud-provider]
gcp-project = test-project
gcp-region = test-region
gcp-zone = test-zone

[cluster]
pd-size = 3000000G

[blast]
program = blastp
db = gs://test-bucket/testdb
queries = gs://test-bucket/test-query.fa
results = gs://test-results
"""

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()

    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Traceback' not in msg
    assert re.search(r'Requested disk size [\w.]* is larger than allowed', msg)


@mock.patch.dict(os.environ, {'ELB_JANITOR_SCHEDULE': '1'})
def test_invalid_janitor_schedule_gcp(app_mocks):
    """Test that invalid janitor schedule is properly reported"""

    conf = f"""[cloud-provider]
gcp-project = test-project
gcp-region = test-region
gcp-zone = test-zone

[blast]
program = blastp
db = gs://test-bucket/testdb
queries = gs://test-bucket/test-query.fa
results = gs://test-results
"""

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()

    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Traceback' not in msg
    assert re.search(r'Invalid value of environment variable ELB_JANITOR_SCHEDULE', msg)


@mock.patch.dict(os.environ, {'ELB_JANITOR_SCHEDULE': '@weekly'})
def test_invalid_janitor_schedule_aws(app_mocks):
    """Test that invalid janitor schedule is properly reported"""

    conf = f"""[cloud-provider]
aws-region = test-region

[blast]
program = blastp
db = s3://test-bucket/testdb
queries = s3://test-bucket/test-query.fa
results = s3://test-results
"""

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()

    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Traceback' not in msg
    assert re.search(r'Invalid value of environment variable ELB_JANITOR_SCHEDULE', msg)


def test_no_gcp_project(app_mocks):
    """Test that missing GCP project is properly reported"""

    conf = f"""[blast]
program = blastp
db = gs://test-bucket/testdb
queries = gs://test-bucket/test-query.fa
results = gs://test-results
"""

    # make GCP project unset in the mocked gcloud config
    app_mocks.gke_mock.cloud.conf['project'] = None

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()

    print(stderr.getvalue())
    assert returncode == constants.INPUT_ERROR
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Traceback' not in msg
    assert re.search(r'GCP project is unset', msg)


@patch(target='elastic_blast.elasticblast_factory.ElasticBlastGcp', new=MagicMock(return_value=MagicMock()))
def test_default_cloud_provider_config(app_mocks):
    """Test that default cloud provider parameters can be determined from the
    results bucket."""

    conf = f"""[blast]
program = blastp
db = gs://test-bucket/testdb
queries = gs://test-bucket/test-query.fa
results = gs://test-results
"""

    with NamedTemporaryFile() as f:
        f.write(conf.encode())
        f.flush()
        f.seek(0)

        cmd = f'elastic-blast submit --cfg {f.name}'
        with patch.object(sys, 'argv', shlex.split(cmd)):
            with contextlib.redirect_stderr(io.StringIO()) as stderr:
                returncode = main()

    assert returncode == 0
    assert 'Traceback' not in stderr.getvalue()

    msg = app_mocks.caplog.text
    assert 'Traceback' not in msg
    assert 'ERROR' not in msg
