/****
DIAMOND protein sequence aligner
Copyright (C) 2012-2026 Benjamin J. Buchfink

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
****/
// SPDX-License-Identifier: GPL-3.0-or-later

#include <algorithm>
#include <iostream>
#include <thread>
#define _REENTRANT
#include "ips4o/ips4o.hpp"
#include "util/parallel/simple_thread_pool.h"
#include "blastdb/blastdb.h"
#include "sequence_file.h"
#include "masking/masking.h"
#include "legacy/dmnd/dmnd.h"
#include "util/system/system.h"
#include "util/sequence/sequence.h"
#include "util/parallel/multiprocessing.h"
#include "basic/config.h"
#include "fasta/fasta_file.h"
#include "fasta/volumed_fasta_file.h"
#include "util/string/tokenizer.h"
#include "util/log_stream.h"
#include "util/data_structures/queue.h"

using std::string;
using std::endl;
using std::setw;
using std::thread;
using std::pair;
using std::tie;
using std::runtime_error;
using std::ofstream;
using std::greater;
using std::numeric_limits;
using std::tuple;
using std::get;
using std::unique_ptr;
using std::function;
using std::vector;
using std::set;
using std::unordered_map;
using std::map;
using std::atomic;

const uint64_t SequenceFile::DEFAULT_LOAD_SIZE = 4 * GIGABYTES;
static constexpr int64_t CHECK_FOR_DNA_COUNT = 10;
const char* const SequenceFile::SEQID_HDR = "seqid";
const DictId SequenceFile::DICT_EMPTY = std::numeric_limits<DictId>::max();

const EMap<SequenceFile::Type> EnumTraits<SequenceFile::Type>::to_string = {
	{SequenceFile::Type::DMND, "Diamond database" },
	{SequenceFile::Type::BLAST, "BLAST database"},
	{SequenceFile::Type::FASTA, "FASTA file"},
	{SequenceFile::Type::BLOCK, ""},
	{SequenceFile::Type::VOLUMED, "FASTA volume list"}
};

static string dict_file_name(const size_t query_block, const size_t target_block) {
	const string file_name = append_label("ref_dict_", query_block) + append_label("_", target_block);
	return join_path(config.parallel_tmpdir, file_name);
}

static size_t single_oid(const SequenceFile* f, const string& acc) {
	const vector<OId> oid = f->accession_to_oid(acc);
	if (oid.empty())
		throw AccessionNotFound();
	if (oid.size() > 1)
		throw std::runtime_error("Multiple oids for target accession: " + acc);
	return oid.front();
}

std::string SequenceFile::taxon_scientific_name(TaxId taxid) const {
	throw OperationNotSupported();
}

void SequenceFile::add_taxid_mapping(const vector<pair<OId, TaxId>>& taxids) {
	throw OperationNotSupported();
}

int SequenceFile::raw_chunk_no() const {
	throw OperationNotSupported();
}

void SequenceFile::advance_seq_count(OId n) {
	throw OperationNotSupported();
}

pair<Block*, int64_t> SequenceFile::load_parallel(const uint64_t max_letters, const BitVector* filter, unordered_map<string, bool>* accs, const Chunk& chunk, bool load_taxids) {
	const bool verbose_logging = false;
	const Loc SEQ_LEN_EST = 200;
	const uint64_t ID_EST = config.oid_title_max > 0 ? 20 : 100, MAX_RESERVE = 8 * GIGABYTES; // TODO
	assert(chunk.n_seqs == 0);
	assert(config.threads_ > 0);
	const uint64_t letters = config.minichunk;
	const int t = std::min(config.load_threads, config.threads_), p = t >= 3 ? t - 2 : 1;
	const int raw_chunk_start = raw_chunk_no();
	int64_t seqs = 0;
	const bool load_seqs = flag_any(flags(), SequenceFile::Flags::SEQS),
		load_titles = flag_any(flags(), SequenceFile::Flags::TITLES);
	Block* block = new Block();
	const uint64_t size_est = std::min(max_letters, disk_size_);
	const BlockId entry_est = (size_est + SEQ_LEN_EST - 1) / SEQ_LEN_EST;
	if (load_seqs)
		block->seqs().reserve(entry_est, size_est, verbose_logging);
	if (load_titles) {
		block->reserve_ids(std::min(ID_EST * entry_est, size_est), entry_est, verbose_logging);
	}
	Queue<RawChunk*> queue(p * 4, 1, p, nullptr);
	Queue<DecodedPackage*> output_queue(p * 4, p, 1, nullptr);
	const bool byte_limited = type() == Type::FASTA || type() == Type::VOLUMED, have_oids = type() == Type::BLAST;
	uint64_t backlog_size = 0;
	SimpleThreadPool pool;
	auto worker = [&](const atomic<bool>& stop) {
		while (!stop) {
			RawChunk* c;
			if (!queue.wait_and_dequeue(c))
				break;
			DecodedPackage* pkg = c->decode(flags(), filter, accs, value_traits_.seq_type);
			output_queue.enqueue(pkg);
			delete c;
		}
		output_queue.close();
		};
	auto writer = [&](const atomic<bool>& stop) {
		map<int, DecodedPackage*> backlog;
		uint64_t bytes_processed = 0;
		int next = raw_chunk_start;
		while (!stop) {
			DecodedPackage* pkg;
			if (!output_queue.wait_and_dequeue(pkg))
				break;
			bytes_processed += pkg->byte_size();
			backlog.emplace(pkg->no, pkg);
			backlog_size += pkg->byte_size();
			while (!backlog.empty() && backlog.begin()->first == next) {
				pkg = backlog.begin()->second;
				const size_t n_seqs = pkg->seq_count;
				seqs += n_seqs;
				block->append(pkg->block, false, load_seqs, load_titles, have_oids);
				if (load_taxids) {
					add_taxid_mapping(pkg->taxids);
				}
				backlog_size -= pkg->byte_size();
				delete pkg;				
				backlog.erase(backlog.begin());
				++next;
			}
		}
		if (!backlog.empty())
			throw runtime_error("SequenceFile::load_parallel");
		};
	for (int i = 0; i < p; ++i)
		pool.spawn(worker);
	pool.spawn(writer);
	uint64_t block_letters = 0;
	RawChunk* rc;
	uint64_t bytes = 0;
	const Flags load_flags = flags() | (accs ? Flags::TITLES : Flags::NONE);
	/*Heartbeat h([&]() {
		uint64_t b = 0;
		*log_stream << "bytes=" << bytes << " backlog=" << backlog_size << endl;
		});*/
	do {
		rc = raw_chunk(std::min(letters, max_letters - block_letters), load_flags);
		if (rc->empty()) {
			delete rc;
			break;
		}
		block_letters += byte_limited ? rc->bytes() : rc->letters();
		bytes += rc->bytes();
		queue.enqueue(rc);
	} while (!pool.stop() && block_letters < max_letters);
	queue.close();
	pool.join_all();
	log_rss();
	if (byte_limited && seqs > 0) {
		const OId oid_begin = tell_seq();
		block->block2oid_.reserve((size_t)seqs);
		for (int64_t i = 0; i < seqs; ++i)
			block->block2oid_.push_back(oid_begin + (OId)i);
		advance_seq_count((OId)seqs);
	}
	if (seqs > 0) {
		if (load_seqs) {
			block->source_seqs().finish_reserve(verbose_logging);
			block->seqs().finish_reserve(verbose_logging);
		}
		if (load_titles)
			block->ids().finish_reserve(verbose_logging);
		if (value_traits_.seq_type == SequenceType::nucleotide)
			block->source_seqs_.finish_reserve(verbose_logging);
	}
	block->raw_bytes_ = bytes;
	return { block, seqs };
}

pair<Block*, uint64_t> SequenceFile::load_twopass(const uint64_t max_letters, const BitVector* filter, const Chunk& chunk) {
	init_seqinfo_access();

	OId database_id = tell_seq();
	uint64_t letters = 0, seqs = 0, id_letters = 0, seqs_processed = 0, filtered_seq_count = 0;
	vector<uint64_t> filtered_pos;
	Block* block = new Block();

	SeqInfo r = read_seqinfo();
	size_t offset = r.pos;
	bool last = false;
	const bool use_filter = filter && !filter->empty();

	auto goon = [&]() {
		if (max_letters > 0)
			return (r.seq_len > 0 && letters < max_letters);
		else
			return (seqs < chunk.n_seqs);
	};

	while (goon()) {
		SeqInfo r_next = read_seqinfo();
		if (!use_filter || filter->get(database_id)) {
			letters += r.seq_len;
			if (flag_any(flags(), SequenceFile::Flags::SEQS)) {
				block->seqs_.reserve(r.seq_len);
			}
			if (flag_any(flags(), SequenceFile::Flags::TITLES)) {
				const size_t id_len = this->id_len(r, r_next);
				id_letters += id_len;
				if (flag_any(flags(), SequenceFile::Flags::SEQS))
					block->ids_.reserve(id_len);
			}
			++filtered_seq_count;
			block->block2oid_.push_back(database_id);
			if (use_filter) {
				filtered_pos.push_back(last ? 0 : r.pos);
			}
			last = true;
		}
		else {
			last = false;
		}
		++database_id;
		++seqs_processed;
		r = r_next;
		++seqs;
	}

	putback_seqinfo();

	if (seqs == 0 || filtered_seq_count == 0)
		return { block, seqs_processed };
	const bool full_titles = bool(flags() & SequenceFile::Flags::FULL_TITLES);
	const bool all_seqids = bool(flags() & SequenceFile::Flags::ALL_SEQIDS);

	if (flag_any(flags(), SequenceFile::Flags::SEQS)) {
		block->seqs_.finish_reserve();
		if (flag_any(flags(), SequenceFile::Flags::TITLES)) block->ids_.finish_reserve();

		static const size_t MAX_LOAD_SIZE = 2 * GIGABYTES;
		if (use_filter && !flag_all(format_flags_, FormatFlags::SEEKABLE))
			throw OperationNotSupported();
		seek_offset(offset);
		for (BlockId i = 0; i < filtered_seq_count; ++i) {
			bool seek = false;
			if (use_filter && filtered_pos[i]) {
				offset = filtered_pos[i];
				seek = true;
			}
			const size_t l = block->seqs_.length(i);
			read_seq_data(block->seqs_.ptr(i), l, offset, seek);
			if (flag_any(flags(), SequenceFile::Flags::TITLES))
				read_id_data(block->block2oid_[i], block->ids_.ptr(i), block->ids_.length(i), all_seqids, full_titles);
			else
				skip_id_data();
			if (type_ == Type::DMND)
				Masking::get().remove_bit_mask(block->seqs_.ptr(i), block->seqs_.length(i));
		}
		
	}
	return { block, seqs_processed };
}

int frame_mask() {
	if (config.query_strands == "both")
		return (1 << 6) - 1;
	else if (config.query_strands == "plus")
		return (1 << 3) - 1;
	else if (config.query_strands == "minus")
		return ((1 << 3) - 1) << 3;
	throw runtime_error("frame_mask");
}

pair<Block*, uint64_t> SequenceFile::load_onepass(const uint64_t max_letters, OId max_seqs, const BitVector* filter) {
	static const char* DNA_ERR = "The sequences are expected to be proteins but only contain DNA letters. Use the option --ignore-warnings to proceed.";
	vector<Letter> seq;
	string id;
	vector<char> qual;
	uint64_t letters = 0, seq_count = 0;
	const bool load_seqs = flag_any(flags(), SequenceFile::Flags::SEQS),
		load_titles = flag_any(flags(), SequenceFile::Flags::TITLES),
		preserve_dna = flag_any(flags(), SequenceFile::Flags::DNA_PRESERVATION);
	Block* block = new Block();
	vector<char>* q = flag_any(flags(), SequenceFile::Flags::QUALITY) ? &qual : nullptr;
	OId oid = tell_seq();
	const bool first_block = oid == 0;
	const int frame_mask = ::frame_mask();
	const int64_t modulo = file_count();
	unsigned int looks_like_dna = 0;
	auto goon = [&]() { return max_seqs > 0 ? seq_count < max_seqs : letters < max_letters; };
	do {
		if (!read_seq(seq, id, q))
			break;
		if (seq.size() == 0)
			continue;
		if (filter && !filter->get(oid)) {
			++oid;
			continue;
		}

		letters += block->push_back(Sequence(seq), load_titles ? id.c_str() : nullptr, q, oid++, this->value_traits_.seq_type, frame_mask, !preserve_dna);

        ++seq_count;
		if (first_block && seq_count <= CHECK_FOR_DNA_COUNT && value_traits_.seq_type == SequenceType::amino_acid && Util::Seq::looks_like_dna(Sequence(seq)) && !config.ignore_warnings) {
			++looks_like_dna;
			if (looks_like_dna >= CHECK_FOR_DNA_COUNT)
				throw runtime_error(DNA_ERR);
		}
	} while (goon() || seq_count % modulo != 0);
	if (seq_count > 0 && looks_like_dna == seq_count)
		throw runtime_error(DNA_ERR);
	// TODO
	//if (file_count() == 2 && !files_synced())
		//throw runtime_error("Unequal number of sequences in paired read files.");
	block->seqs_.finish_reserve();
	if (value_traits_.seq_type == SequenceType::nucleotide)
		block->source_seqs_.finish_reserve();
	if (load_titles)
		block->ids_.finish_reserve();
	if (q)
		block->qual_.finish_reserve();
	return { block, seq_count };
}

size_t SequenceFile::dict_block(const size_t ref_block)
{
	return config.multiprocessing ? ref_block : 0;
}

Block* SequenceFile::load_seqs(const int64_t max_letters, OId max_seqs, const BitVector* filter, const Chunk& chunk)
{
	if (max_letters == 0)
		seek_chunk(chunk);

	Block* block;
	int64_t seqs_processed;
	if (type_ == Type::BLAST) {
		tie(block, seqs_processed) = load_parallel(max_letters, filter, nullptr, chunk, false);
	}
	else if (type_ == Type::FASTA && filter == nullptr && file_count() == 1 && static_cast<FastaFile*>(this)->is_fasta() && max_seqs == 0 && chunk.n_seqs == 0 && !flag_any(flags_, Flags::QUALITY | Flags::DNA_PRESERVATION) && config.command != Config::regression_test) {
		tie(block, seqs_processed) = load_parallel(max_letters, filter, nullptr, chunk, false);
	}
	else if (type_ == Type::VOLUMED && filter == nullptr && max_seqs == 0 && chunk.n_seqs == 0 && !flag_any(flags_, Flags::QUALITY | Flags::DNA_PRESERVATION)) {
		tie(block, seqs_processed) = load_parallel(max_letters, filter, nullptr, chunk, false);
	}
	else if (flag_any(format_flags_, FormatFlags::LENGTH_LOOKUP))
		tie(block, seqs_processed) = load_twopass(max_letters, filter, chunk);
	else {
		if (chunk.n_seqs)
			throw OperationNotSupported();
		tie(block, seqs_processed) = load_onepass(max_letters, max_seqs, filter);
	}

	if (block->empty())
		return block;

	if (flag_any(flags(), SequenceFile::Flags::LAZY_MASKING))
		block->masked_.resize(block->seqs_.size(), false);
	//block->seqs_.print_stats();
	return block;
}

void SequenceFile::get_seq()
{
	std::map<string, string> seq_titles;
	if (!config.query_file.empty()) {
		File list(config.single_query_file(), "rb", File::Flags::DETECT_COMPRESSION);
		const char* l;
		while (l = list.getline(), !list.eof() || l[0] != '\0') {
			const vector<string> t(Util::String::tokenize(l, "\t"));
			if (t.size() != 2)
				throw std::runtime_error("Query file format error.");
			seq_titles[t[0]] = t[1];
		}
		list.close();
	}

	vector<Letter> seq;
	string id;
	bool all = config.seq_no.size() == 0 && seq_titles.empty() && config.oid_list.empty();

	std::set<size_t> seqs;
	if (!all)
		for (vector<string>::const_iterator i = config.seq_no.begin(); i != config.seq_no.end(); ++i)
			seqs.insert(atoi(i->c_str()) - 1);
	if (!config.oid_list.empty()) {
		File f(config.oid_list, "rb", File::Flags::DETECT_COMPRESSION);
		OId oid;
		const char* l;
		while (l = f.getline(), !f.eof() || l[0] != '\0') {
			Util::String::Tokenizer<Util::String::CharDelimiter>(l, Util::String::CharDelimiter('\t')) >> oid;
			seqs.insert(oid);
		}
		f.close();
	}
	if (!seqs.empty())
		*message_stream << "#Selected sequences: " << seqs.size() << endl;

	const size_t max_letters = config.chunk_size == 0.0 ? std::numeric_limits<size_t>::max() : (size_t)(config.chunk_size * 1e9);
	size_t letters = 0;
	TextBuffer buf;
	File out(config.output_file, "wb");
	for (uint64_t n = 0; n < sequence_count().value(); ++n) {
		read_seq(seq, id);
		std::map<string, string>::const_iterator mapped_title = seq_titles.find(Util::Seq::seqid(id.c_str()));
		if (all || seqs.find(n) != seqs.end() || mapped_title != seq_titles.end()) {
			if (config.reverse) {
				buf << '>' << (mapped_title != seq_titles.end() ? mapped_title->second : id) << '\n';
				Sequence(seq).print(buf, value_traits, Sequence::Reversed());
				buf << '\n';
			}
			else if (config.hardmasked) {
				buf << '>' << (mapped_title != seq_titles.end() ? mapped_title->second : id) << '\n';
				Sequence(seq).print(buf, value_traits, Sequence::Hardmasked());
				buf << '\n';
			}
			else {
				Util::Seq::format(seq, id.c_str(), nullptr, buf, "fasta", amino_acid_traits, 80);
			}
		}
		out.write(buf.data(), buf.size());
		letters += seq.size();
		if (letters >= max_letters)
			break;
		seq.clear();
		id.clear();
		buf.clear();
	}

	out.close();
}

SequenceFile::~SequenceFile()
{
	if (dict_file_) {
		dict_file_->close();
		dict_file_.reset();
	}
}

static bool is_blast_db(const string& path) {
	if (exists(path + ".pin") || exists(path + ".pal") || ends_with(path, ".pal")) {
		if (config.multiprocessing)
			throw runtime_error("--multiprocessing is not compatible with BLAST databases.");
		if (config.target_indexed)
			throw runtime_error("--target-indexed is not compatible with BLAST databases.");
		return true;
	}
	return false;
}

SequenceFile* SequenceFile::auto_create(const vector<string>& path, Flags flags, const ValueTraits& value_traits) {
	if (path.size() == 1) {
		if (is_blast_db(path.front()))
			return new BlastDB(path.front(), flags, value_traits);
		const string a = auto_append_extension_if_exists(path.front(), DatabaseFile::FILE_EXTENSION);
		if (DatabaseFile::is_diamond_db(a))
			return new DatabaseFile(a, flags, value_traits);
	}
	if (!flag_any(flags, Flags::NO_FASTA)) {
		if (path.size() == 1 && VolumedFastaFile::is_volume_list(path.front()))
			return new VolumedFastaFile(path.front(), flags, value_traits);
		//message_stream << "Database file is not a DIAMOND or BLAST database, treating as FASTA." << std::endl;
		return new FastaFile(path, flags, value_traits);
	}
	throw FormatDetectionError("Sequence file does not have a supported format.");
}

void SequenceFile::load_dict_block(File* f, const size_t ref_block)
{
	while (load_dict_entry(*f, ref_block));
}

void SequenceFile::load_dictionary(const size_t query_block, const size_t ref_blocks)
{
	if (!dict_file_ && !config.multiprocessing)
		return;
	TaskTimer timer("Loading dictionary", 3);
	if (config.multiprocessing) {
		dict_oid_ = vector<vector<OId>>(ref_blocks);
		if (flag_any(flags_, Flags::SELF_ALN_SCORES))
			dict_self_aln_score_ = vector<vector<double>>(ref_blocks);
		reserve_dict(ref_blocks);
		for (size_t i = 0; i < ref_blocks; ++i) {
			File f(dict_file_name(query_block, i), "rb");
			load_dict_block(&f, i);
			f.close();
		}
	}
	else {
		File* t = dynamic_cast<File*>(dict_file_.get());
		if (!t)
			throw std::runtime_error("Failed to load dictionary file.");
		dict_oid_ = { {} };
		dict_oid_.front().reserve(next_dict_id_);
		if (flag_any(flags_, Flags::SELF_ALN_SCORES)) {
			dict_self_aln_score_ = { {} };
			dict_self_aln_score_.front().reserve(next_dict_id_);
		}
		reserve_dict(0);
		t->rewind();
		load_dict_block(t, 0);
		if ((DictId)dict_oid_.front().size() != next_dict_id_)
			throw std::runtime_error("Dictionary corrupted.");
		t->close();
		dict_file_.reset();
	}
}

void SequenceFile::free_dictionary()
{
	dict_oid_.clear();
	dict_oid_.shrink_to_fit();
	dict_len_.clear();
	dict_len_.shrink_to_fit();
	dict_title_.clear();
	dict_title_.shrink_to_fit();
	dict_seq_.clear();
	dict_seq_.shrink_to_fit();
	dict_self_aln_score_.clear();
	dict_self_aln_score_.shrink_to_fit();
	block_to_dict_id_.clear();
}

optional<uint64_t> SequenceFile::total_blocks() const {
	if (!letters())
		return nullopt;
	const size_t c = (size_t)(config.chunk_size * 1e9);
	return (this->letters().value() + c - 1) / c;
}

SequenceSet SequenceFile::seqs_by_accession(const std::vector<std::string>::const_iterator begin, const std::vector<std::string>::const_iterator end)
{
	SequenceSet out;
	vector<size_t> oids;
	oids.reserve(end - begin);
	for (auto it = begin; it != end; ++it) {
		try {
			const size_t oid = single_oid(this, *it);
			oids.push_back(oid);
			out.reserve(seq_length(oid));
		}
		catch (AccessionNotFound&) {
			out.reserve(0);
			oids.push_back(SIZE_MAX);
		}		
	}
	out.finish_reserve();
	vector<Letter> seq;
	for (size_t i = 0; i < oids.size(); ++i) {
		if (oids[i] == SIZE_MAX)
			continue;
		seq_data(oids[i], seq);
		out.assign(i, seq.begin(), seq.end());
	}
	return out;
}

std::vector<Letter> SequenceFile::seq_by_accession(const std::string& acc)
{
	const size_t oid = single_oid(this, acc);
	vector<Letter> seq;
	seq_data(oid, seq);
	return seq;
}

void SequenceFile::init_dict(const size_t query_block, const size_t target_block)
{
	if (dict_file_)
		dict_file_->close();
	dict_file_.reset(config.multiprocessing ? new File(dict_file_name(query_block, target_block), "wb") : new File(Temporary()));
	next_dict_id_ = 0;
	dict_alloc_size_ = 0;
	block_to_dict_id_.clear();
}

void SequenceFile::init_dict_block(size_t block, size_t seq_count, bool persist)
{
	if(!persist)
		block_to_dict_id_.clear();
	if(block_to_dict_id_.find(block) == block_to_dict_id_.end())
		block_to_dict_id_[block] = vector<DictId>(seq_count, DICT_EMPTY);
}

void SequenceFile::close_dict_block(bool persist)
{
	if (config.multiprocessing) {
		dict_file_->close();
		dict_file_.reset();
	}
	if (!persist)
		block_to_dict_id_.clear();
}

DictId SequenceFile::dict_id(size_t block, size_t block_id, size_t oid, size_t len, const char* id, const Letter* seq, const double self_aln_score)
{
	auto it = block_to_dict_id_.find(block);
	if (it == block_to_dict_id_.end() || block_id >= it->second.size())
		throw std::runtime_error("Dictionary not initialized.");
	vector<DictId>& v = it->second;
	DictId n = v[block_id];
	if (n != DICT_EMPTY)
		return n;
	{
		std::lock_guard<std::mutex> lock(dict_mtx_);
		n = v[block_id];
		if (n != DICT_EMPTY)
			return n;
		n = next_dict_id_++;
		v[block_id] = n;
		write_dict_entry(block, oid, len, id, seq, self_aln_score);
		return n;
	}
}

size_t SequenceFile::oid(DictId dict_id, const size_t ref_block) const
{
	const size_t b = dict_block(ref_block);
	if (b >= dict_oid_.size() || dict_id >= (DictId)dict_oid_[b].size())
		throw std::runtime_error("Dictionary not loaded.");
	return dict_oid_[b][dict_id];
}

double SequenceFile::dict_self_aln_score(const size_t dict_id, const size_t ref_block) const
{
	const size_t b = dict_block(ref_block);
	if (b >= dict_self_aln_score_.size() || dict_id >= dict_self_aln_score_[b].size())
		throw std::runtime_error("Dictionary not loaded.");
	return dict_self_aln_score_[b][dict_id];
}

SequenceFile::SequenceFile(Type type, Flags flags, FormatFlags format_flags, const ValueTraits& value_traits):
	flags_(flags),
	format_flags_(format_flags),
	value_traits_(value_traits),
	disk_size_(0),
	type_(type)
{
	if (flag_any(flags_, Flags::OID_TO_ACC_MAPPING))
		throw runtime_error("Not implemented");
}

uint64_t SequenceFile::disk_size() const {
	return disk_size_;
}

void SequenceFile::write_dict_entry(size_t block, size_t oid, size_t len, const char* id, const Letter* seq, const double self_aln_score)
{
	File& f = *dict_file_;
	f.write((uint32_t)oid);
	if (flag_any(format_flags_, FormatFlags::DICT_LENGTHS))
		f.write((uint32_t)len);
	if (flag_any(format_flags_, FormatFlags::DICT_SEQIDS)) {
		f.write_c_str(id);
		dict_alloc_size_ += strlen(id);
	}
	if (flag_any(flags_, Flags::TARGET_SEQS))
		f.write(seq, len);
	if (flag_any(flags_, Flags::SELF_ALN_SCORES))
		f.write(self_aln_score);
}

bool SequenceFile::load_dict_entry(File& f, size_t ref_block)
{
	const size_t b = dict_block(ref_block);
	uint32_t oid, len;
	string title;
	if(f.read_max(&oid, sizeof(oid)) == 0)
		return false;
	dict_oid_[b].push_back(oid);
	if (flag_any(format_flags_, FormatFlags::DICT_LENGTHS)) {
		f.read(len);
		dict_len_[b].push_back(len);
	}
	if (flag_any(format_flags_, FormatFlags::DICT_SEQIDS)) {
		f.read_c_str(title);
		dict_title_[b].push_back(title.begin(), title.end());
	}
	if (flag_any(flags_, Flags::TARGET_SEQS)) {
		vector<Letter> v(len);
		f.read(v.data(), len);
		dict_seq_[b].push_back(v.begin(), v.end());
	}
	if (flag_any(flags_, Flags::SELF_ALN_SCORES)) {
		double self_aln_score;
		f.read(self_aln_score);
		dict_self_aln_score_[b].push_back(self_aln_score);
	}
	return true;
}

void SequenceFile::reserve_dict(const size_t ref_blocks)
{
	if (config.multiprocessing) {
		if (flag_any(format_flags_, FormatFlags::DICT_LENGTHS))
			dict_len_ = std::vector<std::vector<uint32_t>>(ref_blocks);
		if (flag_any(format_flags_, FormatFlags::DICT_SEQIDS))
			dict_title_ = std::vector<StringSet>(ref_blocks);
		if (flag_any(flags_, Flags::TARGET_SEQS))
			dict_seq_ = std::vector<SequenceSet>(ref_blocks);
	}
	else {
		if (flag_any(format_flags_, FormatFlags::DICT_LENGTHS)) {
			dict_len_ = { {} };
			dict_len_[0].reserve(next_dict_id_);
		}
		if (flag_any(format_flags_, FormatFlags::DICT_SEQIDS)) {
			dict_title_ = { {} };
			dict_title_[0].reserve(next_dict_id_, dict_alloc_size_);
		}
		if (flag_any(flags_, Flags::TARGET_SEQS)) {
			dict_seq_ = { {} };
			dict_seq_[0].reserve(next_dict_id_, 0);
		}
	}
}

std::string SequenceFile::dict_title(DictId dict_id, const size_t ref_block) const
{
	const size_t b = dict_block(ref_block);
	if (b >= dict_title_.size() || dict_id >= (DictId)dict_title_[b].size())
		throw std::runtime_error("Dictionary not loaded.");
	return dict_title_[b][dict_id];
}

Loc SequenceFile::dict_len(DictId dict_id, const size_t ref_block)
{
	const size_t b = dict_block(ref_block);
	if (b >= dict_len_.size() || dict_id >= (DictId)dict_len_[b].size())
		throw runtime_error("Dictionary not loaded.");
	return dict_len_[b][dict_id];
}

std::vector<Letter> SequenceFile::dict_seq(DictId dict_id, const size_t ref_block)
{
	const size_t b = dict_block(ref_block);
	if (b >= dict_seq_.size() || dict_id >= (DictId)dict_seq_[b].size())
		throw std::runtime_error("Dictionary not loaded.");
	Sequence s = dict_seq_[b][dict_id];
	return vector<Letter>(s.data(), s.end());
}

DbFilter* SequenceFile::filter_by_taxonomy(std::istream& filter, char delimiter, bool exclude)
{
	DbFilter *f = new DbFilter(sequence_count().value());
	set<TaxId> taxon_filter_list;
	string token;
	while (std::getline(filter, token, delimiter)) {
		taxon_filter_list.insert(std::stoi(token));
	}
	if (taxon_filter_list.empty())
		throw runtime_error("Option --taxonlist/--taxon-exclude used with empty list.");
	if (taxon_filter_list.find(1) != taxon_filter_list.end() || taxon_filter_list.find(0) != taxon_filter_list.end())
		throw runtime_error("Option --taxonlist/--taxon-exclude used with invalid argument (0 or 1).");
	for (OId i = 0; i < sequence_count().value(); ++i) {
		const bool c = contained(taxids(i), taxon_filter_list, exclude, exclude);
		if (c ^ exclude) {
			f->oid_filter.set(i);
			f->letter_count += seq_length(i);
		}
	}
	return f;
}

void SequenceFile::build_acc_to_oid() {
	acc2oid_.reserve(sequence_count().value());
	set_seqinfo_ptr(0);
	vector<Letter> seq;
	string id;
	for (OId i = 0; i < sequence_count().value(); ++i) {
		read_seq(seq, id);
		acc2oid_[Util::Seq::seqid(id.c_str())] = i;
	}
}

vector<OId> SequenceFile::accession_to_oid(const string& accession) const {
	try {
		return { acc2oid_.at(accession) };
	}
	catch (std::out_of_range&) {
		throw runtime_error("Accession not found in database: " + accession);
	}
}

void SequenceFile::init_random_access(const size_t query_block, const size_t ref_blocks, bool dictionary)
{
	if (dictionary)
		load_dictionary(query_block, ref_blocks);
}

std::string SequenceFile::seqid(OId oid, bool all, bool full_titles) {
	throw std::runtime_error("seqid");
}

template<typename It>
std::vector<int64_t> SequenceFile::seq_offsets(It begin, It end) {
	assert(std::is_sorted(begin, end));
	vector<int64_t> r;
	if (end <= begin)
		return r;
	r.reserve(end - begin);
	set_seqinfo_ptr(0);
	init_seqinfo_access();
	const OId end_oid = *(end - 1) + 1;
	if (end_oid > sequence_count().value())
		throw runtime_error("OId out of bounds.");
	It it = begin;
	for (OId i = 0; i < end_oid; ++i) {
		SeqInfo info = read_seqinfo();
		if (i == *it) {
			if (it != begin && i - 1 == *(it - 1))
				r.push_back(-1);
			else
				r.push_back(info.pos);
			++it;
		}
	}
	return r;
}

pair<int64_t, int64_t> SequenceFile::read_fai_file(const string& file_name, int64_t seqs, int64_t letters) {
	string acc;
	Loc len;
	File fai(file_name, "rb", File::Flags::DETECT_COMPRESSION);
	const char* l;
	while (l = fai.getline(), !fai.eof() || l[0] != '\0') {
		Util::String::Tokenizer<Util::String::CharDelimiter>(l, Util::String::CharDelimiter('\t')) >> acc >> len;
		if (flag_any(flags_, Flags::ACC_TO_OID_MAPPING))
			acc2oid_[acc] = seqs;
		++seqs;
		letters += len;
	}
	fai.close();
	return { seqs, letters };
}

void db_info() {
	using std::cout;
	if (config.database.empty())
		throw std::runtime_error("Missing option for database file: --db/-d.");
	SequenceFile* db;
	try {
		db = SequenceFile::auto_create({ config.database }, SequenceFile::Flags::NO_FASTA | SequenceFile::Flags::NO_COMPATIBILITY_CHECK);
	} catch (const FormatDetectionError& e) {
		throw std::runtime_error("Error opening the database: " + std::string(e.what()));
	}
	const std::streamsize w = 25;
	cout << setw(w) << "Database type  " << to_string(db->type()) << endl;
	cout << setw(w) << "Database format version  " << db->db_version() << endl;
	if(db->type() == SequenceFile::Type::DMND)
		cout << setw(w) << "Diamond build  " << db->program_build_version() << endl;
	cout << setw(w) << "Sequences  " << db->sequence_count().value() << endl;
	/*if (db->type() == SequenceFile::Type::BLAST && db->sequence_count() != db->sparse_sequence_count())
		cout << setw(w) << "Sequences (filtered) " << db->sparse_sequence_count() << endl;*/
	cout << setw(w) << "Letters  " << db->letters().value() << endl;
	db->close();
	delete db;
}

void SequenceFile::add_seqid_mapping(const std::string& id, OId oid) {
	const string acc = Util::Seq::seqid(id.c_str());
	if (flag_any(flags_, Flags::ACC_TO_OID_MAPPING)) {
		if (oid != (OId)acc2oid_.size())
			throw runtime_error("add_seqid_mapping");
		auto r = acc2oid_.emplace(acc, oid);
		if (!r.second)
			throw runtime_error("Accession is not unique in database file: " + acc);
	}
}

size_t SequenceFile::letters_filtered(const DbFilter& v) {
	size_t n = 0;
	for (OId i = 0; i < v.oid_filter.size(); ++i)
		if (v.oid_filter.get(i))
			n += seq_length(i);
	return n;
}

set<TaxId> SequenceFile::rank_taxid(const std::vector<TaxId>& taxid, int rank) {
	set<TaxId> r;
	for (TaxId i : taxid)
		r.insert(rank_taxid(i, rank));
	return r;
}

TaxId SequenceFile::get_parent(TaxId taxid) {
	throw OperationNotSupported();
}

int SequenceFile::rank(TaxId taxid) const {
	throw OperationNotSupported();
}

TaxId SequenceFile::rank_taxid(TaxId taxid, int rank) {
	static const int max = 64;
	int n = 0;
	while (true) {
		if (this->rank(taxid) == rank)
			return taxid;
		if (taxid <= 1)
			return 0;
		if (++n > max)
			throw runtime_error("Path in taxonomy too long (rank_taxid).");
		taxid = get_parent(taxid);
	}
	return 0;
}

vector<TaxId> SequenceFile::lineage(TaxId taxid) {
	vector<TaxId> out;
	int n = 0;
	while (true) {
		if (taxid <= 0)
			return {};
		if (taxid == 1)
			break;
		if (++n > MAX_LINEAGE)
			throw runtime_error("Path in taxonomy too long (TaxonomyNodes::lineage).");
		out.push_back(taxid);
		taxid = get_parent(taxid);
	}
	reverse(out.begin(), out.end());
	return out;
}

TaxId SequenceFile::get_lca(TaxId t1, TaxId t2)
{
	if (t1 == t2 || t2 <= 0)
		return t1;
	if (t1 <= 0)
		return t2;
	TaxId p = t2;
	set<TaxId> l;
	l.insert(p);
	int n = 0;
	do {
		p = get_parent(p);
		if (p <= 0)
			return t1;
		l.insert(p);
		if (++n > MAX_LINEAGE)
			throw runtime_error("Path in taxonomy too long (get_lca).");
	} while (p != t1 && p != 1);
	if (p == t1)
		return p;
	p = t1;
	n = 0;
	while (l.find(p) == l.end()) {
		p = get_parent(p);
		if (p <= 0)
			return t2;
		if (++n > MAX_LINEAGE)
			throw runtime_error("Path in taxonomy too long (get_lca).");
	}
	return p;
}

TaxId SequenceFile::max_taxid() const {
	throw OperationNotSupported();
}

bool SequenceFile::contained(TaxId query, const set<TaxId>& filter, bool include_invalid)
{
	static const int max = 64;
	if (get_parent(query) < 0)
		return include_invalid;
	if (cached_[query])
		return contained_[query];
	if (filter.find(1) != filter.end())
		return true;
	int n = 0;
	TaxId p = query;
	while (p > 1 && filter.find(p) == filter.end()) {
		p = get_parent(p);
		if (p <= 0)
			return include_invalid;
		if (++n > max)
			throw runtime_error("Path in taxonomy too long (contained).");
	}
	const bool contained = p > 1;
	TaxId q = query;
	while (set_cached(q, contained), q != p)
		q = get_parent(q);
	return contained;
}

bool SequenceFile::contained(const vector<TaxId>& query, const set<TaxId>& filter, bool all, bool include_invalid)
{
	static const int max = 64;
	if (filter.find(1) != filter.end())
		return true;
	for (vector<TaxId>::const_iterator i = query.begin(); i != query.end(); ++i) {
		const bool c = contained(*i, filter, include_invalid);
		if (c && !all)
			return true;
		if (!c && all)
			return false;
	}
	return all;
}

void SequenceFile::init_cache() {
	const TaxId m = max_taxid();
	cached_.insert(cached_.end(), m + 1, false);
	contained_.insert(contained_.end(), m + 1, false);
}

void SequenceFile::print_info() const {
	*message_stream << "Database: " << config.database << ' ';
	*message_stream << "(type: " << to_string(type()) << ", ";
	*message_stream << "sequences: " << sequence_count().value() << ", ";
	*message_stream << "letters: " << letters().value() << ')' << endl;
}

void SequenceFile::init_taxon_output_fields() {
	using namespace std::placeholders;
	auto callback = [](const TabularFormat&, const HspContext& r, Output::Info& info, Rank rank) {
		set<TaxId> tax_id = info.db->rank_taxid(info.db->taxids(r.subject_oid), rank);
		tax_id.erase(-1);
		tax_id.erase(0);
		tax_id.erase(1);
		if (tax_id.empty())
			info.out << "N/A";
		else
			print_taxon_names(tax_id.begin(), tax_id.end(), *info.db, info.out);
		};
	auto star = [](const TabularFormat&, Output::Info& info) {
		info.out << '*';
		};
	for (int i = 1; i < Rank::count; ++i) {
		const FieldId next = FieldId((int)TabularFormat::field_def.rbegin()->first + 1);
		OutputField f;
		f.description = "Unique subject ";
		f.description += Rank::names[i];
		f.description += "(s), separated by a ';'";
		f.flags = Output::Flags::IS_ARRAY | Output::Flags::NO_REALIGN;
		f.hsp_values = HspValues::NONE;
		f.id = next;
		f.key = "s";
		f.key += Rank::names[i];
		std::replace(f.key.begin(), f.key.end(), ' ', '_');
		if (!TabularFormat::field_def.emplace(next, f).second)
			throw runtime_error("Duplicate taxonomic rank.");
		TabularFormat::field_callbacks[next].match = std::bind(callback, _1, _2, _3, Rank(i));
		TabularFormat::field_callbacks[next].query_intro = star;
	}
}

RawChunk* SequenceFile::raw_chunk(size_t letters, SequenceFile::Flags flags) {
	throw OperationNotSupported();
}

pair<vector<OId>, vector<uint64_t>> SequenceFile::partition(uint64_t max_block_size, uint64_t start_size) const {
	pair<vector<OId>, vector<uint64_t>> result;
	result.first.push_back(0);
	uint64_t block_size = start_size;
	for (OId i = 0; i < (OId)seq_length_.size(); ++i) {
		if (block_size + seq_length_[i] > max_block_size && block_size > 0) {
			result.first.push_back(i);
			result.second.push_back(block_size - (result.second.empty() ? start_size : 0));
			block_size = 0;
		}
		block_size += seq_length_[i];
	}
	result.first.push_back((OId)seq_length_.size());
	result.second.push_back(block_size - (result.second.empty() ? start_size : 0));
	return result;
}