/*
 * The MIT License
 *
 * Copyright (c) 2010 The Broad Institute
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package htsjdk.samtools;

import htsjdk.HtsjdkTest;
import htsjdk.samtools.cram.ref.ReferenceSource;
import htsjdk.samtools.cram.structure.CRAMEncodingStrategy;
import htsjdk.samtools.reference.InMemoryReferenceSequenceFile;
import htsjdk.samtools.reference.ReferenceSequenceFileFactory;
import htsjdk.samtools.util.IOUtil;
import htsjdk.samtools.util.Log.LogLevel;
import htsjdk.samtools.util.SequenceUtil;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import org.testng.Assert;
import org.testng.annotations.Test;

public class CRAMFileWriterTest extends HtsjdkTest {

    LogLevel globalLogLevel;
    final File SAM_TOOLS_TEST_DIR = new File("src/test/resources/htsjdk/samtools");

    @Test(description = "Test for lossy CRAM compression invariants.")
    public void lossyCramInvariantsTest() {
        doTest(createRecords(1000));
    }

    @Test(description = "Tests a writing records with null SAMFileHeaders")
    public void writeRecordsWithNullHeader() {
        final List<SAMRecord> samRecs = createRecords(50);
        for (final SAMRecord rec : samRecs) {
            rec.setHeader(null);
        }
        doTest(samRecs);
    }

    @Test(description = "Tests a unmapped record with sequence and quality fields")
    public void unmappedWithSequenceAndQualityField() throws Exception {
        unmappedSequenceAndQualityFieldHelper(true);
    }

    @Test(description = "Tests a unmapped record with no sequence or quality fields")
    public void unmappedWithNoSequenceAndQualityField() throws Exception {
        unmappedSequenceAndQualityFieldHelper(false);
    }

    private void unmappedSequenceAndQualityFieldHelper(boolean unmappedHasBasesAndQualities) throws Exception {
        final List<SAMRecord> list = new ArrayList<>(2);
        final SAMRecordSetBuilder builder = new SAMRecordSetBuilder();

        if (builder.getHeader().getReadGroups().isEmpty()) {
            throw new Exception("Read group expected in the header");
        }

        builder.setUnmappedHasBasesAndQualities(unmappedHasBasesAndQualities);
        builder.addUnmappedFragment("test1");
        builder.addUnmappedPair("test2");

        list.addAll(builder.getRecords());
        list.sort(new SAMRecordCoordinateComparator());

        doTest(list);
    }

    private List<SAMRecord> createRecords(int count) {
        final List<SAMRecord> list = new ArrayList<>(count);
        final SAMRecordSetBuilder builder = new SAMRecordSetBuilder();
        if (builder.getHeader().getReadGroups().isEmpty()) {
            throw new IllegalStateException("Read group expected in the header");
        }

        int posInRef = 1;
        for (int i = 0; i < count / 2; i++) {
            builder.addPair(Integer.toString(i), 0, posInRef += 1, posInRef += 3);
        }
        list.addAll(builder.getRecords());
        list.sort(new SAMRecordCoordinateComparator());

        return list;
    }

    private SAMFileHeader createSAMHeader(SAMFileHeader.SortOrder sortOrder) {
        final SAMFileHeader header = new SAMFileHeader();
        header.setSortOrder(sortOrder);
        header.addSequence(new SAMSequenceRecord("chr1", 123));
        SAMReadGroupRecord readGroupRecord = new SAMReadGroupRecord("1");
        header.addReadGroup(readGroupRecord);
        return header;
    }

    private ReferenceSource createReferenceSource() {
        final byte[] refBases = new byte[1024 * 1024];
        Arrays.fill(refBases, (byte) 'A');
        InMemoryReferenceSequenceFile rsf = new InMemoryReferenceSequenceFile();
        rsf.add("chr1", refBases);
        return new ReferenceSource(rsf);
    }

    private void writeRecordsToCRAM(CRAMFileWriter writer, List<SAMRecord> samRecords) {
        for (SAMRecord record : samRecords) {
            writer.addAlignment(record);
        }
    }

    private void validateRecords(
            final List<SAMRecord> expectedRecords, ByteArrayInputStream is, ReferenceSource referenceSource) {
        // CRAM decode regenerates missing NM/MD tags (matching htslib behavior), so add them
        // to expected records before comparison
        addMissingNmMd(expectedRecords, referenceSource);

        try (CRAMFileReader cReader = new CRAMFileReader(null, is, referenceSource)) {

            SAMRecordIterator iterator2 = cReader.getIterator();
            int index = 0;
            while (iterator2.hasNext()) {
                SAMRecord actualRecord = iterator2.next();
                SAMRecord expectedRecord = expectedRecords.get(index++);

                Assert.assertEquals(actualRecord.getReadName(), expectedRecord.getReadName());
                Assert.assertEquals(actualRecord.getFlags(), expectedRecord.getFlags());
                Assert.assertEquals(actualRecord.getAlignmentStart(), expectedRecord.getAlignmentStart());
                Assert.assertEquals(actualRecord.getAlignmentEnd(), expectedRecord.getAlignmentEnd());
                Assert.assertEquals(actualRecord.getReferenceName(), expectedRecord.getReferenceName());
                Assert.assertEquals(actualRecord.getMateAlignmentStart(), expectedRecord.getMateAlignmentStart());
                Assert.assertEquals(actualRecord.getMateReferenceName(), expectedRecord.getMateReferenceName());
                Assert.assertEquals(actualRecord.getReadBases(), expectedRecord.getReadBases());
                Assert.assertEquals(actualRecord.getBaseQualities(), expectedRecord.getBaseQualities());

                Assert.assertEquals(
                        actualRecord.getAttributes().stream().map(s -> s.tag).collect(Collectors.toSet()),
                        expectedRecord.getAttributes().stream().map(s -> s.tag).collect(Collectors.toSet()),
                        expectedRecord.getReadName());

                actualRecord.getAttributes().forEach(tv -> {
                    Assert.assertEquals(tv.value, expectedRecord.getAttribute(tv.tag));
                });
            }
        }
    }

    /** Add NM/MD to mapped records that are missing them, matching what CRAM decode will produce. */
    private static void addMissingNmMd(final List<SAMRecord> records, final ReferenceSource refSource) {
        for (final SAMRecord rec : records) {
            if (!rec.getReadUnmappedFlag() && rec.getReferenceIndex() >= 0 && rec.getAttribute(SAMTag.MD) == null) {
                final byte[] refBases =
                        refSource.getReferenceBases(rec.getHeader().getSequence(rec.getReferenceIndex()), false);
                if (refBases != null) {
                    SequenceUtil.calculateMdAndNmTags(rec, refBases, true, true);
                }
            }
        }
    }

    private void doTest(final List<SAMRecord> samRecords) {
        final SAMFileHeader header = createSAMHeader(SAMFileHeader.SortOrder.coordinate);
        final ReferenceSource refSource = createReferenceSource();
        final ByteArrayOutputStream os = new ByteArrayOutputStream();

        try (CRAMFileWriter writer = new CRAMFileWriter(os, refSource, header, null)) {
            writeRecordsToCRAM(writer, samRecords);
        }

        validateRecords(samRecords, new ByteArrayInputStream(os.toByteArray()), refSource);
    }

    @Test(description = "Test CRAMWriter constructor with index stream")
    public void testCRAMWriterWithIndex() {
        final SAMFileHeader header = createSAMHeader(SAMFileHeader.SortOrder.coordinate);
        final ReferenceSource refSource = createReferenceSource();
        final ByteArrayOutputStream outStream = new ByteArrayOutputStream();
        final ByteArrayOutputStream indexStream = new ByteArrayOutputStream();

        final List<SAMRecord> samRecords = createRecords(100);
        try (CRAMFileWriter writer = new CRAMFileWriter(outStream, indexStream, refSource, header, null)) {
            writeRecordsToCRAM(writer, samRecords);
        }

        validateRecords(samRecords, new ByteArrayInputStream(outStream.toByteArray()), refSource);
        Assert.assertTrue(indexStream.size() != 0);
    }

    @Test(description = "Test CRAMWriter constructor with presorted==false")
    public void testCRAMWriterNotPresorted() {
        final SAMFileHeader header = createSAMHeader(SAMFileHeader.SortOrder.coordinate);
        final ReferenceSource refSource = createReferenceSource();
        final ByteArrayOutputStream outStream = new ByteArrayOutputStream();
        final ByteArrayOutputStream indexStream = new ByteArrayOutputStream();

        final List<SAMRecord> samRecords = createRecords(100);
        try (CRAMFileWriter writer = new CRAMFileWriter(outStream, indexStream, false, refSource, header, null)) {

            // force records to not be coordinate sorted to ensure we're relying on presorted=false
            samRecords.sort(new SAMRecordCoordinateComparator().reversed());
            writeRecordsToCRAM(writer, samRecords);
        }
        // for validation, restore the sort order of the expected records so they match the order of the written records
        samRecords.sort(new SAMRecordCoordinateComparator());
        validateRecords(samRecords, new ByteArrayInputStream(outStream.toByteArray()), refSource);
        Assert.assertTrue(indexStream.size() != 0);
    }

    @Test(
            description = "Test CRAMWriter constructor reference required 1",
            expectedExceptions = IllegalArgumentException.class)
    public void testCRAMWriterConstructorRequiredReference_1() {
        final SAMFileHeader header = createSAMHeader(SAMFileHeader.SortOrder.coordinate);
        final ByteArrayOutputStream outStream = new ByteArrayOutputStream();
        try (CRAMFileWriter ignored = new CRAMFileWriter(outStream, null, header, null)) {}
    }

    @Test(
            description = "Test CRAMWriter constructor reference required 2",
            expectedExceptions = IllegalArgumentException.class)
    public void testCRAMWriterConstructorRequiredReference_2() {
        final SAMFileHeader header = createSAMHeader(SAMFileHeader.SortOrder.coordinate);
        final ByteArrayOutputStream outStream = new ByteArrayOutputStream();
        try (CRAMFileWriter ignored = new CRAMFileWriter(outStream, null, null, header, null)) {}
    }

    @Test(
            description = "Test CRAMWriter constructor reference required 3",
            expectedExceptions = IllegalArgumentException.class)
    public void testCRAMWriterConstructorRequiredReference_3() {
        final SAMFileHeader header = createSAMHeader(SAMFileHeader.SortOrder.coordinate);
        final ByteArrayOutputStream outStream = new ByteArrayOutputStream();
        try (CRAMFileWriter ignored = new CRAMFileWriter(outStream, null, true, null, header, null)) {}
    }

    @Test
    public void test_roundtrip_tlen_preserved() throws IOException {
        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
        final ReferenceSource source = new ReferenceSource(new File(SAM_TOOLS_TEST_DIR, "cram_tlen.fasta"));
        final List<SAMRecord> records = new ArrayList<>();

        try (SamReader reader =
                        SamReaderFactory.make().open(new File(SAM_TOOLS_TEST_DIR, "cram_tlen_reads.sorted.sam"));
                CRAMFileWriter writer = new CRAMFileWriter(baos, source, reader.getFileHeader(), "test.cram")) {
            for (final SAMRecord record : reader) {
                writer.addAlignment(record);
                records.add(record);
            }
        }

        // CRAM decode regenerates NM/MD, so add them to expected records
        addMissingNmMd(records, source);

        try (CRAMFileReader cramReader = new CRAMFileReader(
                new ByteArrayInputStream(baos.toByteArray()), (File) null, source, ValidationStringency.STRICT)) {
            final SAMRecordIterator iterator = cramReader.getIterator();
            int i = 0;
            while (iterator.hasNext()) {
                final SAMRecord record1 = iterator.next();
                final SAMRecord record2 = records.get(i++);
                Assert.assertEquals(
                        record1.getInferredInsertSize(), record2.getInferredInsertSize(), record1.getReadName());
                Assert.assertEquals(record1, record2, record1.getReadName());
            }
            Assert.assertEquals(records.size(), i);
        }
    }

    @Test
    public void test_roundtrip_many_reads() throws IOException {

        // Create the input file
        final File outputDir =
                IOUtil.createTempDir(this.getClass().getSimpleName() + ".tmp").toFile();
        outputDir.deleteOnExit();
        final Path output = new File(outputDir, "input.cram").toPath();
        IOUtil.deleteOnExit(output);
        final Path fastaDir = IOUtil.createTempDir("CRAMFileWriterTest");
        IOUtil.deleteOnExit(fastaDir);
        final Path newFasta = fastaDir.resolve("input.fasta");
        IOUtil.deleteOnExit(newFasta);
        IOUtil.deleteOnExit(ReferenceSequenceFileFactory.getFastaIndexFileName(newFasta));
        IOUtil.deleteOnExit(ReferenceSequenceFileFactory.getDefaultDictionaryForReferenceSequence(newFasta));

        final SAMRecordSetBuilder samRecordSetBuilder = new SAMRecordSetBuilder();
        samRecordSetBuilder.setHeader(
                SAMRecordSetBuilder.makeDefaultHeader(SAMFileHeader.SortOrder.coordinate, 10_000, true));
        samRecordSetBuilder.writeRandomReference(newFasta);

        final List<SAMRecord> records = new ArrayList<>();

        try (SAMFileWriter writer =
                new SAMFileWriterFactory().makeWriter(samRecordSetBuilder.getHeader(), true, output, newFasta)) {

            // make sure we don't write reads that go off the end of the reference, which is 10,000 bases
            for (int position = 1; position <= 9900; position += 1) {
                samRecordSetBuilder.addFrag("read_" + position, 0, position, false);
            }
            samRecordSetBuilder.getRecords().forEach(r -> {
                writer.addAlignment(r);
                records.add(r);
            });
        }

        // CRAM decode regenerates NM/MD, so add them to expected records
        final ReferenceSource refSourceForValidation = new ReferenceSource(newFasta);
        addMissingNmMd(records, refSourceForValidation);

        try (SamReader cramReader =
                SamReaderFactory.make().referenceSequence(newFasta).open(output)) {
            final SAMRecordIterator iterator = cramReader.iterator();
            int i = 0;
            while (iterator.hasNext()) {
                final SAMRecord record1 = iterator.next();
                final SAMRecord record2 = records.get(i++);
                Assert.assertEquals(
                        record1.getInferredInsertSize(), record2.getInferredInsertSize(), record1.getReadName());
                Assert.assertEquals(record1, record2, record1.getReadName());
            }
            Assert.assertEquals(records.size(), i);
        }
    }

    @Test
    public void testCRAMQuerySort() throws IOException {
        final File input = new File(SAM_TOOLS_TEST_DIR, "cram_query_sorted.cram");
        final File reference = new File(SAM_TOOLS_TEST_DIR, "cram_query_sorted.fasta");
        final File outputFile = File.createTempFile("tmp.", ".cram");
        final SamReaderFactory samReaderFactory = SamReaderFactory.makeDefault().referenceSequence(reference);

        try (final SamReader reader = samReaderFactory.open(input);
                final SAMFileWriter writer = new SAMFileWriterFactory()
                        .makeWriter(reader.getFileHeader().clone(), false, outputFile, reference)) {
            for (final SAMRecord rec : reader) {
                writer.addAlignment(rec);
            }
        }

        try (final SamReader outReader = samReaderFactory.open(outputFile)) {
            String prevName = null;
            for (final SAMRecord rec : outReader) {
                if (prevName == null) {
                    prevName = rec.getReadName();
                    continue;
                }
                // test if the read names are sorted alphabetically:
                Assert.assertTrue(rec.getReadName().compareTo(prevName) >= 0);
            }
        }
    }

    /**
     * Verify that records with NM and MD tags survive a CRAM round-trip: tags are stripped during
     * encoding and regenerated during decoding.
     */
    @Test
    public void testNmMdRegenerationRoundTrip() {
        final SAMFileHeader header = createSAMHeader(SAMFileHeader.SortOrder.coordinate);
        final ReferenceSource refSource = createReferenceSource();

        // Create a mapped record with NM and MD
        final SAMRecord record = new SAMRecord(header);
        record.setReadName("testRead");
        record.setReferenceName("chr1");
        record.setAlignmentStart(1);
        record.setCigarString("10M");
        record.setReadBases("AAAAAAAAAA".getBytes());
        record.setBaseQualities("IIIIIIIIII".getBytes());
        record.setReadPairedFlag(false);
        record.setReadUnmappedFlag(false);
        record.setAttribute(SAMTag.NM, 0);
        record.setAttribute(SAMTag.MD, "10");
        record.setAttribute(SAMTag.RG, header.getReadGroups().get(0).getId());

        final ByteArrayOutputStream os = new ByteArrayOutputStream();
        try (CRAMFileWriter writer = new CRAMFileWriter(os, refSource, header, null)) {
            writer.addAlignment(record);
        }

        try (CRAMFileReader reader = new CRAMFileReader(null, new ByteArrayInputStream(os.toByteArray()), refSource)) {
            final SAMRecord decoded = reader.getIterator().next();
            Assert.assertEquals(
                    decoded.getIntegerAttribute(SAMTag.NM),
                    Integer.valueOf(0),
                    "NM should be regenerated during CRAM decode");
            Assert.assertEquals(
                    decoded.getStringAttribute(SAMTag.MD), "10", "MD should be regenerated during CRAM decode");
            Assert.assertNull(decoded.getAttribute("cF"), "Internal cF tag should be stripped during decode");
        }
    }

    /**
     * Verify that storeNM=true/storeMD=true preserves NM and MD verbatim without cF tag.
     */
    @Test
    public void testStoreNmMdVerbatim() {
        final SAMFileHeader header = createSAMHeader(SAMFileHeader.SortOrder.coordinate);
        final ReferenceSource refSource = createReferenceSource();
        final CRAMEncodingStrategy strategy =
                new CRAMEncodingStrategy().setStoreNM(true).setStoreMD(true);

        // Create a mapped record with NM and MD
        final SAMRecord record = new SAMRecord(header);
        record.setReadName("verbatimNmMd");
        record.setReferenceName("chr1");
        record.setAlignmentStart(1);
        record.setCigarString("10M");
        record.setReadBases("AAAAAAAAAA".getBytes());
        record.setBaseQualities("IIIIIIIIII".getBytes());
        record.setReadPairedFlag(false);
        record.setReadUnmappedFlag(false);
        record.setAttribute(SAMTag.NM, 0);
        record.setAttribute(SAMTag.MD, "10");
        record.setAttribute(SAMTag.RG, header.getReadGroups().get(0).getId());

        final ByteArrayOutputStream os = new ByteArrayOutputStream();
        try (CRAMFileWriter writer = new CRAMFileWriter(strategy, os, null, true, refSource, header, null)) {
            writer.addAlignment(record);
        }

        try (CRAMFileReader reader = new CRAMFileReader(null, new ByteArrayInputStream(os.toByteArray()), refSource)) {
            final SAMRecord decoded = reader.getIterator().next();
            Assert.assertEquals(
                    decoded.getIntegerAttribute(SAMTag.NM), Integer.valueOf(0), "NM should be preserved verbatim");
            Assert.assertEquals(decoded.getStringAttribute(SAMTag.MD), "10", "MD should be preserved verbatim");
            Assert.assertNull(decoded.getAttribute("cF"), "cF tag should not exist when NM/MD are stored verbatim");
        }
    }
}
