/**
 * Distribution License:
 * JSword is free software; you can redistribute it and/or modify it under
 * the terms of the GNU Lesser General Public License, version 2.1 or later
 * as published by the Free Software Foundation. This program is distributed
 * in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even
 * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU Lesser General Public License for more details.
 *
 * The License is available on the internet at:
 *       http://www.gnu.org/copyleft/lgpl.html
 * or by writing to:
 *      Free Software Foundation, Inc.
 *      59 Temple Place - Suite 330
 *      Boston, MA 02111-1307, USA
 *
 * Copyright: 2005-2013
 *     The copyright to this program is held by its authors.
 *
 */
package org.crosswire.jsword.book.sword;

import java.io.IOException;
import java.io.RandomAccessFile;

import org.crosswire.common.compress.CompressorType;
import org.crosswire.jsword.JSMsg;
import org.crosswire.jsword.book.BookException;
import org.crosswire.jsword.book.BookMetaData;
import org.crosswire.jsword.book.sword.state.OpenFileStateManager;
import org.crosswire.jsword.book.sword.state.ZVerseBackendState;
import org.crosswire.jsword.passage.BitwisePassage;
import org.crosswire.jsword.passage.Key;
import org.crosswire.jsword.passage.KeyUtil;
import org.crosswire.jsword.passage.RocketPassage;
import org.crosswire.jsword.passage.Verse;
import org.crosswire.jsword.versification.Testament;
import org.crosswire.jsword.versification.Versification;
import org.crosswire.jsword.versification.system.Versifications;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A backend to read compressed data verse based files. While the text file
 * contains data compressed with ZIP or LZSS, it cannot be uncompressed using a
 * stand alone zip utility, such as WinZip or gzip. The reason for this is that
 * the data file is a concatenation of blocks of compressed data.
 * 
 * <p>
 * The blocks can either be "b", book (aka testament); "c", chapter or "v",
 * verse. The choice is a matter of trade offs. The program needs to uncompress
 * a block into memory. Having it at the book level is very memory expensive.
 * Having it at the verse level is very disk expensive, but takes the least
 * amount of memory. The most common is chapter.
 * </p>
 * 
 * <p>
 * In order to find the data in the text file, we need to find the block. The
 * first index (idx) is used for this. Each verse is indexed to a tuple (block
 * number, verse start, verse size). This data allows us to find the correct
 * block, and to extract the verse from the uncompressed block, but it does not
 * help us uncompress the block.
 * </p>
 * 
 * <p>
 * Once the block is known, then the next index (comp) gives the location of the
 * compressed block, its compressed size and its uncompressed size.
 * </p>
 * 
 * <p>
 * There are 3 files for each testament, 2 (idx and comp) are indexes into the
 * third (text) which contains the data. The key into each index is the verse
 * index within that testament, which is determined by book, chapter and verse
 * of that key.
 * </p>
 * 
 * <p>
 * All unsigned numbers are stored 2-complement, little endian.
 * </p>
 * <p>
 * Then proceed as follows, at all times working on the set of files for the
 * testament in question:
 * </p>
 * 
 * The three files are laid out in the following fashion:
 * <ul>
 * <li>The idx file has one entry per verse in the versification. The number
 * of verses varies by versification and testament. Each entry describes the
 * compressed block in which it is found, the start of the verse in the
 * uncompressed block and the length of the verse.
 * <ul>
 * <li>Block number - 32-bit/4-bytes - the number of the entry in the comp file.</li>
 * <li>Verse start - 32 bit/4-bytes - the start of the verse in the uncompressed block in the dat file.</li>
 * <li>Verse length - 16 bit/4-bytes - the length of the verse in the uncompressed block from the dat file.</li>
 * </ul>
 * Algorithm:
 * <ul>
 * <li>Given the ordinal value of the verse, seek to the ordinal * 10 and read 10 bytes.
 * <li>Decode the 10 bytes as Block Number, Verse start and length</li>
 * </ul>
 * </li>
 * <li>The comp file has one entry per block.
 * Each entry describes the location of a compressed block,
 * giving its start and size in the next file.
 * <ul>
 * <li>Block Start - 32-bit/4-byte - the start of the block in the dat file</li>
 * <li>Compressed Block Size - 32-bit/4-byte - the size of the compressed block in the dat file</li>
 * <li>Uncompressed Block Size - 32-bit/4-byte - the size of the block after uncompressing</li>
 * </ul>
 * Algorithm:
 * <ul>
 * <li>Given a block number, seek to block-index * 12 and read 12 bytes</li>
 * <li>Decode the 12 bytes as Block Start, Compressed Block Size and Uncompressed Block Size</li>
 * </ul>
 * </li>
 * <li> The dat file is compressed blocks of verses.
 * <br>
 * Algorithm:
 * <ul>
 * <li>Given the entry from the comp file, seek to the start and read the indicated compressed block size</li>
 * <li>If the book is enciphered it, decipher it.</li>
 * <li>Uncompress the block, using the uncompressed size as an optimization.</li>
 * <li>Using the verse start, seek to that location in the uncompressed block and read the indicated verse size.</li>
 * <li>Convert the bytes to a String using the books indicated charset.</li>
 * </ul>
 * </li>
 * </ul>
 * 
 * @see gnu.lgpl.License The GNU Lesser General Public License for details.
 * @author Joe Walker
 * @author DM Smith
 */
public class ZVerseBackend extends AbstractBackend<ZVerseBackendState> {
    /**
     * Simple ctor
     * @param sbmd 
     * @param blockType 
     */
    public ZVerseBackend(SwordBookMetaData sbmd, BlockType blockType) {
        super(sbmd);
        this.blockType = blockType;
    }

    /* This method assumes single keys. It is the responsibility of the caller to provide the iteration. 
     * 
     * FIXME: this could be refactored to push the iterations down, but no performance benefit would be gained since we have a manager that keeps the file accesses open
     * (non-Javadoc)
     * @see org.crosswire.jsword.book.sword.AbstractBackend#contains(org.crosswire.jsword.passage.Key)
     */
    @Override
    public boolean contains(Key key) {
        return getRawTextLength(key) > 0;
    }

    /* (non-Javadoc)
     * @see org.crosswire.jsword.book.sword.AbstractBackend#size(org.crosswire.jsword.passage.Key)
     */
    @Override
    public int getRawTextLength(Key key) {
        ZVerseBackendState rafBook = null;
        try {
            rafBook = initState();

            String v11nName = getBookMetaData().getProperty(BookMetaData.KEY_VERSIFICATION);
            Versification v11n = Versifications.instance().getVersification(v11nName);
            Verse verse = KeyUtil.getVerse(key);

            int index = verse.getOrdinal();
            Testament testament = v11n.getTestament(index);
            index = v11n.getTestamentOrdinal(index);

            RandomAccessFile idxRaf = rafBook.getIdxRaf(testament);

            // If Bible does not contain the desired testament, then false
            if (idxRaf == null) {
                return 0;
            }

            // 10 because the index is 10 bytes long for each verse
            byte[] temp = SwordUtil.readRAF(idxRaf, 1L * index * IDX_ENTRY_SIZE, IDX_ENTRY_SIZE);

            // If the Bible does not contain the desired verse, return nothing.
            // Some Bibles have different versification, so the requested verse
            // may not exist.
            if (temp == null || temp.length == 0) {
                return 0;
            }

            // The data is little endian - extract the verseSize
            return SwordUtil.decodeLittleEndian16(temp, 8);

        } catch (IOException e) {
            return 0;
        } catch (BookException e) {
            // FIXME(CJB): fail silently as before, but i don't think this is
            // correct behaviour - would cause API changes
            log.error("Unable to ascertain key validity", e);
            return 0;
        } finally {
            OpenFileStateManager.instance().release(rafBook);
        }
    }

    /* (non-Javadoc)
     * @see org.crosswire.jsword.book.sword.AbstractBackend#getGlobalKeyList()
     */
    @Override
    public Key getGlobalKeyList() throws BookException {
        ZVerseBackendState rafBook = null;
        try {
            rafBook = initState();

            String v11nName = getBookMetaData().getProperty(BookMetaData.KEY_VERSIFICATION);
            Versification v11n = Versifications.instance().getVersification(v11nName);

            Testament[] testaments = new Testament[] {
                    Testament.OLD, Testament.NEW
            };

            BitwisePassage passage = new RocketPassage(v11n);
            passage.raiseEventSuppresion();
            passage.raiseNormalizeProtection();

            for (Testament currentTestament : testaments) {
                RandomAccessFile idxRaf = rafBook.getIdxRaf(currentTestament);

                // If Bible does not contain the desired testament, then false
                if (idxRaf == null) {
                    // no keys in this testament
                    continue;
                }

                int maxIndex = v11n.getCount(currentTestament) - 1;

                // Read in the whole index, a few hundred Kb at most.
                byte[] temp = SwordUtil.readRAF(idxRaf, 0, IDX_ENTRY_SIZE * maxIndex);

                // For each entry of 10 bytes, the length of the verse in bytes
                // is in the last 2 bytes. If both bytes are 0, then there is no content.
                for (int ii = 0; ii < temp.length; ii += IDX_ENTRY_SIZE) {
                    // This can be simplified to temp[ii + 8] == 0 && temp[ii + 9] == 0.
                    // int verseSize = SwordUtil.decodeLittleEndian16(temp, ii + 8);
                    // if (verseSize > 0) {
                    if (temp[ii + 8] != 0 || temp[ii + 9] != 0) {
                        int ordinal = ii / IDX_ENTRY_SIZE;
                        passage.addVersifiedOrdinal(v11n.getOrdinal(currentTestament, ordinal));
                    }
                }
            }

            passage.lowerNormalizeProtection();
            passage.lowerEventSuppressionAndTest();

            return passage;
        } catch (IOException e) {
            throw new BookException(JSMsg.gettext("Unable to read key list from book."));
        } finally {
            OpenFileStateManager.instance().release(rafBook);
        }
    }

    /* (non-Javadoc)
     * @see org.crosswire.jsword.book.sword.StatefulFileBackedBackend#initState()
     */
    public ZVerseBackendState initState() throws BookException {
        return OpenFileStateManager.instance().getZVerseBackendState(getBookMetaData(), blockType);
    }

    /* (non-Javadoc)
     * @see org.crosswire.jsword.book.sword.StatefulFileBackedBackend#readRawContent(org.crosswire.jsword.book.sword.state.OpenFileState, org.crosswire.jsword.passage.Key)
     */
    public String readRawContent(ZVerseBackendState rafBook, Key key) throws IOException {

        SwordBookMetaData bookMetaData = getBookMetaData();
        final String charset = bookMetaData.getBookCharset();
        final String compressType = bookMetaData.getProperty(SwordBookMetaData.KEY_COMPRESS_TYPE);

        final String v11nName = getBookMetaData().getProperty(BookMetaData.KEY_VERSIFICATION);
        final Versification v11n = Versifications.instance().getVersification(v11nName);
        Verse verse = KeyUtil.getVerse(key);

        int index = verse.getOrdinal();
        final Testament testament = v11n.getTestament(index);
        index = v11n.getTestamentOrdinal(index);
        final RandomAccessFile idxRaf;
        final RandomAccessFile compRaf;
        final RandomAccessFile textRaf;

        idxRaf = rafBook.getIdxRaf(testament);
        compRaf = rafBook.getCompRaf(testament);
        textRaf = rafBook.getTextRaf(testament);

        // If Bible does not contain the desired testament, return nothing.
        if (idxRaf == null) {
            return "";
        }

        //dumpIdxRaf(v11n, 0, compRaf);
        //dumpCompRaf(idxRaf);
        // 10 because the index is 10 bytes long for each verse
        byte[] temp = SwordUtil.readRAF(idxRaf, 1L * index * IDX_ENTRY_SIZE, IDX_ENTRY_SIZE);

        // If the Bible does not contain the desired verse, return nothing.
        // Some Bibles have different versification, so the requested verse
        // may not exist.
        if (temp == null || temp.length == 0) {
            return "";
        }

        // The data is little endian - extract the blockNum, verseStart
        // and
        // verseSize
        final long blockNum = SwordUtil.decodeLittleEndian32(temp, 0);
        final int verseStart = SwordUtil.decodeLittleEndian32(temp, 4);
        final int verseSize = SwordUtil.decodeLittleEndian16(temp, 8);

        // Can we get the data from the cache
        byte[] uncompressed = null;
        if (blockNum == rafBook.getLastBlockNum() && testament == rafBook.getLastTestament()) {
            uncompressed = rafBook.getLastUncompressed();
        } else {
            // Then seek using this index into the idx file
            temp = SwordUtil.readRAF(compRaf, blockNum * COMP_ENTRY_SIZE, COMP_ENTRY_SIZE);
            if (temp == null || temp.length == 0) {
                return "";
            }

            final int blockStart = SwordUtil.decodeLittleEndian32(temp, 0);
            final int blockSize = SwordUtil.decodeLittleEndian32(temp, 4);
            final int uncompressedSize = SwordUtil.decodeLittleEndian32(temp, 8);

            // Read from the data file.
            final byte[] data = SwordUtil.readRAF(textRaf, blockStart, blockSize);

            decipher(data);

            uncompressed = CompressorType.fromString(compressType).getCompressor(data).uncompress(uncompressedSize).toByteArray();

            // cache the uncompressed data for next time
            rafBook.setLastBlockNum(blockNum);
            rafBook.setLastTestament(testament);
            rafBook.setLastUncompressed(uncompressed);
        }

        // and cut out the required section.
        final byte[] chopped = new byte[verseSize];
        System.arraycopy(uncompressed, verseStart, chopped, 0, verseSize);

        return SwordUtil.decode(key.getName(), chopped, charset);

    }

    /* (non-Javadoc)
     * @see org.crosswire.jsword.book.sword.AbstractBackend#setAliasKey(org.crosswire.jsword.passage.Key, org.crosswire.jsword.passage.Key)
     */
    public void setAliasKey(ZVerseBackendState rafBook, Key alias, Key source) throws IOException {
        throw new UnsupportedOperationException();
    }

    /* (non-Javadoc)
     * @see org.crosswire.jsword.book.sword.AbstractBackend#setRawText(org.crosswire.jsword.passage.Key, java.lang.String)
     */
    public void setRawText(ZVerseBackendState rafBook, Key key, String text) throws BookException, IOException {
        throw new UnsupportedOperationException();
    }

    /** 
     * Experimental code.
     * 
     * @param v11n
     * @param ordinalStart
     * @param raf
     */
    public void dumpIdxRaf(Versification v11n, int ordinalStart, RandomAccessFile raf) {
        long end = -1;
        try {
            end = raf.length();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

        int i = ordinalStart;
        StringBuilder buf = new StringBuilder();
        System.out.println("osisID\tblock\tstart\tsize");
        for (long offset = 0; offset < end; offset += IDX_ENTRY_SIZE) {
            // 10 because the index is 10 bytes long for each verse
            byte[] temp = null;
            try {
                temp = SwordUtil.readRAF(raf, offset, IDX_ENTRY_SIZE);
            } catch (IOException e) {
                e.printStackTrace();
            }

            // If the Bible does not contain the desired verse, return nothing.
            // Some Bibles have different versification, so the requested verse
            // may not exist.
            long blockNum = -1;
            int verseStart = -1;
            int verseSize = -1;
            if (temp != null && temp.length > 0) {
                // The data is little endian - extract the blockNum, verseStart and verseSize
                blockNum = SwordUtil.decodeLittleEndian32(temp, 0);
                verseStart = SwordUtil.decodeLittleEndian32(temp, 4);
                verseSize = SwordUtil.decodeLittleEndian16(temp, 8);
            }
            buf.setLength(0);
            buf.append(v11n.decodeOrdinal(i++).getOsisID());
            buf.append('\t');
            buf.append(blockNum);
            buf.append('\t');
            buf.append(verseStart);
            buf.append('\t');
            buf.append(verseSize);
            System.out.println(buf.toString());
        }
    }

    /**
     * Experimental code.
     * 
     * @param raf
     */
    public void dumpCompRaf(RandomAccessFile raf) {
        long end = -1;
        try {
            end = raf.length();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

        int blockNum = 0;
        StringBuilder buf = new StringBuilder();
        System.out.println("block\tstart\tsize\tuncompressed");
        for (long offset = 0; offset < end; offset += COMP_ENTRY_SIZE) {
            // 12 because the index is 12 bytes long for each verse
            byte[] temp = null;
            try {
                temp = SwordUtil.readRAF(raf, offset, COMP_ENTRY_SIZE);
            } catch (IOException e) {
                e.printStackTrace();
            }

            // If the Bible does not contain the desired verse, return nothing.
            // Some Bibles have different versification, so the requested verse
            // may not exist.
            int blockStart = -1;
            int blockSize = -1;
            int uncompressedSize = -1;
            if (temp != null && temp.length > 0) {
                // The data is little endian - extract the blockNum, verseStar and verseSize
                 blockStart = SwordUtil.decodeLittleEndian32(temp, 0);
                 blockSize = SwordUtil.decodeLittleEndian32(temp, 4);
                 uncompressedSize = SwordUtil.decodeLittleEndian32(temp, 8);
            }
            buf.setLength(0);
            buf.append(blockNum);
            buf.append('\t');
            buf.append(blockStart);
            buf.append('\t');
            buf.append(blockSize);
            buf.append('\t');
            buf.append(uncompressedSize);
            System.out.println(buf.toString());
        }
    }

    /**
     * Whether the book is blocked by Book, Chapter or Verse.
     */
    private final BlockType blockType;

    /**
     * How many bytes in the idx index?
     */
    private static final int IDX_ENTRY_SIZE = 10;

    /**
     * How many bytes in the comp index?
     */
    private static final int COMP_ENTRY_SIZE = 12;

    /**
     * The log stream
     */
    private static final Logger log = LoggerFactory.getLogger(ZVerseBackend.class);
}
