/*
 * Copyright (C) 2007-2009 KenD00
 * 
 * This file is part of DumpHD.
 * 
 * DumpHD is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package dumphd.core;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.*;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
import java.util.regex.*;

import dumphd.util.Utils;

/**
 * Class for accessing the key database in the format DumpHD 1.3.
 * For reading there is backward compatibility to BackupHDDVD 1.0 but not BackupBluRay 0.21.
 * All data gets written in DumpHD 1.3 format.
 * 
 * The file is stored as UTF-8 without BOM and uses Windows line terminators (CR+LF). However, a file with only LF can be read,
 * but editing such lines will convert these line terminators to CR+LF.
 * 
 * TODO: Current setKeyData can damage the database if not enough diskspace is left (nothing can be done about this), optional allow 2 file writing?
 * TODO: Support ignoring of unknown ENTRY ID's?
 * 
 * @author KenD00
 */
public class KeyDataFile {


   /**
    * Size of the buffers used for reading the key database file. This value must be at least as big to hold one decoded entry line.
    */
   private final static int LINEBUFFER_SIZE = 16 * 1024;
   //private final static int LINEBUFFER_SIZE = 5;
   /**
    * This pattern matches one line, line terminator is system dependent, CR+LF and LF are supported
    */
   private final static Pattern linePattern = Pattern.compile("(.*)\r?\n");
   /*
    * The following Strings identify various data type identifiers and the corresponding data.
    * They always start with the separator char | and go just before the next separator char.
    * Some of them may be applied multiple times, e.g. Title Keys
    */
   /**
    * This String identifies the data type id's
    * 
    * Group 1: The identifier
    */
   private final static String dataIdPatternString = "\\|[ \\t]*([DMIBVPTU])[ \\t]*";
   /**
    * This String identifies the date data, old and new format
    * 
    * Group 1: New date format (nd)
    * Group 2: nd year
    * Group 3: nd month
    * Group 4: nd day
    * Group 5: Old date format (od)
    * Group 6: od month
    * Group 7: od day
    * Group 8: od year
    */
   private final static String dPatternString = "\\|[ \\t]*(?:(([0-9]{4})[ \\t]*-[ \\t]*([0-9]{2})[ \\t]*-[ \\t]*([0-9]{2}))|(([0-9M]{2})[ \\t]*/[ \\t]*([0-9D]{2})[ \\t]*/[ \\t]*([0-9Y]{2})))[ \\t]*";
   /**
    * This String identifies the data for the MEK, VID and VUK
    * 
    * Group 1: The key
    */
   private final static String ivPatternString = "\\|[ \\t]*([0-9ABCDEF]{32})[ \\t]*";
   /**
    * This String identifies the data for the Title Key and CPS Unit Key. There may be more than entry.
    * This String is also used for Binding Nonces and Protected Area Keys.
    * 
    * Group 1: The key number
    * Group 2: The key
    */
   private final static String tuPatternString = "\\|[ \\t]*([0-9]{1,5})[ \\t]*-[ \\t]*([0-9ABCDEF]{32})[ \\t]*";
   /*
    * End of data (type) identifiers
    */
   /**
    * This Pattern identifies the beginning of the dataset line, that is the DiscID, Title, the first Content Type identifier and optional the old-style date field following it
    * 
    * Group 1        : The DiscID
    * Group 2        : The title
    * Group x        : Groups from dataIdPatternString
    * Group x + 1    : Date data
    * Group x + 1 + y: Groups from dPatternString
    */
   //private final static Pattern datasetIdPattern = Pattern.compile("[ \\t]*([0-9ABCDEF]{40})[ \\t]*=[ \\t]*(.*?)[ \\t]*" + dataIdPatternString + "(" + dPatternString + ")?", Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
   // TODO: This expression "adds" all whitespace after the title up to the first pipe to the title but the previous expression
   //       will "add" broken Data Entries to the title until it finds a correct one
   private final static Pattern datasetIdPattern = Pattern.compile("[ \\t]*([0-9ABCDEF]{40})[ \\t]*=[ \\t]*([^|]*)" + dataIdPatternString + "(" + dPatternString + ")?", Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
   /**
    * This Pattern identifies the data type id
    * 
    * Groups are from dataIdPatternString
    */
   private final static Pattern dataIdPattern = Pattern.compile("(?:;(.*))|(?:" + dataIdPatternString + ")", Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
   /**
    * This Pattern identifies the data
    * 
    * Groups are from the following Strings in the following order:
    * dPatternString
    * ivPatternString
    * tuPatternString
    */
   private final static Pattern dataPattern = Pattern.compile("(?:" + dPatternString + ")|(?:" + ivPatternString + ")|(?:" + tuPatternString + ")", Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE); 

   /**
    * The file object used to open the physical file
    */
   private RandomAccessFile raf = null;
   /**
    * Acquired FileChannel from raf. All reading and writing to the file is done through this channel.
    */
   private FileChannel fc = null;
   /**
    * Direct buffer to read into from file
    */
   private ByteBuffer bb = null;
   /**
    * Helper buffer, used for temporary encoding of cb's contents to determine its byte size
    */
   private ByteBuffer bbh = null;
   /**
    * Buffer used for encoding and decoding chars from / to file
    */
   private CharBuffer cb = null;
   /**
    * Decodes bytes to chars
    */
   private CharsetDecoder decoder = null;
   /**
    * Encodes chars to bytes
    */
   private CharsetEncoder encoder = null;


   /**
    * Opens the specified file and initializes the object.
    * 
    * @param src The key database file to open. If the file is not present, it gets created.
    * @throws IOException An I/O error occurred
    */
   public KeyDataFile(File src) throws IOException {
      raf = new RandomAccessFile(src, "rw");
      fc = raf.getChannel();
      bb = ByteBuffer.allocateDirect(LINEBUFFER_SIZE);
      bbh = ByteBuffer.allocate(LINEBUFFER_SIZE);
      cb = CharBuffer.allocate(LINEBUFFER_SIZE);
      decoder = Charset.forName("UTF-8").newDecoder();
      encoder = Charset.forName("UTF-8").newEncoder();
   }

   /**
    * Determines which parameters should be written. Avoids redundant data and uses the highest level of data.
    * E.g. if a VUK is present the VUK gets written instead of the TUKs.
    * 
    * TODO: Make this private?
    *  
    * @param kd The KeyData object to determine the mode for
    * @return The write mode
    */
   public int getSmartMode(KeyData kd) {
      int mode = 0;
      // Always write the title
      mode |= KeyData.DATA_TITLE;
      // Always write the date
      mode |= KeyData.DATA_DATE;
      // Always write the comment
      mode |= KeyData.DATA_COMMENT;
      // Write VUK if present, otherwise write the MEK/VID or TUKs
      if ((kd.getVuk() != null) || (kd.pakCount() > 0)) {
         mode |= KeyData.DATA_VUKPAK;
      } else if ((kd.getMek() != null) && ((kd.getVid() != null) || (kd.bnCount() > 0))) {
         mode |= KeyData.DATA_MEK | KeyData.DATA_VIDBN;
      } else {
         mode |= KeyData.DATA_TUK;
      }
      return mode;
   }

   /**
    * Builds an entry string for the key database file from the given KeyData object. This string includes the line terminator.
    * 
    * TODO: Make this private?
    * 
    * @param kd The KeyData to build the entry for
    * @param bluRay Set this to true if the KeyData is from a BD
    * @param recordable Set this to true if the KeyData is from a recordable
    * @param mode The write mode to use. This is a binary OR of KeyData.DATA_*-Types, every supplied entry type gets written if present (except date, an empty date will be written if none is present)
    * @return The entry for the key database, the string contains the trailing newline
    */
   public String buildEntry(KeyData kd, boolean bluRay, boolean recordable, int mode) {
      StringBuffer sb = new StringBuffer(8192);
      byte[] temp = kd.getDiscId();
      sb.append(Utils.toHexString(temp, 0, temp.length));
      sb.append(" = ");
      String title = null;
      if ((mode & KeyData.DATA_TITLE) != 0) {
         title = kd.getTitle();
         if (title != null) {
            // Remove all | and cr/lf characters, they will break the entry because they have a special meaning
            title = title.replaceAll("(?:\\|)|(?:\r?\n)", "");
            // Finally remove leading and trailing whitespace
            // This must be done after the special characters have been removed to ensure that not a whitespace-only string remains
            title = title.trim();
         }
      }
      if (title == null) {
         title = "";
      }
      // Format the title that it has a minimum width
      //TODO: Longer minwidth?
      if (title.length() < 48) {
         title = String.format("%1$-48s", title);
      }
      sb.append(title);
      // Special case for date, allow an empty value
      if ((mode & KeyData.DATA_DATE) != 0) {
         sb.append(" | D | ");
         Date date = kd.getDate();
         if (date != null) {
            sb.append(String.format("%1$tY-%1$tm-%1$td", date));
         }
         else {
            sb.append("0000-00-00");
         }
      }
      // For all other entries, write them only if data is available for them
      if ((mode & KeyData.DATA_MEK) != 0) {
         temp = kd.getMek();
         if (temp != null) {
            sb.append(" | M | ");
            sb.append(Utils.toHexString(temp, 0, temp.length));
         }
      }
      if ((mode & KeyData.DATA_VIDBN) != 0) {
         if (recordable) {
            if (kd.bnCount() > 0) {
               sb.append(" | B");
               Iterator<Integer> it = kd.bnIdx().iterator();
               while (it.hasNext()) {
                  int bnIndex = it.next();
                  temp = kd.getBn(bnIndex);
                  sb.append(" | ");
                  sb.append(bnIndex);
                  sb.append("-");
                  sb.append(Utils.toHexString(temp, 0, temp.length));
               }
            }
         } else {
            temp = kd.getVid();
            if (temp != null) {
               sb.append(" | I | ");
               sb.append(Utils.toHexString(temp, 0, temp.length));
            }
         }
      }
      if ((mode & KeyData.DATA_VUKPAK) != 0) {
         if (recordable) {
            if (kd.pakCount() > 0) {
               sb.append(" | P");
               Iterator<Integer> it = kd.pakIdx().iterator();
               while (it.hasNext()) {
                  int pakIndex = it.next();
                  temp = kd.getPak(pakIndex);
                  sb.append(" | ");
                  sb.append(pakIndex);
                  sb.append("-");
                  sb.append(Utils.toHexString(temp, 0, temp.length));
               }
            }
         } else {
            temp = kd.getVuk();
            if (temp != null) {
               sb.append(" | V | ");
               sb.append(Utils.toHexString(temp, 0, temp.length));
            }
         }
      }
      if ((mode & KeyData.DATA_TUK) != 0) {
         if (kd.tukCount() > 0) {
            if (bluRay) {
               sb.append(" | U");
            } else {
               sb.append(" | T");
            }
            Iterator<Integer> it = kd.tukIdx().iterator();
            while (it.hasNext()) {
               int tukIndex = it.next();
               temp = kd.getTuk(tukIndex);
               sb.append(" | ");
               sb.append(tukIndex);
               sb.append("-");
               sb.append(Utils.toHexString(temp, 0, temp.length));
            }
         }
      }
      if ((mode & KeyData.DATA_COMMENT) != 0) {
         String comment = kd.getComment();
         if (comment != null) {
            sb.append(" ;");
            sb.append(comment);
         }
      }
      // Because most people will use Windows and will use stupid editors which will mess up the line endings write Windows-Newlines
      sb.append("\r\n");
      return sb.toString();
   }

   /**
    * Searches the key database for the given entry and returns it if found.
    *  
    * @param discId The DiscID to search for
    * @param offset Offset of the DiscID in the given array
    * @return The KeyData object for the queried DiscID, null if no data was found
    * @throws IOException An I/O error occurred
    */
   public KeyData getKeyData(byte[] discId, int offset) throws IOException {
      return getSetKeyData(discId, offset, null, false, false, false, 0);
   }

   /**
    * Updates the entry denoted by the given KeyData object in the key database with the given KeyData object.
    * The parameters to written are automatically determined by the method getSmartMode(KeyData).
    * 
    * @param kd The entry with the DiscId from this object gets updated with the data from this object
    * @param bluRay Set this to true if the KeyData is from a BD
    * @param recordable Set this to true if the KeyData is from a recordable
    * @param keepDataTypes If true, all present data types of the entry in the key database will be preserved (if possible, they must be present in kd)
    * @return If the entry has been found and was updated the old KeyData object from the key database is returned, otherwise null 
    * @throws IOException An I/O error occurred
    */
   public KeyData setKeyData(KeyData kd, boolean bluRay, boolean recordable, boolean keepDataTypes) throws IOException {
      return getSetKeyData(kd.getDiscId(), 0, kd, bluRay, recordable, keepDataTypes, getSmartMode(kd));
   }

   /**
    * Updates the entry denoted by the given KeyData object in the key database with the given KeyData object.
    * 
    * @param kd The entry with the DiscId from this object gets updated with the data from this object
    * @param bluRay Set this to true if the KeyData is from a BD
    * @param recordable Set this to true if the KeyData is from a recordable
    * @param keepDataTypes If true, all present data types of the entry in the key database will be preserved (if possible, they must be present in kd)
    * @param mode The write mode to use
    * @return If the entry has been found and was updated the old KeyData object from the key database is returned, otherwise null
    * @throws IOException An I/O error occurred
    */
   public KeyData setKeyData(KeyData kd, boolean bluRay, boolean recordable, boolean keepDataTypes, int mode) throws IOException {
      return getSetKeyData(kd.getDiscId(), 0, kd, bluRay, recordable, keepDataTypes, mode);
   }

   /**
    * Appends the given KeyData object to the key database. It is not checked if the entry is already present.
    * The parameters to written are automatically determined by the method getSmartMode(KeyData).
    * 
    * @param kd The entry with the DiscId from this object gets updated with the data from this object
    * @param bluRay Set this to true if the KeyData is from a BD
    * @param recordable Set this to true if the KeyData is from a recordable
    * @return kd in case of success, null otherwise 
    * @throws IOException An I/O error occurred
    */
   public KeyData appendKeyData(KeyData kd, boolean bluRay, boolean recordable) throws IOException {
      return appendKeyDataImpl(kd, bluRay, recordable, getSmartMode(kd));
   }

   /**
    * Appends the given KeyData object to the key database. It is not checked if the entry is already present.
    * 
    * @param kd The entry with the DiscId from this object gets updated with the data from this object
    * @param bluRay Set this to true if the KeyData is from a BD
    * @param recordable Set this to true if the KeyData is from a recordable
    * @param mode The write mode to use
    * @return kd in case of success, null otherwise 
    * @throws IOException An I/O error occurred
    */
   public KeyData appendKeyData(KeyData kd, boolean bluRay, boolean recordable, int mode) throws IOException {
      return appendKeyDataImpl(kd, bluRay, recordable, mode);
   }

   /**
    * Closes the KeyDataFile.
    * 
    * @throws IOException An I/O error occurred
    */
   public void close() throws IOException {
      raf.close();
   }

   /**
    * Actual implementation that appends the given KeyData.
    * 
    * @param kd KeyData to append to the key database file
    * @param bluRay Set this to true if the KeyData is from a BD
    * @param recordable Set this to true if the KeyData is from a recordable
    * @param mode The write mode to use
    * @return kd in case of success, null otherwise
    * @throws IOException An I/O error occurred
    */
   private KeyData appendKeyDataImpl(KeyData kd, boolean bluRay, boolean recordable, int mode) throws IOException {
      // Clear cb to store the new value
      cb.clear();
      // Try to put the new entry in the line buffer, catch the runtime exception to throw it as checked IOException
      try {
         //System.out.println("Entry: " + buildEntry(kd, bluRay, recordable, mode));
         cb.put(buildEntry(kd, bluRay, recordable, mode));
      }
      catch (BufferOverflowException e) {
         Utils.getMessagePrinter().println("Keyentry exceeds linebuffer size");
         return null;
      }
      // Flib cb to read from it
      cb.flip();
      // Set the channels position to the end
      fc.position(fc.size());
      // Write cb
      writeCb();
      // Forces all changes to be physically written
      fc.force(true);
      return kd;
   }

   /**
    * Actual implementation that either retrieves the KeyData for the given DiscID or replaces the entry with the given replacement string.
    * 
    * TODO: Don't calculate the bufferOffset when retrieving the KeyData?
    * 
    * @param discId The DiscID of the entry to retrieve or replace
    * @param offset The offset into discId to start reading from. 20 bytes get read.
    * @param replacement If not null, the found entry gets replaced with this
    * @param bluRay Set this to true if the KeyData is from a BD. Only used if replacement != null.
    * @param recordable Set this to true if the KeyData is from a recordable
    * @param keepDataTypes If true, all present data types of the entry in the key database will be preserved (if possible, they must be present in replacement). Only used if replacement != null.
    * @param mode The write mode to use. Only used if replacement != null.
    * @return The found entry in the key database, null if it was not found / replaced
    * @throws IOException An I/O error occurred
    */
   private KeyData getSetKeyData(byte[] discId, int offset, KeyData replacement, boolean bluRay, boolean recordable, boolean keepDataTypes, int mode) throws IOException {
      //System.out.println("GetSetKeyData");
      // ***************************************************
      // *** Variables for reading and decoding the file ***
      // ***************************************************
      // The first position of cb is this position inside the file
      long bufferOffset = 0;
      // Counts the number of bytes from which the current contents of cb was decoded
      long bufferBytes = 0;
      // If true, then EOF of the input file has been reached
      boolean eofReached = false;
      // If true, then the final deocde operation and EOF has been done
      boolean eofProcessed = false;
      // If true, the decoder has been finally flushed
      boolean flushed = false;
      // CoderResult object used by the decoder / encoder
      CoderResult coderResult = null;
      // *************************************************************
      // *** Variables for processing the decoded input (== lines) ***
      // *************************************************************
      // Matches a line in the key database file 
      Matcher lineMatcher = linePattern.matcher(cb);
      // Matches the beginning of a dataset
      Matcher datasetIdMatcher = datasetIdPattern.matcher(cb);
      // Matches an ENTRY ID
      Matcher dataIdMatcher = dataIdPattern.matcher(cb);
      // Matches an ENTRY DATA
      Matcher dataMatcher = dataPattern.matcher(cb);
      // Number of the current line, only used for printing error messages
      int lineCounter = 0;
      // Offset of the current line in cb, -1 if no line was found
      int lineStart = -1;
      // End of the current line in cb (one position behind the last character)
      int lineEnd = 0;
      // If true, the parser ignores the next line. Used to discard the remainings of an overflowed line.
      boolean ignoreNextLine = false;
      // Backup of cb's position, created before the Inner Loop because this will modify cb's position and limit to the found lines
      int positionBackup = 0;
      // Backup of cb's limit, created before the Inner Loop because this will modify cb's position and limit to the found lines
      int limitBackup = 0;
      // **************************************************
      // *** Variables for processing the detected data ***
      // **************************************************
      // Stores the date object created by the dataParser
      Date date = null;
      // Used to parse the datestring created from the found date values
      SimpleDateFormat dateParser = new SimpleDateFormat("yyyyMMdd");
      dateParser.setLenient(false);
      // Temporary buffer to decode the keystrings into
      byte[] temp = new byte[20];
      // ************************************************
      // *** Initialize everything for the outer loop ***
      // ************************************************
      // Reset all used objects
      decoder.reset();
      bb.clear();
      cb.clear();
      // Jump to the beginning of the file
      fc.position(0L);
      // ****************************************
      // *** Outer loop for reading from file ***
      // ****************************************
      while (!flushed) {
         // Fill the complete buffer
         //TODO: Read incremental?
         while (!eofReached && bb.hasRemaining()) {
            //System.out.println("Reading");
            if (fc.read(bb) == -1) {
               //System.out.println("Reading: reached EOF");
               eofReached = true;
               break;
            }
         }
         // Flip bb for reading from it!!
         bb.flip();
         // If EOF has been reached, check if the final decode operation has been already done
         if (eofReached) {
            if (!eofProcessed) {
               // Final decode operation pending, do it
               //System.out.println("Decoder: Processing EOF");
               coderResult = decoder.decode(bb, cb, true);
               if (coderResult.isUnderflow()) {
                  // The final decode (not flush!) operation was successful, now set the flag so that the decoder can get flushed
                  eofProcessed = true;
               }
            }
            // Check again if eof has been processed to flush at once (saves an iteration)
            if (eofProcessed) {
               // Finally flush the buffer
               //System.out.println("Decoder: Flushing EOF");
               coderResult = decoder.flush(cb);
               if (coderResult.isUnderflow()) {
                  // Buffer has been successfully flushed, no more processing required, set the exit flag to true
                  flushed = true;
               }
            }
         } else {
            // EOF has not been reached, normal, incremental decode operation
            //System.out.println("Decoder: decoding");
            coderResult = decoder.decode(bb, cb, false);
         }
         // Check the decoder result, throw an exception in case of an error
         if (coderResult.isError()) {
            coderResult.throwException();
         }
         // The current position inside bb reflects the number of bytes which have been decoded to cb
         // It is possible that bb has been compacted without the bufferOffset beeing changed, therefor the postion must be added to the bufferCount
         bufferBytes += (long)bb.position();
         // *************************************************
         // *** Prepare objects for inner processing loop ***
         // *************************************************
         // Every time the outer loop has finished reading, the beginning of cb contains the next unprocessed (partial) line
         // Flip the cb for reading from it!!
         cb.flip();
         //System.out.println("cb contents: " + cb.toString());
         lineStart = -1;
         lineEnd = 0;
         lineMatcher.reset();
         positionBackup = cb.position();
         limitBackup = cb.limit();
         //System.out.println("Saved position: " + positionBackup + ", limit: " + limitBackup);
         // ************************************************
         // *** Inner loop for processing decoding input ***
         // ************************************************
         while (lineMatcher.find()) {
            lineCounter++;
            //System.out.println("Line " + lineCounter + ": " + lineMatcher.group(1));
            // Update the line offsets
            lineStart = lineMatcher.start();
            lineEnd = lineMatcher.end();
            // End of the line contents (== end of line excluding line terminator)
            // ATTENTION: Here this value is in cb coordinates!
            int lineContentEnd = lineMatcher.end(1);
            // Check if this line is the end of an overflowed line and skip it in that case
            // This MUST be the first check to be made or the ignore flag my not be reset correctly if by chance one of the other conditions is also true
            if (ignoreNextLine) {
               //System.out.println("End of overflowed line found, ignoring line");
               ignoreNextLine = false;
               continue;
            }
            // Check if the line is a comment line and skip it if it is
            if (cb.charAt(lineStart) == ';') {
               //System.out.println("Comment line found, skipping line");
               continue;
            }
            // Check if an empty line is found and skip it if it is
            if (lineStart == lineContentEnd) {
               //System.out.println("Empty line found, skipping line");
               continue;
            }
            // Limit and position cb to the found line
            // ATTENTION! After these lines of code the backup restore code at the end of the while body MUST be reached to restore the original values!
            //System.out.println("Setting line " + lineCounter + " position: " + lineStart + ", limit: " + lineContentEnd);
            cb.limit(lineContentEnd);
            cb.position(lineStart);
            // ATTENTION: Here the value is changed to the following matchers coordinates!
            lineContentEnd -= lineStart;
            //System.out.println("Converted lineContentEnd: " + lineContentEnd);
            // Reset the matchers
            datasetIdMatcher.reset();
            dataIdMatcher.reset();
            dataMatcher.reset();
            // Check if the line is the one we are looking for
            if (datasetIdMatcher.lookingAt()) {
               //System.out.println("datasetIdMatcher-result: " + datasetIdMatcher.group());
               // Check if this is the entry we are looking for and retrieve it
               // ******************************
               // *** Process datasetIdMatch ***
               // ******************************
               // Decode the DiscID to compare it with the one we are looking for
               Utils.decodeHexString(datasetIdMatcher.group(1), temp, 0);
               // Compare the decoded DiscID with the one we are looking for
               boolean discIdMatch = true;
               for (int i = 0; i < 20; i++) {
                  if (temp[i] != discId[offset + i]) {
                     discIdMatch = false;
                     break;
                  }
               }
               if (discIdMatch) {
                  // ********************
                  // *** DiscID match ***
                  // ********************
                  // The KeyData to build and return
                  KeyData kd = new KeyData(discId, offset);
                  //System.out.println("Title: " + datasetIdMatcher.group(2));
                  // The title must be trimmed because of the changed regular expression
                  //kd.setTitle(datasetIdMatcher.group(2));
                  kd.setTitle(datasetIdMatcher.group(2).trim());
                  // Set the current id to the found one
                  String currentDataId = datasetIdMatcher.group(3).toUpperCase();
                  // Check if the old BackupHDDVD format is present, that is the date is following the id char which identifies the data after the date field
                  if (datasetIdMatcher.group(4) != null) {
                     // Old BackupHDDVD format or new DumpHD format with the date data as first entry
                     if (!currentDataId.equals("D")) {
                        // Old BackupHDVD format, parse the date data
                        date = null;
                        try {
                           if (datasetIdMatcher.group(5) != null) {
                              // New date format
                              date = dateParser.parse(datasetIdMatcher.group(6) + datasetIdMatcher.group(7) + datasetIdMatcher.group(8));
                           } else {
                              // Old date format
                              date = dateParser.parse("20" + datasetIdMatcher.group(12) + datasetIdMatcher.group(10) + datasetIdMatcher.group(11));
                           }
                        }
                        catch (ParseException e) {
                           // Ignore the exception, the date parser allows "invalid" dates
                        }
                        if (date != null) {
                           kd.setDate(date);
                        }
                        // Set region of dataMatcher to start just behind the datasetIdMatcher match
                        dataMatcher.region(datasetIdMatcher.end(), lineContentEnd);
                     } else {
                        // New DumpHD format, set region of dataMatcher just behind the id so that the date data will be found again
                        dataMatcher.region(datasetIdMatcher.start(4), lineContentEnd);
                     }
                  } else {
                     // New DumpHD format, set the region of dataMatcher to start just behind this match
                     dataMatcher.region(datasetIdMatcher.end(), lineContentEnd);
                  }
                  // ***************************************
                  // *** Parse all following data entrys ***
                  // ***************************************
                  // For-ever, the loop will get a appropriate break
                  for(;;) {
                     // *************************
                     // *** Process dataMatch ***
                     // *************************
                     //System.out.println("Current dataId: " + currentDataId);
                     if (dataMatcher.lookingAt()) {
                        // End offset of the current dataMatch
                        int dataEnd = dataMatcher.end();
                        //System.out.println("dataMatcher-result: " + dataMatcher.group());
                        // In case of an error kd is set to null as an error flag
                        if (currentDataId.equals("D")) {
                           //System.out.println("Date: " + dataMatcher.group(1) + ", " + dataMatcher.group(5));
                           date = null;
                           try {
                              if (dataMatcher.group(1) != null) {
                                 // New date format
                                 date = dateParser.parse(dataMatcher.group(2) + dataMatcher.group(3) + dataMatcher.group(4));
                              } else if (dataMatcher.group(5) != null) {
                                 // Old date format
                                 date = dateParser.parse("20" + dataMatcher.group(8) + dataMatcher.group(6) + dataMatcher.group(7));
                              } else {
                                 // Invalid data
                                 Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start() + 1) + ": Invalid DATE data");
                                 kd = null;
                              }
                           }
                           catch (ParseException e) {
                              // Ignore the exception, the date parser allows "invalid" dates
                           }
                           // In case of error (kd == null) date is also null, therefor this does work without an extra check
                           if (date != null) {
                              kd.setDate(date);
                           }
                        } else if (currentDataId.equals("M")) {
                           //System.out.println("MEK: " + dataMatcher.group(9));
                           if (dataMatcher.group(9) != null) {
                              Utils.decodeHexString(dataMatcher.group(9), temp, 0);
                              kd.setMek(temp, 0);
                           } else {
                              // Invalid data
                              Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start() + 1) + ": Invalid MEK data");
                              kd = null;
                           }
                        } else if (currentDataId.equals("I")) {
                           //System.out.println("VID: " + dataMatcher.group(9));
                           if (dataMatcher.group(9) != null) {
                              Utils.decodeHexString(dataMatcher.group(9), temp, 0);
                              kd.setVid(temp, 0);
                           } else {
                              // Invalid data
                              Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start() + 1) + ": Invalid VID data");
                              kd = null;
                           }
                        } else if (currentDataId.equals("B")) {
                           do {
                              //System.out.println("BN: " + dataMatcher.group(10) + ", " + dataMatcher.group(11));
                              dataEnd = dataMatcher.end();
                              if (dataMatcher.group(10) != null) {
                                 //TODO: Catch exception?
                                 int keyNumber = Integer.parseInt(dataMatcher.group(10));
                                 //TODO: More checks?
                                 if (keyNumber >= 0) {
                                    // Valid BN number
                                    Utils.decodeHexString(dataMatcher.group(11), temp, 0);
                                    kd.setBn(keyNumber, temp, 0);
                                 } else {
                                    // Invalid key number
                                    Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start(10) + 1) + ": Invalid BN number");
                                    kd = null;
                                    break;
                                 }
                                 //TODO: Check for line end?
                                 dataMatcher.region(dataMatcher.end(), lineContentEnd);
                              } else {
                                 // Invalid data
                                 Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start() + 1) + ": Invalid BN data");
                                 kd = null;
                                 break;
                              }
                           } while (dataMatcher.lookingAt());
                        } else if (currentDataId.equals("V")) {
                           //System.out.println("VUK: " + dataMatcher.group(9));
                           if (dataMatcher.group(9) != null) {
                              Utils.decodeHexString(dataMatcher.group(9), temp, 0);
                              kd.setVuk(temp, 0);
                           } else {
                              // Invalid data
                              Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start() + 1) + ": Invalid VUK data");
                              kd = null;
                           }
                        } else if (currentDataId.equals("P")) {
                           do {
                              //System.out.println("PAK: " + dataMatcher.group(10) + ", " + dataMatcher.group(11));
                              dataEnd = dataMatcher.end();
                              if (dataMatcher.group(10) != null) {
                                 //TODO: Catch exception?
                                 int keyNumber = Integer.parseInt(dataMatcher.group(10));
                                 //TODO: More checks?
                                 if (keyNumber >= 0) {
                                    // Valid PAK number
                                    Utils.decodeHexString(dataMatcher.group(11), temp, 0);
                                    kd.setPak(keyNumber, temp, 0);
                                 } else {
                                    // Invalid key number
                                    Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start(10) + 1) + ": Invalid PAK number");
                                    kd = null;
                                    break;
                                 }
                                 //TODO: Check for line end?
                                 dataMatcher.region(dataMatcher.end(), lineContentEnd);
                              } else {
                                 // Invalid data
                                 Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start() + 1) + ": Invalid PAK data");
                                 kd = null;
                                 break;
                              }
                           } while (dataMatcher.lookingAt());
                        } else if (currentDataId.equals("T")) {
                           do {
                              //System.out.println("TK: " + dataMatcher.group(10) + ", " + dataMatcher.group(11));
                              dataEnd = dataMatcher.end();
                              if (dataMatcher.group(10) != null) {
                                 //TODO: Catch exception?
                                 int keyNumber = Integer.parseInt(dataMatcher.group(10));
                                 //TODO: More checks?
                                 if (keyNumber > 0) {
                                    // Valid TK number
                                    Utils.decodeHexString(dataMatcher.group(11), temp, 0);
                                    kd.setTuk(keyNumber, temp, 0);
                                 } else {
                                    // Invalid key number
                                    Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start(10) + 1) + ": Invalid TK number");
                                    kd = null;
                                    break;
                                 }
                                 //TODO: Check for line end?
                                 dataMatcher.region(dataMatcher.end(), lineContentEnd);
                              } else {
                                 // Invalid data
                                 Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start() + 1) + ": Invalid TK data");
                                 kd = null;
                                 break;
                              }
                           } while (dataMatcher.lookingAt());
                        } else if (currentDataId.equals("U")) {
                           do {
                              //System.out.println("UK: " + dataMatcher.group(10) + ", " + dataMatcher.group(11));
                              dataEnd = dataMatcher.end();
                              if (dataMatcher.group(10) != null) {
                                 //TODO: Catch exception?
                                 int keyNumber = Integer.parseInt(dataMatcher.group(10));
                                 //TODO: More checks?
                                 if (keyNumber > 0) {
                                    // Valid UK number
                                    Utils.decodeHexString(dataMatcher.group(11), temp, 0);
                                    kd.setTuk(keyNumber, temp, 0);
                                 } else {
                                    // Invalid key number
                                    Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start(10) + 1) + ": Invalid UK number");
                                    kd = null;
                                    break;
                                 }
                                 //TODO: Check for line end?
                                 dataMatcher.region(dataMatcher.end(), lineContentEnd);
                              } else {
                                 // Invalid data
                                 Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.start() + 1) + ": Invalid UK data");
                                 kd = null;
                                 break;
                              }
                           } while (dataMatcher.lookingAt());
                        }
                        // Check if error flag is set, break out if it is
                        if (kd == null) {
                           break;
                        }
                        // **************************************************
                        // *** dataMatch processed, check for new data id ***
                        // **************************************************
                        if (dataEnd != lineContentEnd) {
                           // There is more data to parse
                           // Set the region of the dataIdMatcher to start just behind the last dataMatch
                           dataIdMatcher.region(dataEnd, lineContentEnd);
                           if (dataIdMatcher.lookingAt()) {
                              // New data id or comment found
                              String comment = dataIdMatcher.group(1);
                              if (comment == null) {
                                 // New data id found
                                 currentDataId = dataIdMatcher.group(2).toUpperCase();
                                 // Set the region of the dataMatcher to start just behind the data id
                                 dataMatcher.region(dataIdMatcher.end(), lineContentEnd);
                                 // A next run in the loop
                                 continue;
                              } else {
                                 // Comment found, the parsing is finished
                                 kd.setComment(comment);
                                 // Fall through to continue with the action after the parsing
                              }
                           } else {
                              // Invalid data
                              Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataIdMatcher.regionStart() + 1) + ": Invalid ENTRY ID");
                              // Set kd to null to be consistent
                              kd = null;
                              break;
                           }
                        }
                        // Parsing finished
                        // Check if the result should be returned or replaced
                        if (replacement != null) {
                           // *******************************
                           // *** Replace the found entry ***
                           // *******************************
                           // WARNING!! The current content of cb and bb will get lost, this section MUST return from the method, the outer loop MUST NOT take a next loop
                           // Determine the bytes from the beginning of the buffer to the line start to calculate the offset of the line into the file
                           cb.position(0);
                           cb.limit(lineStart);
                           // Position in file to start writing the replacement
                           long writeOffset = bufferOffset + (long)getRemainingCbBytes();
                           // Determine the size in bytes of the line 
                           cb.position(lineStart);
                           cb.limit(lineEnd);
                           // The length in bytes of the present entry
                           int entrySize = getRemainingCbBytes();
                           // If the present data types should be preserved add them to mode
                           if (keepDataTypes) {
                              mode |= kd.dataMask();
                           }
                           // Encode the replacement just to know how many bytes it consumes
                           //TODO: Make bb big enough to guarantee that a full cb would fit in it?
                           cb.position(0);
                           cb.limit(cb.capacity());
                           try {
                              cb.put(buildEntry(replacement, bluRay, recordable, mode));
                           }
                           catch (BufferOverflowException e) {
                              Utils.getMessagePrinter().println("Keyentry exceeds linebuffer size");
                              return null;
                           }
                           cb.flip();
                           int replacementSize = getRemainingCbBytes();
                           // Reset cb's position
                           cb.position(0);
                           // Depending on the size difference of found entry and replacement either write only the replacement or also shift the present data
                           if (replacementSize == entrySize) {
                              // Old and new entry have the same size
                              //System.out.println("Replace entry: Both entrys have the same size, old: " + entrySize + ", new: " + replacementSize);
                              // Simply write the new entry
                              // Set fc's positon to the write position
                              fc.position(writeOffset);
                              writeCb();
                           } else if (replacementSize < entrySize) {
                              // New entry is smaller than old entry
                              //System.out.println("Replace entry: New entry is smaller than old entry, old: " + entrySize + ", new: " + replacementSize);
                              // Write new entry
                              fc.position(writeOffset);
                              writeCb();
                              // Shift the remaining data left
                              // The position where the next read should be made from, initially the end of the present entry
                              long readPos = writeOffset + (long)entrySize;
                              // The position where the next write should be made to, initially the end of the new entry (which has just been written)
                              long writePos = fc.position();
                              // Set fc's position to readPos
                              fc.position(readPos);
                              // Use bb as read / write buffer
                              bb.clear();
                              // Read / write incrementally until EOF
                              while (fc.read(bb) != -1) {
                                 // New readPos starts at current position
                                 readPos = fc.position();
                                 // flip buffer to read from it
                                 bb.flip();
                                 // Jump to the writePos and write the buffer
                                 fc.position(writePos);
                                 fc.write(bb);
                                 // New writePos starts at current position
                                 writePos = fc.position();
                                 // Compact buffer to read into it
                                 bb.compact();
                                 // Jump to readPos and start next read-loop
                                 fc.position(readPos);
                              }
                              // Shifting finished
                              // Because the data was shifted left, truncate after the last written position
                              fc.truncate(writePos);
                           } else {
                              // New entry is bigger than old entry
                              //System.out.println("Replace entry: New entry is bigger than old entry, old: " + entrySize + ", new: " + replacementSize);
                              // First shift the data
                              // Use bb as read / write buffer
                              bb.clear();
                              // Store the fileSize to have a fixed value for it to use it multiple times during the initial position calculation
                              // Do this to avoid unpredictable behavior if the filesize gets modified during the calculation from outside
                              // (this itself is not allowed but at least this code will not run amok)
                              long fileSize = fc.size();
                              // The end of the entry to be replaced
                              long entryEnd = writeOffset + (long)entrySize;
                              // Check if the file has been illegally truncated
                              if (fileSize < entryEnd) {
                                 Utils.getMessagePrinter().println("Fatal error: Key database has been illegally truncated from outside beyond the entry to update");
                                 return null;
                              }
                              // Set readPos just one bb left from the end of file to read up a maximum of data
                              long readPos = fileSize - (long)bb.capacity();
                              // If readPos is left of the end postion of the entry (there is less data to shift than a full bb) set readPos
                              // to this position and limit bb accordingly
                              if (readPos < entryEnd) {
                                 bb.limit(bb.capacity() - (int)(entryEnd - readPos));
                                 readPos = entryEnd;
                              }
                              // Set writePos the shift difference behind the end of file
                              // The buffer limit (== buffer size) must be subtracted because writePos is the starting write position!
                              long writePos = fileSize + (long)(replacementSize - entrySize) - (long)bb.limit();
                              // As long as bb has a limit > 0 there is data to move
                              while (bb.limit() > 0) {
                                 //System.out.println("Shift loop");
                                 // Jump to readPos and fill bb completely
                                 fc.position(readPos);
                                 while (bb.hasRemaining()) {
                                    fc.read(bb);
                                 }
                                 // Flip bb to read from it!
                                 bb.flip();
                                 // Jump to writePos and write bb fully
                                 fc.position(writePos);
                                 while (bb.hasRemaining()) {
                                    fc.write(bb);
                                 }
                                 // Clear bb for a new run
                                 bb.clear();
                                 // Move readPos one bb capacity to the left
                                 readPos -= (long)bb.capacity();
                                 // Check if readPos is left of entryEnd and move it to there and adjust bb's limit
                                 if (readPos < entryEnd) {
                                    bb.limit(bb.capacity() - (int)(entryEnd - readPos));
                                    readPos = entryEnd;
                                 }
                                 // Shift writePos by the amount that will get written
                                 writePos -= (long)bb.limit();
                              }
                              // Now write the entry
                              fc.position(writeOffset);
                              writeCb();
                           }
                           // Forces all changes to be physically written
                           fc.force(true);
                           // Return the found entry
                           return kd;
                        } else {
                           // ******************************
                           // *** Return the found entry ***
                           // ******************************
                           return kd;
                        }
                     } else {
                        // No data present
                        Utils.getMessagePrinter().println("Error at line " + lineCounter + ", position " + (dataMatcher.regionStart() + 1) + ": Invalid ENTRY DATA");
                        // Set kd to null to be consistent
                        kd = null;
                        break;
                     } // End if dataMatcher.lookingAt()
                  } // End for ever entry parse loop
               } // End if discIdMatch
            } else {
               // Line is not a key entry
               Utils.getMessagePrinter().println("Error at line " + lineCounter + ": Line is not a key entry");
            } // End if datasetIdMatcher.loockingAt()
            // ************************************
            // *** Restore saved cb constraints ***
            // ************************************
            cb.position(positionBackup);
            cb.limit(limitBackup);
         } // End while lineMatcher.find()
         //System.out.println("After lineMatcher bb.remaining(): " + bb.remaining() + ", cb.reamining(): " + cb.remaining() + ", cb.position(): " + cb.position());
         // *************************************
         // *** Prepare buffers for a new run ***
         // *************************************
         if (!flushed) {
            // Buffer not flushed, prepare for a new loop
            if (lineStart == -1) {
               // No line found
               // Because the buffer was flipped, the limit marks the max position the buffer is filled up to
               if (cb.limit() == cb.capacity()) {
                  // Line buffer overflow
                  Utils.getMessagePrinter().println("Error at line " + (lineCounter + 1) + ": Linebuffer overflow, ignoring line");
                  // Clear cb and compact bb to continue reading
                  // -> the next found line will be the end of the too big line, set flag to ignore the next found line
                  // lineCounter + 1 because the overflowed line has not been counted
                  // Because cb gets cleared, the current bufferOffset is advanced by the current bufferBytes which must be reset to 0
                  bufferOffset += bufferBytes;
                  bufferBytes = 0L;
                  cb.clear();
                  bb.compact();
                  ignoreNextLine = true;
               } else {
                  // There is still space left in cb, set position and limit to allow appending, compact bb and let more data be read
                  //System.out.println("No line found, adding more data to cb");
                  // No need to update bufferOffset because cb gets only appended
                  cb.position(cb.limit());
                  cb.limit(cb.capacity());
                  bb.compact();
               }
            } else {
               // At least one line found
               //System.out.println("Line(s) found, starting new run");
               // Set cb's position to the end of the last line so that after compacting the buffer it starts with the beginning of the new line
               cb.position(lineEnd);
               // Now cb's position is the correct one to determine the size of the remaining chars
               long remainingBytes = (long)getRemainingCbBytes();
               // Reset cb's position
               cb.position(lineEnd);
               // Subtract the remaining bytes from the current bufferBytes to get the size of the already processed lines to update the bufferOffset correctly
               bufferOffset += bufferBytes - remainingBytes;
               // The buffer contains already the remaining bytes
               bufferBytes = remainingBytes;
               //System.out.println("New run, bufferOffset: " + bufferOffset);
               cb.compact();
               bb.compact();
            }
         } else {
            // Buffer flushed, check for remaining (invalid) data
            if (lineEnd != cb.limit()) {
               // There is data behind the last found line
               Utils.getMessagePrinter().println("Error behind line " + lineCounter + ": Invalid data at end");
            }
         }
      } // End while file processing loop
      return null;
   }

   /**
    * Helper method!
    * 
    * Encodes the chars between cb's current position and its limit and returns the size of the encoding in bytes.
    * Uses encoder and bbh, cb's position is advanced to its limit.
    *  
    * @return The number of bytes which are needed to store the chars between cb's current position and its limit
    * @throws IOException An error occured during encoding
    */
   private int getRemainingCbBytes() throws IOException {
      // Total number of bytes the encoded chars need
      int encodedSize = 0;
      // Flag indicating if the data has been fully encoded
      boolean encoded = false;
      // Flag indicating if the encoder has been flushed
      boolean flushed = false;
      // Return value of the encoder
      CoderResult coderResult = null;
      // Reset all used objects
      encoder.reset();
      bbh.clear();
      // Encode the data
      while (!flushed) {
         if (!encoded) {
            // Output has not been fully encoded yet
            coderResult = encoder.encode(cb, bbh, true);
            if (coderResult.isUnderflow()) {
               // The encode was fully done
               encoded = true;
            }
         } else {
            // We have everything encoded, now flush the encoder
            // No need to check if we have already flushed, we wont get here if we had
            coderResult = encoder.flush(bbh);
            if (coderResult.isUnderflow()) {
               flushed = true;
            }
         }
         if (coderResult.isError()) {
            coderResult.throwException();
         }
         // The current position reflects how many bytes the current encode loop produced
         encodedSize += bbh.position();
         // Just clear bbh, the encoded output is not needed
         bbh.clear();
      }
      return encodedSize;
   }

   /**
    * Helper method!
    * 
    * Writes the contents of cb using its current position and limit to the current position of fc.
    * Uses encoder and bb, cb's position is advanced to its limit and fc's position is advanced by the bytes written.
    *  
    * @throws IOException An I/O error occured
    */
   private void writeCb() throws IOException {
      // Flag indicating if the data has been fully encoded
      boolean encoded = false;
      // Flag indicating if the encoder has been flushed
      boolean flushed = false;
      // Return value of the encoder
      CoderResult coderResult = null;
      // Reset all used objects
      encoder.reset();
      bb.clear();
      // Encode the data
      while (!flushed) {
         if (!encoded) {
            // Output has not been fully encoded yet
            coderResult = encoder.encode(cb, bb, true);
            if (coderResult.isUnderflow()) {
               // The encode was fully done
               encoded = true;
            }
         } else {
            // We have everything encoded, now flush the encoder
            // No need to check if we have already flushed, we wont get here if we had
            coderResult = encoder.flush(bb);
            if (coderResult.isUnderflow()) {
               flushed = true;
            }
         }
         if (coderResult.isError()) {
            coderResult.throwException();
         }
         // Flip bb to read from it
         bb.flip();
         // Write bb fully and reset it for a new run
         //TODO: Write incremental?
         while (bb.hasRemaining()) {
            fc.write(bb);
         }
         bb.clear();
      }
   }

}
