/***************************************************************************
**
**  This file is part of GeopsyCore.
**
**  GeopsyCore is free software: you can redistribute it and/or modify
**  it under the terms of the GNU General Public License as published by
**  the Free Software Foundation, either version 3 of the License, or
**  (at your option) any later version.
**
**  GeopsyCore is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with Foobar.  If not, see <http://www.gnu.org/licenses/>
**
**  See http://www.geopsy.org for more information.
**
**  Created: 2009-08-14
**  Copyright: 2009-2019
**    Marc Wathelet
**    Marc Wathelet (LGIT, Grenoble, France)
**
***************************************************************************/

#include "AsciiSignalFormat.h"

namespace GeopsyCore {

  /*!
    \class AsciiSignalFormat AsciiSignalFormat.h
    \brief Brief description of class still missing

    Description of plain ascii parsing: how to determine header size, how to get information from header.
    Patterns are all regular expressions. Patterns related to data fields must enclose interesting information
    between '(' and ')' and specify the index.
  */

  const QString AsciiSignalFormat::xmlAsciiSignalFormatTag="AsciiSignalFormat";
  bool AsciiSignalFormat::_verbose=false;

  /*!
    Constructs a basic ascii format without header
  */
  AsciiSignalFormat::AsciiSignalFormat()
  {
    _headerType=NoHeader;
    _headerLineCount=0;
    _headerPattern="#";
    _timeFormat="yyyy-MM-dd hh:mm:ss";
    _separators=AsciiLineParser::Tab | AsciiLineParser::Space;
    _emptyValues=false;
    _quotes=false;
    _hexadecimal=false;
  }

  /*!
    Copy constructor
  */
  AsciiSignalFormat::AsciiSignalFormat(const AsciiSignalFormat& o)
      : AbstractFileFormat(o), _rules(o._rules)
  {
    TRACE;
    _headerType=o._headerType;
    _headerLineCount=o._headerLineCount;
    _headerPattern=o._headerPattern;
    _timeFormat=o._timeFormat;
    _components=o._components;
    _rules=o._rules;
    _separators=o._separators;
    _emptyValues=o._emptyValues;
    _quotes=o._quotes;
    _hexadecimal=o._hexadecimal;
  }

  void AsciiSignalFormat::operator=(const AsciiSignalFormat& o)
  {
    TRACE;
    AbstractFileFormat::operator=(o);
    _headerType=o._headerType;
    _headerLineCount=o._headerLineCount;
    _headerPattern=o._headerPattern;
    _timeFormat=o._timeFormat;
    _components=o._components;
    _rules=o._rules;
    _separators=o._separators;
    _emptyValues=o._emptyValues;
    _quotes=o._quotes;
    _hexadecimal=o._hexadecimal;
  }

  /*!
    Must be re-implemented in sub-classes
  */
  bool AsciiSignalFormat::xml_inherits(const QString& tagName) const
  {
    if(tagName==xmlAsciiSignalFormatTag) {
      return true;
    } else {
      return AbstractFileFormat::xml_inherits(tagName);
    }
  }

  /*!
    Count the number of mandatory rules. This is used to determine format from
    header content. The formats with more rules (more complex) are first checked to avoid less
    restrictive formats to be selected instead.
  */
  int AsciiSignalFormat::complexity() const
  {
    TRACE;
    int n=0;
    for(QList<AsciiSignalFormatRule>::const_iterator it=_rules.begin();it!=_rules.end();it++) {
      if(it->mandatory()) {
        n++;
      }
    }
    return n;
  }

  /*!
    Returns true if the file \a fileName contains a header that can be correctly parsed.
  */
  bool AsciiSignalFormat::isValid(const QString& fileName) const
  {
    TRACE;
    QFile f(fileName);
    if(!f.open(QIODevice::ReadOnly)) {
      return false;
    }
    QTextStream s(&f);
    int lineNumber=0;
    AsciiSignalFormat fileFormat(*this);
    QString h=fileFormat.header(s, lineNumber);
    return !h.isEmpty() && fileFormat.parseHeader(h);
  }

  void AsciiSignalFormat::setHeaderType(const QString& t)
  {
    TRACE;
    if(t.isEmpty()) return;
    switch(t[0].unicode()) {
    case 'N':
      _headerType=NoHeader;
      break;
    case 'F':
      _headerType=FixedHeader;
      break;
    case 'H':
      _headerType=HeaderPattern;
      break;
    case 'E':
      _headerType=EndHeaderPattern;
      break;
    default:
      break;
    }
  }

  QString AsciiSignalFormat::headerTypeString() const
  {
    TRACE;
    switch(_headerType) {
    case NoHeader:
      break;
    case FixedHeader:
      return "FixedHeader";
    case HeaderPattern:
      return "HeaderPattern";
    case EndHeaderPattern:
      return "EndHeaderPattern";
    }
    return "NoHeader";
  }

  void AsciiSignalFormat::xml_writeProperties(XML_WRITEPROPERTIES_ARGS) const
  {
    TRACE;
    AbstractFileFormat::xml_writeProperties(s, context);
    writeProperty(s, "headerType", headerTypeString());
    writeProperty(s, "headerPattern", _headerPattern);
    writeProperty(s, "headerLineCount", _headerLineCount);
    writeProperty(s, "timeFormat", _timeFormat);
    writeProperty(s, "separators", AsciiLineParser::separatorsToString(_separators));
    writeProperty(s, "emptyValues", _emptyValues);
    writeProperty(s, "quotes", _quotes);
    writeProperty(s, "hexadecimal", _hexadecimal);
  }

  void AsciiSignalFormat::xml_writeChildren(XML_WRITECHILDREN_ARGS) const
  {
    TRACE;
    for(QList<AsciiSignalFormatComponent>::const_iterator it=_components.begin();it!=_components.end();it++) {
      it->xml_save(s, context);
    }
    for(QList<AsciiSignalFormatRule>::const_iterator it=_rules.begin();it!=_rules.end();it++) {
      it->xml_save(s, context);
    }
  }

  XMLMember AsciiSignalFormat::xml_member(XML_MEMBER_ARGS)
  {
    TRACE;
    if(tag==AsciiSignalFormatRule::xmlRuleTag) {
      _rules.append(AsciiSignalFormatRule());
      return XMLMember(&_rules.last());
    } else if(tag==AsciiSignalFormatComponent::xmlComponentTag) {
      _components.append(AsciiSignalFormatComponent());
      return XMLMember(&_components.last());
    }  else if(tag=="headerType") return XMLMember(0);
    else if(tag=="headerPattern") return XMLMember(1);
    else if(tag=="headerLineCount") return XMLMember(2);
    else if(tag=="timeFormat") return XMLMember(3);
    else if(tag=="separators") return XMLMember(4);
    else if(tag=="emptyValues") return XMLMember(5);
    else if(tag=="quotes") return XMLMember(6);
    else if(tag=="hexadecimal") return XMLMember(7);
    else return AbstractFileFormat::xml_member(tag, attributes, context)+8;
  }

  bool AsciiSignalFormat::xml_setProperty(XML_SETPROPERTY_ARGS)
  {
    TRACE;
    switch (memberID) {
    case 0: setHeaderType(content.toString()); return true;
    case 1: _headerPattern=content.toString(); return true;
    case 2: _headerLineCount=content.toInt(); return true;
    case 3: _timeFormat=content.toString(); return true;
    case 4: _separators=AsciiLineParser::stringToSeparators(content.toString()); return true;
    case 5: _emptyValues=content.toBool(); return true;
    case 6: _quotes=content.toBool(); return true;
    case 7: _hexadecimal=content.toBool(); return true;
    default:
      break;
    }
    return AbstractFileFormat::xml_setProperty(memberID-8, tag, attributes, content, context);
  }

  DateTime AsciiSignalFormat::startTime() const
  {
    TRACE;
    bool ok=true;
    for(int i=_rules.count()-1; i>=0; i--) {
      const AsciiSignalFormatRule& r=_rules.at(i);
      if(r.data().id()==MetaDataFactory::StartTime) {
        DateTime t;
        ok=t.fromString(r.value().toString(), _timeFormat);
        if(ok) {
          if(_verbose) {
            App::log(tr("StartTime rule:\n  value=%1\n  startTime=%2\n")
                             .arg(r.value().toString())
                             .arg(t.toString()));
          }
          return t;
        } else {
          if(_verbose) {
            App::log(tr("StartTime rule:\n  value='%1'\n  startTime=n/a\n")
                             .arg(r.value().toString()));
          }
        }
      }
    }
    if(_verbose) {
      App::log(tr("Found no valid rule with T0\n"));
    }
    return DateTime::null;
  }

  /*!
    Returns the full header and stream (\a s) pos is set just after the end of header.
    Optionaly \a terminate is a flag that can be turned to true to stop the parser.
  */
  QString AsciiSignalFormat::header(QTextStream& s, int& lineNumber, QAtomicInt * terminate) const
  {
    TRACE;
    QString headerText;
    // Identify the whole header before extracting information
    switch(_headerType) {
    case NoHeader:
      return QString();
    case FixedHeader:
      while(!s.atEnd() && lineNumber<_headerLineCount) {
        lineNumber++;
        headerText+=s.readLine()+"\n";
        if(terminate && terminate->testAndSetOrdered(true, true)) {
          return QString();
        }
      }
      if(lineNumber<_headerLineCount) {
        App::log(tr("File contains %1 lines while header must be %2 lines long\n").arg(lineNumber).arg(_headerLineCount) );
        return QString();
      }
      break;
    case HeaderPattern: {
        QRegExp rexp(_headerPattern);
        qint64 offset=s.pos();
        QString line=s.readLine();
        while(!s.atEnd() && rexp.indexIn(line)>-1) {
          headerText+=line+"\n";
          lineNumber++;
          offset=s.pos();
          line=s.readLine();
          if(lineNumber>1000) {
            App::log(tr("Header has more than 1000 lines, aborting\n") );
            return QString();
          }
          if(terminate && terminate->testAndSetOrdered(true, true)) {
            return QString();
          }
        }

        s.seek(offset); // back to last line which matched regexp
      }
      break;
    case EndHeaderPattern: { // Line just after match contains data
        QRegExp rexp(_headerPattern);
        QString line=s.readLine();
        lineNumber++;
        while(!s.atEnd() && rexp.indexIn(line)==-1) {
          headerText+=line+"\n";
          line=s.readLine();
          lineNumber++;
          if(lineNumber>1000) {
            App::log(tr("Reading 1000 lines without finding the End Header Pattern\n") );
            return QString();
          }
          if(terminate && terminate->testAndSetOrdered(true, true)) {
            return QString();
          }
        }
        headerText+=line+"\n";
        // Check if next lines match EndHeaderPattern
        qint64 offset=s.pos();
        line=s.readLine();
        lineNumber++;
        while(!s.atEnd() && rexp.indexIn(line)>-1) {
          offset=s.pos();
          line=s.readLine();
          lineNumber++;
          if(lineNumber>1000) {
            App::log(tr("Header has more than 1000 lines, aborting\n") );
            return QString();
          }
          if(terminate && terminate->testAndSetOrdered(true, true)) {
            return QString();
          }
        }
        s.seek(offset); // back to last line which matched regexp
      }
      break;
    }
    return headerText;
  }

  /*!
    Optionaly \a terminate is a flag that can be turned to true to stop the parser.
  */
  bool AsciiSignalFormat::parseHeader(const QString& headerText, QAtomicInt * terminate)
  {
    TRACE;
    if(!_verbose && App::verbosity()>5) {
      _verbose=true;
    }
    if(_verbose) {
      App::log(tr("Parsing header:\n") );
    }
    for(QList<AsciiSignalFormatRule>::iterator it=_rules.begin();it!=_rules.end();it++) {
      AsciiSignalFormatRule& r=*it;
      if(!r.constant()) {
        QRegExp rexp(r.pattern());
        if(rexp.indexIn(headerText)>-1) {
          if(_verbose) {
            App::log(tr("  Rule '%1' matched with value '%2'\n")
                          .arg(r.pattern()).arg(rexp.cap(r.patternIndex())));
          }
          r.setValue(rexp.cap(r.patternIndex()));
        } else {
          if(_verbose) {
            App::log(tr("  Rule '%1' NOT matched\n")
                          .arg(r.pattern()));
          }
          r.setValue(QVariant());
          if(r.mandatory()) {
            App::log(tr("Cannot identify '%1' in ascii file header from regular expression '%2'\n")
                          .arg(MetaDataFactory::instance()->name(r.data()))
                          .arg(r.pattern()));
            return false;
          }
        }
      }
      if(terminate && terminate->testAndSetOrdered(true, true)) {
        return false;
      }
    }
    return true;
  }

  /*!
    Sets header information to signal \a sig.
  */
  void AsciiSignalFormat::assign(Signal * sig)
  {
    TRACE;
    if(_verbose) {
      App::log(tr("Assigning values to signal (NumberInFile=%1):\n")
                       .arg(sig->numberInFile()));
    }
    for(int i=0; i<_rules.count(); i++) {
      const AsciiSignalFormatRule& r=_rules.at(i);
      if(_verbose) {
        App::log(tr("  Rule[%1] for setting '%2'\n").arg(i)
                         .arg(MetaDataFactory::instance()->name(r.data())));
      }
      if((r.channel()==-1 || sig->numberInFile()==r.channel()) && r.value().isValid()) {
        switch(r.data().id()) {
        case MetaDataFactory::Component: {
            if(_verbose) {
              App::log(tr("    setting Component from value '%1'\n").arg(r.value().toString()));
            }
            for(QList<AsciiSignalFormatComponent>::const_iterator it=_components.begin();it!=_components.end();it++) {
              QRegExp exp(it->pattern());
              if(exp.exactMatch(r.value().toString())) {
                sig->setComponent(it->component());
                if(_verbose) {
                  App::log(tr("    found match in Component table: '%1'\n").arg(sig->componentStandardName()) );
                }
              }
            }
          }
          break;
        case MetaDataFactory::StartTime:
           // Does nothing here get it directly with startTime()
          break;
        default: {
            bool ret=false;
            if(r.value().convert(QVariant::Double)) {
              double v0=sig->header(r.data()).toDouble();
              double v=r.value().toDouble()*r.factor();
              switch(r.operation()) {
              case AsciiSignalFormatRule::Assign:
                ret=sig->setHeader(r.data(), v);
                break;
              case AsciiSignalFormatRule::Add:
                ret=sig->setHeader(r.data(), v0+v);
                break;
              case AsciiSignalFormatRule::Subtract:
                ret=sig->setHeader(r.data(), v0-v);
                break;
              case AsciiSignalFormatRule::Multiply:
                ret=sig->setHeader(r.data(), v0*v);
                break;
              case AsciiSignalFormatRule::Divide:
                ret=sig->setHeader(r.data(), v0/v);
                break;
              }
            } else {
              switch(r.operation()) {
              case AsciiSignalFormatRule::Assign:
                ret=sig->setHeader(r.data(), r.value());
                break;
              case AsciiSignalFormatRule::Add: {
                  QString v0=sig->header(r.data()).toString();
                  if(v0.isEmpty()) {
                    ret=sig->setHeader(r.data(), r.value().toString());
                  } else {
                    ret=sig->setHeader(r.data(), v0+"\n"+r.value().toString());
                  }
                }
                break;
              case AsciiSignalFormatRule::Subtract:
              case AsciiSignalFormatRule::Multiply:
              case AsciiSignalFormatRule::Divide:
                if(_verbose) {
                  App::log(tr("    operation '%1' not supported for strings.\n")
                                   .arg(r.operationString()));
                }
                break;
              }
            }
            if(ret) {
              if(_verbose) {
                App::log(tr("    setting '%1' from value '%2'\n")
                                 .arg(MetaDataFactory::instance()->name(r.data()))
                                 .arg(sig->header(r.data()).toString()));
              }
            } else {
              if(_verbose) {
                App::log(tr("    cannot set '%1' from value '%2'\n")
                                 .arg(MetaDataFactory::instance()->name(r.data()))
                                 .arg(r.value().toString()));
              } else {
                App::log(tr("Cannot set '%1' from ascii file header\n")
                    .arg(MetaDataFactory::instance()->name(r.data())));
              }
            }
          }
          break;
        }
      }
    }
  }

  /*!
    Get start time and sampling frequency from signal \a sig,
    assumed to be time stamped.

    Returns false is time information is not consistent.
  */
  bool AsciiSignalFormat::readTimeColumn(QByteArray& buffer, int columnCount, int lineNumber)
  {
    TRACE;
    // Check if there is one time column
    int columnIndex=-1;
    for(int i=0; i<columnCount; i++) {
      for(QList<AsciiSignalFormatRule>::const_iterator it=_rules.begin();it!=_rules.end();it++) {
        const AsciiSignalFormatRule& r=*it;
        if((r.channel()==-1 || i==r.channel()) && r.value().isValid() && r.data().id()==MetaDataFactory::Component) {
          for(QList<AsciiSignalFormatComponent>::const_iterator it=_components.begin();it!=_components.end();it++) {
            QRegExp exp(it->pattern());
            if(it->component()==Signal::Time && exp.exactMatch(r.value().toString())) {
              columnIndex=i;
              i=columnCount;
              break;
            }
          }
        }
      }
    }
    if(columnIndex==-1) {
      if(_verbose) {
        App::log(tr("Found no Time column\n") );
      }
      return true;
    }
    // Found Time column, now read it, check it and deduce TimeReference and T0
    if(_verbose) {
      App::log(tr("Found Time at column %1\n").arg(columnIndex) );
    }
    AsciiLineParser p(&buffer);
    p.setSeparators(_separators);
    p.setEmptyValues(_emptyValues);
    p.setQuotes(_quotes);
    DateTime t1, t2;
    p.readColumn(columnIndex);
    lineNumber++;
    if(_verbose) {
      App::log(tr("Converting time '%1'\n").arg(p.toString()) );
    }
    t1.fromString(p.toString(), _timeFormat);
    if(p.atEnd()) {
      Message::warning(MSG_ID, tr("Loading ASCII time column"),
                       tr("Only one line of samples found, cannot compute sampling frequency."), true);
      return false;
    }
    p.readColumn(columnIndex);
    lineNumber++;
    if(_verbose) {
      App::log(tr("Converting time '%1'\n").arg(p.toString()) );
    }
    t2.fromString(p.toString(), _timeFormat);
    double samplingPeriod=t1.secondsTo(t2);
    if(samplingPeriod<=0.0) {
      Message::warning(MSG_ID, tr("Loading ASCII time column"),
                       tr("Negative or null sampling period at line %1").arg(lineNumber), true);
      return false;
    }
    AsciiSignalFormatRule r;
    _rules.append(r);
    r.setValue(t1.toString());
    r.setData(MetaDataFactory::StartTime);
    t1=t2;
    while(!p.atEnd()) {
      p.readColumn(columnIndex);
      lineNumber++;
      if(_verbose) {
        App::log(tr("Converting time '%1'\n").arg(p.toString()) );
      }
      t2.fromString(p.toString(), _timeFormat);
      if(fabs(t1.secondsTo(t2)-samplingPeriod)>1e-6*samplingPeriod) {
        Message::warning(MSG_ID, tr("Loading ASCII time column"),
                         tr("Sampling period is not constant while reading time stamps at line %1.\n"
                            "Sampling period=%2\n"
                            "t1=%3\n"
                            "t2=%4\n"
                            "dt=%5\n"
                            "SamplingPeriod-dt=%6")
                         .arg(lineNumber)
                         .arg(samplingPeriod)
                         .arg(t1.toString())
                         .arg(t2.toString())
                         .arg(t1.secondsTo(t2))
                         .arg(samplingPeriod-t1.secondsTo(t2)), true);
        return false;
      }
      t1=t2;
    }
    r.setConstant(true);
    r.setValue(QString::number(samplingPeriod));
    r.setData(MetaDataFactory::SamplingPeriod);
    _rules.append(r);
    return true;
  }

  /*!
    Returns true if column \a columnIndex can be ignored while reading data.
  */
  bool AsciiSignalFormat::isIgnored(int columnIndex) const
  {
    TRACE;
    for(QList<AsciiSignalFormatRule>::const_iterator it=_rules.begin();it!=_rules.end();it++) {
      const AsciiSignalFormatRule& r=*it;
      if((r.channel()==-1 || columnIndex==r.channel()) && r.value().isValid() && r.data().id()==MetaDataFactory::Component) {
        for(QList<AsciiSignalFormatComponent>::const_iterator it=_components.begin();it!=_components.end();it++) {
          QRegExp exp(it->pattern());
          switch(it->component()) {
          case Signal::Time:
          case Signal::Ignore:
            if(exp.exactMatch(r.value().toString())) {
              return true;
            }
          default:
            break;
          }
        }
      }
    }
    return false;
  }

  void AsciiSignalFormat::moveRuleUp(int index)
  {
    TRACE;
    if(index>0) {
      AsciiSignalFormatRule r=_rules[index-1];
      _rules[index-1]=_rules[index];
      _rules[index]=r;
    }
  }

  void AsciiSignalFormat::moveRuleDown(int index)
  {
    TRACE;
    if(index<_rules.count()-1) {
      AsciiSignalFormatRule r=_rules[index+1];
      _rules[index+1]=_rules[index];
      _rules[index]=r;
    }
  }

} // namespace GeopsyCore
