/***************************************************************************
**
**  This file is part of QGpCoreTools.
**
**  This library is free software; you can redistribute it and/or
**  modify it under the terms of the GNU Lesser General Public
**  License as published by the Free Software Foundation; either
**  version 2.1 of the License, or (at your option) any later version.
**
**  This file is distributed in the hope that it will be useful, but WITHOUT
**  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
**  FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
**  License for more details.
**
**  You should have received a copy of the GNU Lesser General Public
**  License along with this library; if not, write to the Free Software
**  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
**
**  See http://www.geopsy.org for more information.
**
**  Created: 2008-07-11
**  Copyright: 2008-2019
**    Marc Wathelet
**    Marc Wathelet (LGIT, Grenoble, France)
**
***************************************************************************/

#include "ColumnTextParser.h"
#include "StringSection.h"
#include "XMLMap.h"
#include "CoreApplication.h"
#include "MemoryChecker.h"

namespace QGpCoreTools {

  /*!
    \class ColumnTextParser ColumnTextParser.h
    \brief Brief description of class still missing

    Full description of class still missing
  */

  const QString ColumnTextParser::_nullCell;
  const QString ColumnTextParser::xmlColumnTextParserTag="ColumnTextParser";

  /*!
    Description of constructor still missing
  */
  ColumnTextParser::ColumnTextParser(QObject * parent)
      : Thread(parent)
  {
    TRACE;
    setObjectName("ColumnTextParser");
    _text=nullptr;
    _terminate=false;

    // Init at least one standard type
    setStandardTypes(QStringList());

    _startLine=0;
    _endLine=0;
    _acceptPattern="";
    _rejectPattern="^#";
    _hasComments=true;

    _delimiters=" \t";
    _maximumColumnCount=99;
    _columnCount=-1;
    _acceptNullColumns=false;
    _fixedWidth=false;
    _columnDescriptions.resize(1); // At least one default column is mandatory

    _isSectionLinePattern=true;
    _sectionLinePattern="^#";
    _isSectionLineCount=false;
    _sectionLineCount=0;
    _isSectionRowCount=false;
    _sectionRowCount=0;
    _isSectionCellPattern=false;
    _sectionCellColumn=0;
    _sectionCellPattern="";
    connect(this, SIGNAL(finished()), this, SIGNAL(rowsParsed()));
  }

  /*!
    Copy constructor, constructed without parent.
    Keep only properties to run on another file.
  */
  ColumnTextParser::ColumnTextParser(const ColumnTextParser& o)
      : Thread(), XMLClass()
  {
    setObjectName("ColumnTextParser");
    _text=nullptr;
    _terminate=false;

    _standardTypes=o._standardTypes;
    _columnDescriptions=o._columnDescriptions;

    _startLine=o._startLine;
    _endLine=o._endLine;
    _acceptPattern=o._acceptPattern;
    _rejectPattern=o._rejectPattern;
    _hasComments=o._hasComments;

    _delimiters=o._delimiters;
    _maximumColumnCount=o._maximumColumnCount;
    _columnWidths=o._columnWidths;
    _fixedWidth=o._fixedWidth;
    _acceptNullColumns=o._acceptNullColumns;

    _isSectionLinePattern=o._isSectionLinePattern;
    _sectionLinePattern=o._sectionLinePattern;
    _isSectionLineCount=o._isSectionLineCount;
    _sectionLineCount=o._sectionLineCount;
    _isSectionRowCount=o._isSectionRowCount;
    _sectionRowCount=o._sectionRowCount;
    _isSectionCellPattern=o._isSectionCellPattern;
    _sectionCellColumn=o._sectionCellColumn;
    _sectionCellPattern=o._sectionCellPattern;
    connect(this, SIGNAL(finished()), this, SIGNAL(rowsParsed()));
  }

  /*!
    Description of destructor still missing
  */
  ColumnTextParser::~ColumnTextParser()
  {
    TRACE;
    delete _text;
  }

  void ColumnTextParser::setTypes(const QVector<int>& types)
  {
    TRACE;
    int n=types.count();
    for(int i=0; i<n; i++) {
      setType(i, types[i]);
    }
  }

  /*!
    Set test stream to parse. \a s is deleted by the destructor or when calling this
    function several times. It is safe to call this function even when the parser is
    running. In this case it is forced to stop updates before switching to the new stream.
  */
  void ColumnTextParser::setText(QTextStream * s)
  {
    TRACE;
    if(isRunning()) {
      stopUpdates();
    }
    delete _text;
    _text=s;
  }

  void ColumnTextParser::stopUpdates()
  {
    TRACE;
    if(isRunning()) {
      _terminate.fetchAndStoreOrdered(true);
      wait();
    }
  }

  void ColumnTextParser::startUpdates()
  {
    TRACE;
    _terminate=false;
    emit rowsAboutToBeParsed();
    start();
  }

  /*!
    Set the list of standard types. Used when restoring from an xml stream and by setType().
    The first item of \a standardTypes has type index 1. A default "Skipped" type is always
    prepended, hence there is no need to include one in \a standardTypes.
  */
  void ColumnTextParser::setStandardTypes(const QStringList& standardTypes)
  {
    TRACE;
    _standardTypes=standardTypes;
    _standardTypes.prepend(tr("Skipped"));
  }

  /*!
    Returns column description of column \a column. Size of internal vector is eventually
    increased if \a column does not exist yet.
  */
  ColumnTextDescription& ColumnTextParser::columnDescription(int column)
  {
    if(column>=_columnDescriptions.count()) {
      if(column>_maximumColumnCount) {
        App::log(tr("Maximum column index is %1\n").arg(_maximumColumnCount) );
        column=_maximumColumnCount;
      } else {
        _columnDescriptions.resize(column+1);
      }
    }
    return _columnDescriptions[column];
  }

  /*!
    Returns column description of column \a column. If \a column does not exist yet, the
    description of the last column is returned.
  */
  const ColumnTextDescription& ColumnTextParser::columnDescription(int column) const
  {
    if(column>=_columnDescriptions.count()) {
      return _columnDescriptions.last();
    } else {
      return _columnDescriptions[column];
    }
  }

  /*!
    Sets type of column index \a column to \a type.

    setTypeName() is called only from ColumnTextItem, in response to user action.
    The list of standard type is never huge, hence we search in a QStringList.
  */
  void ColumnTextParser::setTypeName(int column, QString type)
  {
    int i=_standardTypes.indexOf(type);
    if(i>-1) {
      columnDescription(column).setType(i);
    }
  }

  /*!
    \fn QString ColumnTextParser::type(int column) const
    Returns the type of column \a column.
  */

  /*!
    \fn QString ColumnTextParser::typeName(int column) const
    Returns the type of column \a column.
  */

  /*!
    \fn double ColumnTextParser::factor(int column) const
    Returns the factor of column \a column.
  */

  /*!
    \fn QString ColumnTextParser::replaceRx(int column) const
    Returns the regular expression of column \a column.
  */

  /*!
    \fn QString ColumnTextParser::replaceAfter(int column) const
    Returns the replacement string of column \a column.
  */


  /*!
    \fn void ColumnTextParser::setStartLine(int i)
    Lines start at 1, If \a i is 0, no limit on start line is set.
  */

  /*!
    \fn void ColumnTextParser::setEndLine(int i)
    Lines start at 1, If \a i is 0, no limit on end line is set.
  */

  QVector<int> ColumnTextParser::columnWidths(const QString& ws)
  {
    QVector<int> wi;
    QStringList wStrList=ws.split(",");
    for(QStringList::iterator it=wStrList.begin();it!=wStrList.end();it++) {
      wi.append(it->trimmed().toInt());
    }
    return wi;
  }

  QString ColumnTextParser::columnWidths(const QVector<int>& wi)
  {
    QString ws;
    QVector<int>::const_iterator it=wi.begin();
    if(it!=wi.end()) {
      ws=QString::number(*it);
      for(it++;it!=wi.end();it++) {
        ws += ", " + QString::number(*it);
      }
    }
    return ws;
  }

  int ColumnTextParser::rowCount() const
  {
    TRACE;
    QMutexLocker ml(&_mutex);
    return _cells.count();
  }

  int ColumnTextParser::columnCount() const
  {
    TRACE;
    QMutexLocker ml(&_mutex);
    return _columnCount;
  }

  int ColumnTextParser::lineNumber(int row) const
  {
    QMutexLocker ml(&_mutex);
    if(row<_cells.count()) {
      return _lines2rows[row];
    } else {
      return 0;
    }
  }

  int ColumnTextParser::sectionIndexOf(int row) const
  {
    QMutexLocker ml(&_mutex);
    QList<int>::const_iterator it=std::lower_bound(_sectionMarkers.begin(), _sectionMarkers.end(), row);
    return it-_sectionMarkers.begin();
  }

  int ColumnTextParser::sectionBeginRow(int index) const
  {
    QMutexLocker ml(&_mutex);
    if(index<_sectionMarkers.count()) {
      return _sectionMarkers.at(index);
    } else {
      return 0;
    }
  }

  int ColumnTextParser::sectionEndRow(int index) const
  {
    QMutexLocker ml(&_mutex);
    index++;
    if(index<_sectionMarkers.count()) {
      return _sectionMarkers.at(index);
    } else {
      return _cells.count();
    }

  }

  const QString& ColumnTextParser::sectionComments(int index) const
  {
    QMutexLocker ml(&_mutex);
    if(index<_sectionComments.count()) {
      return _sectionComments.at(index);
    } else {
      return _nullCell;
    }
  }

  const QString& ColumnTextParser::text(int row, int column) const
  {
    QMutexLocker ml(&_mutex);
    if(row<_cells.count()) {
      const QStringList& cols=_cells.at(row);
      if(column<cols.count()) {
        return cols.at(column);
      }
    }
    return _nullCell;
  }

  void ColumnTextParser::run()
  {
    TRACE;
    _mutex.MUTEX_LOCK;
    _columnCount=-1;
    _cells.clear();
    _sectionMarkers.clear();
    _sectionComments.clear();
    _lines2rows.clear();
    _mutex.unlock();
    parseRows();
  }

  void ColumnTextParser::addSection(int row)
  {
    if(_sectionMarkers.isEmpty() || _sectionMarkers.last()<row) {
      _sectionMarkers.append(row);
      _sectionComments.append(QString());
    }
  }

  void ColumnTextParser::parseRows()
  {
    _mutex.MUTEX_LOCK;
    _lineNumber=0;
    _lastSectionMarkerLine=0;
    _lastSectionMarkerRow=0;
    addSection(0); // add at least one section
    _mutex.unlock();
    if(!_text || !_text->seek(0)) return;
    QRegExp * acceptRegExp, * rejectRegExp, * sectionLineRegExp, * sectionCellRegExp;
    if(_acceptPattern.isEmpty()) {
      acceptRegExp=nullptr;
    } else {
      acceptRegExp=new QRegExp(_acceptPattern);
    }
    if(_rejectPattern.isEmpty()) {
      rejectRegExp=nullptr;
    } else {
      rejectRegExp=new QRegExp(_rejectPattern);
    }
    if(_isSectionLinePattern) {
      sectionLineRegExp=new QRegExp(_sectionLinePattern);
    } else {
      sectionLineRegExp=nullptr;
    }
    if(_isSectionCellPattern) {
      sectionCellRegExp=new QRegExp(_sectionCellPattern);
    } else {
      sectionCellRegExp=nullptr;
    }
    while(!_text->atEnd()) {
      QString rowStr=_text->readLine();
      _mutex.MUTEX_LOCK;
      _lineNumber++;
      if(_isSectionLinePattern && rowStr.contains(*sectionLineRegExp)) {
        addSection(_cells.count());
        _lastSectionMarkerLine=_lineNumber;
        _lastSectionMarkerRow=_cells.count();
      }
      if(_isSectionLineCount && _lineNumber >= (_lastSectionMarkerLine+_sectionLineCount)) {
        addSection(_cells.count());
        _lastSectionMarkerLine=_lineNumber;
        _lastSectionMarkerRow=_cells.count();
      }
      _mutex.unlock();
      if((_startLine==0 || _lineNumber>=_startLine) &&
         (_endLine==0 || _lineNumber<=_endLine) &&
         (!acceptRegExp || rowStr.contains(*acceptRegExp)) &&
         (!rejectRegExp || !rowStr.contains(*rejectRegExp))) {
        _mutex.MUTEX_LOCK;
        if(_isSectionRowCount && _cells.count()>=(_lastSectionMarkerRow+_sectionRowCount)) {
          addSection(_cells.count());
          _lastSectionMarkerLine=_lineNumber;
          _lastSectionMarkerRow=_cells.count();
        }

        _lines2rows.insert(_cells.count(), _lineNumber);
        parseColumns(rowStr, sectionCellRegExp);
        _mutex.unlock();
      } else if(_hasComments) { // Rejected lines are considered as comments
        _sectionComments.last()+=rowStr+"\n";
      }
      if(_endLine>0 && _lineNumber>_endLine) break;
      if(_terminate.testAndSetOrdered(true, true)) break;
    }
    delete sectionLineRegExp;
    delete sectionCellRegExp;
    delete acceptRegExp;
    delete rejectRegExp;
  }

  void ColumnTextParser::parseColumns(const QString& rowStr, QRegExp * sectionCellRegExp)
  {
    QStringList cols;
    if(_fixedWidth) {
      int pos=0;
      for(QVector<int>::const_iterator it=_columnWidths.begin(); it!=_columnWidths.end();it++) {
        if(_terminate.testAndSetOrdered(true, true)) return;
        if(pos>=rowStr.count()) break;
        cols.append(cellContent(cols.count(), rowStr.mid(pos, *it)));
        pos+=*it;
      }
      if(pos<rowStr.count()) {
        cols.append(cellContent(cols.count(), rowStr.mid(pos)));
      }
    } else {
      StringSection rowStrSec(rowStr);
      StringSection colStrSec;
      const QChar * ptr=nullptr;
      int lastColumnIndex=_maximumColumnCount-1;
      while(true) {
        if(_terminate.testAndSetOrdered(true, true)) return;
        if(cols.count()==lastColumnIndex) {
          colStrSec=rowStrSec.nextField(ptr, "");
          if(!colStrSec.isValid()) break;
          cols.append(cellContent(cols.count(), colStrSec.toString()));
          break;
        } else {
          colStrSec=rowStrSec.nextField(ptr, _delimiters, !_acceptNullColumns);
          if(!colStrSec.isValid()) break;
          cols.append(cellContent(cols.count(), colStrSec.toString()));
        }
      }

    }
    if(_isSectionCellPattern && cols.count()>_sectionCellColumn) {
      if(cols.at(_sectionCellColumn).contains(*sectionCellRegExp)) {
        addSection(_cells.count()-1); // include this row as well
        _lastSectionMarkerLine=_lineNumber;
        _lastSectionMarkerRow=_cells.count()-1;
      }
    }
    _cells.append(cols);
    if(cols.count()>_columnCount) {
      _columnCount=cols.count();
    }
  }

  /*!
    Edits cell content according to replaceRx, replaceAfter and factor.
  */
  QString ColumnTextParser::cellContent(int column, QString s) const
  {
    QString rxStr=replaceRx(column);
    if(!rxStr.isEmpty()) {
      QRegExp rx(rxStr);
      s.replace(rx, replaceAfter(column));
    }
    double fac=factor(column);
    if(fac!=1.0) {
      return QString::number(s.toDouble()*fac);
    }
    return s;
  }

  void ColumnTextParser::xml_writeProperties(XML_WRITEPROPERTIES_ARGS) const
  {
    TRACE;
    Q_UNUSED(context)
    writeProperty(s, "startLine", _startLine);
    writeProperty(s, "endLine", _endLine);
    writeProperty(s, "acceptPattern", _acceptPattern);
    writeProperty(s, "rejectPattern", _rejectPattern);
    writeProperty(s, "hasComments", static_cast<bool>(_hasComments));

    writeProperty(s, "fixedWidth", static_cast<bool>(_fixedWidth));
    writeProperty(s, "acceptNullColumns", static_cast<bool>(_acceptNullColumns));
    writeProperty(s, "delimiters", _delimiters);
    writeProperty(s, "maximumColumnCount", _maximumColumnCount);
    writeProperty(s, "columnWidths", columnWidths(_columnWidths));

    writeProperty(s, "isSectionLinePattern", static_cast<bool>(_isSectionLinePattern));
    writeProperty(s, "sectionLinePattern", _sectionLinePattern);
    writeProperty(s, "isSectionLineCount", static_cast<bool>(_isSectionLineCount));
    writeProperty(s, "sectionLineCount", _sectionLineCount);
    writeProperty(s, "isSectionRowCount", static_cast<bool>(_isSectionRowCount));
    writeProperty(s, "sectionRowCount", _sectionRowCount);
    writeProperty(s, "isSectionCellPattern", static_cast<bool>(_isSectionCellPattern));
    writeProperty(s, "sectionCellColumn", _sectionCellColumn);
    writeProperty(s, "sectionCellPattern", _sectionCellPattern);
  }

  void ColumnTextParser::xml_writeChildren(XML_WRITECHILDREN_ARGS) const
  {
    TRACE;
    Q_UNUSED(context)
    static const QString key("column");
    XMLSaveAttributes att;
    QString& value=att.add(key);
    int n=_columnDescriptions.count();
    for(int i=0; i<n; i++) {
      value=QString::number(i);
      _columnDescriptions[i].xml_save(s, context, att);
    }
  }

  XMLMember ColumnTextParser::xml_member(XML_MEMBER_ARGS)
  {
    TRACE;
    Q_UNUSED(context)
    if(tag=="ColumnTextDescription") {
      static const QString tmp("column");
      XMLRestoreAttributeIterator it=attributes.find(tmp);
      if(it!=attributes.end()) {
        return XMLMember(&columnDescription(it.value().toInt()));
      } else {
        App::log(tr("No column index defined for description\n") );
        return XMLMember(XMLMember::Unknown);
      }
    } else if(tag=="startLine") return XMLMember(0);
    else if(tag=="endLine") return XMLMember(1);
    else if(tag=="acceptPattern") return XMLMember(2);
    else if(tag=="rejectPattern") return XMLMember(3);
    else if(tag=="hasComments") return XMLMember(4);

    else if(tag=="fixedWidth") return XMLMember(5);
    else if(tag=="acceptNullColumns") return XMLMember(6);
    else if(tag=="delimiters") return XMLMember(7);
    else if(tag=="maximumColumnCount") return XMLMember(8);
    else if(tag=="columnWidths") return XMLMember(9);

    else if(tag=="isSectionLinePattern") return XMLMember(10);
    else if(tag=="sectionLinePattern") return XMLMember(11);
    else if(tag=="isSectionLineCount") return XMLMember(12);
    else if(tag=="sectionLineCount") return XMLMember(13);
    else if(tag=="isSectionRowCount") return XMLMember(14);
    else if(tag=="sectionRowCount") return XMLMember(15);
    else if(tag=="isSectionCellPattern") return XMLMember(16);
    else if(tag=="sectionCellColumn") return XMLMember(17);
    else if(tag=="sectionCellPattern") return XMLMember(18);
    return XMLMember(XMLMember::Unknown);
  }

  bool ColumnTextParser::xml_setProperty(XML_SETPROPERTY_ARGS)
  {
    TRACE;
    Q_UNUSED(tag)
    Q_UNUSED(attributes)
    Q_UNUSED(context)
    switch (memberID) {
    case 0: _startLine=content.toInt(); return true;
    case 1: _endLine=content.toInt(); return true;
    case 2: _acceptPattern=content.toString(); return true;
    case 3: _rejectPattern=content.toString(); return true;
    case 4: _hasComments=content.toBool(); return true;

    case 5: _fixedWidth=content.toBool(); return true;
    case 6: _acceptNullColumns=content.toBool(); return true;
    case 7: _delimiters=content.toString(); return true;
    case 8: _maximumColumnCount=content.toInt(); return true;
    case 9: _columnWidths=columnWidths(content.toString()); return true;

    case 10: _isSectionLinePattern=content.toBool(); return true;
    case 11: _sectionLinePattern=content.toString(); return true;
    case 12: _isSectionLineCount=content.toBool(); return true;
    case 13: _sectionLineCount=content.toInt(); return true;
    case 14: _isSectionRowCount=content.toBool(); return true;
    case 15: _sectionRowCount=content.toInt(); return true;
    case 16: _isSectionCellPattern=content.toBool(); return true;
    case 17: _sectionCellColumn=content.toInt(); return true;
    case 18: _sectionCellPattern=content.toString(); return true;
    default: return false;
    }
  }

  /*!
    After restoring from an xml stream there might be some non-standard column types, remove them.
  */
  bool ColumnTextParser::xml_polish(XML_POLISH_ARGS)
  {
    TRACE;
    Q_UNUSED(context)
    int standardTypeCount=_standardTypes.count();
    int nColumns=_columnDescriptions.count();
    for(int i=0; i<nColumns; i++) {
      if(_columnDescriptions[i].type()>=standardTypeCount) {
        _columnDescriptions[i].setType(0);
      }
    }
    return true;
  }

  /*!
    Print a convenient message when detecting an error
  */
  void ColumnTextParser::errorParsingColumn(int iCol, int iRow) const
  {
    TRACE;
    App::log(tr("error parsing '%1' from '%2' at column %3 and line %4\n")
             .arg(typeName(iCol))
             .arg(text(iRow, iCol))
             .arg(iCol)
             .arg(lineNumber(iRow)));
  }

} // namespace QGpCoreTools
