/***************************************************************************
**
**  This file is part of DinverCore.
**
**  DinverCore is free software: you can redistribute it and/or modify
**  it under the terms of the GNU General Public License as published by
**  the Free Software Foundation, either version 3 of the License, or
**  (at your option) any later version.
**
**  DinverCore is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with Foobar.  If not, see <http://www.gnu.org/licenses/>
**
**  See http://www.geopsy.org for more information.
**
**  Created: 2010-12-22
**  Copyright: 2010-2019
**    Marc Wathelet (LGIT, Grenoble, France)
**
***************************************************************************/

#include "ModelRepository.h"
#include "GeneratorModels.h"
#include "AbstractForward.h"
#include "ParentIndex.h"
#include "NewModel.h"
#include "ModelSet.h"
#include "BestModels.h"
#include "ReportWriter.h"
#include "ReportReader.h"

namespace DinverCore {

  /*!
    \class ModelRepository ModelRepository.h
    \brief Brief description of class still missing

    Full description of class still missing
  */

  /*!
    Description of constructor still missing
  */
  ModelRepository::ModelRepository(QObject *parent)
    : QObject(parent), _terminated(false)
  {
    TRACE;

    _maximumModelCount=10000;         // Similar to Ns0+Itmax*Ns in old releases
    _maximumBestModelCount=INT_MAX;   // by default no limit
    _maximumQueueLenght=50;           // Similar to ns=50 in old releases
    _maximumSavedMisfit=std::numeric_limits<double>::infinity();     // by default no limit
    _stableMisfitCount=INT_MAX;       // by default do not check misfit stability
    _significantMisfitReduction=0.01; // Defines what's a significant misfit reduction (for misfit stability assessment)
                                      // Default=1%
    _lastBestMisfit=std::numeric_limits<double>::infinity();         // Misfit increases when many atomic cells are detected
                                      // Best cell has already been found, it start exploring elsewhere
    _lastBestIndex=INT_MAX;          // Prevent stopping too early
    _giveUp=0.9;
    _walkCount=2;

    _processingQueue=false;
    _report=nullptr;
    _forward=nullptr;
    _parameterSpaceChecksum=0;
    _allModels=nullptr;
    _generatorModels=nullptr;
  }

  /*!
    Description of destructor still missing
  */
  ModelRepository::~ModelRepository()
  {
    TRACE;
    wait();
    delete _forward;
    // Not owner of forward
    delete _allModels;
    if(_generatorModels) {
      GeneratorModels::removeReference(_generatorModels);
    }
    delete _report;
  }

  /*!
    Returns true if the two repositories have the same parameterization.
  */
  bool ModelRepository::operator==(const ModelRepository& o) const
  {
    return _forward->parameterSpace()==o._forward->parameterSpace();
  }

  void ModelRepository::setMaximumModelCount(int n)
  {
    TRACE;
    _maximumModelCount=n;
    if(_maximumQueueLenght>_maximumModelCount) {
      App::log(tr("Queue length reduced to %1\n").arg(_maximumModelCount) );
      _maximumQueueLenght=_maximumModelCount;
    }
  }

  /*!
    Sets forward solver. Ownership is not taken over \a f. It can be
    deleted right after this function call.
  */
  bool ModelRepository::setForward(const AbstractForward * f)
  {
    TRACE;
    _forward=f->clone(); // Even the "mother" forward is created with clone
                         // It makes sure that all forwards are created the same way
                         // This is critical for parameter grids when limits are
                         // modified by adjustRanges()
    _forward->parameterSpace().setVariableParameters();
    if(_forward->parameterSpace().variableParameterCount()<=0) {
      App::log(tr("No variable parameters, check parameterization.\n") );
      return false;
    }
    if(!_forward->parameterSpace().adjustRanges()) {
      return false;
    }
    _forward->parameterSpace().humanInfo();
    return true;
  }

  void ModelRepository::setStorage()
  {
    if(_allModels) {
      ASSERT(_generatorModels);
      delete _allModels;
      GeneratorModels::removeReference(_generatorModels);
    }
    _parameterSpaceChecksum=_forward->parameterSpace().checksum();
    int ndVar=_forward->parameterSpace().variableParameterCount();
    _allModels=new ModelSet(ndVar, 1);
    _generatorModels=new GeneratorModels(_allModels);
    _generatorModels->addReference();
  }
  void ModelRepository::setBestModelCount(int nr)
  {
    TRACE;
    _generatorModels->setBestModelCount(nr);
  }

  /*!
    Initializes report.
  */
  bool ModelRepository::openReport(const QString& fileName)
  {
    TRACE;
    if(!fileName.isEmpty()) {
      delete _report;
      _report=new ReportWriter(fileName);
      if(!_report->open()) {
        App::log(tr("Cannot open report %1 for writing\n").arg(fileName) );
        delete _report;
        _report=nullptr;
        return false;
      }
    }
    return true;
  }

  void ModelRepository::init()
  {
    TRACE;
    _terminated.setValue(false);
  }

  bool ModelRepository::start(int nThreads, Generator::Type type)
  {
    TRACE;
    // Adjust number of threads
    if(nThreads>CoreApplication::instance()->maximumThreadCount()) {
      nThreads=CoreApplication::instance()->maximumThreadCount();
    }
    if(nThreads>_forward->maximumThreadCount()) {
      nThreads=_forward->maximumThreadCount();
    }
    if(nThreads<1) nThreads=1;
    App::log(tr("Starting %1 generator thread(s)\n").arg(nThreads) );

    // Calculate seed model
    Random randomNumber;
    if(_generatorModels->activeModels()->count()==0) {
      if(!_forward->firstModel(randomNumber, _terminated)) {
        return false;
      }
    } else {
      // Load one of the best models into _forward
      RealSpace& parameterSpace=_forward->parameterSpace();
      int ndVar=parameterSpace.variableParameterCount();
      const int * model=_generatorModels->bestModels()->model(randomNumber.uniform(0, _generatorModels->bestModels()->count()-1));
      for(int ip=0; ip<ndVar; ip++) {
        parameterSpace.variableParameter(ip)->setGridValue(model[ip]);
      }
      // Some parametrizations may need some updates before proceeding
      _forward->valueChanged();
      // Check validity of existing best models
      if(!parameterSpace.isOkDebug()) {
        return false;
      }
    }

    // Init all forwards with this first model
    qDeleteAll(_generators);
    _generators.clear();
    for(int i=0; i<nThreads; i++) {
      AbstractForward * f=_forward->clone();
      RealSpace& parameterSpace=f->parameterSpace();
      parameterSpace.setVariableParameters();
      parameterSpace.adjustRanges(); // Assumed to be ok, because checked before for the first one
      // Init first model for f
      f->copyValues(*_forward);
      Generator * g=new Generator(this, f, randomNumber.uniform(0, INT_MAX));
      g->setObjectName(QString("Generator_%1").arg(i));
      App::setStream(App::stream(nullptr), g);
      g->setType(type);
      g->setWalkCount(_walkCount);
      _generators.append(g);
    }
    _chrono.start();
    for(QList<Generator *>::iterator it=_generators.begin(); it!=_generators.end(); it++) {
      Generator * g=*it;
      connect(g, SIGNAL(finished()), this, SLOT(generatorFinished()));
      g->start();
    }
    return true;
  }

  void ModelRepository::stop()
  {
    TRACE;
    _terminated.setValue(true);
    for(QList<Generator *>::iterator it=_generators.begin(); it!=_generators.end(); it++) {
      (*it)->stop();
    }
  }

  void ModelRepository::generatorFinished()
  {
    TRACE;
    for(QList<Generator *>::iterator it=_generators.begin(); it!=_generators.end(); it++) {
      if((*it)->isRunning()) {
        return;
      }
    }
    // Emit only if all generators are done
    emit finished();
  }

  void ModelRepository::wait()
  {
    TRACE;
    for(QList<Generator *>::iterator it=_generators.begin(); it!=_generators.end(); it++) {
      if((*it)->isRunning()) {
        (*it)->wait();
      }
    }
    // Flush model queue
    _queueLock.lock();
    processQueue();
    _queueLock.unlock();
    // Clean generators
    qDeleteAll(_generators);
    _generators.clear();
  }

  void ModelRepository::clear()
  {
    TRACE;
    _allModels->clear();
    _generatorModels->clear();
  }

  void ModelRepository::printActiveModels()
  {
    static int index=0;
    const ActiveModels * ms=_generatorModels->activeModels();
    for(int i=0; i<ms->count(); i++) {
      const ActiveModel& m=ms->at(i);
      App::log(1, tr("navigator hit %4 %1 %2 %3 %5\n")
                       .arg(i)
                       .arg(m.navigatorHitCount())
                       .arg(m.navigatorHitTime())
                       .arg(index)
                       .arg(m.isQuarantined() ? 1 : 0));
    }
    index++;
  }

  /*!
    Called by all Generator before creating one new model.
    Thread-safe.

    \a randomValue must be between 0 and 1, generated by Random::ran2().

    Returns a ParentIndex ready for model generation. \a parameterSpace is
    populated with the values of the selected parent model.
  */
  ParentIndex * ModelRepository::createModel(double randomValue, RealSpace& parameterSpace) const
  {
    TRACE;
    int ndVar=parameterSpace.variableParameterCount();
    // Selects randomly one of the best cells
    _generatorModelLock.lockForRead();
    const BestModels * bestModels=_generatorModels->bestModels();
    int bestModelIndex=qRound((bestModels->count()-1)*randomValue);
    ParentIndex * parentIndex=new ParentIndex(_generatorModels, bestModels->modelIndex(bestModelIndex));
    // Copy model to parameter structure to enable condition testing
    _allModels->lock();
    const int * v=parentIndex->activeModels()->model(parentIndex->activeIndex());
    for(int ip=0; ip<ndVar; ip++) {
      parameterSpace.variableParameter(ip)->setGridValue(v[ip]);
    }
    _allModels->unlock();
    _generatorModelLock.unlock();
    return parentIndex;
  }

  /*!
    Called by all ModelGenerator after creating one new model.
    Thread-safe.
  */
  void ModelRepository::addModel(NewModel * m)
  {
    TRACE;
    _queueLock.lock();
    _newModels.enqueue(m);
    if(!_processingQueue && _newModels.count()>=_maximumQueueLenght) {
      processQueue();
    }
    _queueLock.unlock();
  }

  /*!
    Process model queue. _queueLock must be locked on entry.
  */
  void ModelRepository::processQueue()
  {
    TRACE;
    _processingQueue=true; // Prevent dequeue by several threads
    int generationTime=_chrono.elapsed();
    _chrono.start();

    // Counts navigator hits and accepted/rejected models
    // No need to lock because only this thread can touch the current GeneratorModels
    QList<NewModel *> modelList;
    while(!_newModels.isEmpty()) {
      NewModel * m=_newModels.dequeue();
      _queueLock.unlock();
      _generatorModels->commitStatistics(m, _giveUp);
      modelList.append(m);
      _queueLock.lock();
    }
    //int statisticsTime=_chrono.elapsed();

    if(!modelList.isEmpty()) {
      _queueLock.unlock();
      // Gets a clean new GeneratorModels with no dead cells
      // No need to lock for read because only this thread can lock for write.
      GeneratorModels * newGenModels=_generatorModels->removeDeadModels();
      newGenModels->addReference();
      //int removeDeadTime=_chrono.elapsed();

      _allModels->lock();  // update() makes use of misfit()
                           // This lock may be blocked by model generation in other threads
                           // during reallocation only, which is likely to occur seldomly.

      // Update best models if there are less than nr models in best models
      newGenModels->bestModels()->update();

      // Adds new models
      for(QList<NewModel *>::iterator it=modelList.begin(); it!=modelList.end(); ++it) {
        NewModel * m=*it;
        if(m->isValidParent() && m->isValidMisfit()) {
          _allModels->setMisfit(m->modelIndex(), 0, m->misfit());
          newGenModels->add(m->modelIndex());
        }
        delete m;
      }
      //int addNewTime=_chrono.elapsed();

      // Regenerates an up-to-date ScaledModels
      newGenModels->setScaledModels(_forward->parameterSpace());
      int nGenerated=_allModels->count();
      //int scaleTime=_chrono.elapsed();

      _allModels->unlock();

      if(nGenerated==0) {
        stop();
      } else {
        double bestMisfit=newGenModels->bestModels()->misfit(0)[0];
        App::log(tr("[stat] %1 models, %2 models/s, %3 active models, %4 rejected, %5 quarantined, %6 best models, best misfit %7 last best %8 at index %9\n")
            .arg(nGenerated)
            .arg(1000.0*modelList.count()/generationTime)
            .arg(newGenModels->activeModels()->count())
            .arg(newGenModels->rejectedCount())
            .arg(newGenModels->quarantinedCount())
            .arg(newGenModels->bestModels()->count())
            .arg(bestMisfit).arg(_lastBestMisfit).arg(_lastBestIndex));

        int effectiveModelCount=nGenerated-newGenModels->rejectedCount();
        if(effectiveModelCount>=_maximumModelCount ||
           newGenModels->bestModels()->count()>=_maximumBestModelCount) {
          stop();
        }
        // Monitor best misfit values
        if(bestMisfit<=_lastBestMisfit) { // If misfit increases because of atomic cells
                                          // we want to keep track of the very best misfit
                                          // and not just the current one
          if(_lastBestMisfit-bestMisfit>_significantMisfitReduction*_lastBestMisfit) {
            _lastBestIndex=newGenModels->activeModels()->at(newGenModels->bestModels()->modelIndex(0)).modelIndex();
          }
          _lastBestMisfit=bestMisfit;
        }
        if(effectiveModelCount>_stableMisfitCount) {
          if(effectiveModelCount>_lastBestIndex+_stableMisfitCount) {
            stop();
          }
        }
      }

      // Switches to new active models
      _generatorModelLock.lockForWrite();
      GeneratorModels::removeReference(_generatorModels);
      _generatorModels=newGenModels;
      _generatorModelLock.unlock();

      //int totalQueueProcessTime=_chrono.elapsed();
      _queueLock.lock();
      /*App::log(tr("Queue length %1 queue process time %2: %3 %4 %5 %6").arg(_newModels.count())
          .arg(totalQueueProcessTime-generationTime)
          .arg(statisticsTime-generationTime)
          .arg(removeDeadTime-statisticsTime)
          .arg(addNewTime-removeDeadTime)
          .arg(scaleTime-addNewTime)
          << endl;*/
    }

    _processingQueue=false;
  }

  void ModelRepository::addReport(NewModel * m, AbstractForward * forward)
  {
    TRACE;
    if(_report &&
       m->isValidParent() &&
       m->isValidMisfit() &&
       m->misfit()<=_maximumSavedMisfit) {
      _reportLock.lock();
      _report->addModel(m->misfit(), _parameterSpaceChecksum, forward->parameterSpace());
      forward->writeReport(_report);
      _reportLock.unlock();
    }
  }

  /*!
    If \a strict is true, the parameter space checksum must exactly match
  */
  bool ModelRepository::importModels(QString fileName, bool strict)
  {
    TRACE;
    int i=_allModels->count();

    if(ReportReader::isReportFormat(fileName)) {
      ReportReader report(fileName);
      if(!report.open()) {
        App::log(tr("Error opening report %1\n").arg(fileName) );
        return false;
      }
      report.synchronize();
      _maximumModelCount=report.nModels();

      if(!_allModels->importModelsFromReport(_forward->parameterSpace(), report, strict)) {
        return false;
      }
    } else {
      if(!_allModels->importModelsFromAscii(_forward->parameterSpace(), fileName)) {
        return false;
      }
    }
    int n=_allModels->count();
    for(;i<n;i++) {
      _generatorModels->add(i);
    }
    return true;
  }

  /*!
    Used only by external objects to acces generated models (read only)
  */
  void ModelRepository::lock() const
  {
    if(_allModels) {
      _allModels->lock();
    }
    _generatorModelLock.lockForRead();
  }

  /*!
    Used only by external objects to acces generated models (read only)
  */
  void ModelRepository::unlock() const
  {
    if(_allModels) {
      _allModels->unlock();
    }
    _generatorModelLock.unlock();
  }

  int ModelRepository::validModelCount() const
  {
    return _allModels->count()-_generatorModels->rejectedCount();
  }

  int ModelRepository::activeModelCount() const
  {
    return _generatorModels->activeModels()->count();
  }

  int ModelRepository::bestModelCount() const
  {
    return _generatorModels->bestModels()->count();
  }

  int ModelRepository::rejectedCount() const
  {
    return _generatorModels->rejectedCount();
  }

  int ModelRepository::giveUpCount() const
  {
    return _generatorModels->giveUpCount();
  }

  int ModelRepository::visitedModelCount() const
  {
    if(!_allModels) printf("Blank all models\n");
    return _allModels ? _allModels->count() : 0;
  }

  int ModelRepository::variableParameterCount() const
  {
    return _allModels ? _allModels->parameterCount() : 0;
  }

  double ModelRepository::misfit(int modelIndex) const
  {
    // TODO: multi-dimensional misfit
    return _allModels ? _allModels->misfit(modelIndex)[0] : 0;
  }

  const int * ModelRepository::model(int modelIndex) const
  {
    return _allModels->model(modelIndex);
  }

  int ModelRepository::bestModelIndex() const
  {
    // TODO: multi-dimensional misfit
    return _allModels->bestModel();
  }

  const Parameter * ModelRepository::variableParameter(int paramIndex) const
  {
    const RealSpace& parameterSpace=_forward->parameterSpace();
    return parameterSpace.variableParameter(paramIndex);
  }

  double ModelRepository::variableParameterValue(int modelIndex, int paramIndex) const
  {
    return variableParameter(paramIndex)->realValue(_allModels->model(modelIndex)[paramIndex]);
  }

} // namespace DinverCore
