src/convModel/gc_datastore_files.cpp

// $Id: gc_datastore_files.cpp,v 1.36 2011/06/22 18:22:22 jmcgill Exp $

/*
  Copyright 2002  Mary Kuhner, Jon Yamato, and Joseph Felsenstein

  This software is distributed free of charge for non-commercial use
  and is copyrighted.  Of course, we do not guarantee that the software
  works, and are not responsible for any damage you may cause or have.
*/

#include <cassert>

#include "gc_creation_info.h"
#include "gc_data.h"
#include "gc_datastore.h"
#include "gc_file.h"
#include "gc_infile_err.h"
#include "gc_loci_match.h"
#include "gc_locus_err.h"
#include "gc_parse_block.h"
#include "gc_parse_locus.h"
#include "gc_parse_sample.h"
#include "gc_pop_match.h"
#include "gc_strings.h"
#include "gc_structures_err.h"

#include "wx/file.h"
#include "wx/filename.h"
#include "wx/log.h"

//------------------------------------------------------------------------------------

GCFile &
GCDataStore::AddDataFile(wxString           fullPathFileName,
                         GCFileFormat    format,
                         gcGeneralDataType   dataType,
                         GCInterleaving  interleaving)
// version to use when reading from batch command file
{
    // test for readability
    if(! ::wxFileExists(fullPathFileName))
    {
        throw missing_file_error(fullPathFileName.c_str());
    }

    GCFile * file = new GCFile(*this,fullPathFileName);
    GCParseVec * parseVecP = new GCParseVec();

    try
    {
        GCParse * parse = OneParse(*file,format,dataType,interleaving);
        parseVecP->push_back(parse);
    }
    catch(const gc_ex& e)
    {
        parseVecP->NukeContents();
        delete parseVecP;
        delete file;
        throw;
    }

    file->SetParses(parseVecP);
    m_structures.AddFile(*file);

    m_dataFiles.insert(file);
    wxLogVerbose(gcverbose::addedFile,fullPathFileName.c_str());
    return *file;
}

GCFile &
GCDataStore::AddDataFile(wxString fullPathFileName)
{
    // test for readability
    if(! ::wxFileExists(fullPathFileName))
    {
        throw missing_file_error (fullPathFileName.c_str());
    }

    GCFile * file = new GCFile(*this,fullPathFileName);

    GCParseVec * parseVecP = AllParsesForFile(*file);
    assert(parseVecP != NULL);
    if (parseVecP->size() == 0)
    {
        delete parseVecP;
        delete file;
        throw unparsable_file_error(fullPathFileName.c_str());
    }

    file->SetParses(parseVecP);
    m_structures.AddFile(*file);
    m_dataFiles.insert(file);
    wxLogVerbose(gcverbose::addedFile,fullPathFileName.c_str());

    try
    {
        if(file->GetParseCount() == 1)
        {
            GCPopMatcher p(popmatch_DEFAULT);
            GCLocusMatcher l(locmatch_DEFAULT);
            SetParseChoice(file->GetParse(0),p,l);
        }
    }
    catch (const duplicate_file_error& e)
    {
        GCWarning(e.what());
    }
    catch (const missing_file_error& e)
    {
        GCWarning(e.what());
    }
    catch (const unparsable_file_error& e)
    {
        GCWarning(e.what());
    }

    return *file;
}

const GCParseBlock *
GCDataStore::GetParseBlock(size_t blockId) const
{
    for(dataFileSet::const_iterator iter=m_dataFiles.begin();
        iter != m_dataFiles.end(); iter++)
    {
        const GCFile & fileRef = **iter;
        for(size_t index=0; index < fileRef.GetParseCount(); index++)
        {
            const GCParse & parseRef = fileRef.GetParse(index);
            constBlockVector blocks = parseRef.GetBlocks();
            for(constBlockVector::const_iterator biter = blocks.begin();
                biter != blocks.end(); biter++)
            {
                const GCParseBlock * blockP = *biter;
                if(blockId == blockP->GetId())
                {
                    return blockP;
                }
            }
        }

    }
    assert(false);
    return NULL;
}

constBlockVector
GCDataStore::GetBlocks(size_t popId, size_t locusId) const
{
    gcIdSet ids = m_structures.GetBlockIds(popId,locusId);
    constBlockVector retVal;

    for(dataFileSet::const_iterator iter=m_dataFiles.begin();
        iter != m_dataFiles.end(); iter++)
    {
        const GCFile & fileRef = **iter;
        for(size_t index=0; index < fileRef.GetParseCount(); index++)
        {
            const GCParse & parseRef = fileRef.GetParse(index);
            constBlockVector blocks = parseRef.GetBlocks();
            for(constBlockVector::const_iterator biter = blocks.begin();
                biter != blocks.end(); biter++)
            {
                const GCParseBlock * blockP = *biter;
                size_t blockId = blockP->GetId();
                if(ids.find(blockId) != ids.end())
                {
                    retVal.push_back(blockP);
                }
            }
        }

    }
    return retVal;
}

constBlockVector
GCDataStore::GetBlocksForLocus(size_t locusId) const
{
    gcIdSet ids = m_structures.GetBlocksForLocus(locusId);
    constBlockVector retVal;

    for(dataFileSet::const_iterator iter=m_dataFiles.begin();
        iter != m_dataFiles.end(); iter++)
    {
        const GCFile & fileRef = **iter;
        for(size_t index=0; index < fileRef.GetParseCount(); index++)
        {
            const GCParse & parseRef = fileRef.GetParse(index);
            constBlockVector blocks = parseRef.GetBlocks();
            for(constBlockVector::const_iterator biter = blocks.begin();
                biter != blocks.end(); biter++)
            {
                const GCParseBlock * blockP = *biter;
                size_t blockId = blockP->GetId();
                if(ids.find(blockId) != ids.end())
                {
                    retVal.push_back(blockP);
                }
            }
        }

    }
    return retVal;
}
size_t
GCDataStore::GetDataFileCount() const
{
    return m_dataFiles.size();
}

GCFile &
GCDataStore::GetDataFile(size_t fileIndex)
{
    for(dataFileSet::iterator i = m_dataFiles.begin(); i != m_dataFiles.end(); i++)
    {
        GCFile & dataFile = **i;
        if(dataFile.GetId() == fileIndex)
        {
            return dataFile;
        }
    }
    wxString msg = wxString::Format(gcerr::missingFileId,(int)fileIndex);
    throw gc_implementation_error (msg.c_str());
}

const GCFile &
GCDataStore::GetDataFile(size_t fileIndex) const
{
    for(dataFileSet::const_iterator i = m_dataFiles.begin(); i != m_dataFiles.end(); i++)
    {
        const GCFile & dataFile = **i;
        if(dataFile.GetId() == fileIndex)
        {
            return dataFile;
        }
    }
    wxString msg = wxString::Format(gcerr::missingFileId,(int)fileIndex);
    throw gc_implementation_error (msg.c_str());
}

const GCParse &
GCDataStore::GetParse(size_t parseIndex) const
{
    for(dataFileSet::const_iterator i = m_dataFiles.begin(); i != m_dataFiles.end(); i++)
    {
        const GCFile & dataFile = **i;
        size_t numParses = dataFile.GetParseCount();
        for(size_t p=0; p < numParses; p++)
        {
            const GCParse & parseRef = dataFile.GetParse(p);
            if(parseRef.GetId() == parseIndex)
            {
                return parseRef;
            }
        }
    }
    wxString msg = wxString::Format(gcerr::missingParseId,(int)parseIndex);
    throw gc_implementation_error(msg.c_str());
}

const dataFileSet &
GCDataStore::GetDataFiles() const
{
    return m_dataFiles;
}

size_t
GCDataStore::GetSelectedDataFileCount() const
{
    return m_structures.SelectedFileCount();
}

bool
GCDataStore::HasNoDataFiles() const
{
    return m_dataFiles.empty();
}

bool
GCDataStore::HasUnparsedFiles() const
{
    return m_structures.HasUnparsedFiles();
}

void
GCDataStore::RemoveDataFile(GCFile & fileRef)
{
    dataFileSet::iterator matching = m_dataFiles.find(&fileRef);
    if(matching != m_dataFiles.end())
    {
        wxLogVerbose(gcverbose::removedFile,fileRef.GetName().c_str());
        gcIdSet blocks = fileRef.IdsOfAllBlocks();
        m_structures.RemoveBlocks(blocks);
        m_dataFiles.erase(matching);
        delete &fileRef;
        // EWFIX.P3 -- need an ignore method?? causes UNDO/REDO problem
    }
}

void
GCDataStore::RemoveFiles(bool selectedFilesOnly)
{
    for(dataFileSet::iterator iter=m_dataFiles.begin();
        iter != m_dataFiles.end(); iter++)
    {
        GCFile & fileRef = **iter;
        size_t fileId = fileRef.GetId();
        if(!selectedFilesOnly || m_structures.GetFileSelection(fileId))
        {
            m_structures.RemoveFile(fileId);
            RemoveDataFile(fileRef);
        }

    }
}

void
GCDataStore::SelectAllFiles()
{
    m_structures.AllFileSelectionsTo(true);
}

void
GCDataStore::UnselectAllFiles()
{
    m_structures.AllFileSelectionsTo(false);
}

bool
GCDataStore::GetSelected(const GCFile & fileRef) const
{
    return m_structures.GetFileSelection(fileRef.GetId());
}

void
GCDataStore::SetSelected(const GCFile & fileRef, bool selected)
{
    m_structures.SetFileSelection(fileRef.GetId(),selected);
}

locVector
GCDataStore::GetLociFor(const GCParse & parseRef, const GCLocusMatcher & locMatch)
{
    locVector loci;

    for(size_t i=0; i < parseRef.GetLociCount(); i++)
    {
        GCLocusSpec spec = locMatch.GetLocSpec(i,parseRef);
        try
        {
            gcLocus & loc = m_structures.GetLocus(spec.GetLocusName());
            loci.push_back(&loc);
        }
        catch(const gc_missing_locus& e)
        {
            loc_match locmatchType = locMatch.GetLocMatchType();
            if(locmatchType == locmatch_DEFAULT || locmatchType == locmatch_LINKED)
                // EWFIX.BUG674 -- take out single region
            {
                size_t lineNumber = parseRef.GetParseLocus(i).GetLineNumber();
                wxString fileName = parseRef.GetFileRef().GetName();
                gcCreationInfo creationInfo = gcCreationInfo::MakeDataFileCreationInfo(lineNumber,fileName);
                try
                {
                    gcRegion & region = m_structures.GetRegion(spec.GetRegionName());
                    gcLocus & loc = m_structures.MakeLocus(region,spec.GetLocusName(),spec.GetBlessedLocus(),creationInfo);
                    gcGeneralDataType dType = parseRef.GetDataType();
                    if(dType.size()== 1)
                    {
                        loc.SetDataType(*dType.begin());
                    }
                    loci.push_back(&loc);
                }
                catch(const missing_region& r)
                {
                    gcRegion & region = m_structures.MakeRegion(spec.GetRegionName(),spec.GetBlessedRegion());
                    gcLocus & loc = m_structures.MakeLocus(region,spec.GetLocusName(),spec.GetBlessedLocus(),creationInfo);
                    gcGeneralDataType dType = parseRef.GetDataType();
                    if(dType.size() == 1)
                    {
                        loc.SetDataType(*dType.begin());
                    }
                    loci.push_back(&loc);
                }
            }
            else
            {
                throw;
            }
        }
    }

    return loci;
}

popVector
GCDataStore::GetPopsFor(const GCParse & parseRef, const GCPopMatcher & popMatch)
{
    popVector pops;

    for(size_t i=0; i < parseRef.GetPopCount(); i++)
    {
        GCPopSpec spec = popMatch.GetPopSpec(i,parseRef);
        try
        {
            gcPopulation & pop = m_structures.GetPop(spec.GetName());
            pops.push_back(&pop);
        }
        catch(const gc_missing_population& e)
        {
            if(popMatch.GetPopMatchType() == popmatch_DEFAULT)
            {
                gcPopulation & pop = m_structures.MakePop(spec.GetName(),spec.GetBlessed());
                pops.push_back(&pop);
            }
            else
            {
                throw;
            }
        }
    }

    return pops;
}

bool
GCDataStore::CanAssignParseLocus(const GCParseLocus & pLocus, const gcLocus & locus) const
{
    // EWFIX.P3 -- refactor with AssignParseLocus
    gcSpecificDataType  locusType   = locus.GetDataType();
    gcGeneralDataType   parseType   = pLocus.GetDataType();

    if(locusType != sdatatype_NONE_SET)
    {
        if(parseType.find(locusType) == parseType.end())
        {
            return false;
        }
    }
    if(locus.HasNumMarkers())
    {
        if(locus.GetNumMarkers() != pLocus.GetNumMarkers())
        {
            return false;
        }
    }
    return true;
}

void
GCDataStore::AssignParseLocus(const GCParseLocus & pLocus, gcLocus & locus)
{
    gcSpecificDataType  locusType   = locus.GetDataType();
    gcGeneralDataType   parseType   = pLocus.GetDataType();

    if(locusType != sdatatype_NONE_SET)
    {
        if(parseType.find(locusType) == parseType.end())
        {
            throw gc_locus_types_mismatch(pLocus.GetName(),locus.GetName(),ToWxString(parseType),ToWxString(locusType));
        }
    }
    else
    {
        if(parseType.size() == 1)
        {
            locus.SetDataType(*(parseType.begin()));
        }
    }

    // number of markers
    if(locus.HasNumMarkers())
    {
        if(locus.GetNumMarkers() != pLocus.GetNumMarkers())
        {
            throw gc_locus_site_count_mismatch(pLocus.GetLongName(),locus.GetLongName(),pLocus.GetNumMarkers(),locus.GetNumMarkers());
        }
    }
    else
    {
        locus.SetNumMarkers(pLocus.GetNumMarkers());
    }
}

void
GCDataStore::AssignPop(const GCParsePop & pPop, gcPopulation & pop)
{
    // EWFIX.P5 LATER -- right now all pops are mergeable
}

void
GCDataStore::SetParseChoice(const GCParse &         parseRef,
                            const GCPopMatcher &    popMatch,
                            const GCLocusMatcher &  locMatch)
{
    // remove assignments to old parse
    const GCFile & fileRef = parseRef.GetFileRef();
    if(m_structures.HasParse(fileRef))
    {
        const GCParse & oldParse = m_structures.GetParse(fileRef);
        m_structures.UnsetParse(oldParse);
        UnsetParseChoice(m_structures.GetParse(fileRef));
    }

    // check we can assign to these pops
    popVector pops = GetPopsFor(parseRef,popMatch);
    assert (pops.size() == parseRef.GetPopCount());
    for(size_t i=0; i < pops.size(); i++)
    {
        const GCParsePop & parsePop = parseRef.GetParsePop(i);
        gcPopulation & pop = *(pops[i]);
        AssignPop(parsePop,pop);
    }

    locVector locs = GetLociFor(parseRef,locMatch);
    assert (locs.size() == parseRef.GetLociCount());

    for(size_t lIndex = 0; lIndex < locs.size(); lIndex++)
    {
        const GCParseLocus & parseLocus = parseRef.GetParseLocus(lIndex);
        gcLocus & locus = *(locs[lIndex]);
        AssignParseLocus(parseLocus,locus);

        for(size_t pIndex = 0; pIndex < pops.size(); pIndex++)
        {
            const GCParseBlock & block = parseRef.GetBlock(pIndex,lIndex);
            size_t blockId = block.GetId();
            m_structures.AssignBlock(blockId,pops[pIndex]->GetId(),locs[lIndex]->GetId());
        }
    }

    // if we make it this far, update the parse in structures
    m_structures.SetParse(parseRef);

    // and document the pop and locus matchers in file info
    GetStructures().SetPopMatcher(fileRef,popMatch);
    GetStructures().SetLocusMatcher(fileRef,locMatch);

}

void
GCDataStore::UnsetParseChoice(const GCParse & parseRef)
{
    //
    constBlockVector blocks = parseRef.GetBlocks();
    for(constBlockVector::iterator i=blocks.begin(); i != blocks.end(); i++)
    {
        const GCParseBlock & block = *(*i);
        m_structures.RemoveBlockAssignment(block.GetId());
    }
    m_structures.UnsetParse(parseRef);
}

const GCParse &
GCDataStore::GetParse(const GCFile & file) const
{
    return m_structures.GetParse(file);
}

const GCParse &
GCDataStore::GetParse(const GCFile & file, size_t index) const
{
    return file.GetParse(index);
}

bool
GCDataStore::HasParse(const GCFile & file) const
{
    return m_structures.HasParse(file);
}

gcGeneralDataType
GCDataStore::GetLegalLocusTypes(size_t locusId) const
{
    gcGeneralDataType allowedTypes = gcdata::allDataTypes();
    constBlockVector blocks = GetBlocksForLocus(locusId);

    for(constBlockVector::const_iterator i = blocks.begin(); i != blocks.end(); i++)
    {
        const GCParseBlock & pb = **i;
        const GCParse & parse = pb.GetParse();
        gcGeneralDataType thisType = parse.GetDataType();
        allowedTypes.Intersect(thisType);
    }
    return allowedTypes;
}

bool
GCDataStore::FileInducesHaps(size_t fileId) const
{
    const GCFile & fileRef = GetDataFile(fileId);

    for(size_t index=0; index < fileRef.GetParseCount(); index++)
    {
        const GCParse & parseRef = fileRef.GetParse(index);
        constBlockVector blocks = parseRef.GetBlocks();
        for(constBlockVector::const_iterator biter = blocks.begin();
            biter != blocks.end(); biter++)
        {
            const GCParseBlock & pb = **biter;
            const GCParseSamples & samples = pb.GetSamples();
            for(size_t j = 0; j < samples.size(); j++)
            {
                const GCParseSample & s = *(samples[j]);
                if(s.GetSequencesPerLabel() > 1)
                {
                    return true;
                }
            }
        }
    }
    return false;
}

//____________________________________________________________________________________