Source code for lsst.sims.maf.db.trackingDb

from __future__ import print_function
from builtins import str
from builtins import object
import os
from sqlalchemy import create_engine, Column, Integer, String
from sqlalchemy.engine import url
from sqlalchemy.orm import sessionmaker

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.exc import DatabaseError

Base = declarative_base()

__all__ = ['TrackingDb', 'addRunToDatabase']


class RunRow(Base):
    """
    Define contents and format of run list table.

    Table to list all available MAF results, along with their opsim run and some comment info.
    """
    __tablename__ = "runs"
    # Define columns in metric list table.
    mafRunId = Column(Integer, primary_key=True)
    opsimGroup = Column(String)
    opsimRun = Column(String)
    opsimComment = Column(String)
    opsimVersion = Column(String)
    opsimDate = Column(String)
    dbFile = Column(String)
    mafComment = Column(String)
    mafVersion = Column(String)
    mafDate = Column(String)
    mafDir = Column(String)
    def __repr__(self):
        rstr = "<Run(mafRunId='%d', opsimGroup='%s', opsimRun='%s', opsimComment='%s', " \
               "opsimVersion='%s', opsimDate='%s', mafComment='%s', " \
               "mafVersion='%s', mafDate='%s', mafDir='%s', dbFile='%s'>" \
                % (self.mafRunId, self.opsimGroup, self.opsimRun, self.opsimComment,
                   self.opsimVersion, self.opsimDate, self.mafComment, self.mafVersion, self.mafDate,
                   self.mafDir, self.dbFile)
        return rstr


[docs]class TrackingDb(object): """Sqlite database to track MAF output runs and their locations, for showMaf.py """ def __init__(self, database=None, trackingDbverbose=False): """ Instantiate the results database, creating metrics, plots and summarystats tables. """ self.verbose = trackingDbverbose self.driver = 'sqlite' # Connect to database # for sqlite, connecting to non-existent database creates it automatically if database is None: # Default is a file in the current directory. self.database = os.path.join(os.getcwd(), 'trackingDb_sqlite.db') else: self.database = database # only sqlite dbAddress = url.URL(drivername=self.driver, database=self.database) engine = create_engine(dbAddress, echo=self.verbose) if self.verbose: print('Created or connected to MAF tracking %s database at %s' %(self.driver, self.database)) self.Session = sessionmaker(bind=engine) self.session = self.Session() # Create the tables, if they don't already exist. try: Base.metadata.create_all(engine) except DatabaseError: raise DatabaseError("Cannot create a %s database at %s. Check directory exists." %(self.driver, self.database))
[docs] def close(self): self.session.close()
[docs] def addRun(self, opsimGroup=None, opsimRun=None, opsimComment=None, opsimVersion=None, opsimDate=None, mafComment=None, mafVersion=None, mafDate=None, mafDir=None, dbFile=None, mafRunId=None): """Add a run to the tracking database. Parameters ---------- opsimGroup : str, opt Set a name to group this run with (eg. "Tier 1, 2016"). opsimRun : str, opt Set a name for the opsim run. opsimComment : str, opt Set a comment describing the opsim run. opsimVersion : str, opt Set the version of opsim. opsimDate : str, opt Set the date the opsim run was created. mafComment : str, opt Set a comment to describe the MAF analysis. mafVersion : str, opt Set the version of MAF used for analysis. mafDate : str, opt Set the date the MAF analysis was run. mafDir : str, opt The relative path to the MAF directory. dbFile : str, opt The relative path to the Opsim SQLite database file. mafRunId : int, opt The MafRunID to assign to this record in the database (note this is a primary key!). If this run (ie the mafDir) exists in the database already, this will be ignored. Returns ------- int The mafRunID stored in the database. """ if opsimGroup is None: opsimGroup = 'NULL' if opsimRun is None: opsimRun = 'NULL' if opsimComment is None: opsimComment = 'NULL' if opsimVersion is None: opsimVersion = 'NULL' if opsimDate is None: opsimDate = 'NULL' if mafComment is None: mafComment = 'NULL' if mafVersion is None: mafVersion = 'NULL' if mafDate is None: mafDate = 'NULL' if mafDir is None: mafDir = 'NULL' if dbFile is None: dbFile = 'NULL' # Test if mafDir already exists in database. prevrun = self.session.query(RunRow).filter_by(mafDir=mafDir).all() if len(prevrun) > 0: runIds = [] for run in prevrun: runIds.append(run.mafRunId) print('Updating information in tracking database - %s already present with runId %s.' % (mafDir, runIds)) for run in prevrun: self.session.delete(run) self.session.commit() runinfo = RunRow(mafRunId=runIds[0], opsimGroup=opsimGroup, opsimRun=opsimRun, opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, mafDir=mafDir, dbFile=dbFile) else: if mafRunId is not None: # Check if mafRunId exists already. existing = self.session.query(RunRow).filter_by(mafRunId=mafRunId).all() if len(existing) > 0: raise ValueError('MafRunId %d already exists in database, for %s. ' \ 'Record must be deleted first.' % (mafRunId, existing[0].mafDir)) runinfo = RunRow(mafRunId=mafRunId, opsimGroup=opsimGroup, opsimRun=opsimRun, opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, mafDir=mafDir, dbFile=dbFile) else: runinfo = RunRow(opsimGroup=opsimGroup, opsimRun=opsimRun, opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, mafDir=mafDir, dbFile=dbFile) self.session.add(runinfo) self.session.commit() return runinfo.mafRunId
[docs] def delRun(self, runId): """ Remove a run from the tracking database. """ runinfo = self.session.query(RunRow).filter_by(mafRunId=runId).all() if len(runinfo) == 0: raise Exception('Could not find run with mafRunId %d' %(runId)) if len(runinfo) > 1: raise Exception('Found more than one run with mafRunId %d' %(runId)) print('Removing run info for runId %d ' %(runId)) print(' ', runinfo) self.session.delete(runinfo[0]) self.session.commit()
[docs]def addRunToDatabase(mafDir, trackingDbFile, opsimGroup=None, opsimRun=None, opsimComment=None, mafComment=None, dbFile=None): """Adds information about a MAF analysis run to a MAF tracking database. Parameters ---------- mafDir : str Path to the directory where the MAF results are located. trackingDb : str or lsst.sims.maf.TrackingDb Full filename (+path) to the tracking database storing the MAF run information or a TrackingDb object. opsimGroup: str, opt Name to use to group this run with other opsim runs. Default None. opsimRun : str, opt Name of the opsim run. If not provided, will attempt to use runName from confSummary.txt. opsimComment : str, opt Comment about the opsim run. If not provided, will attempt to use runComment from confSummary.txt. mafComment : str, opt Comment about the MAF analysis. If not provided, no comment will be recorded. dbFile : str, opt Relative path + name of the opsim database file. If not provided, no location will be recorded. """ mafDir = os.path.abspath(mafDir) if not os.path.isdir(mafDir): raise ValueError('There is no directory containing MAF outputs at %s.' % (mafDir)) trackingDb = TrackingDb(database=trackingDbFile) autoOpsimRun = None autoOpsimComment = None opsimVersion = None opsimDate = None mafVersion = None mafDate = None if os.path.isfile(os.path.join(mafDir, 'configSummary.txt')): file = open(os.path.join(mafDir, 'configSummary.txt')) for line in file: tmp = line.split() if tmp[0].startswith('RunName'): autoOpsimRun = ' '.join(tmp[1:]) if tmp[0].startswith('RunComment'): autoOpsimComment = ' '.join(tmp[1:]) # MAF Date may be in a line with "MafDate" (new configs) # or at the end of "MAFVersion" (old configs). if tmp[0].startswith('MAFDate'): mafDate = tmp[-1] if tmp[0].startswith('MAFVersion'): mafVersion = tmp[1] if len(tmp) > 2: mafDate = tmp[-1] if tmp[0].startswith('OpsimDate'): opsimDate = tmp[-1] if len(tmp) > 2: opsimDate = tmp[-2] if tmp[0].startswith('OpsimVersion'): opsimVersion = tmp[1] if len(tmp) > 2: opsimDate = tmp[-2] # And convert formats to '-' (again, multiple versions of configs). if mafDate is not None: if len(mafDate.split('/')) > 1: t = mafDate.split('/') if len(t[2]) == 2: t[2] = '20' + t[2] mafDate = '-'.join([t[2], t[1], t[0]]) if opsimDate is not None: if len(opsimDate.split('/')) > 1: t = opsimDate.split('/') if len(t[2]) == 2: t[2] = '20' + t[2] opsimDate = '-'.join([t[2], t[1], t[0]]) if opsimRun is None: opsimRun = autoOpsimRun if opsimComment is None: opsimComment = autoOpsimComment print('Adding to tracking database at %s:' % (trackingDbFile)) print(' MafDir = %s' % (mafDir)) print(' MafComment = %s' % (mafComment)) print(' OpsimGroup = %s' % (opsimGroup)) print(' OpsimRun = %s' % (opsimRun)) print(' OpsimComment = %s' % (opsimComment)) print(' OpsimVersion = %s' % (opsimVersion)) print(' OpsimDate = %s' % (opsimDate)) print(' MafVersion = %s' % (mafVersion)) print(' MafDate = %s' % (mafDate)) print(' Opsim dbFile = %s' % (dbFile)) runId = trackingDb.addRun(opsimGroup=opsimGroup, opsimRun=opsimRun, opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, mafDir=mafDir, dbFile=dbFile) print('Used MAF RunID %d' % (runId)) trackingDb.close()