omatrix.py

SIMO optimization data matrix:

>>> import numpy
>>> import datetime
>>> import os

>>> from minimock import Mock
>>> epsilon = 0.00001

class OMatrix(object):

Class for in-memory storage of SIMO optimization data structures. Get the wanted data from input database, stores it into multidimensional numpy arrays for fast access during optimization process.

Data is stored for each unique subobjective and constraint operand into dictionaries where keys are the unique operand indices and values are threedimensional Numpy double arrays.

The data arrays have dimensions: (object, branch, timestep)

def __init__(self):

>>> taskdef1 = Mock('TaskDef')
>>> taskdef1.main_level = u'comp_unit'
>>> taskdef1.num_of_subobjs = 1
>>> taskdef1.num_of_consts = 1
>>> taskdef1.unique_operands = {('data',
...                              (u'comp_unit', 'PV'),
...                              None,
...                              False): 0,
...                             ('op',
...                              'cash_flow',
...                              None,
...                              False): 1}
>>> taskdef1.operand_index = {0: 0, 1: 1}
>>> taskdef1.one_aggr_level = {0: True, 1:False}
>>> taskdef1.date_ranges = {0: (datetime.date(1998,1,1),None),
...                         1: (datetime.date(1998,1,1),None)}
>>> taskdef1.discount_rate = 10.0
>>> taskdef1.conditional_discounting = \
...     [('comp_unit:SC lt 3 and comp_unit:SC lt 3',
...       (('data', ('comp_unit', 'SC')), ('value', 3),
...       ('lt', lambda x,y: x < y)),
...       15.0),]
>>> taskdef1.data_filter = None
>>> taskdef1.init_date = datetime.date(1998,1,1)
>>> taskdef1.subobjectives = [Mock('Subobjective')]
>>> taskdef1.subobjectives[0].expr = [('data',
...                                   (u'comp_unit', 'PV'),
...                                   (datetime.date(1998,1,1),None),
...                                   None,
...                                   False, (0, 0))]
>>> taskdef1.constraints = [Mock('Constraint')]
>>> taskdef1.constraints[0].expr = [('op',
...                                  'cash_flow',
...                                  (datetime.date(1998,1,1),None),
...                                  None,
...                                  False, (1, 1))]
>>> add_error = Mock('add_error')

Initialize OMatrix:

>>> from simo.optimization.tools.omatrix import OMatrix
>>> run_id = 'test'
>>> keep_data = False
>>> use_existing= False
>>> limit2branches = {'include': [], 'exclude': []}
>>> om = OMatrix(taskdef1, run_id, use_existing, keep_data, limit2branches,
...              add_error)

def analyze_data(self, db, add_info):

Parameters

db -- input database handle
add_info -- logger function object

Go through subobjectives, constraints and input database and analyze some data properties

>>> db = Mock('DB')
>>> db.get_ids.mock_returns = [u'001', u'002', u'003']
>>> db.get_max_iter.mock_returns = 1
>>> db.get_max_branch.mock_returns = 1
>>> db.get_dates.mock_returns = [datetime.date(1998,12,31),
...                              datetime.date(1999,12,31)]
>>> db.get_data_from_level.mock_returns_iter = iter([
...     [['001', 0, datetime.date(1998,12,31), 4.0],
...      ['001', 1, datetime.date(1998,12,31), 3.0],
...      ['001', 0, datetime.date(1999,12,31), 2.0],
...      ['001', 1, datetime.date(1999,12,31), 5.0],
...      ['002', 0, datetime.date(1998,12,31), 1.0]],
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['001', 1, datetime.date(1998,12,31), 1010.0],
...      ['001', 0, datetime.date(1999,12,31), 1001.0],
...      ['001', 1, datetime.date(1999,12,31), 1011.0],
...      ['002', 0, datetime.date(1998,12,31), 2000.0]],
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['001', 1, datetime.date(1998,12,31), 1010.0],
...      ['001', 0, datetime.date(1999,12,31), 1001.0],
...      ['001', 1, datetime.date(1999,12,31), 1011.0],
...      ['002', 0, datetime.date(1998,12,31), 2000.0]],
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['001', 1, datetime.date(1998,12,31), 1010.0],
...      ['001', 0, datetime.date(1999,12,31), 1001.0],
...      ['001', 1, datetime.date(1999,12,31), 1011.0],
...      ['002', 0, datetime.date(1998,12,31), 2000.0]],
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['001', 1, datetime.date(1998,12,31), 1010.0],
...      ['001', 0, datetime.date(1999,12,31), 1001.0],
...      ['001', 1, datetime.date(1999,12,31), 1011.0],
...      ['002', 0, datetime.date(1998,12,31), 2000.0]],
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['001', 1, datetime.date(1998,12,31), 1010.0],
...      ['001', 0, datetime.date(1999,12,31), 1001.0],
...      ['001', 1, datetime.date(1999,12,31), 1011.0],
...      ['002', 0, datetime.date(1998,12,31), 2000.0]],
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['001', 1, datetime.date(1998,12,31), 1010.0],
...      ['001', 0, datetime.date(1999,12,31), 1001.0],
...      ['001', 1, datetime.date(1999,12,31), 1011.0],
...      ['002', 0, datetime.date(1998,12,31), 2000.0]],
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['001', 1, datetime.date(1998,12,31), 1010.0],
...      ['001', 0, datetime.date(1999,12,31), 1001.0],
...      ['001', 1, datetime.date(1999,12,31), 1011.0],
...      ['002', 0, datetime.date(1998,12,31), 2000.0]],
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['001', 1, datetime.date(1998,12,31), 1010.0],
...      ['001', 0, datetime.date(1999,12,31), 1001.0],
...      ['001', 1, datetime.date(1999,12,31), 1011.0],
...      ['002', 0, datetime.date(1998,12,31), 2000.0]],
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['001', 1, datetime.date(1998,12,31), 1010.0],
...      ['001', 0, datetime.date(1999,12,31), 1001.0],
...      ['001', 1, datetime.date(1999,12,31), 1011.0],
...      ['002', 0, datetime.date(1998,12,31), 2000.0]],
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['001', 1, datetime.date(1998,12,31), 1010.0],
...      ['001', 0, datetime.date(1999,12,31), 1001.0],
...      ['001', 1, datetime.date(1999,12,31), 1011.0],
...      ['002', 0, datetime.date(1998,12,31), 2000.0]]])
>>> add_info = Mock('logger.add_message')

>>> om.analyze_data(db, add_info) 
Called logger.add_message('  analysing input database')
Called logger.add_message('    processing subobjectives')
Called logger.add_message('    processing constraints')
Called logger.add_message('    calculating problem size')
Called DB.get_ids('data', u'comp_unit')
Called DB.get_ids('op_res', u'comp_unit')
Called DB.get_max_iter('data')
Called DB.get_max_iter('op_res')
Called DB.get_max_branch('data')
Called DB.get_max_branch('op_res')
Called logger.add_message('    size calculated in 0.0 sec')
Called logger.add_message('  number of simulated iterations: 2')
True

def construct_data(self, iteration):

Parameters:

iteration -- current simulated iteration indice as int

Construct optimization data matrix and fill it with data from input databases

>>> evaluator = Mock('Evaluator')
>>> evaluator.evaluate.mock_returns = True
>>> om._evaluator = evaluator  # substitute the real evaluator with a mock
>>> _add_info = Mock('_add_info')

>>> om.construct_data(0)  
Called logger.add_message('Constructing optimization data set for iteration 0')
Called logger.add_message('  creating optimisation tables')
Called DB.get_dates(...)
Called DB.get_dates(...)
Called DB.get_dates(...)
Called logger.add_message(
    '  number of unique subobjective/constraint operands: 2')
Called logger.add_message('  total size for optimization data arrays: ...')
Called DB.prepare_db(
    <simo.optimization.tools.omatrix.OMatrix instance at ...>)
Called logger.add_message('  filling tables')
Called DB.get_data_from_level(...)
Called logger.add_message('    collecting values for operand ...')
Called logger.add_message('      getting ... results from db')
Called DB.get_data_from_level(...)
Called logger.add_message('    collecting values for operand ...')
Called logger.add_message('      getting ... results from db')
Called DB.get_data_from_level(...)
Called Evaluator.evaluate(
    array([0, 1, 2]),
    0,
    [('data', (u'comp_unit', 'PV'), (datetime.date(1998, 1, 1), None), None, False, (0, 0))],
    {0: (0, 1), 1: (0, 1)},
    {0: array([[[..., ...],
        [..., ...]],

       [[..., ...],
        [..., ...]],

       [[..., ...],
        [..., ...]]], dtype=float32), 1: array([[[..., ...],
        [..., ...]],

       [[..., ...],
        [..., ...]],

       [[..., ...],
        [..., ...]]], dtype=float32)},
    3,
    False)
Called Evaluator.evaluate(
    array([0, 1, 2]),
    1,
    [('data', (u'comp_unit', 'PV'), (datetime.date(1998, 1, 1), None), None, False, (0, 0))],
    {0: (0, 1), 1: (0, 1)},
    {0: array([[[..., ...],
        [..., ...]],

       [[..., ...],
        [..., ...]],

       [[..., ...],
        [..., ...]]], dtype=float32), 1: array([[[..., ...],
        [..., ...]],

       [[..., ...],
        [..., ...]],

       [[..., ...],
        [..., ...]]], dtype=float32)},
    3,
    False)
Called logger.add_message('  optimization data arrays filled in ...')
Called logger.add_message('  data set ready for optimization')
True


>>> om.data_level
u'comp_unit'
>>> om._earliest_date
datetime.date(1998, 1, 1)
>>> om._latest_date
>>> om.max_iterations
2
>>> om.num_of_units
3
>>> om._max_branches
2

>> dict(om.branches)
{1: [0], 2: [0, 1]}
>>> om._date_index[0][datetime.date(1998,12,31)]
0
>>> om._date_index[0][datetime.date(1999,12,31)]
1
>>> om._date_index[1][datetime.date(1998,12,31)]
0
>>> om._date_index[1][datetime.date(1999,12,31)]
1
>>> om.unit_index[u'001']
2
>>> om.unit_index[u'002']
1
>>> om.unit_index[u'003']
0
>>> om._unit_range
array([0, 1, 2])
>>> data = om.data['operands']
>>> data[0]
array([[[    0.,     0.],
        [    0.,     0.]],

       [[ 2000.,     0.],
        [    0.,     0.]],

       [[ 1000.,  1001.],
        [ 1010.,  1011.]]], dtype=float32)
>>> data[1]
array([[[    0.,     0.],
        [    0.,     0.]],

       [[ 2000.,     0.],
        [    0.,     0.]],

       [[ 1000.,  1001.],
        [ 1010.,  1011.]]], dtype=float32)
>>> data[0].shape
(3, 2, 2)
>>> data[1].shape
(3, 2, 2)

Additional tests for initialization

Initialize to keep the optimization data:

>>> run_id = 'test2'
>>> keep_data = True
>>> use_existing = False
>>> limit2branches = {'include': [], 'exclude': []}
>>> om2 = OMatrix(taskdef1, run_id, use_existing, keep_data,
...               limit2branches,add_error)
>>> om2._evaluator = evaluator
>>> om2.analyze_data(db, add_info) 
Called ...
True
>>> om2.construct_data(0) 
Called ...
True
>>> os.path.exists('test2.dat')
False
>>> om2.close()
Called logger.add_message('Writing optimization data structure to file...')
>>> del om2
>>> os.path.exists('test2.dat')
True

Initialize to reuse existing optimization data file:

>>> keep_data = False
>>> use_existing = True

NB! The commented out section should work but doesnt’t... test2.h5 exists up to call to om2.construct_data(0) when it just disappears...

# >>> om2 = OMatrix(taskdef1, run_id, use_existing, keep_data, add_error) # >>> om2._evaluator = evaluator # >>> om2.analyze_data(db, add_info) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE # Called ... # True # >>> om2.construct_data(0) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE # Called ... # True # >>> os.path.exists(‘test2.data’) # True # >>> om2.close() # >>> os.path.exists(‘test2.data’) # False

>>> om2 = OMatrix(taskdef1, run_id, use_existing, keep_data,
...               limit2branches, add_error)
>>> om2._evaluator = evaluator  # substitute the real evaluator with a mock
>>> om2.analyze_data(db, add_info) 
Called ...
True
>>> os.remove(om2._data_file)
>>> om2.construct_data(0) 
Called ...
...'Reuse optimization data file set, but test2.dat does not exist'...
False

def _create_tables(self):

Create optimization data tables, using Numpy arrays for the unique operand arrays.

The data is stored in a dictionary where keys are unique operand indices and values are unique operand data arrays.

Unique operand (subobjective or constraint) array shape: (units, branches, timesteps)

def _process_exprs(self, exprs, db, exprtype, subobj=True):

Go through the expressions and try to deduce data level and earliest and latest dates

def _fill_tables(self, it):

Fill subobjective and constraint arrays with values from input databases.

Parameters

it -- current simulated iteration indice, int

def _collect_values(self, it, operand, array, ui, forced_dates=None):

Get values for a single expression operand from either data or operation result database.

Parameters

it -- current simulated iteration indice as int
operand -- subobjective or constraint operand
array -- target Numpy array
ui -- unique subobjective/constraint operand indice as int
forced_dates -- forced date info

Collect values for a simple expression with only ‘data’ type operands

>>> om._db = db
>>> array1 = numpy.zeros([3,2,2], dtype=float)
>>> array2 = numpy.zeros([3,2,2], dtype=float)

>>> operand1 = ('data', (u'comp_unit', 'PV'), None, False)
>>> operand2 = ('data', (u'comp_unit', 'AREA'), None, True)

>>> om._collect_values(0, operand1, array1, 0)
Called logger.add_message('    collecting values for operand 0')
Called logger.add_message('      getting simulated results from db')
Called DB.get_data_from_level(
    'data',
    ['id', 'branch', 'data_date', 'PV'],
    u'comp_unit',
    constraints=[('iteration', 'eq', 0)],
    dates=(datetime.date(1998, 1, 1), None),
    limit2branches={'exclude': [], 'include': []})
True

>>> om._collect_values(0, operand2, array2, 0)
Called logger.add_message('    collecting values for operand 0')
Called logger.add_message('      getting simulated results from db')
Called DB.get_data_from_level(
    'data',
    ['id', 'branch', 'data_date', 'AREA'],
    u'comp_unit',
    constraints=[('iteration', 'eq', 0)],
    dates=(datetime.date(1998, 1, 1), None),
    limit2branches={'exclude': [], 'include': []})
True

>>> array1[:,:,0]
array([[    0.,     0.],
       [ 2000.,     0.],
       [ 1000.,  1010.]])

>>> array2[:,:,0]
array([[    0.        ,     0.        ],
       [ 1739.79649507,     0.        ],
       [  869.89824753,   918.42156907]])

Collect values for an expression with ‘op’ type operands

>>> array1 = numpy.zeros([3,2,2], dtype=float)
>>> array2 = numpy.zeros([3,2,2], dtype=float)
>>> operand1 = ('op', u'cash_flow', None, True)
>>> operand2 = ('data', (u'comp_unit', 'AREA'), None, False)

>>> om._collect_values(0, operand1, array1, 0)  
Called logger.add_message('    collecting values for operand 0')
Called logger.add_message('      getting operation results from db')
Called DB.get_data_from_level(
    'op_res',
    ['id', 'branch', 'op_date', u'cash_flow'],
    0,
    constraints=[('iteration', 'eq', 0)],
    dates=(datetime.date(1998, 1, 1), None),
    limit2branches={'exclude': [], 'include': []},
    required=[u'cash_flow'])
True

>>> om._collect_values(0, operand2, array2, 0)  
Called logger.add_message('    collecting values for operand 0')
Called logger.add_message('      getting simulated results from db')
Called DB.get_data_from_level(
    'data',
    ['id', 'branch', 'data_date', 'AREA'],
    u'comp_unit',
    constraints=[('iteration', 'eq', 0)],
    dates=(datetime.date(1998, 1, 1), None),
    limit2branches={'exclude': [], 'include': []})
True

Collect values for an expression with conditions in ‘data’ type operands

>>> array = numpy.zeros([3,2,2], dtype=float)
>>> dt = (datetime.date(1998,12,31), None)
>>> cond = [('data', u'comp_unit', u'SP'), ('value', 1000), ('gt',(lambda a,b: a>b), 'gt')]
>>> operand = ('data', (u'comp_unit', 'PV'), cond, False)
>>> om._collect_values(0, operand, array, 0)  
Called logger.add_message('    collecting values for operand 0')
Called logger.add_message('      getting simulated results from db')
Called DB.get_data_from_level(
    'data',
    ['id', 'branch', 'data_date', 'PV'],
    u'comp_unit',
    constraints=[('iteration', 'eq', 0, 'and'), (u'SP', 'gt', 1000)],
    dates=(datetime.date(1998, 1, 1), None),
    limit2branches={'exclude': [], 'include': []})
True

Collect values for an expression with conditions in ‘op’ type operands

>>> operand = ('op', u'cash_flow',
...            [('op', 'op', u'operation_name'),
...             ('value', u'thinning'),
...              ('eq', (lambda a,b: a == b), 'eq')], True)

>>> db = Mock('DB')
>>> db.get_data_from_level.mock_returns = \
...     [['001', 0, datetime.date(1998,12,31), 1000.0],
...      ['002', 0, datetime.date(1999,12,31), 3000.0],
...      ['002', 1, datetime.date(1999,12,31), 4000.0]]
>>> om._db = db
>>> om._evaluator = Mock('Evaluator')
>>> om._evaluator.evaluate.mock_returns_iter = iter([True, False, True, True])
>>> array[...] = 0.0
>>> om._collect_values(0, operand, array, 0)  
Called logger.add_message('    collecting values for operand 0')
Called logger.add_message('      getting operation results from db')
Called DB.get_data_from_level(
    'op_res',
    ['id', 'branch', 'op_date', u'cash_flow'],
    0,
    constraints=[('iteration', 'eq', 0, 'and'), (u'operation_name', 'eq', u'thinning')],
    dates=(datetime.date(1998, 1, 1), None),
    limit2branches={'exclude': [], 'include': []},
    required=[u'cash_flow'])
True

>>> array[...] 
array([[[    0.        ,     0.        ],
        [    0.        ,     0.        ]],

       [[    0.        ,  2269...],
        [    0.        ,  3306... ]],

       [[    869...,     0.        ],
        [    0.        ,     0.        ]]])

def _get_values_from_db(self, it, operand, dates, norm, condition=None):

Get value(s) for a single operand from data or operation result database

Parameters:

it – current simulated iteration indice as int operand – current operand definition dates – start and end dates norm – boolean indicating whether the operand is normal condition – operand condition/constraint

def _eval_constraint(self, it, dates, array, operand, ui, norm):

Evaluate operand condition and store only those objects for which the condition evaluated True.

Parameters:

it – current simulated iteration indice as int dates – start and end dates array – target Numpy array operand – operand structure ui – unique subobjective/constraint location indice norm – boolean indicating whether the operand is normal

def _prepare_db(self):

Makes sure the proper indices exist and that their content is up to date.

def _set_value(self, array, uid, br, ui, date, value):

Store a single value into a correct location in a matrix and update branch number container.

Parameters:

value – Numpy array uid – object id, str br – branch, int ui – unique subobjective/constraint indice, int date – datetime object value – value to store, float

def _filter_data_by_last_date(self, data):

Filter data so that only the values from the last recorded date of each iteration-branch-object combination are selected. This is needed when operand dates are defined as [-1:-1]

Parameters

data -- data generator object from datadb

>>> data = [
...     ('1', 0, datetime.date(2000,12,31), 1000.),
...     ('1', 0, datetime.date(2005,12,31), 1005.),
...     ('1', 0, datetime.date(2010,12,31), 1010.),
...     ('1', 1, datetime.date(2000,12,31), 1100.),
...     ('1', 1, datetime.date(2005,12,31), 1105.),
...     ('1', 1, datetime.date(2010,12,31), 1110.),
...     ('2', 0, datetime.date(2000,12,31), 2000.),
...     ('2', 0, datetime.date(2005,12,31), 2005.),
...     ('2', 0, datetime.date(2010,12,31), 2010.),
...     ('2', 1, datetime.date(2000,12,31), 2100.),
...     ('2', 1, datetime.date(2005,12,31), 2105.),
...     ('2', 1, datetime.date(2010,12,31), 2110.)]
>>> data = om._filter_data_by_last_date(data)
>>> for item in data: print item
('1', 1, datetime.date(2010, 12, 31), 1110.0)
('2', 0, datetime.date(2010, 12, 31), 2010.0)
('1', 0, datetime.date(2010, 12, 31), 1010.0)
('2', 1, datetime.date(2010, 12, 31), 2110.0)

def solution_utility(self, solution):

Check objective function utility value in given solution

def _value2utility(self, iso, value, maxvalue):

Transform real values to utilities, a Cython extension function

def solution_feasibility(self, solution):

Check given solution feasibility. For the initial seek for feasible solution goes through all the constraints in order to be able to report back to the user the constraint feasible/not feasible ratio in case of not finding any feasible solutions.

def compare_utilities(self, best, candidate):

Compare utility values of two solutions: is candidate better than the current best solution?

>>> om.task_def.objective_func.target = 'max'
>>> om.compare_utilities(0.0, 1.0)
True
>>> om.compare_utilities(2.0, 1.0)
False

>>> om.task_def.objective_func.target = 'min'
>>> om.compare_utilities(0.0, 1.0)
False
>>> om.compare_utilities(2.0, 1.0)
True

def get_random_solution(self):

Get a random solution without checkin it’s feasibility

>>> om.get_random_solution().shape
(3,)

def get_status(self, solution):

Get a data structure with string representations and current values of all subobjectives and constraints.

Parameters:

iteration – current iteration solution – current solution