exprparser.py

SIMO optimization expression parser:

>>> epsilon = 0.00001
>>> from lxml import etree
>>> import datetime

class ExpressionParser(object):

Optimization task expression parser

Parameters:

• validator: SIMO lexicon validator

def __init__(self):

>>> from simo.builder.optimization.exprparser import ExpressionParser
>>> parser = ExpressionParser()

def parse(self, expr, initdate, step, aggregator_constraint=None):

Parse SIMO optimization subobjective or constraint expression into a nested list and further into a Reverse Polish notation / Postfix stack.

Parameters

expr -- expression definition in a list structure
initdate -- initial date as datetime object
step -- time step unit, eg. 'year', 'month', 'day'
aggregator_constraint -- valid aggregator function constraint, iterable eg. ['sum', 'min']

Expression is stored as a list of tuples. Each tuple is one of: operation or data variable definition, value definition, or function object. The first item of the tuple defines the type of the expression item. The expression is stored in postfix notation (reverse polish notation).

Variable and operation tuple structure:

• 0: ‘data’ or ‘op’
• 1: (level, variable) tuple if ‘data’, operation result variable if ‘op’
• 2: (start date, end date) tuple, None if date is -1 (stands for last possible date in the data)
• 3: condition expression, given also in postfix notation, None if no condition
• 4: discount, True if variable values should be discounted, False if not

Value tuple structure: - 0: ‘value’ - 1: value as float

Function tuple structure: - 0: ‘eq’, ‘ari’, or ‘aggr’ for equality, arithmetic, and aggregation functions respectively - 1: function object:

>>> initdate = datetime.date(2000, 1, 1)
>>> step = 'year'

Parse: sum[period](X):

>>> expr = u'sum[1:10](comp_unit:PV)'
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'PV'), (datetime.date(2000, 1, 1),
 datetime.date(2009, 12, 31)), None, False)
('aggr', <function sum at ...>)

Parse: sum[period](X:discount):

>>> expr = u'sum[1:10](comp_unit:PV:discount)'
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'PV'), (datetime.date(2000, 1, 1),
 datetime.date(2009, 12, 31)), None, True)
('aggr', <function sum at ...>)

Parse: sum(X) * 15:

>>> expr = u'sum[-1:-1](comp_unit:PV) * 15.0'
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'PV'), (None, None), None, False)
('aggr', <function sum at ...>)
('value', 15.0)
('ari', <function multi at ...>, '*')

Parse: sum(X * Y):

>>> expr = u'sum[-1:-1](comp_unit:PV:discount * comp_unit:AREA)'
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'PV'), (None, None), None, True)
('data', (u'comp_unit', u'AREA'), (None, None), None, False)
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)

Parse: sum(X / Y):

>>> expr = u'sum[-1:-1](comp_unit:X:discount / comp_unit:Y)'
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'X'), (None, None), None, True)
('data', (u'comp_unit', u'Y'), (None, None), None, False)
('ari', <function divide at ...>, '/')
('aggr', <function sum at ...>)

Parse: sum(X * Y) + sum(Z):

>>> expr = u'''sum[-1:-1](comp_unit:PV:discount * comp_unit:AREA) +
...            sum[1:-1](operation:cash_flow:discount)'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'PV'), (None, None), None, True)
('data', (u'comp_unit', u'AREA'), (None, None), None, False)
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)
('op', u'cash_flow', (datetime.date(2000, 1, 1), None), None, True)
('aggr', <function sum at ...>)
('ari', <function plus at ...>, '+')

Parse: sum(X * Y):

>>> expr = u'sum[-1:-1](operation:Income:discount * comp_unit:AREA)'
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('op', u'Income', (None, None), None, True)
('data', (u'comp_unit', u'AREA'), (None, None), None, False)
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)

Parse: sum(X) >= sum(Y):

>>> expr = u'''sum[-1:-1](comp_unit:PV:discount) ge
...            sum[-1:-1](comp_unit:PV:discount)'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'PV'), (None, None), None, True)
('aggr', <function sum at ...>)
('data', (u'comp_unit', u'PV'), (None, None), None, True)
('aggr', <function sum at ...>)
('eq', <function gee at ...>, 'ge')

Parse: (sum(X) >= sum(Y)) and (sum(Z) != 0.0):

>>> expr = u'''sum[-1:-1](comp_unit:PV:discount) ge
...            sum[-1:-1](comp_unit:PV:discount)
...            and
...            sum[-1:-1](operation:Volume) ue 0.0'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'PV'), (None, None), None, True)
('aggr', <function sum at ...>)
('data', (u'comp_unit', u'PV'), (None, None), None, True)
('aggr', <function sum at ...>)
('eq', <function gee at ...>, 'ge')
('op', u'Volume', (None, None), None, False)
('aggr', <function sum at ...>)
('value', 0.0)
('eq', <function nee at ...>, 'ue')
('eq', <function and_ at ...>, 'and')

Parse: sum(X * Y + X * Z):

>>> expr = u'''sum[-1:-1](comp_unit:AREA * comp_unit:PV + comp_unit:AREA *
...                       comp_unit:PV_land)'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'AREA'), (None, None), None, False)
('data', (u'comp_unit', u'PV'), (None, None), None, False)
('ari', <function multi at ...>, '*')
('data', (u'comp_unit', u'AREA'), (None, None), None, False)
('data', (u'comp_unit', u'PV_land'), (None, None), None, False)
('ari', <function multi at ...>, '*')
('ari', <function plus at ...>, '+')
('aggr', <function sum at ...>)

Parse: sum(Z * (X + Y)):

>>> expr = u'''sum[-1:-1](comp_unit:AREA *
...                       (comp_unit:PV + comp_unit:PV_land))'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'AREA'), (None, None), None, False)
('data', (u'comp_unit', u'PV'), (None, None), None, False)
('data', (u'comp_unit', u'PV_land'), (None, None), None, False)
('ari', <function plus at ...>, '+')
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)

Parse: sum(X) - sum(Y) > 0:

>>> expr = u'''sum[20:20](comp_unit:V) - sum[10:10](comp_unit:V) gt 0.0'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'V'), (datetime.date(2019, 1, 1), datetime.date(2019, 12, 31)), None, False)
('aggr', <function sum at ...>)
('data', (u'comp_unit', u'V'), (datetime.date(2009, 1, 1), datetime.date(2009, 12, 31)), None, False)
('aggr', <function sum at ...>)
('ari', <function minus at ...>, '-')
('value', 0.0)
('eq', <function gte at ...>, 'gt')

Parse: sum(X) - sum(Y) > 0:

>>> expr = u'''sum[20:-1](operation:Volume) - sum[10:-1](operation:Volume) gt 0.0'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('op', u'Volume', (datetime.date(2019, 1, 1), None), None, False)
('aggr', <function sum at ...>)
('op', u'Volume', (datetime.date(2009, 1, 1), None), None, False)
('aggr', <function sum at ...>)
('ari', <function minus at ...>, '-')
('value', 0.0)
('eq', <function gte at ...>, 'gt')

Parse: sum(X * Z) - sum(Y * Z) > 0:

>>> expr = u'''sum[20:20](comp_unit:V * comp_unit:AREA) -
...            sum[10:10](comp_unit:V * comp_unit:AREA) gt 0.0'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'V'), (datetime.date(2019, 1, 1), datetime.date(2019, 12, 31)), None, False)
('data', (u'comp_unit', u'AREA'), (datetime.date(2019, 1, 1), datetime.date(2019, 12, 31)), None, False)
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)
('data', (u'comp_unit', u'V'), (datetime.date(2009, 1, 1), datetime.date(2009, 12, 31)), None, False)
('data', (u'comp_unit', u'AREA'), (datetime.date(2009, 1, 1), datetime.date(2009, 12, 31)), None, False)
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)
('ari', <function minus at ...>, '-')
('value', 0.0)
('eq', <function gte at ...>, 'gt')

Parse: sum((X + Y) * (Z + K)):

>>> expr = u'''sum[-1:-1]((comp_unit:PV + comp_unit:PV_land) *
...                       (comp_unit:AREA + operation:cash_flow))'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'PV'), (None, None), None, False)
('data', (u'comp_unit', u'PV_land'), (None, None), None, False)
('ari', <function plus at ...>, '+')
('data', (u'comp_unit', u'AREA'), (None, None), None, False)
('op', u'cash_flow', (None, None), None, False)
('ari', <function plus at ...>, '+')
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)

Parse: sum(X * Y) > sum(Z * K):

>>> expr = u'''sum[20:20](comp_unit:V * comp_unit:AREA) gt
...            sum[10:10](comp_unit:V * comp_unit:AREA)'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'V'), (datetime.date(2019, 1, 1),
  datetime.date(2019, 12, 31)), None, False)
('data', (u'comp_unit', u'AREA'), (datetime.date(2019, 1, 1),
  datetime.date(2019, 12, 31)), None, False)
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)
('data', (u'comp_unit', u'V'), (datetime.date(2009, 1, 1),
  datetime.date(2009, 12, 31)), None, False)
('data', (u'comp_unit', u'AREA'), (datetime.date(2009, 1, 1),
  datetime.date(2009, 12, 31)), None, False)
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)
('eq', <function gte at ...>, 'gt')

Parse: sum(X) - 0.05 * 1000 * 25 > 5.0:

>>> expr = u'''sum[1:-1](operation:Volume / comp_unit:AREA) - 0.05 * 1000
...             gt 5.0'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c:
...     if i[0] != 'value':
...         print i
...     else:
...         print (i[0], '%.2f' % i[1])
... 
('op', u'Volume', (datetime.date(2000, 1, 1), None), None, False)
('data', (u'comp_unit', u'AREA'), (datetime.date(2000, 1, 1), None), None, False)
('ari', <function divide at ...>, '/')
('aggr', <function sum at ...>)
('value', '0.05')
('value', '1000.00')
('ari', <function multi at ...>, '*')
('ari', <function minus at ...>, '-')
('value', '5.00')
('eq', <function gte at ...>, 'gt')

Parse: sum((X + Y) * Z) > 1000.0:

>>> expr = u'''sum[-1:-1]((comp_unit:PV + comp_unit:PV_land) *
...            comp_unit:AREA) gt 1000.0'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'PV'), (None, None), None, False)
('data', (u'comp_unit', u'PV_land'), (None, None), None, False)
('ari', <function plus at ...>, '+')
('data', (u'comp_unit', u'AREA'), (None, None), None, False)
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)
('value', 1000.0)
('eq', <function gte at ...>, 'gt')

Parse: sum(X) / sum(Y) / 0.10:

>>> expr = u'''sum[-1:-1](comp_unit:V[comp_unit:SP eq 5]) /
...            sum[-1:-1](comp_unit:V) /
...            0.10'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'V'), (None, None),
 (('data', u'comp_unit', u'SP'), ('value', 5),
  ('eq', <function ee at ...>, 'eq')), False)
('aggr', <function sum at ...>)
('data', (u'comp_unit', u'V'), (None, None), None, False)
('aggr', <function sum at ...>)
('ari', <function divide at ...>, '/')
('value', 0.1...)
('ari', <function divide at ...>, '/')

Parse: sum(X) + sum(Y) / 0.10:

>>> expr = u'''sum[-1:-1](comp_unit:V[comp_unit:SP eq 5]) +
...            sum[-1:-1](comp_unit:V) /
...            0.10'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'V'), (None, None),
 (('data', u'comp_unit', u'SP'), ('value', 5), ('eq', <function ee at ...>, 'eq')),
 False)
('aggr', <function sum at ...>)
('data', (u'comp_unit', u'V'), (None, None), None, False)
('aggr', <function sum at ...>)
('value', 0.1...)
('ari', <function divide at ...>, '/')
('ari', <function plus at ...>, '+')

Parse: (sum(X) + sum(Y)) / 0.10:

>>> expr = u'''(sum[-1:-1](comp_unit:V[comp_unit:SP eq 5]) +
...            sum[-1:-1](comp_unit:V)) /
...            0.10'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('data', (u'comp_unit', u'V'), (None, None),
 (('data', u'comp_unit', u'SP'), ('value', 5), ('eq', <function ee at ...>, 'eq')),
 False)
('aggr', <function sum at ...>)
('data', (u'comp_unit', u'V'), (None, None), None, False)
('aggr', <function sum at ...>)
('ari', <function plus at ...>, '+')
('value', 0.1...)
('ari', <function divide at ...>, '/')

Parse: (sum(X) / sum(Y) > 0.05) and (sum(Z) / sum(K) < 0.15):

>>> expr = u'''(sum[-1:-1](operation:Volume[operation:SP eq 1]) /
...             sum[-1:-1](operation:Volume) gt 0.05)
...            and
...            (sum[-1:-1](operation:Volume[operation:SP eq 1]) /
...             sum[-1:-1](operation:Volume) lt 0.15)'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c:
...     if i[0] != 'value':
...         print i
...     else:
...         print (i[0], '%.2f' % i[1])
... 
('op', u'Volume', (None, None), (('op', 'op', u'SP'),
    ('value', 1), ('eq', <function ee at ...>, 'eq')), False)
('aggr', <function sum at ...>)
('op', u'Volume', (None, None), None, False)
('aggr', <function sum at ...>)
('ari', <function divide at ...>, '/')
('value', '0.05')
('eq', <function gte at ...>, 'gt')
('op', u'Volume', (None, None), (('op', 'op', u'SP'),
    ('value', 1), ('eq', <function ee at ...>, 'eq')), False)
('aggr', <function sum at ...>)
('op', u'Volume', (None, None), None, False)
('aggr', <function sum at ...>)
('ari', <function divide at ...>, '/')
('value', '0.15')
('eq', <function lte at ...>, 'lt')
('eq', <function and_ at ...>, 'and')

Parse: sum(X) with simple conditions for variable X:

>>> expr = u'''sum[-1:-1](operation:Income[operation:SP eq 1]:discount)'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('op', u'Income', (None, None), (('op', 'op', u'SP'),
    ('value', 1), ('eq', <function ee at ...>, 'eq')), True)
('aggr', <function sum at ...>)

>>> expr = u'''sum[-1:-1](operation:Income[operation:op_name eq clearcut])'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('op', u'Income', (None, None), (('op', 'op', u'op_name'),
    ('value', u'clearcut'), ('eq', <function ee at ...>, 'eq')), False)
('aggr', <function sum at ...>)

>>> expr = u'''sum[-1:-1](operation:Income[operation:op_group eq final_harvest
...                                        and comp_unit:MAIN_SP eq 1])'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('combined', u'Income', (None, None),
 (('combined', 'op', u'op_group'),
  ('value', u'final_harvest'),
  ('eq', <function ee at ...>, 'eq'),
  ('combined', u'comp_unit', u'MAIN_SP'),
  ('value', 1),
  ('eq', <function ee at ...>, 'eq'),
  ('eq', <function and_ at ...>, 'and')), False)
('aggr', <function sum at ...>)

>>> expr = u'''sum[-1:-1](operation:Income[comp_unit:MAIN_GROUP eq 1]:discount)'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('combined', u'Income', (None, None), (('combined', u'comp_unit', u'MAIN_GROUP'),
 ('value', 1), ('eq', <function ee at ...>, 'eq')), True)
('aggr', <function sum at ...>)

Parse: sum(X) with a condition with multiple conditions for variable X:

>>> expr = u'''sum[-1:-1](operation:Income[operation:op_name eq thinning and
...                       operation:SP eq 1 and operation:assortment eq 1]:discount)'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 
('op', u'Income', (None, None),
    (('op', 'op', u'op_name'), ('value', u'thinning'),
     ('eq', <function ee at ...>, 'eq'),
     ('op', 'op', u'SP'), ('value', 1), ('eq', <function ee at ...>, 'eq'),
     ('eq', <function and_ at ...>, 'and'),
     ('op', 'op', u'assortment'), ('value', 1),
      ('eq', <function ee at ...>, 'eq'),
     ('eq', <function and_ at ...>, 'and')),
     True)
('aggr', <function sum at ...>)

Parse: (sum(X * Y) / sum(X * Y) - 0.1) >= 0, with some conditions:

>>> expr = u'''sum[1:1](comp_unit:V[comp_unit:MAIN_SP in (3, 4, 5, 6, 7, 9)] *
...                     comp_unit:AREA) /
...            sum[1:1](comp_unit:V * comp_unit:AREA) - 0.1 ge 0'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c:
...     if i[0] != 'value':
...         print i
...     else:
...         print (i[0], '%.2f' % i[1])
... 
('data', (u'comp_unit', u'V'), (datetime.date(2000, 1, 1),
  datetime.date(2000, 12, 31)),
  (('data', u'comp_unit', u'MAIN_SP'), ('value', (3, 4, 5, 6, 7, 9)),
   ('eq', <function in_ at ...>, 'in')), False)
('data', (u'comp_unit', u'AREA'), (datetime.date(2000, 1, 1),
  datetime.date(2000, 12, 31)), None, False)
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)
('data', (u'comp_unit', u'V'), (datetime.date(2000, 1, 1),
  datetime.date(2000, 12, 31)), None, False)
('data', (u'comp_unit', u'AREA'), (datetime.date(2000, 1, 1),
  datetime.date(2000, 12, 31)), None, False)
('ari', <function multi at ...>, '*')
('aggr', <function sum at ...>)
('ari', <function divide at ...>, '/')
('value', '0.10')
('ari', <function minus at ...>, '-')
('value', '0.00')
('eq', <function gee at ...>, 'ge')

Parse expressions with invalid syntax:

>>> expr = u'''comp_unit:PV[-1:-1]'''
>>> c,e = parser.parse(expr, initdate, step)
>>> print e
(0, "Expected one of 'sum', 'mean', 'min', 'max', 'avg'")

>>> expr = u'''sum[-1:-1](comp_unit:PV * 10.0)'''
>>> c,e = parser.parse(expr, initdate, step)
>>> print e
(28, 'Expected ":"')

>>> expr = u'''sum[-1:-1](comp_unit:PV * comp_unit:AREA > 1000.0'''
>>> c,e = parser.parse(expr, initdate, step)
>>> print e
(41, 'Expected ")"')

>>> expr = u'''sum[1:-1](comp_unit:V[operation:SP eq 1])'''
>>> c,e = parser.parse(expr, initdate, step)
>>> for i in c: print i 

Parse expressions using aggregation constraints

>>> expr = u'sum[10:10](comp_unit:PV)'
>>> c,e = parser.parse(expr, initdate, step, ('sum','min','max'))
>>> for i in c: print i 
('data', (u'comp_unit', u'PV'), (datetime.date(2009, 1, 1), datetime.date(2009, 12, 31)), None, False)
('aggr', <function sum at ...>)

Test the error handling in the case of invalid aggregation function

>>> c,e = parser.parse(expr, initdate, step, ('min','max'))
>>> print e
(0, "Invalid aggregator function 'sum'")