1#!/usr/bin/env python
2# -*- coding: utf-8; py-indent-offset:4 -*-
3###############################################################################
4#
5# Copyright (C) 2015, 2016, 2017 Daniel Rodriguez
6#
7# This program is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation, either version 3 of the License, or
10# (at your option) any later version.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with this program.  If not, see <http://www.gnu.org/licenses/>.
19#
20###############################################################################
21from __future__ import (absolute_import, division, print_function,
22                        unicode_literals)
23
24import collections
25import datetime
26
27import backtrader as bt
28from backtrader.comminfo import CommInfoBase
29from backtrader.order import Order, BuyOrder, SellOrder
30from backtrader.position import Position
31from backtrader.utils.py3 import string_types, integer_types
32
33__all__ = ['BackBroker', 'BrokerBack']
34
35
36class BackBroker(bt.BrokerBase):
37    '''Broker Simulator
38
39      The simulation supports different order types, checking a submitted order
40      cash requirements against current cash, keeping track of cash and value
41      for each iteration of ``cerebro`` and keeping the current position on
42      different datas.
43
44      *cash* is adjusted on each iteration for instruments like ``futures`` for
45       which a price change implies in real brokers the addition/substracion of
46       cash.
47
48      Supported order types:
49
50        - ``Market``: to be executed with the 1st tick of the next bar (namely
51          the ``open`` price)
52
53        - ``Close``: meant for intraday in which the order is executed with the
54          closing price of the last bar of the session
55
56        - ``Limit``: executes if the given limit price is seen during the
57          session
58
59        - ``Stop``: executes a ``Market`` order if the given stop price is seen
60
61        - ``StopLimit``: sets a ``Limit`` order in motion if the given stop
62          price is seen
63
64      Because the broker is instantiated by ``Cerebro`` and there should be
65      (mostly) no reason to replace the broker, the params are not controlled
66      by the user for the instance.  To change this there are two options:
67
68        1. Manually create an instance of this class with the desired params
69           and use ``cerebro.broker = instance`` to set the instance as the
70           broker for the ``run`` execution
71
72        2. Use the ``set_xxx`` to set the value using
73           ``cerebro.broker.set_xxx`` where ```xxx`` stands for the name of the
74           parameter to set
75
76        .. note::
77
78           ``cerebro.broker`` is a *property* supported by the ``getbroker``
79           and ``setbroker`` methods of ``Cerebro``
80
81      Params:
82
83        - ``cash`` (default: ``10000``): starting cash
84
85        - ``commission`` (default: ``CommInfoBase(percabs=True)``)
86          base commission scheme which applies to all assets
87
88        - ``checksubmit`` (default: ``True``)
89          check margin/cash before accepting an order into the system
90
91        - ``eosbar`` (default: ``False``):
92          With intraday bars consider a bar with the same ``time`` as the end
93          of session to be the end of the session. This is not usually the
94          case, because some bars (final auction) are produced by many
95          exchanges for many products for a couple of minutes after the end of
96          the session
97
98        - ``eosbar`` (default: ``False``):
99          With intraday bars consider a bar with the same ``time`` as the end
100          of session to be the end of the session. This is not usually the
101          case, because some bars (final auction) are produced by many
102          exchanges for many products for a couple of minutes after the end of
103          the session
104
105        - ``filler`` (default: ``None``)
106
107          A callable with signature: ``callable(order, price, ago)``
108
109            - ``order``: obviously the order in execution. This provides access
110              to the *data* (and with it the *ohlc* and *volume* values), the
111              *execution type*, remaining size (``order.executed.remsize``) and
112              others.
113
114              Please check the ``Order`` documentation and reference for things
115              available inside an ``Order`` instance
116
117            - ``price`` the price at which the order is going to be executed in
118              the ``ago`` bar
119
120            - ``ago``: index meant to be used with ``order.data`` for the
121              extraction of the *ohlc* and *volume* prices. In most cases this
122              will be ``0`` but on a corner case for ``Close`` orders, this
123              will be ``-1``.
124
125              In order to get the bar volume (for example) do: ``volume =
126              order.data.voluume[ago]``
127
128          The callable must return the *executed size* (a value >= 0)
129
130          The callable may of course be an object with ``__call__`` matching
131          the aforementioned signature
132
133          With the default ``None`` orders will be completely executed in a
134          single shot
135
136        - ``slip_perc`` (default: ``0.0``) Percentage in absolute termns (and
137          positive) that should be used to slip prices up/down for buy/sell
138          orders
139
140          Note:
141
142            - ``0.01`` is ``1%``
143
144            - ``0.001`` is ``0.1%``
145
146        - ``slip_fixed`` (default: ``0.0``) Percentage in units (and positive)
147          that should be used to slip prices up/down for buy/sell orders
148
149          Note: if ``slip_perc`` is non zero, it takes precendence over this.
150
151        - ``slip_open`` (default: ``False``) whether to slip prices for order
152          execution which would specifically used the *opening* price of the
153          next bar. An example would be ``Market`` order which is executed with
154          the next available tick, i.e: the opening price of the bar.
155
156          This also applies to some of the other executions, because the logic
157          tries to detect if the *opening* price would match the requested
158          price/execution type when moving to a new bar.
159
160        - ``slip_match`` (default: ``True``)
161
162          If ``True`` the broker will offer a match by capping slippage at
163          ``high/low`` prices in case they would be exceeded.
164
165          If ``False`` the broker will not match the order with the current
166          prices and will try execution during the next iteration
167
168        - ``slip_limit`` (default: ``True``)
169
170          ``Limit`` orders, given the exact match price requested, will be
171          matched even if ``slip_match`` is ``False``.
172
173          This option controls that behavior.
174
175          If ``True``, then ``Limit`` orders will be matched by capping prices
176          to the ``limit`` / ``high/low`` prices
177
178          If ``False`` and slippage exceeds the cap, then there will be no
179          match
180
181        - ``slip_out`` (default: ``False``)
182
183          Provide *slippage* even if the price falls outside the ``high`` -
184          ``low`` range.
185
186        - ``coc`` (default: ``False``)
187
188          *Cheat-On-Close* Setting this to ``True`` with ``set_coc`` enables
189           matching a ``Market`` order to the closing price of the bar in which
190           the order was issued. This is actually *cheating*, because the bar
191           is *closed* and any order should first be matched against the prices
192           in the next bar
193
194        - ``coo`` (default: ``False``)
195
196          *Cheat-On-Open* Setting this to ``True`` with ``set_coo`` enables
197           matching a ``Market`` order to the opening price, by for example
198           using a timer with ``cheat`` set to ``True``, because such a timer
199           gets executed before the broker has evaluated
200
201        - ``int2pnl`` (default: ``True``)
202
203          Assign generated interest (if any) to the profit and loss of
204          operation that reduces a position (be it long or short). There may be
205          cases in which this is undesired, because different strategies are
206          competing and the interest would be assigned on a non-deterministic
207          basis to any of them.
208
209        - ``shortcash`` (default: ``True``)
210
211          If True then cash will be increased when a stocklike asset is shorted
212          and the calculated value for the asset will be negative.
213
214          If ``False`` then the cash will be deducted as operation cost and the
215          calculated value will be positive to end up with the same amount
216
217        - ``fundstartval`` (default: ``100.0``)
218
219          This parameter controls the start value for measuring the performance
220          in a fund-like way, i.e.: cash can be added and deducted increasing
221          the amount of shares. Performance is not measured using the net
222          asset value of the porftoflio but using the value of the fund
223
224        - ``fundmode`` (default: ``False``)
225
226          If this is set to ``True`` analyzers like ``TimeReturn`` can
227          automatically calculate returns based on the fund value and not on
228          the total net asset value
229
230    '''
231    params = (
232        ('cash', 10000.0),
233        ('checksubmit', True),
234        ('eosbar', False),
235        ('filler', None),
236        # slippage options
237        ('slip_perc', 0.0),
238        ('slip_fixed', 0.0),
239        ('slip_open', False),
240        ('slip_match', True),
241        ('slip_limit', True),
242        ('slip_out', False),
243        ('coc', False),
244        ('coo', False),
245        ('int2pnl', True),
246        ('shortcash', True),
247        ('fundstartval', 100.0),
248        ('fundmode', False),
249    )
250
251    def __init__(self):
252        super(BackBroker, self).__init__()
253        self._userhist = []
254        self._fundhist = []
255        # share_value, net asset value
256        self._fhistlast = [float('NaN'), float('NaN')]
257
258    def init(self):
259        super(BackBroker, self).init()
260        self.startingcash = self.cash = self.p.cash
261        self._value = self.cash
262        self._valuemkt = 0.0  # no open position
263
264        self._valuelever = 0.0  # no open position
265        self._valuemktlever = 0.0  # no open position
266
267        self._leverage = 1.0  # initially nothing is open
268        self._unrealized = 0.0  # no open position
269
270        self.orders = list()  # will only be appending
271        self.pending = collections.deque()  # popleft and append(right)
272        self._toactivate = collections.deque()  # to activate in next cycle
273
274        self.positions = collections.defaultdict(Position)
275        self.d_credit = collections.defaultdict(float)  # credit per data
276        self.notifs = collections.deque()
277
278        self.submitted = collections.deque()
279
280        # to keep dependent orders if needed
281        self._pchildren = collections.defaultdict(collections.deque)
282
283        self._ocos = dict()
284        self._ocol = collections.defaultdict(list)
285
286        self._fundval = self.p.fundstartval
287        self._fundshares = self.p.cash / self._fundval
288        self._cash_addition = collections.deque()
289
290    def get_notification(self):
291        try:
292            return self.notifs.popleft()
293        except IndexError:
294            pass
295
296        return None
297
298    def set_fundmode(self, fundmode, fundstartval=None):
299        '''Set the actual fundmode (True or False)
300
301        If the argument fundstartval is not ``None``, it will used
302        '''
303        self.p.fundmode = fundmode
304        if fundstartval is not None:
305            self.set_fundstartval(fundstartval)
306
307    def get_fundmode(self):
308        '''Returns the actual fundmode (True or False)'''
309        return self.p.fundmode
310
311    fundmode = property(get_fundmode, set_fundmode)
312
313    def set_fundstartval(self, fundstartval):
314        '''Set the starting value of the fund-like performance tracker'''
315        self.p.fundstartval = fundstartval
316
317    def set_int2pnl(self, int2pnl):
318        '''Configure assignment of interest to profit and loss'''
319        self.p.int2pnl = int2pnl
320
321    def set_coc(self, coc):
322        '''Configure the Cheat-On-Close method to buy the close on order bar'''
323        self.p.coc = coc
324
325    def set_coo(self, coo):
326        '''Configure the Cheat-On-Open method to buy the close on order bar'''
327        self.p.coo = coo
328
329    def set_shortcash(self, shortcash):
330        '''Configure the shortcash parameters'''
331        self.p.shortcash = shortcash
332
333    def set_slippage_perc(self, perc,
334                          slip_open=True, slip_limit=True,
335                          slip_match=True, slip_out=False):
336        '''Configure slippage to be percentage based'''
337        self.p.slip_perc = perc
338        self.p.slip_fixed = 0.0
339        self.p.slip_open = slip_open
340        self.p.slip_limit = slip_limit
341        self.p.slip_match = slip_match
342        self.p.slip_out = slip_out
343
344    def set_slippage_fixed(self, fixed,
345                           slip_open=True, slip_limit=True,
346                           slip_match=True, slip_out=False):
347        '''Configure slippage to be fixed points based'''
348        self.p.slip_perc = 0.0
349        self.p.slip_fixed = fixed
350        self.p.slip_open = slip_open
351        self.p.slip_limit = slip_limit
352        self.p.slip_match = slip_match
353        self.p.slip_out = slip_out
354
355    def set_filler(self, filler):
356        '''Sets a volume filler for volume filling execution'''
357        self.p.filler = filler
358
359    def set_checksubmit(self, checksubmit):
360        '''Sets the checksubmit parameter'''
361        self.p.checksubmit = checksubmit
362
363    def set_eosbar(self, eosbar):
364        '''Sets the eosbar parameter (alias: ``seteosbar``'''
365        self.p.eosbar = eosbar
366
367    seteosbar = set_eosbar
368
369    def get_cash(self):
370        '''Returns the current cash (alias: ``getcash``)'''
371        return self.cash
372
373    getcash = get_cash
374
375    def set_cash(self, cash):
376        '''Sets the cash parameter (alias: ``setcash``)'''
377        self.startingcash = self.cash = self.p.cash = cash
378        self._value = cash
379
380    setcash = set_cash
381
382    def add_cash(self, cash):
383        '''Add/Remove cash to the system (use a negative value to remove)'''
384        self._cash_addition.append(cash)
385
386    def get_fundshares(self):
387        '''Returns the current number of shares in the fund-like mode'''
388        return self._fundshares
389
390    fundshares = property(get_fundshares)
391
392    def get_fundvalue(self):
393        '''Returns the Fund-like share value'''
394        return self._fundval
395
396    fundvalue = property(get_fundvalue)
397
398    def cancel(self, order, bracket=False):
399        try:
400            self.pending.remove(order)
401        except ValueError:
402            # If the list didn't have the element we didn't cancel anything
403            return False
404
405        order.cancel()
406        self.notify(order)
407        self._ococheck(order)
408        if not bracket:
409            self._bracketize(order, cancel=True)
410        return True
411
412    def get_value(self, datas=None, mkt=False, lever=False):
413        '''Returns the portfolio value of the given datas (if datas is ``None``, then
414        the total portfolio value will be returned (alias: ``getvalue``)
415        '''
416        if datas is None:
417            if mkt:
418                return self._valuemkt if not lever else self._valuemktlever
419
420            return self._value if not lever else self._valuelever
421
422        return self._get_value(datas=datas, lever=lever)
423
424    getvalue = get_value
425
426    def get_value_lever(self, datas=None, mkt=False):
427        return self.get_value(datas=datas, mkt=mkt)
428
429    def _get_value(self, datas=None, lever=False):
430        pos_value = 0.0
431        pos_value_unlever = 0.0
432        unrealized = 0.0
433
434        while self._cash_addition:
435            c = self._cash_addition.popleft()
436            self._fundshares += c / self._fundval
437            self.cash += c
438
439        for data in datas or self.positions:
440            comminfo = self.getcommissioninfo(data)
441            position = self.positions[data]
442            # use valuesize:  returns raw value, rather than negative adj val
443            if not self.p.shortcash:
444                dvalue = comminfo.getvalue(position, data.close[0])
445            else:
446                dvalue = comminfo.getvaluesize(position.size, data.close[0])
447
448            dunrealized = comminfo.profitandloss(position.size, position.price,
449                                                 data.close[0])
450            if datas and len(datas) == 1:
451                if lever and dvalue > 0:
452                    dvalue -= dunrealized
453                    return (dvalue / comminfo.get_leverage()) + dunrealized
454                return dvalue  # raw data value requested, short selling is neg
455
456            if not self.p.shortcash:
457                dvalue = abs(dvalue)  # short selling adds value in this case
458
459            pos_value += dvalue
460            unrealized += dunrealized
461
462            if dvalue > 0:  # long position - unlever
463                dvalue -= dunrealized
464                pos_value_unlever += (dvalue / comminfo.get_leverage())
465                pos_value_unlever += dunrealized
466            else:
467                pos_value_unlever += dvalue
468
469        if not self._fundhist:
470            self._value = v = self.cash + pos_value_unlever
471            self._fundval = self._value / self._fundshares  # update fundvalue
472        else:
473            # Try to fetch a value
474            fval, fvalue = self._process_fund_history()
475
476            self._value = fvalue
477            self.cash = fvalue - pos_value_unlever
478            self._fundval = fval
479            self._fundshares = fvalue / fval
480            lev = pos_value / (pos_value_unlever or 1.0)
481
482            # update the calculated values above to the historical values
483            pos_value_unlever = fvalue
484            pos_value = fvalue * lev
485
486        self._valuemkt = pos_value_unlever
487
488        self._valuelever = self.cash + pos_value
489        self._valuemktlever = pos_value
490
491        self._leverage = pos_value / (pos_value_unlever or 1.0)
492        self._unrealized = unrealized
493
494        return self._value if not lever else self._valuelever
495
496    def get_leverage(self):
497        return self._leverage
498
499    def get_orders_open(self, safe=False):
500        '''Returns an iterable with the orders which are still open (either not
501        executed or partially executed
502
503        The orders returned must not be touched.
504
505        If order manipulation is needed, set the parameter ``safe`` to True
506        '''
507        if safe:
508            os = [x.clone() for x in self.pending]
509        else:
510            os = [x for x in self.pending]
511
512        return os
513
514    def getposition(self, data):
515        '''Returns the current position status (a ``Position`` instance) for
516        the given ``data``'''
517        return self.positions[data]
518
519    def orderstatus(self, order):
520        try:
521            o = self.orders.index(order)
522        except ValueError:
523            o = order
524
525        return o.status
526
527    def _take_children(self, order):
528        oref = order.ref
529        pref = getattr(order.parent, 'ref', oref)  # parent ref or self
530
531        if oref != pref:
532            if pref not in self._pchildren:
533                order.reject()  # parent not there - may have been rejected
534                self.notify(order)  # reject child, notify
535                return None
536
537        return pref
538
539    def submit(self, order, check=True):
540        pref = self._take_children(order)
541        if pref is None:  # order has not been taken
542            return order
543
544        pc = self._pchildren[pref]
545        pc.append(order)  # store in parent/children queue
546
547        if order.transmit:  # if single order, sent and queue cleared
548            # if parent-child, the parent will be sent, the other kept
549            rets = [self.transmit(x, check=check) for x in pc]
550            return rets[-1]  # last one is the one triggering transmission
551
552        return order
553
554    def transmit(self, order, check=True):
555        if check and self.p.checksubmit:
556            order.submit()
557            self.submitted.append(order)
558            self.orders.append(order)
559            self.notify(order)
560        else:
561            self.submit_accept(order)
562
563        return order
564
565    def check_submitted(self):
566        cash = self.cash
567        positions = dict()
568
569        while self.submitted:
570            order = self.submitted.popleft()
571
572            if self._take_children(order) is None:  # children not taken
573                continue
574
575            comminfo = self.getcommissioninfo(order.data)
576
577            position = positions.setdefault(
578                order.data, self.positions[order.data].clone())
579
580            # pseudo-execute the order to get the remaining cash after exec
581            cash = self._execute(order, cash=cash, position=position)
582
583            if cash >= 0.0:
584                self.submit_accept(order)
585                continue
586
587            order.margin()
588            self.notify(order)
589            self._ococheck(order)
590            self._bracketize(order, cancel=True)
591
592    def submit_accept(self, order):
593        order.pannotated = None
594        order.submit()
595        order.accept()
596        self.pending.append(order)
597        self.notify(order)
598
599    def _bracketize(self, order, cancel=False):
600        oref = order.ref
601        pref = getattr(order.parent, 'ref', oref)
602        parent = oref == pref
603
604        pc = self._pchildren[pref]  # defdict - guaranteed
605        if cancel or not parent:  # cancel left or child exec -> cancel other
606            while pc:
607                self.cancel(pc.popleft(), bracket=True)  # idempotent
608
609            del self._pchildren[pref]  # defdict guaranteed
610
611        else:  # not cancel -> parent exec'd
612            pc.popleft()  # remove parent
613            for o in pc:  # activate childnre
614                self._toactivate.append(o)
615
616    def _ococheck(self, order):
617        # ocoref = self._ocos[order.ref] or order.ref  # a parent or self
618        parentref = self._ocos[order.ref]
619        ocoref = self._ocos.get(parentref, None)
620        ocol = self._ocol.pop(ocoref, None)
621        if ocol:
622            for i in range(len(self.pending) - 1, -1, -1):
623                o = self.pending[i]
624                if o is not None and o.ref in ocol:
625                    del self.pending[i]
626                    o.cancel()
627                    self.notify(o)
628
629    def _ocoize(self, order, oco):
630        oref = order.ref
631        if oco is None:
632            self._ocos[oref] = oref  # current order is parent
633            self._ocol[oref].append(oref)  # create ocogroup
634        else:
635            ocoref = self._ocos[oco.ref]  # ref to group leader
636            self._ocos[oref] = ocoref  # ref to group leader
637            self._ocol[ocoref].append(oref)  # add to group
638
639    def add_order_history(self, orders, notify=True):
640        oiter = iter(orders)
641        o = next(oiter, None)
642        self._userhist.append([o, oiter, notify])
643
644    def set_fund_history(self, fund):
645        # iterable with the following pro item
646        # [datetime, share_value, net asset value]
647        fiter = iter(fund)
648        f = list(next(fiter))  # must not be empty
649        self._fundhist = [f, fiter]
650        # self._fhistlast = f[1:]
651
652        self.set_cash(float(f[2]))
653
654    def buy(self, owner, data,
655            size, price=None, plimit=None,
656            exectype=None, valid=None, tradeid=0, oco=None,
657            trailamount=None, trailpercent=None,
658            parent=None, transmit=True,
659            histnotify=False, _checksubmit=True,
660            **kwargs):
661
662        order = BuyOrder(owner=owner, data=data,
663                         size=size, price=price, pricelimit=plimit,
664                         exectype=exectype, valid=valid, tradeid=tradeid,
665                         trailamount=trailamount, trailpercent=trailpercent,
666                         parent=parent, transmit=transmit,
667                         histnotify=histnotify)
668
669        order.addinfo(**kwargs)
670        self._ocoize(order, oco)
671
672        return self.submit(order, check=_checksubmit)
673
674    def sell(self, owner, data,
675             size, price=None, plimit=None,
676             exectype=None, valid=None, tradeid=0, oco=None,
677             trailamount=None, trailpercent=None,
678             parent=None, transmit=True,
679             histnotify=False, _checksubmit=True,
680             **kwargs):
681
682        order = SellOrder(owner=owner, data=data,
683                          size=size, price=price, pricelimit=plimit,
684                          exectype=exectype, valid=valid, tradeid=tradeid,
685                          trailamount=trailamount, trailpercent=trailpercent,
686                          parent=parent, transmit=transmit,
687                          histnotify=histnotify)
688
689        order.addinfo(**kwargs)
690        self._ocoize(order, oco)
691
692        return self.submit(order, check=_checksubmit)
693
694    def _execute(self, order, ago=None, price=None, cash=None, position=None,
695                 dtcoc=None):
696        # ago = None is used a flag for pseudo execution
697        if ago is not None and price is None:
698            return  # no psuedo exec no price - no execution
699
700        if self.p.filler is None or ago is None:
701            # Order gets full size or pseudo-execution
702            size = order.executed.remsize
703        else:
704            # Execution depends on volume filler
705            size = self.p.filler(order, price, ago)
706            if not order.isbuy():
707                size = -size
708
709        # Get comminfo object for the data
710        comminfo = self.getcommissioninfo(order.data)
711
712        # Check if something has to be compensated
713        if order.data._compensate is not None:
714            data = order.data._compensate
715            cinfocomp = self.getcommissioninfo(data)  # for actual commission
716        else:
717            data = order.data
718            cinfocomp = comminfo
719
720        # Adjust position with operation size
721        if ago is not None:
722            # Real execution with date
723            position = self.positions[data]
724            pprice_orig = position.price
725
726            psize, pprice, opened, closed = position.pseudoupdate(size, price)
727
728            # if part/all of a position has been closed, then there has been
729            # a profitandloss ... record it
730            pnl = comminfo.profitandloss(-closed, pprice_orig, price)
731            cash = self.cash
732        else:
733            pnl = 0
734            if not self.p.coo:
735                price = pprice_orig = order.created.price
736            else:
737                # When doing cheat on open, the price to be considered for a
738                # market order is the opening price and not the default closing
739                # price with which the order was created
740                if order.exectype == Order.Market:
741                    price = pprice_orig = order.data.open[0]
742                else:
743                    price = pprice_orig = order.created.price
744
745            psize, pprice, opened, closed = position.update(size, price)
746
747        # "Closing" totally or partially is possible. Cash may be re-injected
748        if closed:
749            # Adjust to returned value for closed items & acquired opened items
750            if self.p.shortcash:
751                closedvalue = comminfo.getvaluesize(-closed, pprice_orig)
752            else:
753                closedvalue = comminfo.getoperationcost(closed, pprice_orig)
754
755            closecash = closedvalue
756            if closedvalue > 0:  # long position closed
757                closecash /= comminfo.get_leverage()  # inc cash with lever
758
759            cash += closecash + pnl * comminfo.stocklike
760            # Calculate and substract commission
761            closedcomm = comminfo.getcommission(closed, price)
762            cash -= closedcomm
763
764            if ago is not None:
765                # Cashadjust closed contracts: prev close vs exec price
766                # The operation can inject or take cash out
767                cash += comminfo.cashadjust(-closed,
768                                            position.adjbase,
769                                            price)
770
771                # Update system cash
772                self.cash = cash
773        else:
774            closedvalue = closedcomm = 0.0
775
776        popened = opened
777        if opened:
778            if self.p.shortcash:
779                openedvalue = comminfo.getvaluesize(opened, price)
780            else:
781                openedvalue = comminfo.getoperationcost(opened, price)
782
783            opencash = openedvalue
784            if openedvalue > 0:  # long position being opened
785                opencash /= comminfo.get_leverage()  # dec cash with level
786
787            cash -= opencash  # original behavior
788
789            openedcomm = cinfocomp.getcommission(opened, price)
790            cash -= openedcomm
791
792            if cash < 0.0:
793                # execution is not possible - nullify
794                opened = 0
795                openedvalue = openedcomm = 0.0
796
797            elif ago is not None:  # real execution
798                if abs(psize) > abs(opened):
799                    # some futures were opened - adjust the cash of the
800                    # previously existing futures to the operation price and
801                    # use that as new adjustment base, because it already is
802                    # for the new futures At the end of the cycle the
803                    # adjustment to the close price will be done for all open
804                    # futures from a common base price with regards to the
805                    # close price
806                    adjsize = psize - opened
807                    cash += comminfo.cashadjust(adjsize,
808                                                position.adjbase, price)
809
810                # record adjust price base for end of bar cash adjustment
811                position.adjbase = price
812
813                # update system cash - checking if opened is still != 0
814                self.cash = cash
815        else:
816            openedvalue = openedcomm = 0.0
817
818        if ago is None:
819            # return cash from pseudo-execution
820            return cash
821
822        execsize = closed + opened
823
824        if execsize:
825            # Confimrm the operation to the comminfo object
826            comminfo.confirmexec(execsize, price)
827
828            # do a real position update if something was executed
829            position.update(execsize, price, data.datetime.datetime())
830
831            if closed and self.p.int2pnl:  # Assign accumulated interest data
832                closedcomm += self.d_credit.pop(data, 0.0)
833
834            # Execute and notify the order
835            order.execute(dtcoc or data.datetime[ago],
836                          execsize, price,
837                          closed, closedvalue, closedcomm,
838                          opened, openedvalue, openedcomm,
839                          comminfo.margin, pnl,
840                          psize, pprice)
841
842            order.addcomminfo(comminfo)
843
844            self.notify(order)
845            self._ococheck(order)
846
847        if popened and not opened:
848            # opened was not executed - not enough cash
849            order.margin()
850            self.notify(order)
851            self._ococheck(order)
852            self._bracketize(order, cancel=True)
853
854    def notify(self, order):
855        self.notifs.append(order.clone())
856
857    def _try_exec_historical(self, order):
858        self._execute(order, ago=0, price=order.created.price)
859
860    def _try_exec_market(self, order, popen, phigh, plow):
861        ago = 0
862        if self.p.coc and order.info.get('coc', True):
863            dtcoc = order.created.dt
864            exprice = order.created.pclose
865        else:
866            if not self.p.coo and order.data.datetime[0] <= order.created.dt:
867                return    # can only execute after creation time
868
869            dtcoc = None
870            exprice = popen
871
872        if order.isbuy():
873            p = self._slip_up(phigh, exprice, doslip=self.p.slip_open)
874        else:
875            p = self._slip_down(plow, exprice, doslip=self.p.slip_open)
876
877        self._execute(order, ago=0, price=p, dtcoc=dtcoc)
878
879    def _try_exec_close(self, order, pclose):
880        # pannotated allows to keep track of the closing bar if there is no
881        # information which lets us know that the current bar is the closing
882        # bar (like matching end of session bar)
883        # The actual matching will be done one bar afterwards but using the
884        # information from the actual closing bar
885
886        dt0 = order.data.datetime[0]
887        # don't use "len" -> in replay the close can be reached with same len
888        if dt0 > order.created.dt:  # can only execute after creation time
889            # or (self.p.eosbar and dt0 == order.dteos):
890            if dt0 >= order.dteos:
891                # past the end of session or right at it and eosbar is True
892                if order.pannotated and dt0 > order.dteos:
893                    ago = -1
894                    execprice = order.pannotated
895                else:
896                    ago = 0
897                    execprice = pclose
898
899                self._execute(order, ago=ago, price=execprice)
900                return
901
902        # If no exexcution has taken place ... annotate the closing price
903        order.pannotated = pclose
904
905    def _try_exec_limit(self, order, popen, phigh, plow, plimit):
906        if order.isbuy():
907            if plimit >= popen:
908                # open smaller/equal than requested - buy cheaper
909                pmax = min(phigh, plimit)
910                p = self._slip_up(pmax, popen, doslip=self.p.slip_open,
911                                  lim=True)
912                self._execute(order, ago=0, price=p)
913            elif plimit >= plow:
914                # day low below req price ... match limit price
915                self._execute(order, ago=0, price=plimit)
916
917        else:  # Sell
918            if plimit <= popen:
919                # open greater/equal than requested - sell more expensive
920                pmin = max(plow, plimit)
921                p = self._slip_down(plimit, popen, doslip=self.p.slip_open,
922                                    lim=True)
923                self._execute(order, ago=0, price=p)
924            elif plimit <= phigh:
925                # day high above req price ... match limit price
926                self._execute(order, ago=0, price=plimit)
927
928    def _try_exec_stop(self, order, popen, phigh, plow, pcreated, pclose):
929        if order.isbuy():
930            if popen >= pcreated:
931                # price penetrated with an open gap - use open
932                p = self._slip_up(phigh, popen, doslip=self.p.slip_open)
933                self._execute(order, ago=0, price=p)
934            elif phigh >= pcreated:
935                # price penetrated during the session - use trigger price
936                p = self._slip_up(phigh, pcreated)
937                self._execute(order, ago=0, price=p)
938
939        else:  # Sell
940            if popen <= pcreated:
941                # price penetrated with an open gap - use open
942                p = self._slip_down(plow, popen, doslip=self.p.slip_open)
943                self._execute(order, ago=0, price=p)
944            elif plow <= pcreated:
945                # price penetrated during the session - use trigger price
946                p = self._slip_down(plow, pcreated)
947                self._execute(order, ago=0, price=p)
948
949        # not (completely) executed and trailing stop
950        if order.alive() and order.exectype == Order.StopTrail:
951            order.trailadjust(pclose)
952
953    def _try_exec_stoplimit(self, order,
954                            popen, phigh, plow, pclose,
955                            pcreated, plimit):
956        if order.isbuy():
957            if popen >= pcreated:
958                order.triggered = True
959                self._try_exec_limit(order, popen, phigh, plow, plimit)
960
961            elif phigh >= pcreated:
962                # price penetrated upwards during the session
963                order.triggered = True
964                # can calculate execution for a few cases - datetime is fixed
965                if popen > pclose:
966                    if plimit >= pcreated:  # limit above stop trigger
967                        p = self._slip_up(phigh, pcreated, lim=True)
968                        self._execute(order, ago=0, price=p)
969                    elif plimit >= pclose:
970                        self._execute(order, ago=0, price=plimit)
971                else:  # popen < pclose
972                    if plimit >= pcreated:
973                        p = self._slip_up(phigh, pcreated, lim=True)
974                        self._execute(order, ago=0, price=p)
975        else:  # Sell
976            if popen <= pcreated:
977                # price penetrated downwards with an open gap
978                order.triggered = True
979                self._try_exec_limit(order, popen, phigh, plow, plimit)
980
981            elif plow <= pcreated:
982                # price penetrated downwards during the session
983                order.triggered = True
984                # can calculate execution for a few cases - datetime is fixed
985                if popen <= pclose:
986                    if plimit <= pcreated:
987                        p = self._slip_down(plow, pcreated, lim=True)
988                        self._execute(order, ago=0, price=p)
989                    elif plimit <= pclose:
990                        self._execute(order, ago=0, price=plimit)
991                else:
992                    # popen > pclose
993                    if plimit <= pcreated:
994                        p = self._slip_down(plow, pcreated, lim=True)
995                        self._execute(order, ago=0, price=p)
996
997        # not (completely) executed and trailing stop
998        if order.alive() and order.exectype == Order.StopTrailLimit:
999            order.trailadjust(pclose)
1000
1001    def _slip_up(self, pmax, price, doslip=True, lim=False):
1002        if not doslip:
1003            return price
1004
1005        slip_perc = self.p.slip_perc
1006        slip_fixed = self.p.slip_fixed
1007        if slip_perc:
1008            pslip = price * (1 + slip_perc)
1009        elif slip_fixed:
1010            pslip = price + slip_fixed
1011        else:
1012            return price
1013
1014        if pslip <= pmax:  # slipping can return price
1015            return pslip
1016        elif self.p.slip_match or (lim and self.p.slip_limit):
1017            if not self.p.slip_out:
1018                return pmax
1019
1020            return pslip  # non existent price
1021
1022        return None  # no price can be returned
1023
1024    def _slip_down(self, pmin, price, doslip=True, lim=False):
1025        if not doslip:
1026            return price
1027
1028        slip_perc = self.p.slip_perc
1029        slip_fixed = self.p.slip_fixed
1030        if slip_perc:
1031            pslip = price * (1 - slip_perc)
1032        elif slip_fixed:
1033            pslip = price - slip_fixed
1034        else:
1035            return price
1036
1037        if pslip >= pmin:  # slipping can return price
1038            return pslip
1039        elif self.p.slip_match or (lim and self.p.slip_limit):
1040            if not self.p.slip_out:
1041                return pmin
1042
1043            return pslip  # non existent price
1044
1045        return None  # no price can be returned
1046
1047    def _try_exec(self, order):
1048        data = order.data
1049
1050        popen = getattr(data, 'tick_open', None)
1051        if popen is None:
1052            popen = data.open[0]
1053        phigh = getattr(data, 'tick_high', None)
1054        if phigh is None:
1055            phigh = data.high[0]
1056        plow = getattr(data, 'tick_low', None)
1057        if plow is None:
1058            plow = data.low[0]
1059        pclose = getattr(data, 'tick_close', None)
1060        if pclose is None:
1061            pclose = data.close[0]
1062
1063        pcreated = order.created.price
1064        plimit = order.created.pricelimit
1065
1066        if order.exectype == Order.Market:
1067            self._try_exec_market(order, popen, phigh, plow)
1068
1069        elif order.exectype == Order.Close:
1070            self._try_exec_close(order, pclose)
1071
1072        elif order.exectype == Order.Limit:
1073            self._try_exec_limit(order, popen, phigh, plow, pcreated)
1074
1075        elif (order.triggered and
1076              order.exectype in [Order.StopLimit, Order.StopTrailLimit]):
1077            self._try_exec_limit(order, popen, phigh, plow, plimit)
1078
1079        elif order.exectype in [Order.Stop, Order.StopTrail]:
1080            self._try_exec_stop(order, popen, phigh, plow, pcreated, pclose)
1081
1082        elif order.exectype in [Order.StopLimit, Order.StopTrailLimit]:
1083            self._try_exec_stoplimit(order,
1084                                     popen, phigh, plow, pclose,
1085                                     pcreated, plimit)
1086
1087        elif order.exectype == Order.Historical:
1088            self._try_exec_historical(order)
1089
1090    def _process_fund_history(self):
1091        fhist = self._fundhist  # [last element, iterator]
1092        f, funds = fhist
1093        if not f:
1094            return self._fhistlast
1095
1096        dt = f[0]  # date/datetime instance
1097        if isinstance(dt, string_types):
1098            dtfmt = '%Y-%m-%d'
1099            if 'T' in dt:
1100                dtfmt += 'T%H:%M:%S'
1101                if '.' in dt:
1102                    dtfmt += '.%f'
1103            dt = datetime.datetime.strptime(dt, dtfmt)
1104            f[0] = dt  # update value
1105
1106        elif isinstance(dt, datetime.datetime):
1107            pass
1108        elif isinstance(dt, datetime.date):
1109            dt = datetime.datetime(year=dt.year, month=dt.month, day=dt.day)
1110            f[0] = dt  # Update the value
1111
1112        # Synchronization with the strategy is not possible because the broker
1113        # is called before the strategy advances. The 2 lines below would do it
1114        # if possible
1115        # st0 = self.cerebro.runningstrats[0]
1116        # if dt <= st0.datetime.datetime():
1117        if dt <= self.cerebro._dtmaster:
1118            self._fhistlast = f[1:]
1119            fhist[0] = list(next(funds, []))
1120
1121        return self._fhistlast
1122
1123    def _process_order_history(self):
1124        for uhist in self._userhist:
1125            uhorder, uhorders, uhnotify = uhist
1126            while uhorder is not None:
1127                uhorder = list(uhorder)  # to support assignment (if tuple)
1128                try:
1129                    dataidx = uhorder[3]  # 2nd field
1130                except IndexError:
1131                    dataidx = None  # Field not present, use default
1132
1133                if dataidx is None:
1134                    d = self.cerebro.datas[0]
1135                elif isinstance(dataidx, integer_types):
1136                    d = self.cerebro.datas[dataidx]
1137                else:  # assume string
1138                    d = self.cerebro.datasbyname[dataidx]
1139
1140                if not len(d):
1141                    break  # may start later as oter data feeds
1142
1143                dt = uhorder[0]  # date/datetime instance
1144                if isinstance(dt, string_types):
1145                    dtfmt = '%Y-%m-%d'
1146                    if 'T' in dt:
1147                        dtfmt += 'T%H:%M:%S'
1148                        if '.' in dt:
1149                            dtfmt += '.%f'
1150                    dt = datetime.datetime.strptime(dt, dtfmt)
1151                    uhorder[0] = dt
1152                elif isinstance(dt, datetime.datetime):
1153                    pass
1154                elif isinstance(dt, datetime.date):
1155                    dt = datetime.datetime(year=dt.year,
1156                                           month=dt.month,
1157                                           day=dt.day)
1158                    uhorder[0] = dt
1159
1160                if dt > d.datetime.datetime():
1161                    break  # cannot execute yet 1st in queue, stop processing
1162
1163                size = uhorder[1]
1164                price = uhorder[2]
1165                owner = self.cerebro.runningstrats[0]
1166                if size > 0:
1167                    o = self.buy(owner=owner, data=d,
1168                                 size=size, price=price,
1169                                 exectype=Order.Historical,
1170                                 histnotify=uhnotify,
1171                                 _checksubmit=False)
1172
1173                elif size < 0:
1174                    o = self.sell(owner=owner, data=d,
1175                                  size=abs(size), price=price,
1176                                  exectype=Order.Historical,
1177                                  histnotify=uhnotify,
1178                                  _checksubmit=False)
1179
1180                # update to next potential order
1181                uhist[0] = uhorder = next(uhorders, None)
1182
1183    def next(self):
1184        while self._toactivate:
1185            self._toactivate.popleft().activate()
1186
1187        if self.p.checksubmit:
1188            self.check_submitted()
1189
1190        # Discount any cash for positions hold
1191        credit = 0.0
1192        for data, pos in self.positions.items():
1193            if pos:
1194                comminfo = self.getcommissioninfo(data)
1195                dt0 = data.datetime.datetime()
1196                dcredit = comminfo.get_credit_interest(data, pos, dt0)
1197                self.d_credit[data] += dcredit
1198                credit += dcredit
1199                pos.datetime = dt0  # mark last credit operation
1200
1201        self.cash -= credit
1202
1203        self._process_order_history()
1204
1205        # Iterate once over all elements of the pending queue
1206        self.pending.append(None)
1207        while True:
1208            order = self.pending.popleft()
1209            if order is None:
1210                break
1211
1212            if order.expire():
1213                self.notify(order)
1214                self._ococheck(order)
1215                self._bracketize(order, cancel=True)
1216
1217            elif not order.active():
1218                self.pending.append(order)  # cannot yet be processed
1219
1220            else:
1221                self._try_exec(order)
1222                if order.alive():
1223                    self.pending.append(order)
1224
1225                elif order.status == Order.Completed:
1226                    # a bracket parent order may have been executed
1227                    self._bracketize(order)
1228
1229        # Operations have been executed ... adjust cash end of bar
1230        for data, pos in self.positions.items():
1231            # futures change cash every bar
1232            if pos:
1233                comminfo = self.getcommissioninfo(data)
1234                self.cash += comminfo.cashadjust(pos.size,
1235                                                 pos.adjbase,
1236                                                 data.close[0])
1237                # record the last adjustment price
1238                pos.adjbase = data.close[0]
1239
1240        self._get_value()  # update value
1241
1242
1243# Alias
1244BrokerBack = BackBroker
1245