1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#
4# Yahoo! Finance market data downloader (+fix for Pandas Datareader)
5# https://github.com/ranaroussi/yfinance
6#
7# Copyright 2017-2019 Ran Aroussi
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13#     http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21
22from __future__ import print_function
23
24import time as _time
25import multitasking as _multitasking
26import pandas as _pd
27
28from . import Ticker, utils
29from . import shared
30
31
32def download(tickers, start=None, end=None, actions=False, threads=True,
33             group_by='column', auto_adjust=False, back_adjust=False,
34             progress=True, period="max", show_errors=True, interval="1d", prepost=False,
35             proxy=None, rounding=False, timeout=None, **kwargs):
36    """Download yahoo tickers
37    :Parameters:
38        tickers : str, list
39            List of tickers to download
40        period : str
41            Valid periods: 1d,5d,1mo,3mo,6mo,1y,2y,5y,10y,ytd,max
42            Either Use period parameter or use start and end
43        interval : str
44            Valid intervals: 1m,2m,5m,15m,30m,60m,90m,1h,1d,5d,1wk,1mo,3mo
45            Intraday data cannot extend last 60 days
46        start: str
47            Download start date string (YYYY-MM-DD) or _datetime.
48            Default is 1900-01-01
49        end: str
50            Download end date string (YYYY-MM-DD) or _datetime.
51            Default is now
52        group_by : str
53            Group by 'ticker' or 'column' (default)
54        prepost : bool
55            Include Pre and Post market data in results?
56            Default is False
57        auto_adjust: bool
58            Adjust all OHLC automatically? Default is False
59        actions: bool
60            Download dividend + stock splits data. Default is False
61        threads: bool / int
62            How many threads to use for mass downloading. Default is True
63        proxy: str
64            Optional. Proxy server URL scheme. Default is None
65        rounding: bool
66            Optional. Round values to 2 decimal places?
67        show_errors: bool
68            Optional. Doesn't print errors if True
69        timeout: None or float
70            If not None stops waiting for a response after given number of
71            seconds. (Can also be a fraction of a second e.g. 0.01)
72    """
73
74    # create ticker list
75    tickers = tickers if isinstance(
76        tickers, (list, set, tuple)) else tickers.replace(',', ' ').split()
77
78    # accept isin as ticker
79    shared._ISINS = {}
80    _tickers_ = []
81    for ticker in tickers:
82        if utils.is_isin(ticker):
83            isin = ticker
84            ticker = utils.get_ticker_by_isin(ticker, proxy)
85            shared._ISINS[ticker] = isin
86        _tickers_.append(ticker)
87
88    tickers = _tickers_
89
90    tickers = list(set([ticker.upper() for ticker in tickers]))
91
92    if progress:
93        shared._PROGRESS_BAR = utils.ProgressBar(len(tickers), 'completed')
94
95    # reset shared._DFS
96    shared._DFS = {}
97    shared._ERRORS = {}
98
99    # download using threads
100    if threads:
101        if threads is True:
102            threads = min([len(tickers), _multitasking.cpu_count() * 2])
103        _multitasking.set_max_threads(threads)
104        for i, ticker in enumerate(tickers):
105            _download_one_threaded(ticker, period=period, interval=interval,
106                                   start=start, end=end, prepost=prepost,
107                                   actions=actions, auto_adjust=auto_adjust,
108                                   back_adjust=back_adjust,
109                                   progress=(progress and i > 0), proxy=proxy,
110                                   rounding=rounding, timeout=timeout)
111        while len(shared._DFS) < len(tickers):
112            _time.sleep(0.01)
113
114    # download synchronously
115    else:
116        for i, ticker in enumerate(tickers):
117            data = _download_one(ticker, period=period, interval=interval,
118                                 start=start, end=end, prepost=prepost,
119                                 actions=actions, auto_adjust=auto_adjust,
120                                 back_adjust=back_adjust, proxy=proxy,
121                                 rounding=rounding, timeout=timeout)
122            shared._DFS[ticker.upper()] = data
123            if progress:
124                shared._PROGRESS_BAR.animate()
125
126    if progress:
127        shared._PROGRESS_BAR.completed()
128
129    if shared._ERRORS and show_errors:
130        print('\n%.f Failed download%s:' % (
131            len(shared._ERRORS), 's' if len(shared._ERRORS) > 1 else ''))
132        # print(shared._ERRORS)
133        print("\n".join(['- %s: %s' %
134                         v for v in list(shared._ERRORS.items())]))
135
136    if len(tickers) == 1:
137        ticker = tickers[0]
138        return shared._DFS[shared._ISINS.get(ticker, ticker)]
139
140    try:
141        data = _pd.concat(shared._DFS.values(), axis=1,
142                          keys=shared._DFS.keys())
143    except Exception:
144        _realign_dfs()
145        data = _pd.concat(shared._DFS.values(), axis=1,
146                          keys=shared._DFS.keys())
147
148    # switch names back to isins if applicable
149    data.rename(columns=shared._ISINS, inplace=True)
150
151    if group_by == 'column':
152        data.columns = data.columns.swaplevel(0, 1)
153        data.sort_index(level=0, axis=1, inplace=True)
154
155    return data
156
157
158def _realign_dfs():
159    idx_len = 0
160    idx = None
161
162    for df in shared._DFS.values():
163        if len(df) > idx_len:
164            idx_len = len(df)
165            idx = df.index
166
167    for key in shared._DFS.keys():
168        try:
169            shared._DFS[key] = _pd.DataFrame(
170                index=idx, data=shared._DFS[key]).drop_duplicates()
171        except Exception:
172            shared._DFS[key] = _pd.concat([
173                utils.empty_df(idx), shared._DFS[key].dropna()
174            ], axis=0, sort=True)
175
176        # remove duplicate index
177        shared._DFS[key] = shared._DFS[key].loc[
178            ~shared._DFS[key].index.duplicated(keep='last')]
179
180
181@_multitasking.task
182def _download_one_threaded(ticker, start=None, end=None,
183                           auto_adjust=False, back_adjust=False,
184                           actions=False, progress=True, period="max",
185                           interval="1d", prepost=False, proxy=None,
186                           rounding=False, timeout=None):
187
188    data = _download_one(ticker, start, end, auto_adjust, back_adjust,
189                         actions, period, interval, prepost, proxy, rounding,
190                         timeout)
191    shared._DFS[ticker.upper()] = data
192    if progress:
193        shared._PROGRESS_BAR.animate()
194
195
196def _download_one(ticker, start=None, end=None,
197                  auto_adjust=False, back_adjust=False,
198                  actions=False, period="max", interval="1d",
199                  prepost=False, proxy=None, rounding=False,
200                  timeout=None):
201
202    return Ticker(ticker).history(period=period, interval=interval,
203                                  start=start, end=end, prepost=prepost,
204                                  actions=actions, auto_adjust=auto_adjust,
205                                  back_adjust=back_adjust, proxy=proxy,
206                                  rounding=rounding, many=True,
207                                  timeout=timeout)
208