1# -*- coding: utf-8 -*- 2 3############################ Copyrights and license ############################ 4# # 5# Copyright 2012 Vincent Jacques <vincent@vincent-jacques.net> # 6# Copyright 2012 Zearin <zearin@gonk.net> # 7# Copyright 2013 AKFish <akfish@gmail.com> # 8# Copyright 2013 Bill Mill <bill.mill@gmail.com> # 9# Copyright 2013 Vincent Jacques <vincent@vincent-jacques.net> # 10# Copyright 2013 davidbrai <davidbrai@gmail.com> # 11# Copyright 2014 Thialfihar <thi@thialfihar.org> # 12# Copyright 2014 Vincent Jacques <vincent@vincent-jacques.net> # 13# Copyright 2015 Dan Vanderkam <danvdk@gmail.com> # 14# Copyright 2015 Eliot Walker <eliot@lyft.com> # 15# Copyright 2016 Peter Buckley <dx-pbuckley@users.noreply.github.com> # 16# Copyright 2017 Jannis Gebauer <ja.geb@me.com> # 17# Copyright 2018 Gilad Shefer <gshefer@redhat.com> # 18# Copyright 2018 Joel Koglin <JoelKoglin@gmail.com> # 19# Copyright 2018 Wan Liuyang <tsfdye@gmail.com> # 20# Copyright 2018 sfdye <tsfdye@gmail.com> # 21# # 22# This file is part of PyGithub. # 23# http://pygithub.readthedocs.io/ # 24# # 25# PyGithub is free software: you can redistribute it and/or modify it under # 26# the terms of the GNU Lesser General Public License as published by the Free # 27# Software Foundation, either version 3 of the License, or (at your option) # 28# any later version. # 29# # 30# PyGithub is distributed in the hope that it will be useful, but WITHOUT ANY # 31# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # 32# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more # 33# details. # 34# # 35# You should have received a copy of the GNU Lesser General Public License # 36# along with PyGithub. If not, see <http://www.gnu.org/licenses/>. # 37# # 38################################################################################ 39 40from urllib.parse import parse_qs 41 42 43class PaginatedListBase: 44 def __init__(self): 45 self.__elements = list() 46 47 def __getitem__(self, index): 48 assert isinstance(index, (int, slice)) 49 if isinstance(index, int): 50 self.__fetchToIndex(index) 51 return self.__elements[index] 52 else: 53 return self._Slice(self, index) 54 55 def __iter__(self): 56 for element in self.__elements: 57 yield element 58 while self._couldGrow(): 59 newElements = self._grow() 60 for element in newElements: 61 yield element 62 63 def _isBiggerThan(self, index): 64 return len(self.__elements) > index or self._couldGrow() 65 66 def __fetchToIndex(self, index): 67 while len(self.__elements) <= index and self._couldGrow(): 68 self._grow() 69 70 def _grow(self): 71 newElements = self._fetchNextPage() 72 self.__elements += newElements 73 return newElements 74 75 class _Slice: 76 def __init__(self, theList, theSlice): 77 self.__list = theList 78 self.__start = theSlice.start or 0 79 self.__stop = theSlice.stop 80 self.__step = theSlice.step or 1 81 82 def __iter__(self): 83 index = self.__start 84 while not self.__finished(index): 85 if self.__list._isBiggerThan(index): 86 yield self.__list[index] 87 index += self.__step 88 else: 89 return 90 91 def __finished(self, index): 92 return self.__stop is not None and index >= self.__stop 93 94 95class PaginatedList(PaginatedListBase): 96 """ 97 This class abstracts the `pagination of the API <http://developer.github.com/v3/#pagination>`_. 98 99 You can simply enumerate through instances of this class:: 100 101 for repo in user.get_repos(): 102 print(repo.name) 103 104 If you want to know the total number of items in the list:: 105 106 print(user.get_repos().totalCount) 107 108 You can also index them or take slices:: 109 110 second_repo = user.get_repos()[1] 111 first_repos = user.get_repos()[:10] 112 113 If you want to iterate in reversed order, just do:: 114 115 for repo in user.get_repos().reversed: 116 print(repo.name) 117 118 And if you really need it, you can explicitly access a specific page:: 119 120 some_repos = user.get_repos().get_page(0) 121 some_other_repos = user.get_repos().get_page(3) 122 """ 123 124 def __init__( 125 self, 126 contentClass, 127 requester, 128 firstUrl, 129 firstParams, 130 headers=None, 131 list_item="items", 132 ): 133 super().__init__() 134 self.__requester = requester 135 self.__contentClass = contentClass 136 self.__firstUrl = firstUrl 137 self.__firstParams = firstParams or () 138 self.__nextUrl = firstUrl 139 self.__nextParams = firstParams or {} 140 self.__headers = headers 141 self.__list_item = list_item 142 if self.__requester.per_page != 30: 143 self.__nextParams["per_page"] = self.__requester.per_page 144 self._reversed = False 145 self.__totalCount = None 146 147 @property 148 def totalCount(self): 149 if not self.__totalCount: 150 params = {} if self.__nextParams is None else self.__nextParams.copy() 151 # set per_page = 1 so the totalCount is just the number of pages 152 params.update({"per_page": 1}) 153 headers, data = self.__requester.requestJsonAndCheck( 154 "GET", self.__firstUrl, parameters=params, headers=self.__headers 155 ) 156 if "link" not in headers: 157 if data and "total_count" in data: 158 self.__totalCount = data["total_count"] 159 elif data: 160 self.__totalCount = len(data) 161 else: 162 self.__totalCount = 0 163 else: 164 links = self.__parseLinkHeader(headers) 165 lastUrl = links.get("last") 166 if lastUrl: 167 self.__totalCount = int(parse_qs(lastUrl)["page"][0]) 168 else: 169 self.__totalCount = 0 170 return self.__totalCount 171 172 def _getLastPageUrl(self): 173 headers, data = self.__requester.requestJsonAndCheck( 174 "GET", self.__firstUrl, parameters=self.__nextParams, headers=self.__headers 175 ) 176 links = self.__parseLinkHeader(headers) 177 lastUrl = links.get("last") 178 return lastUrl 179 180 @property 181 def reversed(self): 182 r = PaginatedList( 183 self.__contentClass, 184 self.__requester, 185 self.__firstUrl, 186 self.__firstParams, 187 self.__headers, 188 self.__list_item, 189 ) 190 r.__reverse() 191 return r 192 193 def __reverse(self): 194 self._reversed = True 195 lastUrl = self._getLastPageUrl() 196 if lastUrl: 197 self.__nextUrl = lastUrl 198 199 def _couldGrow(self): 200 return self.__nextUrl is not None 201 202 def _fetchNextPage(self): 203 headers, data = self.__requester.requestJsonAndCheck( 204 "GET", self.__nextUrl, parameters=self.__nextParams, headers=self.__headers 205 ) 206 data = data if data else [] 207 208 self.__nextUrl = None 209 if len(data) > 0: 210 links = self.__parseLinkHeader(headers) 211 if self._reversed: 212 if "prev" in links: 213 self.__nextUrl = links["prev"] 214 elif "next" in links: 215 self.__nextUrl = links["next"] 216 self.__nextParams = None 217 218 if self.__list_item in data: 219 self.__totalCount = data.get("total_count") 220 data = data[self.__list_item] 221 222 content = [ 223 self.__contentClass(self.__requester, headers, element, completed=False) 224 for element in data 225 if element is not None 226 ] 227 if self._reversed: 228 return content[::-1] 229 return content 230 231 def __parseLinkHeader(self, headers): 232 links = {} 233 if "link" in headers: 234 linkHeaders = headers["link"].split(", ") 235 for linkHeader in linkHeaders: 236 url, rel, *rest = linkHeader.split("; ") 237 url = url[1:-1] 238 rel = rel[5:-1] 239 links[rel] = url 240 return links 241 242 def get_page(self, page): 243 params = dict(self.__firstParams) 244 if page != 0: 245 params["page"] = page + 1 246 if self.__requester.per_page != 30: 247 params["per_page"] = self.__requester.per_page 248 headers, data = self.__requester.requestJsonAndCheck( 249 "GET", self.__firstUrl, parameters=params, headers=self.__headers 250 ) 251 252 if self.__list_item in data: 253 self.__totalCount = data.get("total_count") 254 data = data[self.__list_item] 255 256 return [ 257 self.__contentClass(self.__requester, headers, element, completed=False) 258 for element in data 259 ] 260