binder.py 10.7 KB
Newer Older
1
# Tweepy
2
3
# Copyright 2009-2010 Joshua Roesslein
# See LICENSE for details.
4

Mark Smith's avatar
Mark Smith committed
5
6
from __future__ import print_function

7
import time
8
import re
9

DarkRedman's avatar
DarkRedman committed
10
from six.moves.urllib.parse import quote, urlencode
11
import requests
12

Mark Smith's avatar
Mark Smith committed
13
14
import logging

obskyr's avatar
obskyr committed
15
from tweepy.error import TweepError, RateLimitError, is_rate_limit_error_message
16
from tweepy.utils import convert_to_utf8_str
17
from tweepy.models import Model
18

19

20
21
re_path_template = re.compile('{\w+}')

Mark Smith's avatar
Mark Smith committed
22
log = logging.getLogger('tweepy.binder')
Josh Roesslein's avatar
Josh Roesslein committed
23

Jordi Riera's avatar
Jordi Riera committed
24
def bind_api(**config):
25
26
27

    class APIMethod(object):

Jordi Riera's avatar
Jordi Riera committed
28
29
30
31
32
33
34
35
        api = config['api']
        path = config['path']
        payload_type = config.get('payload_type', None)
        payload_list = config.get('payload_list', False)
        allowed_param = config.get('allowed_param', [])
        method = config.get('method', 'GET')
        require_auth = config.get('require_auth', False)
        search_api = config.get('search_api', False)
Florent Espanet's avatar
Florent Espanet committed
36
        upload_api = config.get('upload_api', False)
Jordi Riera's avatar
Jordi Riera committed
37
        use_cache = config.get('use_cache', True)
Aaron Hill's avatar
Aaron Hill committed
38
        session = requests.Session()
39

40
        def __init__(self, args, kwargs):
Jordi Riera's avatar
Jordi Riera committed
41
            api = self.api
42
43
44
45
46
            # If authentication is required and no credentials
            # are provided, throw an error.
            if self.require_auth and not api.auth:
                raise TweepError('Authentication required!')

47
            self.post_data = kwargs.pop('post_data', None)
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
48
49
50
51
52
53
54
55
56
57
            self.retry_count = kwargs.pop('retry_count',
                                          api.retry_count)
            self.retry_delay = kwargs.pop('retry_delay',
                                          api.retry_delay)
            self.retry_errors = kwargs.pop('retry_errors',
                                           api.retry_errors)
            self.wait_on_rate_limit = kwargs.pop('wait_on_rate_limit',
                                                 api.wait_on_rate_limit)
            self.wait_on_rate_limit_notify = kwargs.pop('wait_on_rate_limit_notify',
                                                        api.wait_on_rate_limit_notify)
58
59
60
            self.parser = kwargs.pop('parser', api.parser)
            self.session.headers = kwargs.pop('headers', {})
            self.build_parameters(args, kwargs)
61
62
63
64

            # Pick correct URL root to use
            if self.search_api:
                self.api_root = api.search_root
Florent Espanet's avatar
Florent Espanet committed
65
66
            elif self.upload_api:
                self.api_root = api.upload_root
67
68
69
70
71
            else:
                self.api_root = api.api_root

            # Perform any path variable substitution
            self.build_path()
72

73
74
            if self.search_api:
                self.host = api.search_host
Florent Espanet's avatar
Florent Espanet committed
75
76
            elif self.upload_api:
                self.host = api.upload_host
77
            else:
78
79
                self.host = api.host

80
81
82
            # Manually set Host header to fix an issue in python 2.5
            # or older where Host is set including the 443 port.
            # This causes Twitter to issue 301 redirect.
Pablo Castellano's avatar
Pablo Castellano committed
83
            # See Issue https://github.com/tweepy/tweepy/issues/12
Aaron Hill's avatar
Aaron Hill committed
84
            self.session.headers['Host'] = self.host
85
86
87
            # Monitoring rate limits
            self._remaining_calls = None
            self._reset_time = None
88

89
        def build_parameters(self, args, kwargs):
Aaron Hill's avatar
Aaron Hill committed
90
            self.session.params = {}
91
            for idx, arg in enumerate(args):
92
93
                if arg is None:
                    continue
94
                try:
Aaron Hill's avatar
Aaron Hill committed
95
                    self.session.params[self.allowed_param[idx]] = convert_to_utf8_str(arg)
96
97
98
                except IndexError:
                    raise TweepError('Too many parameters supplied!')

99
            for k, arg in kwargs.items():
100
101
                if arg is None:
                    continue
Aaron Hill's avatar
Aaron Hill committed
102
                if k in self.session.params:
103
104
                    raise TweepError('Multiple values for parameter %s supplied!' % k)

Aaron Hill's avatar
Aaron Hill committed
105
                self.session.params[k] = convert_to_utf8_str(arg)
106

Mark Smith's avatar
Mark Smith committed
107
108
            log.info("PARAMS: %r", self.session.params)

109
110
111
112
        def build_path(self):
            for variable in re_path_template.findall(self.path):
                name = variable.strip('{}')

Aaron Hill's avatar
Aaron Hill committed
113
                if name == 'user' and 'user' not in self.session.params and self.api.auth:
114
115
116
117
                    # No 'user' parameter provided, fetch it from Auth instead.
                    value = self.api.auth.get_username()
                else:
                    try:
Mark Smith's avatar
Mark Smith committed
118
                        value = quote(self.session.params[name])
119
120
                    except KeyError:
                        raise TweepError('No parameter value found for path variable: %s' % name)
Aaron Hill's avatar
Aaron Hill committed
121
                    del self.session.params[name]
122

123
124
                self.path = self.path.replace(variable, value)

125
        def execute(self):
Joshua Roesslein's avatar
Joshua Roesslein committed
126
127
            self.api.cached_result = False

128
129
            # Build the request URL
            url = self.api_root + self.path
Joshua Roesslein's avatar
Joshua Roesslein committed
130
            full_url = 'https://' + self.host + url
131
132
133

            # Query the cache if one is available
            # and this request uses a GET method.
134
            if self.use_cache and self.api.cache and self.method == 'GET':
DarkRedman's avatar
DarkRedman committed
135
                cache_result = self.api.cache.get('%s?%s' % (url, urlencode(self.session.params)))
136
137
138
139
140
                # if cache result found and not expired, return it
                if cache_result:
                    # must restore api reference
                    if isinstance(cache_result, list):
                        for result in cache_result:
141
142
                            if isinstance(result, Model):
                                result._api = self.api
143
                    else:
144
145
                        if isinstance(cache_result, Model):
                            cache_result._api = self.api
Joshua Roesslein's avatar
Joshua Roesslein committed
146
                    self.api.cached_result = True
147
148
149
150
151
152
                    return cache_result

            # Continue attempting request until successful
            # or maximum number of retries is reached.
            retries_performed = 0
            while retries_performed < self.retry_count + 1:
153
                # handle running out of api calls
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
154
155
156
157
158
159
160
                if self.wait_on_rate_limit:
                    if self._reset_time is not None:
                        if self._remaining_calls is not None:
                            if self._remaining_calls < 1:
                                sleep_time = self._reset_time - int(time.time())
                                if sleep_time > 0:
                                    if self.wait_on_rate_limit_notify:
Mark Smith's avatar
Mark Smith committed
161
                                        print("Rate limit reached. Sleeping for:", sleep_time)
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
162
                                    time.sleep(sleep_time + 5)  # sleep for few extra sec
163

Mark Smith's avatar
Mark Smith committed
164
165
166
167
168
169
170
171
                # if self.wait_on_rate_limit and self._reset_time is not None and \
                #                 self._remaining_calls is not None and self._remaining_calls < 1:
                #     sleep_time = self._reset_time - int(time.time())
                #     if sleep_time > 0:
                #         if self.wait_on_rate_limit_notify:
                #             print("Rate limit reached. Sleeping for: " + str(sleep_time))
                #         time.sleep(sleep_time + 5)  # sleep for few extra sec

172
173
                # Apply authentication
                if self.api.auth:
Aaron Hill's avatar
Aaron Hill committed
174
                    auth = self.api.auth.apply_auth()
175

Mike's avatar
Mike committed
176
177
                # Request compression if configured
                if self.api.compression:
Aaron Hill's avatar
Aaron Hill committed
178
                    self.session.headers['Accept-encoding'] = 'gzip'
Mike's avatar
Mike committed
179

180
181
                # Execute request
                try:
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
182
183
184
185
186
187
                    resp = self.session.request(self.method,
                                                full_url,
                                                data=self.post_data,
                                                timeout=self.api.timeout,
                                                auth=auth,
                                                proxies=self.api.proxy)
Mark Smith's avatar
Mark Smith committed
188
                except Exception as e:
189
                    raise TweepError('Failed to send request: %s' % e)
190
                rem_calls = resp.headers.get('x-rate-limit-remaining')
191
                if rem_calls is not None:
192
                    self._remaining_calls = int(rem_calls)
193
194
                elif isinstance(self._remaining_calls, int):
                    self._remaining_calls -= 1
195
                reset_time = resp.headers.get('x-rate-limit-reset')
196
                if reset_time is not None:
197
                    self._reset_time = int(reset_time)
198
                if self.wait_on_rate_limit and self._remaining_calls == 0 and (
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
199
                        # if ran out of calls before waiting switching retry last call
Prabeesh K's avatar
Prabeesh K committed
200
                        resp.status_code == 429 or resp.status_code == 420):
201
                    continue
202
                retry_delay = self.retry_delay
203
                # Exit request loop if non-retry error code
Aaron Hill's avatar
Aaron Hill committed
204
                if resp.status_code == 200:
205
                    break
Aaron Hill's avatar
Aaron Hill committed
206
207
208
209
                elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit:
                    if 'retry-after' in resp.headers:
                        retry_delay = float(resp.headers['retry-after'])
                elif self.retry_errors and resp.status_code not in self.retry_errors:
210
                    break
211

212
                # Sleep before retrying request again
213
                time.sleep(retry_delay)
214
                retries_performed += 1
Josh Roesslein's avatar
Josh Roesslein committed
215

216
217
            # If an error was returned, throw an exception
            self.api.last_response = resp
218
            if resp.status_code and not 200 <= resp.status_code < 300:
219
                try:
220
221
                    error_msg, api_error_code = \
                        self.parser.parse_error(resp.text)
222
                except Exception:
Aaron Hill's avatar
Aaron Hill committed
223
                    error_msg = "Twitter error response: status code = %s" % resp.status_code
224
                    api_error_code = None
obskyr's avatar
obskyr committed
225
226
227
228

                if is_rate_limit_error_message(error_msg):
                    raise RateLimitError(error_msg, resp)
                else:
229
                    raise TweepError(error_msg, resp, api_code=api_error_code)
230
231

            # Parse the response payload
232
            result = self.parser.parse(self, resp.text)
233
234

            # Store result into cache if one is available.
235
            if self.use_cache and self.api.cache and self.method == 'GET' and result:
DarkRedman's avatar
DarkRedman committed
236
                self.api.cache.store('%s?%s' % (url, urlencode(self.session.params)), result)
237

238
            return result
Josh Roesslein's avatar
Josh Roesslein committed
239

240
241
242
    def _call(*args, **kwargs):
        method = APIMethod(args, kwargs)
        if kwargs.get('create'):
243
244
245
            return method
        else:
            return method.execute()
Josh Roesslein's avatar
Josh Roesslein committed
246

247
    # Set pagination mode
248
    if 'cursor' in APIMethod.allowed_param:
249
        _call.pagination_mode = 'cursor'
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
250
251
252
    elif 'max_id' in APIMethod.allowed_param:
        if 'since_id' in APIMethod.allowed_param:
            _call.pagination_mode = 'id'
253
    elif 'page' in APIMethod.allowed_param:
254
        _call.pagination_mode = 'page'
255

Josh Roesslein's avatar
Josh Roesslein committed
256
    return _call