binder.py 10.8 KB
Newer Older
1
# Tweepy
2
3
# Copyright 2009-2010 Joshua Roesslein
# See LICENSE for details.
4

Mark Smith's avatar
Mark Smith committed
5
6
from __future__ import print_function

7
import time
8
import re
9

DarkRedman's avatar
DarkRedman committed
10
from six.moves.urllib.parse import quote, urlencode
11
import requests
12

Mark Smith's avatar
Mark Smith committed
13
14
import logging

obskyr's avatar
obskyr committed
15
from tweepy.error import TweepError, RateLimitError, is_rate_limit_error_message
16
from tweepy.utils import convert_to_utf8_str
17
from tweepy.models import Model
18
19
import six
import sys
20

21

22
23
re_path_template = re.compile('{\w+}')

Mark Smith's avatar
Mark Smith committed
24
log = logging.getLogger('tweepy.binder')
Josh Roesslein's avatar
Josh Roesslein committed
25

Jordi Riera's avatar
Jordi Riera committed
26
def bind_api(**config):
27
28
29

    class APIMethod(object):

Jordi Riera's avatar
Jordi Riera committed
30
31
32
33
34
35
36
37
        api = config['api']
        path = config['path']
        payload_type = config.get('payload_type', None)
        payload_list = config.get('payload_list', False)
        allowed_param = config.get('allowed_param', [])
        method = config.get('method', 'GET')
        require_auth = config.get('require_auth', False)
        search_api = config.get('search_api', False)
Florent Espanet's avatar
Florent Espanet committed
38
        upload_api = config.get('upload_api', False)
Jordi Riera's avatar
Jordi Riera committed
39
        use_cache = config.get('use_cache', True)
Aaron Hill's avatar
Aaron Hill committed
40
        session = requests.Session()
41

42
        def __init__(self, args, kwargs):
Jordi Riera's avatar
Jordi Riera committed
43
            api = self.api
44
45
46
47
48
            # If authentication is required and no credentials
            # are provided, throw an error.
            if self.require_auth and not api.auth:
                raise TweepError('Authentication required!')

49
            self.post_data = kwargs.pop('post_data', None)
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
50
51
52
53
54
55
56
57
58
59
            self.retry_count = kwargs.pop('retry_count',
                                          api.retry_count)
            self.retry_delay = kwargs.pop('retry_delay',
                                          api.retry_delay)
            self.retry_errors = kwargs.pop('retry_errors',
                                           api.retry_errors)
            self.wait_on_rate_limit = kwargs.pop('wait_on_rate_limit',
                                                 api.wait_on_rate_limit)
            self.wait_on_rate_limit_notify = kwargs.pop('wait_on_rate_limit_notify',
                                                        api.wait_on_rate_limit_notify)
60
61
62
            self.parser = kwargs.pop('parser', api.parser)
            self.session.headers = kwargs.pop('headers', {})
            self.build_parameters(args, kwargs)
63
64
65
66

            # Pick correct URL root to use
            if self.search_api:
                self.api_root = api.search_root
Florent Espanet's avatar
Florent Espanet committed
67
68
            elif self.upload_api:
                self.api_root = api.upload_root
69
70
71
72
73
            else:
                self.api_root = api.api_root

            # Perform any path variable substitution
            self.build_path()
74

75
76
            if self.search_api:
                self.host = api.search_host
Florent Espanet's avatar
Florent Espanet committed
77
78
            elif self.upload_api:
                self.host = api.upload_host
79
            else:
80
81
                self.host = api.host

82
83
84
            # Manually set Host header to fix an issue in python 2.5
            # or older where Host is set including the 443 port.
            # This causes Twitter to issue 301 redirect.
Pablo Castellano's avatar
Pablo Castellano committed
85
            # See Issue https://github.com/tweepy/tweepy/issues/12
Aaron Hill's avatar
Aaron Hill committed
86
            self.session.headers['Host'] = self.host
87
88
89
            # Monitoring rate limits
            self._remaining_calls = None
            self._reset_time = None
90

91
        def build_parameters(self, args, kwargs):
Aaron Hill's avatar
Aaron Hill committed
92
            self.session.params = {}
93
            for idx, arg in enumerate(args):
94
95
                if arg is None:
                    continue
96
                try:
Aaron Hill's avatar
Aaron Hill committed
97
                    self.session.params[self.allowed_param[idx]] = convert_to_utf8_str(arg)
98
99
100
                except IndexError:
                    raise TweepError('Too many parameters supplied!')

101
            for k, arg in kwargs.items():
102
103
                if arg is None:
                    continue
Aaron Hill's avatar
Aaron Hill committed
104
                if k in self.session.params:
105
106
                    raise TweepError('Multiple values for parameter %s supplied!' % k)

Aaron Hill's avatar
Aaron Hill committed
107
                self.session.params[k] = convert_to_utf8_str(arg)
108

Frank Jania's avatar
Frank Jania committed
109
            log.debug("PARAMS: %r", self.session.params)
Mark Smith's avatar
Mark Smith committed
110

111
112
113
114
        def build_path(self):
            for variable in re_path_template.findall(self.path):
                name = variable.strip('{}')

Aaron Hill's avatar
Aaron Hill committed
115
                if name == 'user' and 'user' not in self.session.params and self.api.auth:
116
117
118
119
                    # No 'user' parameter provided, fetch it from Auth instead.
                    value = self.api.auth.get_username()
                else:
                    try:
Mark Smith's avatar
Mark Smith committed
120
                        value = quote(self.session.params[name])
121
122
                    except KeyError:
                        raise TweepError('No parameter value found for path variable: %s' % name)
Aaron Hill's avatar
Aaron Hill committed
123
                    del self.session.params[name]
124

125
126
                self.path = self.path.replace(variable, value)

127
        def execute(self):
Joshua Roesslein's avatar
Joshua Roesslein committed
128
129
            self.api.cached_result = False

130
131
            # Build the request URL
            url = self.api_root + self.path
Joshua Roesslein's avatar
Joshua Roesslein committed
132
            full_url = 'https://' + self.host + url
133
134
135

            # Query the cache if one is available
            # and this request uses a GET method.
136
            if self.use_cache and self.api.cache and self.method == 'GET':
DarkRedman's avatar
DarkRedman committed
137
                cache_result = self.api.cache.get('%s?%s' % (url, urlencode(self.session.params)))
138
139
140
141
142
                # if cache result found and not expired, return it
                if cache_result:
                    # must restore api reference
                    if isinstance(cache_result, list):
                        for result in cache_result:
143
144
                            if isinstance(result, Model):
                                result._api = self.api
145
                    else:
146
147
                        if isinstance(cache_result, Model):
                            cache_result._api = self.api
Joshua Roesslein's avatar
Joshua Roesslein committed
148
                    self.api.cached_result = True
149
150
151
152
153
154
                    return cache_result

            # Continue attempting request until successful
            # or maximum number of retries is reached.
            retries_performed = 0
            while retries_performed < self.retry_count + 1:
155
                # handle running out of api calls
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
156
157
158
159
160
161
162
                if self.wait_on_rate_limit:
                    if self._reset_time is not None:
                        if self._remaining_calls is not None:
                            if self._remaining_calls < 1:
                                sleep_time = self._reset_time - int(time.time())
                                if sleep_time > 0:
                                    if self.wait_on_rate_limit_notify:
163
                                        log.warning("Rate limit reached. Sleeping for: %d" % sleep_time)
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
164
                                    time.sleep(sleep_time + 5)  # sleep for few extra sec
165

Mark Smith's avatar
Mark Smith committed
166
167
168
169
170
                # if self.wait_on_rate_limit and self._reset_time is not None and \
                #                 self._remaining_calls is not None and self._remaining_calls < 1:
                #     sleep_time = self._reset_time - int(time.time())
                #     if sleep_time > 0:
                #         if self.wait_on_rate_limit_notify:
171
                #             log.warning("Rate limit reached. Sleeping for: %d" % sleep_time)
Mark Smith's avatar
Mark Smith committed
172
173
                #         time.sleep(sleep_time + 5)  # sleep for few extra sec

174
                # Apply authentication
175
                auth = None
176
                if self.api.auth:
Aaron Hill's avatar
Aaron Hill committed
177
                    auth = self.api.auth.apply_auth()
178

Mike's avatar
Mike committed
179
180
                # Request compression if configured
                if self.api.compression:
Aaron Hill's avatar
Aaron Hill committed
181
                    self.session.headers['Accept-encoding'] = 'gzip'
Mike's avatar
Mike committed
182

183
184
                # Execute request
                try:
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
185
186
187
188
189
190
                    resp = self.session.request(self.method,
                                                full_url,
                                                data=self.post_data,
                                                timeout=self.api.timeout,
                                                auth=auth,
                                                proxies=self.api.proxy)
Mark Smith's avatar
Mark Smith committed
191
                except Exception as e:
192
193
                    six.reraise(TweepError, TweepError('Failed to send request: %s' % e), sys.exc_info()[2])

194
                rem_calls = resp.headers.get('x-rate-limit-remaining')
195

196
                if rem_calls is not None:
197
                    self._remaining_calls = int(rem_calls)
198
199
                elif isinstance(self._remaining_calls, int):
                    self._remaining_calls -= 1
200
                reset_time = resp.headers.get('x-rate-limit-reset')
201
                if reset_time is not None:
202
                    self._reset_time = int(reset_time)
203
                if self.wait_on_rate_limit and self._remaining_calls == 0 and (
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
204
                        # if ran out of calls before waiting switching retry last call
Prabeesh K's avatar
Prabeesh K committed
205
                        resp.status_code == 429 or resp.status_code == 420):
206
                    continue
207
                retry_delay = self.retry_delay
208
                # Exit request loop if non-retry error code
Aaron Hill's avatar
Aaron Hill committed
209
                if resp.status_code == 200:
210
                    break
Aaron Hill's avatar
Aaron Hill committed
211
212
213
214
                elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit:
                    if 'retry-after' in resp.headers:
                        retry_delay = float(resp.headers['retry-after'])
                elif self.retry_errors and resp.status_code not in self.retry_errors:
215
                    break
216

217
                # Sleep before retrying request again
218
                time.sleep(retry_delay)
219
                retries_performed += 1
Josh Roesslein's avatar
Josh Roesslein committed
220

221
222
            # If an error was returned, throw an exception
            self.api.last_response = resp
223
            if resp.status_code and not 200 <= resp.status_code < 300:
224
                try:
225
226
                    error_msg, api_error_code = \
                        self.parser.parse_error(resp.text)
227
                except Exception:
Aaron Hill's avatar
Aaron Hill committed
228
                    error_msg = "Twitter error response: status code = %s" % resp.status_code
229
                    api_error_code = None
obskyr's avatar
obskyr committed
230
231
232
233

                if is_rate_limit_error_message(error_msg):
                    raise RateLimitError(error_msg, resp)
                else:
234
                    raise TweepError(error_msg, resp, api_code=api_error_code)
235
236

            # Parse the response payload
237
            result = self.parser.parse(self, resp.text)
238
239

            # Store result into cache if one is available.
240
            if self.use_cache and self.api.cache and self.method == 'GET' and result:
DarkRedman's avatar
DarkRedman committed
241
                self.api.cache.store('%s?%s' % (url, urlencode(self.session.params)), result)
242

243
            return result
Josh Roesslein's avatar
Josh Roesslein committed
244

245
246
247
    def _call(*args, **kwargs):
        method = APIMethod(args, kwargs)
        if kwargs.get('create'):
248
249
250
            return method
        else:
            return method.execute()
Josh Roesslein's avatar
Josh Roesslein committed
251

252
    # Set pagination mode
253
    if 'cursor' in APIMethod.allowed_param:
254
        _call.pagination_mode = 'cursor'
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
255
256
257
    elif 'max_id' in APIMethod.allowed_param:
        if 'since_id' in APIMethod.allowed_param:
            _call.pagination_mode = 'id'
258
    elif 'page' in APIMethod.allowed_param:
259
        _call.pagination_mode = 'page'
260

Josh Roesslein's avatar
Josh Roesslein committed
261
    return _call