binder.py 10.4 KB
Newer Older
1
# Tweepy
2
3
# Copyright 2009-2010 Joshua Roesslein
# See LICENSE for details.
4

Mark Smith's avatar
Mark Smith committed
5
6
from __future__ import print_function

7
import time
8
import re
9

Mark Smith's avatar
Mark Smith committed
10
from six.moves.urllib.parse import quote
11
import requests
12

Mark Smith's avatar
Mark Smith committed
13
14
import logging

Josh Roesslein's avatar
Josh Roesslein committed
15
from tweepy.error import TweepError
16
from tweepy.utils import convert_to_utf8_str
17
from tweepy.models import Model
18

19

20
21
re_path_template = re.compile('{\w+}')

Mark Smith's avatar
Mark Smith committed
22
log = logging.getLogger('tweepy.binder')
Josh Roesslein's avatar
Josh Roesslein committed
23

Jordi Riera's avatar
Jordi Riera committed
24
def bind_api(**config):
25
26
27

    class APIMethod(object):

Jordi Riera's avatar
Jordi Riera committed
28
29
30
31
32
33
34
35
        api = config['api']
        path = config['path']
        payload_type = config.get('payload_type', None)
        payload_list = config.get('payload_list', False)
        allowed_param = config.get('allowed_param', [])
        method = config.get('method', 'GET')
        require_auth = config.get('require_auth', False)
        search_api = config.get('search_api', False)
Florent Espanet's avatar
Florent Espanet committed
36
        upload_api = config.get('upload_api', False)
Jordi Riera's avatar
Jordi Riera committed
37
        use_cache = config.get('use_cache', True)
Aaron Hill's avatar
Aaron Hill committed
38
        session = requests.Session()
39

40
        def __init__(self, args, kwargs):
Jordi Riera's avatar
Jordi Riera committed
41
            api = self.api
42
43
44
45
46
            # If authentication is required and no credentials
            # are provided, throw an error.
            if self.require_auth and not api.auth:
                raise TweepError('Authentication required!')

47
            self.post_data = kwargs.pop('post_data', None)
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
48
49
50
51
52
53
54
55
56
57
            self.retry_count = kwargs.pop('retry_count',
                                          api.retry_count)
            self.retry_delay = kwargs.pop('retry_delay',
                                          api.retry_delay)
            self.retry_errors = kwargs.pop('retry_errors',
                                           api.retry_errors)
            self.wait_on_rate_limit = kwargs.pop('wait_on_rate_limit',
                                                 api.wait_on_rate_limit)
            self.wait_on_rate_limit_notify = kwargs.pop('wait_on_rate_limit_notify',
                                                        api.wait_on_rate_limit_notify)
58
59
60
            self.parser = kwargs.pop('parser', api.parser)
            self.session.headers = kwargs.pop('headers', {})
            self.build_parameters(args, kwargs)
61
62
63
64

            # Pick correct URL root to use
            if self.search_api:
                self.api_root = api.search_root
Florent Espanet's avatar
Florent Espanet committed
65
66
            elif self.upload_api:
                self.api_root = api.upload_root
67
68
69
70
71
            else:
                self.api_root = api.api_root

            # Perform any path variable substitution
            self.build_path()
72

73
74
            if self.search_api:
                self.host = api.search_host
Florent Espanet's avatar
Florent Espanet committed
75
76
            elif self.upload_api:
                self.host = api.upload_host
77
            else:
78
79
                self.host = api.host

80
81
82
            # Manually set Host header to fix an issue in python 2.5
            # or older where Host is set including the 443 port.
            # This causes Twitter to issue 301 redirect.
Pablo Castellano's avatar
Pablo Castellano committed
83
            # See Issue https://github.com/tweepy/tweepy/issues/12
Aaron Hill's avatar
Aaron Hill committed
84
            self.session.headers['Host'] = self.host
85
86
87
            # Monitoring rate limits
            self._remaining_calls = None
            self._reset_time = None
88

89
        def build_parameters(self, args, kwargs):
Aaron Hill's avatar
Aaron Hill committed
90
            self.session.params = {}
91
            for idx, arg in enumerate(args):
92
93
                if arg is None:
                    continue
94
                try:
Aaron Hill's avatar
Aaron Hill committed
95
                    self.session.params[self.allowed_param[idx]] = convert_to_utf8_str(arg)
96
97
98
                except IndexError:
                    raise TweepError('Too many parameters supplied!')

99
            for k, arg in kwargs.items():
100
101
                if arg is None:
                    continue
Aaron Hill's avatar
Aaron Hill committed
102
                if k in self.session.params:
103
104
                    raise TweepError('Multiple values for parameter %s supplied!' % k)

Aaron Hill's avatar
Aaron Hill committed
105
                self.session.params[k] = convert_to_utf8_str(arg)
106

Mark Smith's avatar
Mark Smith committed
107
108
            log.info("PARAMS: %r", self.session.params)

109
110
111
112
        def build_path(self):
            for variable in re_path_template.findall(self.path):
                name = variable.strip('{}')

Aaron Hill's avatar
Aaron Hill committed
113
                if name == 'user' and 'user' not in self.session.params and self.api.auth:
114
115
116
117
                    # No 'user' parameter provided, fetch it from Auth instead.
                    value = self.api.auth.get_username()
                else:
                    try:
Mark Smith's avatar
Mark Smith committed
118
                        value = quote(self.session.params[name])
119
120
                    except KeyError:
                        raise TweepError('No parameter value found for path variable: %s' % name)
Aaron Hill's avatar
Aaron Hill committed
121
                    del self.session.params[name]
122

123
124
                self.path = self.path.replace(variable, value)

125
        def execute(self):
Joshua Roesslein's avatar
Joshua Roesslein committed
126
127
            self.api.cached_result = False

128
129
            # Build the request URL
            url = self.api_root + self.path
Joshua Roesslein's avatar
Joshua Roesslein committed
130
            full_url = 'https://' + self.host + url
131
132
133

            # Query the cache if one is available
            # and this request uses a GET method.
134
            if self.use_cache and self.api.cache and self.method == 'GET':
135
                cache_result = self.api.cache.get(url)
136
137
138
139
140
                # if cache result found and not expired, return it
                if cache_result:
                    # must restore api reference
                    if isinstance(cache_result, list):
                        for result in cache_result:
141
142
                            if isinstance(result, Model):
                                result._api = self.api
143
                    else:
144
145
                        if isinstance(cache_result, Model):
                            cache_result._api = self.api
Joshua Roesslein's avatar
Joshua Roesslein committed
146
                    self.api.cached_result = True
147
148
149
150
151
152
                    return cache_result

            # Continue attempting request until successful
            # or maximum number of retries is reached.
            retries_performed = 0
            while retries_performed < self.retry_count + 1:
153
                # handle running out of api calls
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
154
155
156
157
158
159
160
                if self.wait_on_rate_limit:
                    if self._reset_time is not None:
                        if self._remaining_calls is not None:
                            if self._remaining_calls < 1:
                                sleep_time = self._reset_time - int(time.time())
                                if sleep_time > 0:
                                    if self.wait_on_rate_limit_notify:
Mark Smith's avatar
Mark Smith committed
161
                                        print("Rate limit reached. Sleeping for:", sleep_time)
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
162
                                    time.sleep(sleep_time + 5)  # sleep for few extra sec
163

Mark Smith's avatar
Mark Smith committed
164
165
166
167
168
169
170
171
                # if self.wait_on_rate_limit and self._reset_time is not None and \
                #                 self._remaining_calls is not None and self._remaining_calls < 1:
                #     sleep_time = self._reset_time - int(time.time())
                #     if sleep_time > 0:
                #         if self.wait_on_rate_limit_notify:
                #             print("Rate limit reached. Sleeping for: " + str(sleep_time))
                #         time.sleep(sleep_time + 5)  # sleep for few extra sec

172
173
                # Apply authentication
                if self.api.auth:
Aaron Hill's avatar
Aaron Hill committed
174
                    auth = self.api.auth.apply_auth()
175

Mike's avatar
Mike committed
176
177
                # Request compression if configured
                if self.api.compression:
Aaron Hill's avatar
Aaron Hill committed
178
                    self.session.headers['Accept-encoding'] = 'gzip'
Mike's avatar
Mike committed
179

180
181
                # Execute request
                try:
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
182
183
184
185
186
187
                    resp = self.session.request(self.method,
                                                full_url,
                                                data=self.post_data,
                                                timeout=self.api.timeout,
                                                auth=auth,
                                                proxies=self.api.proxy)
Mark Smith's avatar
Mark Smith committed
188
                except Exception as e:
189
                    raise TweepError('Failed to send request: %s' % e)
190
                rem_calls = resp.headers.get('x-rate-limit-remaining')
191
                if rem_calls is not None:
192
                    self._remaining_calls = int(rem_calls)
193
194
                elif isinstance(self._remaining_calls, int):
                    self._remaining_calls -= 1
195
                reset_time = resp.headers.get('x-rate-limit-reset')
196
                if reset_time is not None:
197
                    self._reset_time = int(reset_time)
198
                if self.wait_on_rate_limit and self._remaining_calls == 0 and (
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
199
                        # if ran out of calls before waiting switching retry last call
Prabeesh K's avatar
Prabeesh K committed
200
                        resp.status_code == 429 or resp.status_code == 420):
201
                    continue
202
                retry_delay = self.retry_delay
203
                # Exit request loop if non-retry error code
Aaron Hill's avatar
Aaron Hill committed
204
                if resp.status_code == 200:
205
                    break
Aaron Hill's avatar
Aaron Hill committed
206
207
208
209
                elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit:
                    if 'retry-after' in resp.headers:
                        retry_delay = float(resp.headers['retry-after'])
                elif self.retry_errors and resp.status_code not in self.retry_errors:
210
                    break
211

212
                # Sleep before retrying request again
213
                time.sleep(retry_delay)
214
                retries_performed += 1
Josh Roesslein's avatar
Josh Roesslein committed
215

216
217
            # If an error was returned, throw an exception
            self.api.last_response = resp
218
            if resp.status_code and not 200 <= resp.status_code < 300:
219
                try:
Aaron Hill's avatar
Aaron Hill committed
220
                    error_msg = self.parser.parse_error(resp.text)
221
                except Exception:
Aaron Hill's avatar
Aaron Hill committed
222
                    error_msg = "Twitter error response: status code = %s" % resp.status_code
223
                raise TweepError(error_msg, resp)
224
225

            # Parse the response payload
226
            result = self.parser.parse(self, resp.text)
227
228

            # Store result into cache if one is available.
229
            if self.use_cache and self.api.cache and self.method == 'GET' and result:
230
                self.api.cache.store(url, result)
231

232
            return result
Josh Roesslein's avatar
Josh Roesslein committed
233

234
235
236
    def _call(*args, **kwargs):
        method = APIMethod(args, kwargs)
        if kwargs.get('create'):
237
238
239
            return method
        else:
            return method.execute()
Josh Roesslein's avatar
Josh Roesslein committed
240

241
    # Set pagination mode
242
    if 'cursor' in APIMethod.allowed_param:
243
        _call.pagination_mode = 'cursor'
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
244
245
246
    elif 'max_id' in APIMethod.allowed_param:
        if 'since_id' in APIMethod.allowed_param:
            _call.pagination_mode = 'id'
247
    elif 'page' in APIMethod.allowed_param:
248
        _call.pagination_mode = 'page'
249

Josh Roesslein's avatar
Josh Roesslein committed
250
    return _call