binder.py 10.2 KB
Newer Older
1
# Tweepy
2
3
# Copyright 2009-2010 Joshua Roesslein
# See LICENSE for details.
4

Mark Smith's avatar
Mark Smith committed
5
6
from __future__ import print_function

7
import time
8
import re
9

Mark Smith's avatar
Mark Smith committed
10
from six.moves.urllib.parse import quote
11
import requests
12

Mark Smith's avatar
Mark Smith committed
13
14
import logging

Josh Roesslein's avatar
Josh Roesslein committed
15
from tweepy.error import TweepError
16
from tweepy.utils import convert_to_utf8_str
17
from tweepy.models import Model
18

19

20
21
re_path_template = re.compile('{\w+}')

Mark Smith's avatar
Mark Smith committed
22
log = logging.getLogger('tweepy.binder')
Josh Roesslein's avatar
Josh Roesslein committed
23

Jordi Riera's avatar
Jordi Riera committed
24
def bind_api(**config):
25
26
27

    class APIMethod(object):

Jordi Riera's avatar
Jordi Riera committed
28
29
30
31
32
33
34
35
36
        api = config['api']
        path = config['path']
        payload_type = config.get('payload_type', None)
        payload_list = config.get('payload_list', False)
        allowed_param = config.get('allowed_param', [])
        method = config.get('method', 'GET')
        require_auth = config.get('require_auth', False)
        search_api = config.get('search_api', False)
        use_cache = config.get('use_cache', True)
Aaron Hill's avatar
Aaron Hill committed
37
        session = requests.Session()
38

39
        def __init__(self, args, kwargs):
Jordi Riera's avatar
Jordi Riera committed
40
            api = self.api
41
42
43
44
45
            # If authentication is required and no credentials
            # are provided, throw an error.
            if self.require_auth and not api.auth:
                raise TweepError('Authentication required!')

46
            self.post_data = kwargs.pop('post_data', None)
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
47
48
49
50
51
52
53
54
55
56
            self.retry_count = kwargs.pop('retry_count',
                                          api.retry_count)
            self.retry_delay = kwargs.pop('retry_delay',
                                          api.retry_delay)
            self.retry_errors = kwargs.pop('retry_errors',
                                           api.retry_errors)
            self.wait_on_rate_limit = kwargs.pop('wait_on_rate_limit',
                                                 api.wait_on_rate_limit)
            self.wait_on_rate_limit_notify = kwargs.pop('wait_on_rate_limit_notify',
                                                        api.wait_on_rate_limit_notify)
57
58
59
            self.parser = kwargs.pop('parser', api.parser)
            self.session.headers = kwargs.pop('headers', {})
            self.build_parameters(args, kwargs)
60
61
62
63
64
65
66
67
68

            # Pick correct URL root to use
            if self.search_api:
                self.api_root = api.search_root
            else:
                self.api_root = api.api_root

            # Perform any path variable substitution
            self.build_path()
69

70
71
            if self.search_api:
                self.host = api.search_host
72
            else:
73
74
                self.host = api.host

75
76
77
            # Manually set Host header to fix an issue in python 2.5
            # or older where Host is set including the 443 port.
            # This causes Twitter to issue 301 redirect.
Pablo Castellano's avatar
Pablo Castellano committed
78
            # See Issue https://github.com/tweepy/tweepy/issues/12
Aaron Hill's avatar
Aaron Hill committed
79
            self.session.headers['Host'] = self.host
80
81
82
            # Monitoring rate limits
            self._remaining_calls = None
            self._reset_time = None
83

84
        def build_parameters(self, args, kwargs):
Aaron Hill's avatar
Aaron Hill committed
85
            self.session.params = {}
86
            for idx, arg in enumerate(args):
87
88
                if arg is None:
                    continue
89
                try:
Aaron Hill's avatar
Aaron Hill committed
90
                    self.session.params[self.allowed_param[idx]] = convert_to_utf8_str(arg)
91
92
93
                except IndexError:
                    raise TweepError('Too many parameters supplied!')

94
            for k, arg in kwargs.items():
95
96
                if arg is None:
                    continue
Aaron Hill's avatar
Aaron Hill committed
97
                if k in self.session.params:
98
99
                    raise TweepError('Multiple values for parameter %s supplied!' % k)

Aaron Hill's avatar
Aaron Hill committed
100
                self.session.params[k] = convert_to_utf8_str(arg)
101

Mark Smith's avatar
Mark Smith committed
102
103
            log.info("PARAMS: %r", self.session.params)

104
105
106
107
        def build_path(self):
            for variable in re_path_template.findall(self.path):
                name = variable.strip('{}')

Aaron Hill's avatar
Aaron Hill committed
108
                if name == 'user' and 'user' not in self.session.params and self.api.auth:
109
110
111
112
                    # No 'user' parameter provided, fetch it from Auth instead.
                    value = self.api.auth.get_username()
                else:
                    try:
Mark Smith's avatar
Mark Smith committed
113
                        value = quote(self.session.params[name])
114
115
                    except KeyError:
                        raise TweepError('No parameter value found for path variable: %s' % name)
Aaron Hill's avatar
Aaron Hill committed
116
                    del self.session.params[name]
117

118
119
                self.path = self.path.replace(variable, value)

120
        def execute(self):
Joshua Roesslein's avatar
Joshua Roesslein committed
121
122
            self.api.cached_result = False

123
124
            # Build the request URL
            url = self.api_root + self.path
Joshua Roesslein's avatar
Joshua Roesslein committed
125
            full_url = 'https://' + self.host + url
126
127
128

            # Query the cache if one is available
            # and this request uses a GET method.
129
            if self.use_cache and self.api.cache and self.method == 'GET':
130
                cache_result = self.api.cache.get(url)
131
132
133
134
135
                # if cache result found and not expired, return it
                if cache_result:
                    # must restore api reference
                    if isinstance(cache_result, list):
                        for result in cache_result:
136
137
                            if isinstance(result, Model):
                                result._api = self.api
138
                    else:
139
140
                        if isinstance(cache_result, Model):
                            cache_result._api = self.api
Joshua Roesslein's avatar
Joshua Roesslein committed
141
                    self.api.cached_result = True
142
143
144
145
146
147
                    return cache_result

            # Continue attempting request until successful
            # or maximum number of retries is reached.
            retries_performed = 0
            while retries_performed < self.retry_count + 1:
148
                # handle running out of api calls
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
149
150
151
152
153
154
155
                if self.wait_on_rate_limit:
                    if self._reset_time is not None:
                        if self._remaining_calls is not None:
                            if self._remaining_calls < 1:
                                sleep_time = self._reset_time - int(time.time())
                                if sleep_time > 0:
                                    if self.wait_on_rate_limit_notify:
Mark Smith's avatar
Mark Smith committed
156
                                        print("Rate limit reached. Sleeping for:", sleep_time)
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
157
                                    time.sleep(sleep_time + 5)  # sleep for few extra sec
158

Mark Smith's avatar
Mark Smith committed
159
160
161
162
163
164
165
166
                # if self.wait_on_rate_limit and self._reset_time is not None and \
                #                 self._remaining_calls is not None and self._remaining_calls < 1:
                #     sleep_time = self._reset_time - int(time.time())
                #     if sleep_time > 0:
                #         if self.wait_on_rate_limit_notify:
                #             print("Rate limit reached. Sleeping for: " + str(sleep_time))
                #         time.sleep(sleep_time + 5)  # sleep for few extra sec

167
168
                # Apply authentication
                if self.api.auth:
Aaron Hill's avatar
Aaron Hill committed
169
                    auth = self.api.auth.apply_auth()
170

Mike's avatar
Mike committed
171
172
                # Request compression if configured
                if self.api.compression:
Aaron Hill's avatar
Aaron Hill committed
173
                    self.session.headers['Accept-encoding'] = 'gzip'
Mike's avatar
Mike committed
174

175
176
                # Execute request
                try:
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
177
178
179
180
181
182
                    resp = self.session.request(self.method,
                                                full_url,
                                                data=self.post_data,
                                                timeout=self.api.timeout,
                                                auth=auth,
                                                proxies=self.api.proxy)
Mark Smith's avatar
Mark Smith committed
183
184
                except Exception as e:

185
                    raise TweepError('Failed to send request: %s' % e)
186
                rem_calls = resp.headers.get('x-rate-limit-remaining')
187
                if rem_calls is not None:
188
                    self._remaining_calls = int(rem_calls)
189
190
                elif isinstance(self._remaining_calls, int):
                    self._remaining_calls -= 1
191
                reset_time = resp.headers.get('x-rate-limit-reset')
192
                if reset_time is not None:
193
                    self._reset_time = int(reset_time)
194
                if self.wait_on_rate_limit and self._remaining_calls == 0 and (
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
195
                        # if ran out of calls before waiting switching retry last call
Prabeesh K's avatar
Prabeesh K committed
196
                        resp.status_code == 429 or resp.status_code == 420):
197
                    continue
198
                retry_delay = self.retry_delay
199
                # Exit request loop if non-retry error code
Aaron Hill's avatar
Aaron Hill committed
200
                if resp.status_code == 200:
201
                    break
Aaron Hill's avatar
Aaron Hill committed
202
203
204
205
                elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit:
                    if 'retry-after' in resp.headers:
                        retry_delay = float(resp.headers['retry-after'])
                elif self.retry_errors and resp.status_code not in self.retry_errors:
206
                    break
207

208
                # Sleep before retrying request again
209
                time.sleep(retry_delay)
210
                retries_performed += 1
Josh Roesslein's avatar
Josh Roesslein committed
211

212
213
            # If an error was returned, throw an exception
            self.api.last_response = resp
214
            if resp.status_code and not 200 <= resp.status_code < 300:
215
                try:
Aaron Hill's avatar
Aaron Hill committed
216
                    error_msg = self.parser.parse_error(resp.text)
217
                except Exception:
Aaron Hill's avatar
Aaron Hill committed
218
                    error_msg = "Twitter error response: status code = %s" % resp.status_code
219
                raise TweepError(error_msg, resp)
220
221

            # Parse the response payload
222
            result = self.parser.parse(self, resp.text)
223
224

            # Store result into cache if one is available.
225
            if self.use_cache and self.api.cache and self.method == 'GET' and result:
226
                self.api.cache.store(url, result)
227

228
            return result
Josh Roesslein's avatar
Josh Roesslein committed
229

230
231
232
    def _call(*args, **kwargs):
        method = APIMethod(args, kwargs)
        if kwargs.get('create'):
233
234
235
            return method
        else:
            return method.execute()
Josh Roesslein's avatar
Josh Roesslein committed
236

237
    # Set pagination mode
238
    if 'cursor' in APIMethod.allowed_param:
239
        _call.pagination_mode = 'cursor'
Omer Murat Yildirim's avatar
Omer Murat Yildirim committed
240
241
242
    elif 'max_id' in APIMethod.allowed_param:
        if 'since_id' in APIMethod.allowed_param:
            _call.pagination_mode = 'id'
243
    elif 'page' in APIMethod.allowed_param:
244
        _call.pagination_mode = 'page'
245

Josh Roesslein's avatar
Josh Roesslein committed
246
    return _call