binder.py 9.13 KB
Newer Older
1
# Tweepy
2
3
# Copyright 2009-2010 Joshua Roesslein
# See LICENSE for details.
4

Josh Roesslein's avatar
Josh Roesslein committed
5
import urllib
6
import time
7
import re
8
9

import requests
10

Josh Roesslein's avatar
Josh Roesslein committed
11
from tweepy.error import TweepError
12
from tweepy.utils import convert_to_utf8_str
13
from tweepy.models import Model
14

15

16
17
re_path_template = re.compile('{\w+}')

Josh Roesslein's avatar
Josh Roesslein committed
18

19
20
21
22
23
24
25
26
27
28
29
def bind_api(api,
             path,
             use_cache=True,
             search_api=False,
             require_auth=False,
             method='GET',
             allowed_param=None,
             payload_list=False,
             payload_type=None):
    # common practice to avoid the bug when a list is in arguments.
    allowed_param = allowed_param or []
30
31
32

    class APIMethod(object):

33
34
35
36
37
38
39
40
        path = path
        payload_type = payload_type
        payload_list = payload_list
        allowed_param = allowed_param
        method = method
        require_auth = require_auth
        search_api = search_api
        use_cache = use_cache
Aaron Hill's avatar
Aaron Hill committed
41
        session = requests.Session()
42
        api = api
43

44
        def __init__(self, args, kwargs):
45
46
47
48
49
            # If authentication is required and no credentials
            # are provided, throw an error.
            if self.require_auth and not api.auth:
                raise TweepError('Authentication required!')

50
51
52
53
54
55
56
57
58
            self.post_data = kwargs.pop('post_data', None)
            self.retry_count = kwargs.pop('retry_count', api.retry_count)
            self.retry_delay = kwargs.pop('retry_delay', api.retry_delay)
            self.retry_errors = kwargs.pop('retry_errors', api.retry_errors)
            self.wait_on_rate_limit = kwargs.pop('wait_on_rate_limit', api.wait_on_rate_limit)
            self.wait_on_rate_limit_notify = kwargs.pop('wait_on_rate_limit_notify', api.wait_on_rate_limit_notify)
            self.parser = kwargs.pop('parser', api.parser)
            self.session.headers = kwargs.pop('headers', {})
            self.build_parameters(args, kwargs)
59
60
61
62
63
64
65
66
67

            # Pick correct URL root to use
            if self.search_api:
                self.api_root = api.search_root
            else:
                self.api_root = api.api_root

            # Perform any path variable substitution
            self.build_path()
68

69
70
            if self.search_api:
                self.host = api.search_host
71
            else:
72
73
                self.host = api.host

74
75
76
            # Manually set Host header to fix an issue in python 2.5
            # or older where Host is set including the 443 port.
            # This causes Twitter to issue 301 redirect.
Pablo Castellano's avatar
Pablo Castellano committed
77
            # See Issue https://github.com/tweepy/tweepy/issues/12
Aaron Hill's avatar
Aaron Hill committed
78
79

            self.session.headers['Host'] = self.host
80
81
82
            # Monitoring rate limits
            self._remaining_calls = None
            self._reset_time = None
83

84
        def build_parameters(self, args, kwargs):
Aaron Hill's avatar
Aaron Hill committed
85
            self.session.params = {}
86
            for idx, arg in enumerate(args):
87
88
                if arg is None:
                    continue
89
                try:
Aaron Hill's avatar
Aaron Hill committed
90
                    self.session.params[self.allowed_param[idx]] = convert_to_utf8_str(arg)
91
92
93
                except IndexError:
                    raise TweepError('Too many parameters supplied!')

94
            for k, arg in kwargs.items():
95
96
                if arg is None:
                    continue
Aaron Hill's avatar
Aaron Hill committed
97
                if k in self.session.params:
98
99
                    raise TweepError('Multiple values for parameter %s supplied!' % k)

Aaron Hill's avatar
Aaron Hill committed
100
                self.session.params[k] = convert_to_utf8_str(arg)
101
102
103
104
105

        def build_path(self):
            for variable in re_path_template.findall(self.path):
                name = variable.strip('{}')

Aaron Hill's avatar
Aaron Hill committed
106
                if name == 'user' and 'user' not in self.session.params and self.api.auth:
107
108
109
110
                    # No 'user' parameter provided, fetch it from Auth instead.
                    value = self.api.auth.get_username()
                else:
                    try:
Aaron Hill's avatar
Aaron Hill committed
111
                        value = urllib.quote(self.session.params[name])
112
113
                    except KeyError:
                        raise TweepError('No parameter value found for path variable: %s' % name)
Aaron Hill's avatar
Aaron Hill committed
114
                    del self.session.params[name]
115

116
117
                self.path = self.path.replace(variable, value)

118
        def execute(self):
Joshua Roesslein's avatar
Joshua Roesslein committed
119
120
            self.api.cached_result = False

121
122
            # Build the request URL
            url = self.api_root + self.path
Joshua Roesslein's avatar
Joshua Roesslein committed
123
            full_url = 'https://' + self.host + url
124
125
126

            # Query the cache if one is available
            # and this request uses a GET method.
127
            if self.use_cache and self.api.cache and self.method == 'GET':
128
                cache_result = self.api.cache.get(url)
129
130
131
132
133
                # if cache result found and not expired, return it
                if cache_result:
                    # must restore api reference
                    if isinstance(cache_result, list):
                        for result in cache_result:
134
135
                            if isinstance(result, Model):
                                result._api = self.api
136
                    else:
137
138
                        if isinstance(cache_result, Model):
                            cache_result._api = self.api
Joshua Roesslein's avatar
Joshua Roesslein committed
139
                    self.api.cached_result = True
140
141
142
143
144
145
                    return cache_result

            # Continue attempting request until successful
            # or maximum number of retries is reached.
            retries_performed = 0
            while retries_performed < self.retry_count + 1:
146
147
                # handle running out of api calls
                if self.wait_on_rate_limit and self._reset_time is not None and \
148
                                self._remaining_calls is not None and self._remaining_calls < 1:
149
150
                    sleep_time = self._reset_time - int(time.time())
                    if sleep_time > 0:
151
152
                        if self.wait_on_rate_limit_notify:
                            print "Max retries reached. Sleeping for: " + str(sleep_time)
153
                        time.sleep(sleep_time + 5)  # sleep for few extra sec
154

155
156
                # Apply authentication
                if self.api.auth:
Aaron Hill's avatar
Aaron Hill committed
157
                    auth = self.api.auth.apply_auth()
158

Mike's avatar
Mike committed
159
160
                # Request compression if configured
                if self.api.compression:
Aaron Hill's avatar
Aaron Hill committed
161
                    self.session.headers['Accept-encoding'] = 'gzip'
Mike's avatar
Mike committed
162

163
164
                # Execute request
                try:
Aaron Hill's avatar
Aaron Hill committed
165
                    resp = self.session.request(self.method, full_url,
166
167
                                                data=self.post_data, timeout=self.api.timeout,
                                                auth=auth, proxies=self.api.proxy)
Aaron Hill's avatar
Aaron Hill committed
168
                except Exception, e:
169
                    raise TweepError('Failed to send request: %s' % e)
170
                rem_calls = resp.headers.get('x-rate-limit-remaining')
171
                if rem_calls is not None:
172
                    self._remaining_calls = int(rem_calls)
173
174
                elif isinstance(self._remaining_calls, int):
                    self._remaining_calls -= 1
175
                reset_time = resp.headers.get('x-rate-limit-reset')
176
                if reset_time is not None:
177
                    self._reset_time = int(reset_time)
178
179
                if self.wait_on_rate_limit and self._remaining_calls == 0 and (
                        resp.status == 429 or resp.status == 420):  # if ran out of calls before waiting switching retry last call
180
                    continue
181
                retry_delay = self.retry_delay
182
                # Exit request loop if non-retry error code
Aaron Hill's avatar
Aaron Hill committed
183
                if resp.status_code == 200:
184
                    break
Aaron Hill's avatar
Aaron Hill committed
185
186
187
188
                elif (resp.status_code == 429 or resp.status_code == 420) and self.wait_on_rate_limit:
                    if 'retry-after' in resp.headers:
                        retry_delay = float(resp.headers['retry-after'])
                elif self.retry_errors and resp.status_code not in self.retry_errors:
189
                    break
190

191
                # Sleep before retrying request again
192
                time.sleep(retry_delay)
193
                retries_performed += 1
Josh Roesslein's avatar
Josh Roesslein committed
194

195
196
            # If an error was returned, throw an exception
            self.api.last_response = resp
197
            if resp.status_code and not 200 <= resp.status_code < 300:
198
                try:
Aaron Hill's avatar
Aaron Hill committed
199
                    error_msg = self.parser.parse_error(resp.text)
200
                except Exception:
Aaron Hill's avatar
Aaron Hill committed
201
                    error_msg = "Twitter error response: status code = %s" % resp.status_code
202
                raise TweepError(error_msg, resp)
203
204

            # Parse the response payload
205
            result = self.parser.parse(self, resp.text)
206
207

            # Store result into cache if one is available.
208
            if self.use_cache and self.api.cache and self.method == 'GET' and result:
209
                self.api.cache.store(url, result)
210

211
            return result
Josh Roesslein's avatar
Josh Roesslein committed
212

213
214
215
    def _call(*args, **kwargs):
        method = APIMethod(args, kwargs)
        if kwargs.get('create'):
216
217
218
            return method
        else:
            return method.execute()
Josh Roesslein's avatar
Josh Roesslein committed
219

220
    # Set pagination mode
221
    if 'cursor' in APIMethod.allowed_param:
222
        _call.pagination_mode = 'cursor'
223
    elif 'max_id' in APIMethod.allowed_param and \
224
                    'since_id' in APIMethod.allowed_param:
225
        _call.pagination_mode = 'id'
226
    elif 'page' in APIMethod.allowed_param:
227
        _call.pagination_mode = 'page'
228

Josh Roesslein's avatar
Josh Roesslein committed
229
    return _call