import base64
class SmartProxyAuthMiddleware(object):
def __init__(self, settings):
self.proxy_url = 'http://%s:%d' % ("http://myip.lunaproxy.io", 1000)
self.proxy_auth = "Basic " + base64.urlsafe_b64encode(bytes(("ada202301:ozGOb1iuY"), "ascii")).decode("utf8")
@classmethod
def from_crawler(cls, crawler):
s = cls(crawler.settings)
return s
def process_request(self, request, spider):
request.meta["proxy"] = self.proxy_url
request.headers["Proxy-Authorization"] = self.proxy_auth
def process_response(self, request, response, spider):
spider.logger.info('%d for url[%s] with smart_proxy' % (response.status, response.url))
if response.status >= 300:
new_req = request.replace(dont_filter=True)
spider.logger.warning('retry url[%s] for status[%d] with smart_proxy' % (response.url, response.status))
return new_req
return response
def process_exception(self, request, exception, spider):
spider.logger.error('url[%s] with error[%s]' % (request.url, exception))