import base64
class SmartProxyAuthMiddleware(object):
    
def __init__(self, settings):
    self.proxy_url = 'http://%s:%d' % ("http://myip.lunaproxy.io", 1000)
    self.proxy_auth = "Basic " + base64.urlsafe_b64encode(bytes(("ada202301:ozGOb1iuY"), "ascii")).decode("utf8")
@classmethod
def from_crawler(cls, crawler):
    s = cls(crawler.settings)
    return s
def process_request(self, request, spider):
    request.meta["proxy"] = self.proxy_url
    request.headers["Proxy-Authorization"] = self.proxy_auth
def process_response(self, request, response, spider):
    spider.logger.info('%d for url[%s] with smart_proxy' % (response.status, response.url))
    if response.status >= 300:
        new_req = request.replace(dont_filter=True)
        spider.logger.warning('retry url[%s] for status[%d] with smart_proxy' % (response.url, response.status))
        return new_req
    return response
def process_exception(self, request, exception, spider):
    spider.logger.error('url[%s] with error[%s]' % (request.url, exception))