Beautifulsoup Redirect Error

By Steason Tee

Every forbes url will be directed to their welcome ads (http://www.forbes.com/forbes/welcome/) first then redirect to the page which user want. May I know how to fix this redirect? Although had inserted br.set_handle_redirect(True) but still doesn’t work.

# !/usr/bin/python
# -*- coding: utf-8 -*-
import mechanize
from bs4 import BeautifulSoup
import urllib2
import sys
import csv
import requests
import time
import datetime
import cookielib
# Browser
br = mechanize.Browser()
br.set_handle_equiv(True)
br.set_handle_gzip(False)
br.set_handle_redirect(True)
br.set_handle_referer(True)
br.set_handle_robots(False)
br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
br.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0')]
# Cookie Jar
cj = cookielib.MozillaCookieJar('forbesc.txt')
#br.set_cookiejar(cj)
cj.load()
opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cj))
r = opener.open("http://www.forbes.com")
cj.save("forbesc.txt")
def scrap():
    for link in ('http://www.forbes.com/companies/apple/'
,'http://www.forbes.com/companies/ally-financial/'):
        html = br.open(link).read()
        soup = BeautifulSoup(html, "html.parser")
        try:
            Company = soup.find("div", class_="forbes-list").text
except AttributeError, IndexError:
            print 'Not Found'
            continue
def start():
    scrap()
start()

Source: Stack Overflow

    

Share it with your friends!

    Fatal error: Uncaught Exception: 12: REST API is deprecated for versions v2.1 and higher (12) thrown in /home/content/19/9652219/html/wp-content/plugins/seo-facebook-comments/facebook/base_facebook.php on line 1273