Build a Scalable Python Web Scraper for 3000+ Companies

Getting started with web scraping is easy; a few lines of Python can fetch data from many pages.

The article first shows a minimal script using pandas to read HTML tables from a financial website and save them to a CSV file.

Basic Environment Setup

Python 3

Windows

Modules: pandas, csv

Target Website

Data is scraped from http://s.askci.com/stock/a/ for the report date 2017‑12‑31.

Implementation Code

import pandas as pd
import csv
for i in range(1, 178):
    # Crawl all pages
    tb = pd.read_html('http://s.askci.com/stock/a/?reportTime=2017-12-31&pageNum=%s' % str(i))[3]
    tb.to_csv(r'1.csv', mode='a', encoding='utf_8_sig', header=1, index=0)

The initial script fetches data for over 3000 listed companies and stores it in Excel.

Enhancements

To make the scraper more robust and flexible, the author adds exception handling, configurable URL parameters, switches storage from Excel to MySQL, and upgrades from a single‑process to a multi‑process approach.

Full Script

import requests
import pandas as pd
from bs4 import BeautifulSoup
from lxml import etree
import time
import pymysql
from sqlalchemy import create_engine
from urllib.parse import urlencode

start_time = time.time()

def get_one_page(i):
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) Chrome/66.0.3359.181 Safari/537.36'}
        paras = {'reportTime': '2017-12-31', 'pageNum': i}
        url = 'http://s.askci.com/stock/a/?' + urlencode(paras)
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('Crawl failed')

def parse_one_page(html):
    soup = BeautifulSoup(html, 'lxml')
    content = soup.select('#myTable04')[0]
    tbl = pd.read_html(content.prettify(), header=0)[0]
    tbl.rename(columns={'序号':'serial_number','股票代码':'stock_code','股票简称':'stock_abbre',
                        '公司名称':'company_name','省份':'province','城市':'city',
                        '主营业务收入(201712)':'main_bussiness_income','净利润(201712)':'net_profit',
                        '员工人数':'employees','上市日期':'listing_date','招股书':'zhaogushu',
                        '公司财报':'financial_report','行业分类':'industry_classification',
                        '产品类型':'industry_type','主营业务':'main_business'}, inplace=True)
    return tbl

def generate_mysql():
    conn = pymysql.connect(host='localhost', user='root', password='', port=3306,
                           charset='utf8', db='wade')
    cursor = conn.cursor()
    sql = '''CREATE TABLE IF NOT EXISTS listed_company (
        serial_number INT(20) NOT NULL,
        stock_code INT(20),
        stock_abbre VARCHAR(20),
        company_name VARCHAR(20),
        province VARCHAR(20),
        city VARCHAR(20),
        main_bussiness_income VARCHAR(20),
        net_profit VARCHAR(20),
        employees INT(20),
        listing_date DATETIME,
        zhaogushu VARCHAR(20),
        financial_report VARCHAR(20),
        industry_classification VARCHAR(20),
        industry_type VARCHAR(100),
        main_business VARCHAR(200),
        PRIMARY KEY (serial_number)
    )'''
    cursor.execute(sql)
    conn.close()

def write_to_sql(tbl, db='wade'):
    engine = create_engine(f'mysql+pymysql://root:@localhost:3306/{db}?charset=utf8')
    try:
        tbl.to_sql('listed_company2', con=engine, if_exists='append', index=False)
    except Exception as e:
        print(e)

def main(page):
    generate_mysql()
    for i in range(1, page):
        html = get_one_page(i)
        tbl = parse_one_page(html)
        write_to_sql(tbl)

if __name__ == '__main__':
    main(178)
    print('Program ran %.2f seconds' % (time.time() - start_time))

# Multiprocessing version
from multiprocessing import Pool
if __name__ == '__main__':
    pool = Pool(4)
    pool.map(main, [i for i in range(1, 178)])
    print('Program ran %.2f seconds' % (time.time() - start_time))

By iteratively adding error handling, URL flexibility, MySQL storage, and multiprocessing, the script grows from a few lines to a robust tool for bulk data collection.

Conclusion

The step‑by‑step approach helps beginners gain confidence and understand the typical workflow of a web scraper.