Master Python Metaclasses: Build Dynamic ORM and Proxy Crawlers with Ease

Metaclasses – The "Dao" of Python

In Python, type is the ultimate "Dao" that gives rise to classes, instances, and their attributes. Understanding how type creates classes ("Yi") and how metaclasses ("Yi") create other classes ("Er") unlocks powerful object‑oriented patterns.

Basic Hello World with Metaclasses

# Create a Hello class using a metaclass (type)
class Hello(object):
    def say_hello(self, name='world'):
        print('Hello, %s.' % name)

hello = Hello()
hello.say_hello()

Using type directly:

# Create a class dynamically with type
Hello = type('Hello', (object,), dict(say_hello=lambda self, value, saying='Hello': print(saying + ',' + value + '!')))
hello = Hello()
hello.say_hello('world')

Creating a Custom Metaclass

A metaclass can automatically add methods to a class based on its name. The example below defines SayMetaClass that generates a say_<em>ClassName</em> method for any class it creates.

class SayMetaClass(type):
    def __new__(cls, name, bases, attrs):
        attrs['say_' + name] = lambda self, value, saying=name: print(saying + ',' + value + '!')
        return type.__new__(cls, name, bases, attrs)

class Hello(object, metaclass=SayMetaClass):
    pass

hello = Hello()
hello.say_Hello('world')

Metaclass‑Based ORM (Model Example)

The following code shows how to build a simple ORM using a metaclass that maps class attributes to database columns.

class Field(object):
    def __init__(self, name, column_type):
        self.name = name
        self.column_type = column_type
    def __str__(self):
        return '<%s:%s>' % (self.__class__.__name__, self.name)

class StringField(Field):
    def __init__(self, name):
        super(StringField, self).__init__(name, 'varchar(100)')

class IntegerField(Field):
    def __init__(self, name):
        super(IntegerField, self).__init__(name, 'bigint')

class ModelMetaclass(type):
    def __new__(cls, name, bases, attrs):
        if name == 'Model':
            return type.__new__(cls, name, bases, attrs)
        print('Found model: %s' % name)
        mappings = {}
        for k, v in attrs.items():
            if isinstance(v, Field):
                print('Found mapping: %s ==> %s' % (k, v))
                mappings[k] = v
        for k in mappings.keys():
            attrs.pop(k)
        attrs['__mappings__'] = mappings
        attrs['__table__'] = name
        return type.__new__(cls, name, bases, attrs)

class Model(dict, metaclass=ModelMetaclass):
    def __init__(self, **kw):
        super(Model, self).__init__(**kw)
    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            raise AttributeError("'Model' object has no attribute '%s'" % key)
    def __setattr__(self, key, value):
        self[key] = value
    def save(self):
        fields = []
        args = []
        for k, v in self.__mappings__.items():
            fields.append(v.name)
            args.append(getattr(self, k, None))
        sql = 'insert into %s (%s) values (%s)' % (
            self.__table__, ','.join(fields), ','.join([str(i) for i in args]))
        print('SQL: %s' % sql)
        print('ARGS: %s' % str(args))

class User(Model):
    id = IntegerField('id')
    name = StringField('username')
    email = StringField('email')
    password = StringField('password')

u = User(id=12345, name='Batman', email='[email protected]', password='iamback')
u.save()

Proxy Crawler Using Metaclasses

A metaclass can collect all methods that start with crawl_ and expose them through a unified interface.

class ProxyMetaclass(type):
    def __new__(cls, name, bases, attrs):
        count = 0
        attrs['__CrawlFunc__'] = []
        attrs['__CrawlName__'] = []
        for k, v in attrs.items():
            if k.startswith('crawl_'):
                attrs['__CrawlName__'].append(k)
                attrs['__CrawlFunc__'].append(v)
                count += 1
        for k in attrs['__CrawlName__']:
            attrs.pop(k)
        attrs['__CrawlFuncCount__'] = count
        return type.__new__(cls, name, bases, attrs)

class ProxyGetter(object, metaclass=ProxyMetaclass):
    def get_raw_proxies(self, site):
        proxies = []
        print('Site', site)
        for func in self.__CrawlFunc__:
            if func.__name__ == site:
                for proxy in func(self):
                    print('Getting', proxy, 'from', site)
                    proxies.append(proxy)
        return proxies

    def crawl_daili66(self, page_count=4):
        start_url = 'http://www.66ip.cn/{}.html'
        urls = [start_url.format(p) for p in range(1, page_count + 1)]
        for url in urls:
            print('Crawling', url)
            html = get_page(url)
            if html:
                doc = pq(html)
                trs = doc('.containerbox table tr:gt(0)').items()
                for tr in trs:
                    ip = tr.find('td:nth-child(1)').text()
                    port = tr.find('td:nth-child(2)').text()
                    yield ':'.join([ip, port])

    def crawl_proxy360(self):
        start_url = 'http://www.proxy360.cn/Region/China'
        print('Crawling', start_url)
        html = get_page(start_url)
        if html:
            doc = pq(html)
            lines = doc('div[name="list_proxy_ip"]').items()
            for line in lines:
                ip = line.find('.tbBottomLine:nth-child(1)').text()
                port = line.find('.tbBottomLine:nth-child(2)').text()
                yield ':'.join([ip, port])

    def crawl_goubanjia(self):
        start_url = 'http://www.goubanjia.com/free/gngn/index.shtml'
        html = get_page(start_url)
        if html:
            doc = pq(html)
            tds = doc('td.ip').items()
            for td in tds:
                td.find('p').remove()
                yield td.text().replace(' ', '')

Usage example:

crawler = ProxyGetter()
print(crawler.__CrawlName__)
for i in range(crawler.__CrawlFuncCount__):
    site = crawler.__CrawlName__[i]
    proxies = crawler.get_raw_proxies(site)
    # process proxies as needed

Python's metaclass mechanism enables you to write concise, reusable, and highly dynamic code for both ORM design and web crawling tasks.