Build a Python Crawler to Auto‑Collect TV Drama Download Links

The author, a fan of American TV series, wanted to simplify downloading episodes from the "天天美剧" site and wrote a Python web crawler to collect all drama links and save them to a text file for easy use with Thunder.

Initially the plan was to crawl the whole site from the homepage, but irregular URLs and many duplicates made it difficult, so the author switched to generating URLs by iterating over numeric IDs (e.g., http://cn163.net/archives/24016/).

Requests is used to check each generated URL; URLs returning 404 are skipped, while others are processed to extract download links.

def get_urls(self):
    try:
        for i in range(2015,25000):
            base_url='http://cn163.net/archives/'
            url=base_url+str(i)+'/'
            if requests.get(url).status_code == 404:
                continue
            else:
                self.save_links(url)
    except Exception as e:
        pass

The extraction relies on regular expressions to find ed2k links and episode titles, storing them in a dictionary keyed by season and episode numbers, then writing them in order to a .txt file.

# -*- coding:utf-8 -*-
import requests
import re
import sys
import threading
import time
reload(sys)
sys.setdefaultencoding('utf-8')
class Archives(object):
    def save_links(self,url):
        try:
            data=requests.get(url,timeout=3)
            content=data.text
            link_pat='"(ed2k://|file|[^\"]+?\.(S\d+)(E\d+)[^\"]+?1024X\d{3}[^\"]+?)"'
            name_pat=re.compile(r'<h2 class="entry_title">(.*?)</h2>',re.S)
            links = set(re.findall(link_pat,content))
            name=re.findall(name_pat,content)
            links_dict = {}
            count=len(links)
        except Exception as e:
            pass
        for i in links:
            links_dict[int(i[1][1:3])*100 + int(i[2][1:3])] = i
        try:
            with open(name[0].replace('/',' ') + '.txt','w') as f:
                print name[0]
                for i in sorted(list(links_dict.keys())):
                    f.write(links_dict[i][0] + '
')
            print "Get links ... ", name[0], count
        except Exception as e:
            pass
    def get_urls(self):
        try:
            for i in range(2015,25000):
                base_url='http://cn163.net/archives/'
                url=base_url+str(i)+'/'
                if requests.get(url).status_code == 404:
                    continue
                else:
                    self.save_links(url)
        except Exception as e:
            pass
    def main(self):
        thread1=threading.Thread(target=self.get_urls())
        thread1.start()
        thread1.join()
if __name__ == '__main__':
    start=time.time()
    a=Archives()
    a.main()
    end=time.time()
    print end-start

The full script runs in under 20 minutes for about 20,000 potential pages, skipping invalid URLs and handling filename restrictions (no slashes or special characters).