Solving Weibo's Four‑Grid Hand‑Drawn Captcha with Python Selenium and Image Matching

The article introduces the four‑grid hand‑drawn captcha that appears on the Weibo mobile client during login and shows what the captcha looks like.

1. Idea

Because the captcha consists of four squares connected by directed lines, there are 24 possible directional patterns. By numbering the squares, all 24 patterns can be enumerated and used as templates for matching.

The 24 templates are generated and displayed as images.

2. Code Implementation (First Version)

The script uses Selenium to open the Weibo mobile login page, fill in credentials, and capture the captcha image. It then crops the screenshot to the captcha region and saves it.

<code># -*- coding:utf-8 -*-
import time
from io import BytesIO
from PIL import Image
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class CrackWeiboSlide():
    def __init__(self):
        self.url = "https://passport.weibo.cn/signin/login?entry=mweibo&r=https://m.weibo.cn/"
        self.browser = webdriver.Chrome(r"D:\chromedriver.exe")
        self.browser.maximize_window()
        self.wait = WebDriverWait(self.browser, 5)

    def __del__(self):
        self.browser.close()

    def open(self):
        self.browser.get(self.url)
        username = self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="loginName"]')))
        password = self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="loginPassword"]')))
        submit = self.wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="loginAction"]')))
        username.send_keys("15612345678")
        password.send_keys("xxxxxxxxxxxx")
        submit.click()

    def get_image(self, name="captcha.png"):
        try:
            img = self.wait.until(EC.presence_of_element_located((By.CLASS_NAME, "patt-shadow")))
            time.sleep(1)
            location = img.location
            size = img.size
            top = location["y"]
            bottom = location["y"] + size["height"]
            left = location["x"]
            right = location["x"] + size["width"]
            print("验证码的位置：", left, top, right, bottom)
            screenshot = self.browser.get_screenshot_as_png()
            screenshot = Image.open(BytesIO(screenshot))
            captcha = screenshot.crop((left, top, right, bottom))
            captcha.save(name)
            print("微博登录验证码保存完成！！！")
            return captcha
        except TimeoutException:
            print("没有出现验证码！！")
            self.open()

    def main(self):
        count = 1
        while True:
            self.open()
            self.get_image(str(count) + ".png")
            count += 1

if __name__ == '__main__':
    crack = CrackWeiboSlide()
    crack.main()
</code>

After obtaining the 24 hand‑drawn captcha templates, the next step is template matching.

3. Template Matching

The script iterates over all template images, compares each pixel with the captured captcha, and calculates a similarity score. When the score exceeds 0.99, the matching template is identified and its sequence of points is returned.

<code>import os
import time
from io import BytesIO
from PIL import Image
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class CrackWeiboSlide():
    def __init__(self):
        self.url = "https://passport.weibo.cn/signin/login?entry=mweibo&r=https://m.weibo.cn/"
        self.browser = webdriver.Chrome(r"D:\chromedriver.exe")
        self.browser.maximize_window()
        self.wait = WebDriverWait(self.browser, 5)

    def __del__(self):
        self.browser.close()

    def open(self):
        self.browser.get(self.url)
        username = self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="loginName"]')))
        password = self.wait.until(EC.presence_of_element_located((By.XPATH, '//*[@id="loginPassword"]')))
        submit = self.wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="loginAction"]')))
        username.send_keys("15612345678")
        password.send_keys("xxxxxxxxxxxx")
        submit.click()

    def get_image(self, name="captcha.png"):
        try:
            img = self.wait.until(EC.presence_of_element_located((By.CLASS_NAME, "patt-shadow")))
            time.sleep(1)
            location = img.location
            size = img.size
            top = location["y"]
            bottom = location["y"] + size["height"]
            left = location["x"]
            right = location["x"] + size["width"]
            print("验证码的位置：", left, top, right, bottom)
            screenshot = self.browser.get_screenshot_as_png()
            screenshot = Image.open(BytesIO(screenshot))
            captcha = screenshot.crop((left, top, right, bottom))
            captcha.save(name)
            print("微博登录验证码保存完成！！！")
            return captcha
        except TimeoutException:
            print("没有出现验证码！！")
            self.open()

    def is_pixel_equal(self, image, template, i, j):
        pixel1 = image.load()[i, j]
        pixel2 = template.load()[i, j]
        threshold = 20
        return all(abs(p1 - p2) < threshold for p1, p2 in zip(pixel1, pixel2))

    def same_image(self, image, template):
        threshold = 0.99
        count = 0
        for i in range(image.width):
            for j in range(image.height):
                if self.is_pixel_equal(image, template, i, j):
                    count += 1
        result = float(count) / (image.width * image.height)
        if result > threshold:
            print("匹配成功！！！")
            return True
        return False

    def detect_image(self, image):
        for template_name in os.listdir(r"D:\photo\templates"):
            print("正在匹配", template_name)
            template = Image.open(r"D:\photo\templates\{}".format(template_name))
            if self.same_image(image, template):
                numbers = [int(n) for n in template_name.split(".")[0]]
                print("按照顺序进行拖动", numbers)
                return numbers

    def move(self, numbers):
        circles = self.browser.find_element_by_css_selector('.patt-wrap .patt-circ')
        dx = dy = 0
        for index in range(4):
            circle = circles[numbers[index] - 1]
            if index == 0:
                action = ActionChains(self.browser).move_to_element_with_offset(circle, circle.size["width"]/2, circle.size["height"]/2)
                action.click_and_hold().perform()
            else:
                times = 30
                for i in range(times):
                    ActionChains(self.browser).move_by_offset(dx/times, dy/times).perform()
                    time.sleep(1/times)
                if index == 3:
                    ActionChains(self.browser).release().perform()
                else:
                    dx = circles[numbers[index + 1] - 1].location['x'] - circle.location['x']
                    dy = circles[numbers[index + 1] - 1].location['y'] - circle.location['y']

    def main(self):
        self.open()
        image = self.get_image("captcha.png")
        numbers = self.detect_image(image)
        self.move(numbers)
        time.sleep(10)
        print('识别结束')

if __name__ == '__main__':
    crack = CrackWeiboSlide()
    crack.main()
</code>

After matching, the script draws four directional lines according to the identified sequence, producing the final solved captcha image.

- END -