Automate JUnit Test Failure Analysis with GPT: A Step‑by‑Step Java Guide

Introduction

This tutorial, aimed at Java engineers, demonstrates how to create a JUnit4 Rule that automatically sends error information to GPT after a test failure, receives analysis, and provides code modification suggestions.

Data Model Classes

package com.jd.jr.parrot.llm;
import lombok.Data;
/**
 * created on 2023/7/25 20:07
 */
@Data
public class Choice {
    private String finish_reason;
    private int index;
    private Message message;
}

package com.jd.jr.parrot.llm;
import lombok.Data;
/**
 * created on 2023/7/25 20:07
 */
@Data
public class Message {
    private String role;
    private String content;
}

package com.jd.jr.parrot.llm;
import lombok.Data;
/**
 * created on 2023/7/25 20:06
 */
@Data
public class Usage {
    private int completion_tokens;
    private int prompt_tokens;
    private int total_tokens;
}

package com.jd.jr.parrot.llm;
import java.util.List;
import lombok.Data;
/**
 * created on 2023/7/25 20:17
 */
@Data
public class OpenAiRequest {
    private String erp;
    private List<Message> messages;
    private String model;
}

package com.jd.jr.parrot.llm;
import java.util.List;
import lombok.Data;
/**
 * created on 2023/7/25 20:04
 */
@Data
public class OpenAiResponse {
    private long created;
    private Usage usage;
    private String model;
    private String id;
    private List<Choice> choices;
    private String object;
}

LLM Utility

package com.jd.jr.parrot.llm;
import com.google.gson.Gson;
import java.util.Arrays;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
import java.nio.charset.Charset;
/**
 * created on 2023/7/25 20:07
 */
@Slf4j
public class LLMUtil {
    private static String url = "Your url";
    private static String apiKey = "Your key";

    public static String sendMsg(String msg){
        OpenAiRequest request = new OpenAiRequest();
        Message message = new Message();
        message.setRole("user");
        message.setContent(msg);
        request.setMessages(Arrays.asList(message));
        request.setModel("gpt-3.5-turbo");
        Gson gson = new Gson();
        String result = postGPT(url, gson.toJson(request));
        OpenAiResponse openAiResponse = gson.fromJson(result, OpenAiResponse.class);
        return openAiResponse.getChoices().get(0).getMessage().getContent();
    }

    /**
     * Send POST request
     * @param url request address
     * @param jsonStr JSON string of the form data
     * @return request result
     */
    public static String postGPT(String url, String jsonStr) {
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpPost httpPost = new HttpPost(url);
        httpPost.setHeader("Content-type", "application/json;charset=utf-8");
        httpPost.setHeader("Authorization", apiKey);
        StringEntity entity = new StringEntity(jsonStr, Charset.forName("UTF-8"));
        entity.setContentEncoding("UTF-8");
        entity.setContentType("application/json");
        httpPost.setEntity(entity);
        CloseableHttpResponse httpResponse;
        String result = null;
        try {
            httpResponse = httpClient.execute(httpPost);
            result = EntityUtils.toString(httpResponse.getEntity(), "UTF-8");
            if (httpResponse.getStatusLine().getStatusCode() != 200) {
                System.out.println("post->return error，result: " + result);
                log.error("post->return error，result: {}", result);
            }
        } catch (IOException e) {
            log.error("postStream->error", e);
            e.printStackTrace();
        }
        return result;
    }

    public static void main(String[] args) {
        log.info(sendMsg("Who are you?"));
    }
}

Exception Utility

package com.jd.jr.parrot;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
 * created on 2023/7/28 15:08
 */
public class ExceptionUtil {
    public static String getFileName(String stackTraceLine) {
        Pattern pattern = Pattern.compile("\\((.*\\.java)");
        Matcher matcher = pattern.matcher(stackTraceLine);
        if (matcher.find()) {
            return matcher.group(1);
        }
        return null;
    }

    public static Path getPath(String stackTraceLine) {
        int firstParenthesisIndex = stackTraceLine.indexOf("(");
        int firstDotIndex = stackTraceLine.lastIndexOf(".", firstParenthesisIndex);
        String packagePath = stackTraceLine.substring(0, firstDotIndex).replace(".", File.separator);
        String rootPath = new File("").getAbsolutePath();
        return Paths.get(rootPath, "src", "main", "java", packagePath + ".java");
    }

    public static String getCodeFile(String stackTraceLine) {
        Path filePath = getPath(stackTraceLine);
        if (filePath.toFile().exists()) {
            try (BufferedReader reader = new BufferedReader(new FileReader(filePath.toFile()))) {
                StringBuilder content = new StringBuilder();
                String line;
                while ((line = reader.readLine()) != null) {
                    content.append(line).append("
");
                }
                return content.toString();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return null;
    }

    public static void main(String[] args) {
        String stackTraceLine = "com.jd.jr.parrot.ExceptionExample.main(ExceptionExample.java:24)";
        System.out.println("fileName = " + getFileName(stackTraceLine));
        System.out.println("path = " + getPath(stackTraceLine).toString());
        System.out.println("codeFile = " + getCodeFile(stackTraceLine));
    }
}

OverWatch Rule

package com.jd.jr.parrot.junit;
import com.jd.jr.parrot.ExceptionUtil;
import com.jd.jr.parrot.llm.LLMUtil;
import java.util.Arrays;
import lombok.extern.slf4j.Slf4j;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
/**
 * created on 2023/7/28 14:07
 */
@Slf4j
public class OverWatch extends TestWatcher {
    private static final String prompt = "\"You are a good Java engineer, please analyze the following error and provide possible solutions." +
        "The error message is : %s, The error stack is : %s. The code file is : %s";

    @Override
    protected void failed(Throwable e, Description description) {
        super.failed(e, description);
        String codeFile = ExceptionUtil.getCodeFile(e.getStackTrace()[0].toString());
        log.info(LLMUtil.sendMsg(String.format(prompt, e.getMessage(), Arrays.toString(e.getStackTrace()), codeFile)));
    }
}

Prompting GPT for Faulty Code

The following conversation with GPT requests a deliberately faulty Java program that depends on external libraries, making the error harder to trace.

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.io.IOException;

public class ExceptionExample {
    public static void main(String[] args) {
        try {
            // Use Jsoup to fetch a web page
            Document doc = Jsoup.connect("https://www.example.com").get();
            // Parse the page title
            Element title = doc.select("title").first();
            System.out.println("Title: " + title.text());
            // Attempt to get a non‑existent element
            Element nonExistentElement = doc.select("#nonexistent-element").first();
            // Call a method on a null element, causing NullPointerException
            nonExistentElement.text();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

JUnit Test Using OverWatch

package com.jd.jr.parrot;
import com.jd.jr.parrot.junit.OverWatch;
import java.io.IOException;
import org.junit.FixMethodOrder;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runners.MethodSorters;
/**
 * created on 2023/7/28 14:14
 */
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class OverWatchTest {
    @Rule
    public OverWatch overWatch = new OverWatch();

    @Test
    public void jsoupTest() throws IOException {
        ExceptionExample.main(new String[]{});
    }
}

When the test fails, the OverWatch rule captures the exception, extracts the source file, sends the error message, stack trace, and code to GPT, and logs the AI‑generated analysis.