Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,8 @@ flakiness using the new "rerunFailingTestsCount" option. It also supports re-run
failed build at the exact failed Git revision.

Finally it aggregates statistics of tests (passes, fails and flakes) over Git revisions.

The build parameter names can be changed via system properties:

* jenkins.deflake.param.test (default to `test`)
* jenkins.deflake.param.rerun_fail_test_count (default to `surefire.rerunFailingTestsCount`)
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,9 @@ public void tally() {
* <p>
* After the data is frozen, more files can be parsed
* and then freeze can be called again.
*
* @param parent parent action
* @param build current build
*/
public void freeze(AbstractTestResultAction parent, Run build) {
this.parentAction = parent;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,12 @@ public class FlakyTestResultAction implements RunAction2 {
* Construct a FlakyTestResultAction object with Run and BuildListener
*
* @param build this build
* @param launcher launcher
* @param listener listener of this build
* @throws IOException when there's communication error
* @throws InterruptedException when not able to obtain channel
*/
public FlakyTestResultAction(AbstractBuild build, Launcher launcher, TaskListener listener) throws IOException, InterruptedException {
public FlakyTestResultAction(AbstractBuild build, Launcher launcher, TaskListener listener) throws InterruptedException, IOException {
this.build = build;
// TODO consider the possibility that there is >1 such action
AbstractTestResultAction action = build.getAction(AbstractTestResultAction.class);
Expand Down Expand Up @@ -170,6 +173,9 @@ synchronized void setFlakyRunStats(FlakyRunStats stats) {

/**
* Overwrites the {@link FlakyRunStats} by a new data set.
*
* @param stats the flaky stats
* @param listener the listener
*/
public synchronized void setFlakyRunStats(FlakyRunStats stats, TaskListener listener) {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import hudson.model.Run;
import net.sf.json.JSONObject;

import org.apache.commons.lang.StringUtils;
import org.kohsuke.stapler.StaplerRequest;
import org.kohsuke.stapler.StaplerResponse;

Expand All @@ -34,6 +35,7 @@
import java.util.Set;
import java.util.logging.Level;

import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import javax.servlet.ServletException;

Expand All @@ -60,7 +62,18 @@ public class DeflakeAction implements Action {

private static final String MAVEN_TEST_PARAM = "testParam";

private static final String MAVEN_TEST = "test";
/**
* The name of the build parameter on the deflaked build which can be changed via system property
*/
private static final String BUILD_PARAM_TEST =
System.getProperty("jenkins.deflake.param.test", "test");

/**
* The name of the build parameter on the deflaked build which can be changed via system property.
* This is useful when using failsafe plugin or for other scenarios.
*/
private static final String BUILD_PARAM_RERUN_FAIL_TEST_COUNT =
System.getProperty("jenkins.deflake.param.rerun_fail_test_count", "surefire.rerunFailingTestsCount");

private static final String COMMA = ",";

Expand All @@ -79,9 +92,15 @@ public String apply(@Nonnull Entry<String, Set<String>> entry) {
};

private final Map<String, Set<String>> failingClassMethodMap;
private final Integer parentBuildNumber;

public DeflakeAction(Map<String, Set<String>> failingClassMethodMap) {
this(failingClassMethodMap, null);
}

public DeflakeAction(Map<String, Set<String>> failingClassMethodMap, Run<?, ?> parentBuild) {
this.failingClassMethodMap = failingClassMethodMap;
this.parentBuildNumber = (parentBuild != null) ? parentBuild.getNumber() : null;
}

@Override
Expand Down Expand Up @@ -110,37 +129,40 @@ public String getUrlName() {
*/
public void doIndex(StaplerRequest request, StaplerResponse response) throws
IOException, ServletException, InterruptedException {
Run currentBuild = request.findAncestorObject(Run.class);
Run<?, ?> currentBuild = request.findAncestorObject(Run.class);
if (currentBuild != null) {

Job job = currentBuild.getParent();
Job<?, ?> job = currentBuild.getParent();
job.checkPermission(AbstractProject.BUILD);
response.sendRedirect(DEFLAKE_CONFIG_URL);
}
}

/**
* Get parameters from submitted form and submit deflake request
*
* @param request request
* @param response response
* @throws ServletException when unable to parse input form
* @throws IOException when unable to redirect
*/
public void doSubmitDeflakeRequest(StaplerRequest request, StaplerResponse response) throws
IOException, ServletException, InterruptedException {
public void doSubmitDeflakeRequest(StaplerRequest request, StaplerResponse response) throws ServletException, IOException {

Run run = request.findAncestorObject(Run.class);
Run<?,?> run = request.findAncestorObject(Run.class);
if (run != null) {
Job job = run.getParent();
Job<?, ?> job = run.getParent();
job.checkPermission(AbstractProject.BUILD);
List<Action> actions = constructDeflakeCause(run);

JSONObject formData = request.getSubmittedForm();
List<ParameterValue> parameterValues = new ArrayList<ParameterValue>();
parameterValues.add(getStringParam(formData, RERUN_FAILING_TESTS_COUNT_PARAM));
String rerunFailTestCount = getStringParam(formData, RERUN_FAILING_TESTS_COUNT_PARAM, "0");
parameterValues.add(new StringParameterValue(BUILD_PARAM_RERUN_FAIL_TEST_COUNT, rerunFailTestCount));

JSONObject paramObj = JSONObject.fromObject(formData.get(MAVEN_TEST_PARAM));
boolean onlyRunFailingTests = paramObj.getBoolean("value");
boolean onlyRunFailingTests = getBooleanParam(formData, MAVEN_TEST_PARAM);
if (onlyRunFailingTests) {
String testParameter = generateMavenTestParams();
if (testParameter != null) {
parameterValues.add(new StringParameterValue(MAVEN_TEST, testParameter));
parameterValues.add(new StringParameterValue(BUILD_PARAM_TEST, testParameter));
}
}

Expand All @@ -149,9 +171,10 @@ public void doSubmitDeflakeRequest(StaplerRequest request, StaplerResponse respo
originalParamAction = new ParametersAction();
}

List<Action> actions = constructDeflakeCause(run);
actions.add(originalParamAction.createUpdated(parameterValues));

Jenkins.getInstance().getQueue().schedule((Queue.Task) run.getParent(), 0, actions);
Jenkins.get().getQueue().schedule2((Queue.Task) run.getParent(), 0, actions);
}

response.sendRedirect("../../");
Expand Down Expand Up @@ -181,19 +204,23 @@ private static List<Action> constructDeflakeCause(Run up) {
return actions;
}

private static ParameterValue getBooleanParam(JSONObject formData, String paramName) {
private static boolean getBooleanParam(JSONObject formData, String paramName) {
JSONObject paramObj = JSONObject.fromObject(formData.get(paramName));
String name = paramObj.getString("name");
FlakyTestResultAction.logger.log(Level.FINE,
"Param: " + name + " with value: " + paramObj.getBoolean("value"));
return new BooleanParameterValue(name, paramObj.getBoolean("value"));
return paramObj.getBoolean("value");
}

private static ParameterValue getStringParam(JSONObject formData, String paramName) {
private static String getStringParam(JSONObject formData, String paramName, String defaultValue) {
JSONObject paramObj = JSONObject.fromObject(formData.get(paramName));
String name = paramObj.getString("name");
FlakyTestResultAction.logger.log(Level.FINE,
"Param: " + name + " with value: " + paramObj.getString("value"));
return new StringParameterValue(name, paramObj.getString("value"));
return StringUtils.defaultIfBlank(paramObj.getString("value"), defaultValue);
}

/**
* The build number of the parent build that was deflaked.
*
* @return build number of the parent build
*/
@CheckForNull
public Integer getParentBuildNumber() {
return parentBuildNumber;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
* @author Qingzhou Luo
*/
public class DeflakeCause extends Cause.UpstreamCause {

/**
* DeflakeCause constructor.
* @param up upstream failing build which is being deflaked
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,16 @@
import java.util.logging.Logger;

import hudson.Extension;
import hudson.model.AbstractBuild;
import hudson.Launcher;
import hudson.model.AbstractBuild;
import hudson.model.BuildListener;
import hudson.model.Action;
import hudson.model.Cause;
import hudson.model.Environment;
import hudson.model.Run;
import hudson.model.TaskListener;
import hudson.model.Run.RunnerAbortedException;
import hudson.model.listeners.RunListener;
import hudson.tasks.junit.CaseResult;
import hudson.tasks.junit.TestResultAction;
Expand Down Expand Up @@ -65,7 +71,7 @@ public void onCompleted(Run run, TaskListener listener) {
if (testResultAction != null && testResultAction.getFailCount() > 0) {
// Only add deflake action if there are test failures
run.addAction(
new DeflakeAction(getFailingTestClassMethodMap(testResultAction.getFailedTests())));
new DeflakeAction(getFailingTestClassMethodMap(testResultAction.getFailedTests()), run));
}
}

Expand Down Expand Up @@ -103,4 +109,29 @@ static Map<String, Set<String>> getFailingTestClassMethodMap(List<CaseResult> ca
}
return classMethodMap;
}

@Override
public Environment setUpEnvironment(@SuppressWarnings("rawtypes") AbstractBuild build, Launcher launcher, BuildListener listener)
throws IOException, InterruptedException, RunnerAbortedException {
AbstractBuild<?, ?> rootBuild = build.getRootBuild();
if (rootBuild == null) {
rootBuild = build;
}

final DeflakeAction action = rootBuild.getAction(DeflakeAction.class);
if (action == null) {
return null;
}

return new Environment() {
@Override
public void buildEnvVars(Map<String, String> env) {
env.put("DEFLAKE_FAIL_TEST", action.generateMavenTestParams());
Integer parentBuildNumber = action.getParentBuildNumber();
if (parentBuildNumber != null) {
env.put("DEFLAKE_BUILD_NUMBER", parentBuildNumber.toString());
}
}
};
}
}