Skip to content

Commit

Permalink
feat: implement exponential retry mechanism for handling network erro…
Browse files Browse the repository at this point in the history
…rs (#468)

* feat: implement exponential backoff to handle network error

* feat: add initialDelay logic

* test: add ExponentialBackOffTest.java

* chore: address sonarcloud warning

Replace java.util.Random with java.security.SecureRandom.
  • Loading branch information
1abhishekpandey authored Jul 29, 2024
1 parent 0711b37 commit e07e2cf
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import static com.rudderstack.android.sdk.core.RudderNetworkManager.addEndPoint;

import com.rudderstack.android.sdk.core.gson.RudderGson;
import com.rudderstack.android.sdk.core.util.ExponentialBackOff;
import com.rudderstack.android.sdk.core.util.MessageUploadLock;
import com.rudderstack.android.sdk.core.util.Utils;

Expand Down Expand Up @@ -41,6 +42,7 @@ public void run() {
Result result = null;
final ArrayList<Integer> messageIds = new ArrayList<>();
final ArrayList<String> messages = new ArrayList<>();
final ExponentialBackOff exponentialBackOff = new ExponentialBackOff(5 * 60); // 5 minutes
while (true) {
// clear lists for reuse
messageIds.clear();
Expand All @@ -63,6 +65,7 @@ public void run() {
ReportManager.incrementCloudModeUploadSuccessCounter(messageIds.size());
dbManager.markCloudModeDone(messageIds);
dbManager.runGcForEvents();
exponentialBackOff.resetBackOff();
upTimeInMillis = Utils.getUpTimeInMillis();
sleepCount = Utils.getSleepDurationInSecond(upTimeInMillis, Utils.getUpTimeInMillis());
} else {
Expand All @@ -87,9 +90,13 @@ public void run() {
deleteEventsWithoutAnonymousId(messages, messageIds);
break;
case ERROR:
long sleepIntervalInMillis = exponentialBackOff.nextDelayInMillis();
RudderLogger.logWarn("CloudModeManager: cloudModeProcessor: Retrying in " + Utils.getTimeInReadableFormat(sleepIntervalInMillis));
Thread.sleep(sleepIntervalInMillis);
break;
case NETWORK_UNAVAILABLE:
RudderLogger.logWarn("CloudModeManager: cloudModeProcessor: Retrying in " + Math.abs(sleepCount - config.getSleepTimeOut()) + "s");
Thread.sleep(Math.abs(sleepCount - config.getSleepTimeOut()) * 1000L);
RudderLogger.logWarn("CloudModeManager: cloudModeProcessor: Retrying in 1s");
Thread.sleep(1000);
break;
default:
RudderLogger.logWarn("CloudModeManager: cloudModeProcessor: Retrying in 1s");
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package com.rudderstack.android.sdk.core.util;

import androidx.annotation.VisibleForTesting;

import java.security.SecureRandom;

/**
* This class implements an exponential backoff strategy with jitter for handling retries.
* It allows for configurable maximum delay and includes methods to calculate the next delay
* with jitter and reset the backoff attempts.
* When the calculated delay reaches or exceeds the maximum delay limit, the backoff resets
* and starts again from beginning.
*/
public class ExponentialBackOff {
private int attempt = 0;
private final int maxDelayInSecs;
private final SecureRandom random;

/**
* Constructor to initialize the ExponentialBackOff with a maximum delay.
*
* @param maxDelayInSecs Maximum delay in seconds for the backoff.
*/
public ExponentialBackOff(int maxDelayInSecs) {
this.maxDelayInSecs = maxDelayInSecs;
this.random = new SecureRandom();
}

/**
* Calculates the next delay with exponential backoff and jitter.
*
* @return The next delay in milliseconds.
*/
public long nextDelayInMillis() {
int base = 2;
int initialDelayInSecs = 3;
long delayInSecs = (long) (initialDelayInSecs * Math.pow(base, attempt++));
long exponentialIntervalInSecs = Math.min(maxDelayInSecs, withJitter(delayInSecs));

// Reset the backoff if the delay reaches or exceeds the maximum limit
if (exponentialIntervalInSecs >= maxDelayInSecs) {
resetBackOff();
}

return exponentialIntervalInSecs * 1000;
}

@VisibleForTesting
protected long withJitter(long delayInSecs) {
long jitter = random.nextInt((int) delayInSecs);
return delayInSecs + jitter;
}

/**
* Resets the backoff attempt counter to 0.
*/
public void resetBackOff() {
attempt = 0;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -327,4 +327,24 @@ public static boolean isEmpty(@Nullable String value) {
public static boolean isEmpty(@Nullable List value) {
return (value == null || value.isEmpty());
}

/**
* Converts time in milliseconds to a readable format of minutes and seconds.
*
* @param timeInMillis The time in milliseconds to be converted.
* @return A string representing the time in minutes and seconds.
*/
public static String getTimeInReadableFormat(long timeInMillis) {
long totalSeconds = timeInMillis / 1000;
long minutes = totalSeconds / 60;
long seconds = totalSeconds % 60;

StringBuilder timeInReadableFormat = new StringBuilder();
if (minutes > 0) {
timeInReadableFormat.append(minutes).append("min ");
}
timeInReadableFormat.append(seconds).append("sec");

return timeInReadableFormat.toString();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package com.rudderstack.android.sdk.core;

import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
import com.rudderstack.android.sdk.core.util.ExponentialBackOff;

public class ExponentialBackOffTest {

private ExponentialBackOff backOff;

@Before
public void setUp() {
backOff = new ExponentialBackOff(5 * 60) {
@Override
protected long withJitter(long delayInSecs) {
// Return a custom value for testing
long jitter = 1; // 1 sec
return delayInSecs + jitter;
}
};
}

@Test
public void testExponentialDelay() {
// Check delays for the first few attempts
long delay1 = backOff.nextDelayInMillis();
long delay2 = backOff.nextDelayInMillis();
long delay3 = backOff.nextDelayInMillis();

assertEquals((3 + 1) * 1000, delay1); // Expected delay: (initialDelayInSecs * Math.pow(base, attempt)) * 1000
assertEquals(((3 * 2) + 1) * 1000, delay2); // Expected delay: (initialDelayInSecs * Math.pow(base, attempt)) * 1000
assertEquals(((3 * 4) + 1) * 1000, delay3); // Expected delay: (initialDelayInSecs * Math.pow(base, attempt)) * 1000
}

@Test
public void testResetBackOff() {
// Advance to exceed the max delay - 1
for (int i = 1; i <= 7; i++) {
backOff.nextDelayInMillis(); // At the 7th attempt, the delay will be 3 * 2^6 + 1 = 193 sec
}

long delayAfterReset = backOff.nextDelayInMillis(); // Should reset backoff
assertEquals(5 * 60 * 1000, delayAfterReset); // Expected delay: (initialDelayInSecs * Math.pow(base, attempt)) * 1000

long delayAfterReset2 = backOff.nextDelayInMillis(); // Should start from initial delay
assertEquals((3 + 1) * 1000, delayAfterReset2); // Expected delay: (initialDelayInSecs * Math.pow(base, attempt)) * 1000
}
}

0 comments on commit e07e2cf

Please sign in to comment.