Skip to content

HDFS-13538. HDFS DiskChecker should handle disk full situation #7915

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: trunk
Choose a base branch
from
+50 −10
Open

HDFS-13538. HDFS DiskChecker should handle disk full situation #7915

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
@InterfaceStability.Unstable
public class DiskChecker {
public static final Logger LOG = LoggerFactory.getLogger(DiskChecker.class);
private static final String LINUX_DISK_FULL_MESSAGE = "No space left on device";
private static final String WINDOWS_DISK_FULL_MESSAGE = "There is not enough space on the disk";

public static class DiskErrorException extends IOException {
public DiskErrorException(String msg) {
Expand Down Expand Up @@ -267,7 +269,11 @@ private static void doDiskIo(File dir) throws DiskErrorException {
ioe = e;
}
}
throw ioe; // Just rethrow the last exception to signal failure.
// Throw the exception only if it's not about disk being full.
if (!ioe.getMessage().contains(LINUX_DISK_FULL_MESSAGE) &&
!ioe.getMessage().contains(WINDOWS_DISK_FULL_MESSAGE)) {
throw ioe; // Just rethrow the last exception to signal failure.
}
} catch(IOException e) {
throw new DiskErrorException("Error checking directory " + dir, e);
}
Expand Down
31 changes: 25 additions & 6 deletions ...project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskCheckerWithDiskIo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;


/**
Expand All @@ -48,7 +49,7 @@ public final class TestDiskCheckerWithDiskIo {
@Test
public final void testDiskIoIgnoresTransientCreateErrors() throws Throwable {
DiskChecker.replaceFileOutputStreamProvider(new TestFileIoProvider(
DiskChecker.DISK_IO_MAX_ITERATIONS - 1, 0));
DiskChecker.DISK_IO_MAX_ITERATIONS - 1, 0, false));
checkDirs(true);
}

Expand All @@ -59,7 +60,7 @@ public final void testDiskIoIgnoresTransientCreateErrors() throws Throwable {
public final void testDiskIoDetectsCreateErrors() throws Throwable {
assertThrows(DiskErrorException.class, () -> {
DiskChecker.replaceFileOutputStreamProvider(new TestFileIoProvider(
DiskChecker.DISK_IO_MAX_ITERATIONS, 0));
DiskChecker.DISK_IO_MAX_ITERATIONS, 0, false));
checkDirs(false);
});
}
Expand All @@ -70,7 +71,7 @@ public final void testDiskIoDetectsCreateErrors() throws Throwable {
@Test
public final void testDiskIoIgnoresTransientWriteErrors() throws Throwable {
DiskChecker.replaceFileOutputStreamProvider(new TestFileIoProvider(
0, DiskChecker.DISK_IO_MAX_ITERATIONS - 1));
0, DiskChecker.DISK_IO_MAX_ITERATIONS - 1, false));
checkDirs(true);
}

Expand All @@ -81,7 +82,7 @@ public final void testDiskIoIgnoresTransientWriteErrors() throws Throwable {
public final void testDiskIoDetectsWriteErrors() throws Throwable {
assertThrows(DiskErrorException.class, ()->{
DiskChecker.replaceFileOutputStreamProvider(new TestFileIoProvider(
0, DiskChecker.DISK_IO_MAX_ITERATIONS));
0, DiskChecker.DISK_IO_MAX_ITERATIONS, false));
checkDirs(false);
});
}
Expand All @@ -104,6 +105,18 @@ public void testDiskIoFileNaming() {
"File name does not match expected pattern: " + guidFile);
}

/**
* Verify DiskChecker doesn't fail on ENOSPC errors.
*/
@Test
public void testDiskIoDetectsENOSPCWriteErrors() {
assertDoesNotThrow(()->{
DiskChecker.replaceFileOutputStreamProvider(new TestFileIoProvider(
0, DiskChecker.DISK_IO_MAX_ITERATIONS, true));
checkDirs(true);
});
}

/**
* A dummy {@link DiskChecker#FileIoProvider} that can throw a programmable
* number of times.
Expand All @@ -114,11 +127,13 @@ private static class TestFileIoProvider implements FileIoProvider {

private final int numTimesToThrowOnCreate;
private final int numTimesToThrowOnWrite;
private final boolean throwENOSPCError;

public TestFileIoProvider(
int numTimesToThrowOnCreate, int numTimesToThrowOnWrite) {
int numTimesToThrowOnCreate, int numTimesToThrowOnWrite, boolean throwENOSPCError) {
this.numTimesToThrowOnCreate = numTimesToThrowOnCreate;
this.numTimesToThrowOnWrite = numTimesToThrowOnWrite;
this.throwENOSPCError = throwENOSPCError;
}

/**
Expand All @@ -139,7 +154,11 @@ public FileOutputStream get(File f) throws FileNotFoundException {
@Override
public void write(FileOutputStream fos, byte[] data) throws IOException {
if (numWriteCalls.getAndIncrement() < numTimesToThrowOnWrite) {
throw new IOException("Dummy exception for testing");
if (!throwENOSPCError) {
throw new IOException("Dummy exception for testing");
} else {
throw new IOException("No space left on device");
}
}
fos.write(data);
}
Expand Down
4 changes: 4 additions & 0 deletions hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_DN_CACHED_DFSUSED_CHECK_INTERVAL_DEFAULT_MS =
600000;

public static final String DFS_DATANODE_CHECK_DIR_WITH_DISKIO =
"dfs.datanode.check.dir.with.diskio";
public static final boolean DFS_DATANODE_CHECK_DIR_WITH_DISKIO_DEFAULT = false;

public static final String DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT =
"dfs.namenode.path.based.cache.block.map.allocation.percent";
public static final float DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT_DEFAULT = 0.25f;
Expand Down
17 changes: 14 additions & 3 deletions ...s/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ public class BlockPoolSlice {
private final long cachedDfsUsedCheckTime;
private final Timer timer;
private final int maxDataLength;
private final boolean checkDirWithDiskIo;
private final FileIoProvider fileIoProvider;
private final Configuration config;
private final File bpDir;
Expand Down Expand Up @@ -179,6 +180,10 @@ public int compare(File f1, File f2) {
CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH,
CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT);

this.checkDirWithDiskIo = conf.getBoolean(
DFSConfigKeys.DFS_DATANODE_CHECK_DIR_WITH_DISKIO,
DFSConfigKeys.DFS_DATANODE_CHECK_DIR_WITH_DISKIO_DEFAULT);

this.timer = timer;

// Files that were being written when the datanode was last shutdown
Expand Down Expand Up @@ -484,9 +489,15 @@ ReplicaInfo activateSavedReplica(ReplicaInfo replicaInfo,
}

void checkDirs() throws DiskErrorException {
DiskChecker.checkDir(finalizedDir);
DiskChecker.checkDir(tmpDir);
DiskChecker.checkDir(rbwDir);
if (checkDirWithDiskIo) {
DiskChecker.checkDirWithDiskIo(finalizedDir);
DiskChecker.checkDirWithDiskIo(tmpDir);
DiskChecker.checkDirWithDiskIo(rbwDir);
} else {
DiskChecker.checkDir(finalizedDir);
DiskChecker.checkDir(tmpDir);
DiskChecker.checkDir(rbwDir);
}
}


Expand Down