From 63651c482f16bd234f49c89f2731b855979d8ca6 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Mon, 19 Aug 2024 13:57:26 +0800 Subject: [PATCH 1/7] test --- .../native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c index e7ea07af4cae5..43446edc6592a 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c @@ -221,6 +221,5 @@ int generateDecodeMatrix(IsalDecoder* pCoder) { } } } - return 0; } From f9f671a7b90b8fe5d13b04e58f2adf227d8ce82e Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Mon, 19 Aug 2024 15:59:47 +0800 Subject: [PATCH 2/7] test --- .../native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c | 1 + .../hadoop/service/launcher/TestServiceInterruptHandling.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c index 43446edc6592a..e7ea07af4cae5 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c @@ -221,5 +221,6 @@ int generateDecodeMatrix(IsalDecoder* pCoder) { } } } + return 0; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java index c21fa8b73073f..488ca5c12133c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java @@ -113,5 +113,4 @@ public void interrupted(IrqHandler.InterruptData data) { this.interruptData = data; } } - } From 37d1e9dd45fe49ae14c126c103b303e6edc39ad3 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Mon, 19 Aug 2024 16:45:27 +0800 Subject: [PATCH 3/7] Revert "HADOOP-19180. EC: Fix calculation errors caused by special index order (#6813). Contributed by zhengchenyu." This reverts commit e5b76dc99fdc7c9a3fc2132873eb4ef3e545bb4f. --- .../io/erasurecode/rawcoder/RSRawDecoder.java | 32 +++--- .../hadoop/io/erasurecode/erasure_coder.c | 36 +++--- .../hadoop/io/erasurecode/erasure_coder.h | 1 + .../hadoop/io/erasurecode/erasure_code_test.c | 80 ++++--------- .../TestErasureCodingEncodeAndDecode.java | 108 ------------------ 5 files changed, 62 insertions(+), 195 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestErasureCodingEncodeAndDecode.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java index 824e701c71fe6..d7f78abc05056 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java @@ -51,6 +51,7 @@ public class RSRawDecoder extends RawErasureDecoder { private byte[] gfTables; private int[] cachedErasedIndexes; private int[] validIndexes; + private int numErasedDataUnits; private boolean[] erasureFlags; public RSRawDecoder(ErasureCoderOptions coderOptions) { @@ -119,10 +120,14 @@ private void processErasures(int[] erasedIndexes) { this.gfTables = new byte[getNumAllUnits() * getNumDataUnits() * 32]; this.erasureFlags = new boolean[getNumAllUnits()]; + this.numErasedDataUnits = 0; for (int i = 0; i < erasedIndexes.length; i++) { int index = erasedIndexes[i]; erasureFlags[index] = true; + if (index < getNumDataUnits()) { + numErasedDataUnits++; + } } generateDecodeMatrix(erasedIndexes); @@ -151,22 +156,21 @@ private void generateDecodeMatrix(int[] erasedIndexes) { GF256.gfInvertMatrix(tmpMatrix, invertMatrix, getNumDataUnits()); - for (p = 0; p < erasedIndexes.length; p++) { - int erasedIndex = erasedIndexes[p]; - if (erasedIndex < getNumDataUnits()) { + for (i = 0; i < numErasedDataUnits; i++) { + for (j = 0; j < getNumDataUnits(); j++) { + decodeMatrix[getNumDataUnits() * i + j] = + invertMatrix[getNumDataUnits() * erasedIndexes[i] + j]; + } + } + + for (p = numErasedDataUnits; p < erasedIndexes.length; p++) { + for (i = 0; i < getNumDataUnits(); i++) { + s = 0; for (j = 0; j < getNumDataUnits(); j++) { - decodeMatrix[getNumDataUnits() * p + j] = - invertMatrix[getNumDataUnits() * erasedIndexes[p] + j]; - } - } else { - for (i = 0; i < getNumDataUnits(); i++) { - s = 0; - for (j = 0; j < getNumDataUnits(); j++) { - s ^= GF256.gfMul(invertMatrix[j * getNumDataUnits() + i], - encodeMatrix[getNumDataUnits() * erasedIndexes[p] + j]); - } - decodeMatrix[getNumDataUnits() * p + i] = s; + s ^= GF256.gfMul(invertMatrix[j * getNumDataUnits() + i], + encodeMatrix[getNumDataUnits() * erasedIndexes[p] + j]); } + decodeMatrix[getNumDataUnits() * p + i] = s; } } } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c index e7ea07af4cae5..b2d856b6f889e 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c @@ -132,6 +132,9 @@ static int processErasures(IsalDecoder* pCoder, unsigned char** inputs, index = erasedIndexes[i]; pCoder->erasedIndexes[i] = index; pCoder->erasureFlags[index] = 1; + if (index < numDataUnits) { + pCoder->numErasedDataUnits++; + } } pCoder->numErased = numErased; @@ -172,6 +175,7 @@ int decode(IsalDecoder* pCoder, unsigned char** inputs, // Clear variables used per decode call void clearDecoder(IsalDecoder* decoder) { + decoder->numErasedDataUnits = 0; decoder->numErased = 0; memset(decoder->gftbls, 0, sizeof(decoder->gftbls)); memset(decoder->decodeMatrix, 0, sizeof(decoder->decodeMatrix)); @@ -201,24 +205,24 @@ int generateDecodeMatrix(IsalDecoder* pCoder) { h_gf_invert_matrix(pCoder->tmpMatrix, pCoder->invertMatrix, numDataUnits); - for (p = 0; p < pCoder->numErased; p++) { + for (i = 0; i < pCoder->numErasedDataUnits; i++) { for (j = 0; j < numDataUnits; j++) { - int erasedIndex = pCoder->erasedIndexes[p]; - if (erasedIndex < numDataUnits) { - pCoder->decodeMatrix[numDataUnits * p + j] = - pCoder->invertMatrix[numDataUnits * - pCoder->erasedIndexes[p] + j]; - } else { - for (i = 0; i < numDataUnits; i++) { - s = 0; - for (j = 0; j < numDataUnits; j++) { - s ^= h_gf_mul(pCoder->invertMatrix[j * numDataUnits + i], - pCoder->encodeMatrix[numDataUnits * - pCoder->erasedIndexes[p] + j]); - } - pCoder->decodeMatrix[numDataUnits * p + i] = s; - } + pCoder->decodeMatrix[numDataUnits * i + j] = + pCoder->invertMatrix[numDataUnits * + pCoder->erasedIndexes[i] + j]; + } + } + + for (p = pCoder->numErasedDataUnits; p < pCoder->numErased; p++) { + for (i = 0; i < numDataUnits; i++) { + s = 0; + for (j = 0; j < numDataUnits; j++) { + s ^= h_gf_mul(pCoder->invertMatrix[j * numDataUnits + i], + pCoder->encodeMatrix[numDataUnits * + pCoder->erasedIndexes[p] + j]); } + + pCoder->decodeMatrix[numDataUnits * p + i] = s; } } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h index d2ab24cc30b1e..8f5bf8a3ca7fe 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h @@ -62,6 +62,7 @@ typedef struct _IsalDecoder { unsigned char erasureFlags[MMAX]; int erasedIndexes[MMAX]; int numErased; + int numErasedDataUnits; unsigned char* realInputs[MMAX]; } IsalDecoder; diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c b/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c index ed439805baedb..331bb219b7faf 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c @@ -27,27 +27,25 @@ #include "erasure_code.h" #include "gf_util.h" #include "erasure_coder.h" -#include "dump.h" #include #include #include int main(int argc, char *argv[]) { - int i, j, k, l; + int i, j; char err[256]; size_t err_len = sizeof(err); int chunkSize = 1024; int numDataUnits = 6; int numParityUnits = 3; - int numTotalUnits = numDataUnits + numParityUnits; unsigned char** dataUnits; unsigned char** parityUnits; IsalEncoder* pEncoder; - int erasedIndexes[3]; + int erasedIndexes[2]; unsigned char* allUnits[MMAX]; IsalDecoder* pDecoder; - unsigned char* decodingOutput[3]; + unsigned char* decodingOutput[2]; unsigned char** backupUnits; if (0 == build_support_erasurecode()) { @@ -84,11 +82,6 @@ int main(int argc, char *argv[]) { } } - // Allocate decode output - for (i = 0; i < 3; i++) { - decodingOutput[i] = malloc(chunkSize); - } - pEncoder = (IsalEncoder*)malloc(sizeof(IsalEncoder)); memset(pEncoder, 0, sizeof(*pEncoder)); initEncoder(pEncoder, numDataUnits, numParityUnits); @@ -102,53 +95,26 @@ int main(int argc, char *argv[]) { memcpy(allUnits + numDataUnits, parityUnits, numParityUnits * (sizeof (unsigned char*))); - for (i = 0; i < numTotalUnits; i++) { - for (j = 0; j < numTotalUnits; j++) { - for (k = 0; k < numTotalUnits; k++) { - int numErased; - if (i == j && j == k) { - erasedIndexes[0] = i; - numErased = 1; - backupUnits[0] = allUnits[i]; - allUnits[i] = NULL; - } else if (i == j) { - erasedIndexes[0] = i; - erasedIndexes[1] = k; - numErased = 2; - backupUnits[0] = allUnits[i]; - backupUnits[1] = allUnits[k]; - allUnits[i] = NULL; - allUnits[k] = NULL; - } else if (i == k || j == k) { - erasedIndexes[0] = i; - erasedIndexes[1] = j; - numErased = 2; - backupUnits[0] = allUnits[i]; - backupUnits[1] = allUnits[j]; - allUnits[i] = NULL; - allUnits[j] = NULL; - } else { - erasedIndexes[0] = i; - erasedIndexes[1] = j; - erasedIndexes[2] = k; - numErased = 3; - backupUnits[0] = allUnits[i]; - backupUnits[1] = allUnits[j]; - backupUnits[2] = allUnits[k]; - allUnits[i] = NULL; - allUnits[j] = NULL; - allUnits[k] = NULL; - } - decode(pDecoder, allUnits, erasedIndexes, numErased, decodingOutput, chunkSize); - for (l = 0; l < pDecoder->numErased; l++) { - if (0 != memcmp(decodingOutput[l], backupUnits[l], chunkSize)) { - printf("Decoding failed\n"); - dumpDecoder(pDecoder); - return -1; - } - allUnits[erasedIndexes[l]] = backupUnits[l]; - } - } + erasedIndexes[0] = 1; + erasedIndexes[1] = 7; + + backupUnits[0] = allUnits[1]; + backupUnits[1] = allUnits[7]; + + allUnits[0] = NULL; // Not to read + allUnits[1] = NULL; + allUnits[7] = NULL; + + decodingOutput[0] = malloc(chunkSize); + decodingOutput[1] = malloc(chunkSize); + + decode(pDecoder, allUnits, erasedIndexes, 2, decodingOutput, chunkSize); + + for (i = 0; i < pDecoder->numErased; i++) { + if (0 != memcmp(decodingOutput[i], backupUnits[i], chunkSize)) { + fprintf(stderr, "Decoding failed\n\n"); + dumpDecoder(pDecoder); + return -1; } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestErasureCodingEncodeAndDecode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestErasureCodingEncodeAndDecode.java deleted file mode 100644 index e61f64e423f30..0000000000000 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestErasureCodingEncodeAndDecode.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.io.erasurecode; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; -import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; -import org.junit.Test; - -import java.util.Random; - -import static org.junit.Assert.assertArrayEquals; - -public class TestErasureCodingEncodeAndDecode { - - private final static int CHUNCK = 1024; - private final static int DATAB_LOCKS = 6; - private final static int PARITY_BLOCKS = 3; - private final static int TOTAL_BLOCKS = DATAB_LOCKS + PARITY_BLOCKS; - - @Test - public void testEncodeAndDecode() throws Exception { - Configuration conf = new Configuration(); - int totalBytes = CHUNCK * DATAB_LOCKS; - Random random = new Random(); - byte[] tmpBytes = new byte[totalBytes]; - random.nextBytes(tmpBytes); - byte[][] data = new byte[DATAB_LOCKS][CHUNCK]; - for (int i = 0; i < DATAB_LOCKS; i++) { - System.arraycopy(tmpBytes, i * CHUNCK, data[i], 0, CHUNCK); - } - ErasureCoderOptions coderOptions = new ErasureCoderOptions(DATAB_LOCKS, PARITY_BLOCKS); - - // 1 Encode - RawErasureEncoder encoder = - CodecUtil.createRawEncoder(conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); - byte[][] parity = new byte[PARITY_BLOCKS][CHUNCK]; - encoder.encode(data, parity); - - // 2 Compose the complete data - byte[][] all = new byte[DATAB_LOCKS + PARITY_BLOCKS][CHUNCK]; - for (int i = 0; i < DATAB_LOCKS; i++) { - System.arraycopy(data[i], 0, all[i], 0, CHUNCK); - } - for (int i = 0; i < PARITY_BLOCKS; i++) { - System.arraycopy(parity[i], 0, all[i + DATAB_LOCKS], 0, CHUNCK); - } - - // 3 Decode - RawErasureDecoder rawDecoder = - CodecUtil.createRawDecoder(conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); - byte[][] backup = new byte[PARITY_BLOCKS][CHUNCK]; - for (int i = 0; i < TOTAL_BLOCKS; i++) { - for (int j = 0; j < TOTAL_BLOCKS; j++) { - for (int k = 0; k < TOTAL_BLOCKS; k++) { - int[] erasedIndexes; - if (i == j && j == k) { - erasedIndexes = new int[]{i}; - backup[0] = all[i]; - all[i] = null; - } else if (i == j) { - erasedIndexes = new int[]{i, k}; - backup[0] = all[i]; - backup[1] = all[k]; - all[i] = null; - all[k] = null; - } else if ((i == k) || ((j == k))) { - erasedIndexes = new int[]{i, j}; - backup[0] = all[i]; - backup[1] = all[j]; - all[i] = null; - all[j] = null; - } else { - erasedIndexes = new int[]{i, j, k}; - backup[0] = all[i]; - backup[1] = all[j]; - backup[2] = all[k]; - all[i] = null; - all[j] = null; - all[k] = null; - } - byte[][] decoded = new byte[erasedIndexes.length][CHUNCK]; - rawDecoder.decode(all, erasedIndexes, decoded); - for (int l = 0; l < erasedIndexes.length; l++) { - assertArrayEquals(backup[l], decoded[l]); - all[erasedIndexes[l]] = backup[l]; - } - } - } - } - } -} From d1851a1821cf3027b23462ac223bfbf8f17aea5e Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Mon, 19 Aug 2024 20:12:41 +0800 Subject: [PATCH 4/7] inc timeout --- .../hadoop/service/launcher/TestServiceInterruptHandling.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java index 488ca5c12133c..9fca60b7d3134 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java @@ -44,7 +44,7 @@ public void testRegisterAndRaise() throws Throwable { assertEquals(0, irqHandler.getSignalCount()); irqHandler.raise(); // allow for an async event - GenericTestUtils.waitFor(() -> catcher.interruptData != null, 100, 10000); + GenericTestUtils.waitFor(() -> catcher.interruptData != null, 100, 100000); assertEquals(name, catcher.interruptData.getName()); assertEquals(1, irqHandler.getSignalCount()); } From f9df346581ca13316fd1db15b68ac3a6b63ecb68 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Tue, 20 Aug 2024 12:01:19 +0800 Subject: [PATCH 5/7] test --- .../hadoop/service/launcher/TestServiceInterruptHandling.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java index 9fca60b7d3134..c8cdd980c6786 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java @@ -42,6 +42,7 @@ public void testRegisterAndRaise() throws Throwable { IrqHandler irqHandler = new IrqHandler(name, catcher); irqHandler.bind(); assertEquals(0, irqHandler.getSignalCount()); + Thread.sleep(3000); irqHandler.raise(); // allow for an async event GenericTestUtils.waitFor(() -> catcher.interruptData != null, 100, 100000); From 57d8f6d20cfaef2a7a6512ff74c7a767297c834d Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Tue, 20 Aug 2024 17:03:40 +0800 Subject: [PATCH 6/7] test code --- .../hadoop-common/src/CMakeLists.txt | 1 + .../org/apache/hadoop/util/SignalUtils.java | 17 +++++++++++++++++ .../src/org/apache/hadoop/util/SignalUtils.c | 12 ++++++++++++ .../launcher/TestServiceInterruptHandling.java | 11 ++++++++++- 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalUtils.java create mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/SignalUtils.c diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index d2ef03645a4ae..c7b39ccb452b4 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -256,6 +256,7 @@ hadoop_add_dual_library(hadoop ${SRC}/util/NativeCodeLoader.c ${SRC}/util/NativeCrc32.c ${SRC}/util/bulk_crc32.c + ${SRC}/util/SignalUtils.c ${BULK_CRC_ARCH_SOURCE_FIlE} ) if(NEED_LINK_DL) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalUtils.java new file mode 100644 index 0000000000000..22d7a2993e260 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalUtils.java @@ -0,0 +1,17 @@ +package org.apache.hadoop.util; + +public class SignalUtils { + + public static boolean nativeCodeLoaded = false; + + static { + try { + System.loadLibrary("hadoop"); + nativeCodeLoaded = true; + } catch (Throwable t) { + // Ignore failure to load + } + } + + public static native boolean isSigIgnored(int sig); +} diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/SignalUtils.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/SignalUtils.c new file mode 100644 index 0000000000000..545006707fd76 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/SignalUtils.c @@ -0,0 +1,12 @@ +#include +#include + +JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_SignalUtils_isSigIgnored + (JNIEnv *env, jclass clazz, jint sig) { + struct sigaction oact; + sigaction(sig, (struct sigaction*)NULL, &oact); + if (oact.sa_sigaction == ((void *) SIG_IGN)) + return JNI_TRUE; + else + return JNI_FALSE; +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java index c8cdd980c6786..3151a08a5d7f3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java @@ -22,9 +22,15 @@ import org.apache.hadoop.service.launcher.testservices.FailureTestService; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.NativeCodeLoader; +import org.apache.hadoop.util.SignalUtils; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sun.misc.Signal; +import sun.misc.SignalHandler; + +import static org.junit.Assume.assumeTrue; /** * Test service launcher interrupt handling. @@ -37,12 +43,15 @@ public class TestServiceInterruptHandling @Test public void testRegisterAndRaise() throws Throwable { + assertTrue("Native must be loaded", SignalUtils.nativeCodeLoaded); InterruptCatcher catcher = new InterruptCatcher(); + // we should use "USR2". If USR2 is ignored, we still override new signal hanlder. But SIGINT can not. + // See: https://github.com/openjdk/jdk/blob/48ad07fd2cacdfcde606b33a369b1bf8df592088/hotspot/src/os/linux/vm/jvm_linux.cpp#L100 String name = IrqHandler.CONTROL_C; + assertFalse("SIGINT have been ignored, so can not set signal handler.", SignalUtils.isSigIgnored(2)); // SIGINT is 2 IrqHandler irqHandler = new IrqHandler(name, catcher); irqHandler.bind(); assertEquals(0, irqHandler.getSignalCount()); - Thread.sleep(3000); irqHandler.raise(); // allow for an async event GenericTestUtils.waitFor(() -> catcher.interruptData != null, 100, 100000); From ee5b7304ccd956b73e1c2d8f5f2c42379ef7c91c Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Tue, 20 Aug 2024 17:49:08 +0800 Subject: [PATCH 7/7] update doc --- .../hadoop/service/launcher/TestServiceInterruptHandling.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java index 3151a08a5d7f3..f6755f6041f64 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/TestServiceInterruptHandling.java @@ -46,7 +46,7 @@ public void testRegisterAndRaise() throws Throwable { assertTrue("Native must be loaded", SignalUtils.nativeCodeLoaded); InterruptCatcher catcher = new InterruptCatcher(); // we should use "USR2". If USR2 is ignored, we still override new signal hanlder. But SIGINT can not. - // See: https://github.com/openjdk/jdk/blob/48ad07fd2cacdfcde606b33a369b1bf8df592088/hotspot/src/os/linux/vm/jvm_linux.cpp#L100 + // See: https://github.com/openjdk/jdk/blob/48ad07fd2cacdfcde606b33a369b1bf8df592088/hotspot/src/os/linux/vm/jvm_linux.cpp#L76 String name = IrqHandler.CONTROL_C; assertFalse("SIGINT have been ignored, so can not set signal handler.", SignalUtils.isSigIgnored(2)); // SIGINT is 2 IrqHandler irqHandler = new IrqHandler(name, catcher);