From 4f3dbc20b01f7c0164ca3c9256f10205011cf3c4 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 2 Jun 2021 19:55:35 +0000 Subject: [PATCH 01/67] Remove readBytes and update testing. --- diskqueue.go | 9 ++++++--- diskqueue_test.go | 40 +++++++++++++++++++++------------------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 3aab8e5..6b8149c 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -302,9 +302,12 @@ func (d *diskQueue) skipToNextRWFile() error { d.nextReadFileNum = d.writeFileNum d.nextReadPos = 0 d.depth = 0 - d.readMessages = 0 - d.writeMessages = 0 - d.writeBytes = 0 + + if d.diskLimitFeatIsOn { + d.writeBytes = 0 + d.readMessages = 0 + d.writeMessages = 0 + } return err } diff --git a/diskqueue_test.go b/diskqueue_test.go index 8d4f78f..26af104 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -363,10 +363,10 @@ func TestDiskQueueSyncAfterReadWithDiskSizeImplementation(t *testing.T) { d.writeBytes == 1004 && d.readFileNum == 0 && d.writeFileNum == 0 && - d.readPos == 0 && - d.writePos == 1004 && d.readMessages == 0 && - d.writeMessages == 1 { + d.writeMessages == 1 && + d.readPos == 0 && + d.writePos == 1004 { // success goto next } @@ -384,10 +384,10 @@ next: d.writeBytes == 2008 && d.readFileNum == 0 && d.writeFileNum == 0 && - d.readPos == 1004 && - d.writePos == 2008 && d.readMessages == 1 && - d.writeMessages == 2 { + d.writeMessages == 2 && + d.readPos == 1004 && + d.writePos == 2008 { // success goto completeWriteFile } @@ -409,13 +409,13 @@ completeWriteFile: // test the writeFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 3 && - d.writeBytes == 3020 && + d.writeBytes == 2048 && d.readFileNum == 0 && d.writeFileNum == 1 && - d.readPos == 1004 && - d.writePos == 0 && d.readMessages == 1 && - d.writeMessages == 0 { + d.writeMessages == 0 && + d.readPos == 1004 && + d.writePos == 0 { // success goto completeReadFile } @@ -439,10 +439,10 @@ completeReadFile: d.writeBytes == 1004 && d.readFileNum == 1 && d.writeFileNum == 1 && - d.readPos == 0 && - d.writePos == 1004 && d.readMessages == 0 && - d.writeMessages == 1 { + d.writeMessages == 1 && + d.readPos == 0 && + d.writePos == 1004 { // success goto completeWriteFileAgain } @@ -467,12 +467,13 @@ completeWriteFileAgain: // test the writeFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 7 && + d.writeBytes == 5068 && d.readFileNum == 1 && d.writeFileNum == 3 && - d.readPos == 0 && - d.writePos == 0 && d.readMessages == 0 && - d.writeMessages == 0 { + d.writeMessages == 0 && + d.readPos == 0 && + d.writePos == 0 { // success goto completeReadFileAgain } @@ -495,12 +496,13 @@ completeReadFileAgain: // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 0 && + d.writeBytes == 0 && d.readFileNum == 3 && d.writeFileNum == 3 && - d.readPos == 0 && - d.writePos == 0 && d.readMessages == 0 && - d.writeMessages == 0 { + d.writeMessages == 0 && + d.readPos == 0 && + d.writePos == 0 { // success goto done } From 0ab12757ff20da364438b3d3b577cb87e5135bd2 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 2 Jun 2021 20:47:12 +0000 Subject: [PATCH 02/67] Add comment for readMsgSize. --- diskqueue.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 6b8149c..2cb4069 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -87,6 +87,10 @@ type diskQueue struct { nextReadPos int64 nextReadFileNum int64 + // keep track of the msg size we have read + // (but not yet sent over readChan) + readMsgSize int32 + readFile *os.File writeFile *os.File reader *bufio.Reader @@ -108,9 +112,6 @@ type diskQueue struct { // disk limit implementation flag diskLimitFeatIsOn bool - - // the size of the - readMsgSize int32 } // New instantiates an instance of diskQueue, retrieving metadata From 844e43bf80fb84bffb876423bcaeb8879f207b60 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 3 Jun 2021 15:00:43 +0000 Subject: [PATCH 03/67] Modify disk size limit features only if disk size limit feature is being used . --- diskqueue.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 2cb4069..8212306 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -649,11 +649,13 @@ func (d *diskQueue) moveForward() { d.readFileNum = d.nextReadFileNum d.readPos = d.nextReadPos d.depth -= 1 - d.readMessages += 1 + + if d.diskLimitFeatIsOn { + d.readMessages += 1 + } // see if we need to clean up the old file if oldReadFileNum != d.nextReadFileNum { - d.readMessages = 0 // sync every time we start reading from a new file d.needSync = true @@ -664,7 +666,10 @@ func (d *diskQueue) moveForward() { d.logf(ERROR, "DISKQUEUE(%s) failed to Remove(%s) - %s", d.name, fn, err) } - d.writeBytes -= readFileLen + if d.diskLimitFeatIsOn { + d.readMessages = 0 + d.writeBytes -= readFileLen + } } d.checkTailCorruption(d.depth) From 5068de1d10cfdb27460602e5aadeb9019b01d8d1 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 3 Jun 2021 15:34:22 +0000 Subject: [PATCH 04/67] Increase disk size limit in testing, and get MetaData file size. --- diskqueue.go | 30 ++++++++++++++++++++++++++++++ diskqueue_test.go | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/diskqueue.go b/diskqueue.go index 8212306..c38493c 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -399,6 +399,31 @@ func (d *diskQueue) readOne() ([]byte, error) { return readBuf, nil } +func (d *diskQueue) metaDataFileSize() int64 { + var err error + var metaDataFile *os.File + + // get the MetaData file size + metaDataFile, err = os.OpenFile(d.metaDataFileName(), os.O_RDONLY, 0600) + + var metaDataFileSize int64 + if err == nil { + var stat os.FileInfo + + stat, err = metaDataFile.Stat() + if err == nil { + metaDataFileSize = stat.Size() + } + } + if err != nil { + // use max file size (8 int64 fields) + metaDataFileSize = 64 + err = nil + } + + return metaDataFileSize +} + // writeOne performs a low level filesystem write for a single []byte // while advancing write positions and rolling files, if necessary func (d *diskQueue) writeOne(data []byte) error { @@ -429,6 +454,11 @@ func (d *diskQueue) writeOne(data []byte) error { return fmt.Errorf("invalid message write size (%d) minMsgSize=%d maxMsgSize=%d", dataLen, d.minMsgSize, d.maxMsgSize) } + // check if we have enough space to write this message + if d.diskLimitFeatIsOn { + var metaDataFileSize := d.metaDataFileSize() + } + // add all data to writeBuf before writing to file // this causes everything to be written to file or nothing d.writeBuf.Reset() diff --git a/diskqueue_test.go b/diskqueue_test.go index 26af104..4155768 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -350,7 +350,7 @@ func TestDiskQueueSyncAfterReadWithDiskSizeImplementation(t *testing.T) { panic(err) } defer os.RemoveAll(tmpDir) - dq := NewWithDiskSpace(dqName, tmpDir, 1<<11, 1<<11, 0, 1<<10, 2500, 50*time.Millisecond, l) + dq := NewWithDiskSpace(dqName, tmpDir, 6000, 1<<11, 0, 1<<10, 2500, 50*time.Millisecond, l) defer dq.Close() msgSize := 1000 From 42686e9206d621fb9afe4edc2f151f3e5b3ae86f Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 3 Jun 2021 15:36:14 +0000 Subject: [PATCH 05/67] Add comments to metaDataFileSize func for better readability. --- diskqueue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diskqueue.go b/diskqueue.go index c38493c..f4c769d 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -399,11 +399,11 @@ func (d *diskQueue) readOne() ([]byte, error) { return readBuf, nil } +// get the size of the metaData file or its max possible size func (d *diskQueue) metaDataFileSize() int64 { var err error var metaDataFile *os.File - // get the MetaData file size metaDataFile, err = os.OpenFile(d.metaDataFileName(), os.O_RDONLY, 0600) var metaDataFileSize int64 From 5093a20e2e70ccb380654e35d81346694260c2b5 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 3 Jun 2021 20:26:50 +0000 Subject: [PATCH 06/67] Add logic to remove readFile if we are going to surpass the Disk Size Limit. --- diskqueue.go | 79 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 13 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index f4c769d..d677b70 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -19,11 +19,12 @@ import ( type LogLevel int const ( - DEBUG = LogLevel(1) - INFO = LogLevel(2) - WARN = LogLevel(3) - ERROR = LogLevel(4) - FATAL = LogLevel(5) + DEBUG = LogLevel(1) + INFO = LogLevel(2) + WARN = LogLevel(3) + ERROR = LogLevel(4) + FATAL = LogLevel(5) + numFileMsgBytes = 8 ) type AppLogFunc func(lvl LogLevel, f string, args ...interface{}) @@ -345,7 +346,7 @@ func (d *diskQueue) readOne() ([]byte, error) { d.maxBytesPerFileRead = stat.Size() if d.diskLimitFeatIsOn { // last 8 bytes are reserved for the number of messages in this file - d.maxBytesPerFileRead -= 8 + d.maxBytesPerFileRead -= numFileMsgBytes } } } @@ -413,12 +414,11 @@ func (d *diskQueue) metaDataFileSize() int64 { stat, err = metaDataFile.Stat() if err == nil { metaDataFileSize = stat.Size() - } + } } if err != nil { // use max file size (8 int64 fields) metaDataFileSize = 64 - err = nil } return metaDataFileSize @@ -455,8 +455,61 @@ func (d *diskQueue) writeOne(data []byte) error { } // check if we have enough space to write this message - if d.diskLimitFeatIsOn { - var metaDataFileSize := d.metaDataFileSize() + if d.diskLimitFeatIsOn && d.writeBytes+d.metaDataFileSize() > d.maxBytesDiskSpace { + // check a .bad file exists if it does, delete that first + + // else delete the read file (makeSpace) + if d.readFileNum == d.writeFileNum { + d.skipToNextRWFile() + } else { + if d.readFile == nil { + curFileName := d.fileName(d.readFileNum) + d.readFile, err = os.OpenFile(curFileName, os.O_RDONLY, 0600) + if err != nil { + return err + } + } + + // get the total messages + _, err = d.readFile.Seek(numFileMsgBytes, 2) + if err != nil { + d.readFile.Close() + d.readFile = nil + return err + } + + var totalMessages int64 + err = binary.Read(d.reader, binary.BigEndian, &totalMessages) + if err != nil { + d.readFile.Close() + d.readFile = nil + return err + } + + // update depth with the remaining number of messages + // moveForward() decrements depth, so counteract that with a +1 + d.depth -= totalMessages - d.readMessages + 1 + + // get the size of the file + stat, err := d.readFile.Stat() + if err != nil { + d.readFile.Close() + d.readFile = nil + return err + } + readFileSize := stat.Size() + + // subtract 12 since moveForward() adds 12 + d.readPos = readFileSize - 12 + d.readMsgSize = 0 + + if d.readFileNum == d.nextReadFileNum { + d.nextReadFileNum++ + d.nextReadPos = 0 + } + + d.moveForward() + } } // add all data to writeBuf before writing to file @@ -475,7 +528,7 @@ func (d *diskQueue) writeOne(data []byte) error { totalBytes := int64(4 + dataLen) // check if we reached the file size limit with this message - if d.diskLimitFeatIsOn && d.writePos+totalBytes+8 >= d.maxBytesPerFile { + if d.diskLimitFeatIsOn && d.writePos+totalBytes+numFileMsgBytes >= d.maxBytesPerFile { // write number of messages in binary to file err = binary.Write(&d.writeBuf, binary.BigEndian, d.writeMessages+1) if err != nil { @@ -498,7 +551,7 @@ func (d *diskQueue) writeOne(data []byte) error { if d.diskLimitFeatIsOn { // save space for the number of messages in this file - fileSize += 8 + fileSize += numFileMsgBytes d.writeBytes += totalBytes d.writeMessages += 1 } @@ -513,7 +566,7 @@ func (d *diskQueue) writeOne(data []byte) error { if d.diskLimitFeatIsOn { // add bytes for the number of messages in the file - d.writeBytes += 8 + d.writeBytes += numFileMsgBytes d.writeMessages = 0 } From 425d8d360642e56203a8f3d9835e1b82632c5d2c Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 4 Jun 2021 15:29:57 +0000 Subject: [PATCH 07/67] Abstract code from moveForward and create makeSpace function to be used when DiskQueue needs to make space. --- diskqueue.go | 140 ++++++++++++++++++++++++++++----------------------- 1 file changed, 76 insertions(+), 64 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index d677b70..1aa8cf5 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -424,6 +424,66 @@ func (d *diskQueue) metaDataFileSize() int64 { return metaDataFileSize } +func (d *diskQueue) makeSpace() error { + var err error + + // check a .bad file exists if it does, delete that first + + // else delete the read file (makeSpace) + if d.readFileNum == d.writeFileNum { + d.skipToNextRWFile() + } else { + if d.readFile == nil { + curFileName := d.fileName(d.readFileNum) + d.readFile, err = os.OpenFile(curFileName, os.O_RDONLY, 0600) + if err != nil { + return err + } + } + + // read total messages number at the end of the file + _, err = d.readFile.Seek(numFileMsgBytes, 2) + if err != nil { + d.readFile.Close() + d.readFile = nil + return err + } + + var totalMessages int64 + err = binary.Read(d.reader, binary.BigEndian, &totalMessages) + if err != nil { + d.readFile.Close() + d.readFile = nil + return err + } + + // update depth with the remaining number of messages + d.depth -= totalMessages - d.readMessages + + // get the size of the file + stat, err := d.readFile.Stat() + if err != nil { + d.readFile.Close() + d.readFile = nil + return err + } + readFileSize := stat.Size() + + d.readFile.Close() + d.readFile = nil + + // we have not finished reading this file + if d.readFileNum == d.nextReadFileNum { + d.nextReadFileNum++ + d.nextReadPos = 0 + } + + d.moveToNextReadFile(readFileSize) + } + + return nil +} + // writeOne performs a low level filesystem write for a single []byte // while advancing write positions and rolling files, if necessary func (d *diskQueue) writeOne(data []byte) error { @@ -455,61 +515,8 @@ func (d *diskQueue) writeOne(data []byte) error { } // check if we have enough space to write this message - if d.diskLimitFeatIsOn && d.writeBytes+d.metaDataFileSize() > d.maxBytesDiskSpace { - // check a .bad file exists if it does, delete that first - - // else delete the read file (makeSpace) - if d.readFileNum == d.writeFileNum { - d.skipToNextRWFile() - } else { - if d.readFile == nil { - curFileName := d.fileName(d.readFileNum) - d.readFile, err = os.OpenFile(curFileName, os.O_RDONLY, 0600) - if err != nil { - return err - } - } - - // get the total messages - _, err = d.readFile.Seek(numFileMsgBytes, 2) - if err != nil { - d.readFile.Close() - d.readFile = nil - return err - } - - var totalMessages int64 - err = binary.Read(d.reader, binary.BigEndian, &totalMessages) - if err != nil { - d.readFile.Close() - d.readFile = nil - return err - } - - // update depth with the remaining number of messages - // moveForward() decrements depth, so counteract that with a +1 - d.depth -= totalMessages - d.readMessages + 1 - - // get the size of the file - stat, err := d.readFile.Stat() - if err != nil { - d.readFile.Close() - d.readFile = nil - return err - } - readFileSize := stat.Size() - - // subtract 12 since moveForward() adds 12 - d.readPos = readFileSize - 12 - d.readMsgSize = 0 - - if d.readFileNum == d.nextReadFileNum { - d.nextReadFileNum++ - d.nextReadPos = 0 - } - - d.moveForward() - } + for d.diskLimitFeatIsOn && d.writeBytes+d.metaDataFileSize() > d.maxBytesDiskSpace { + err = d.makeSpace() } // add all data to writeBuf before writing to file @@ -725,17 +732,10 @@ func (d *diskQueue) checkTailCorruption(depth int64) { } } -func (d *diskQueue) moveForward() { - // add bytes for the number of messages and the size of the message - readFileLen := int64(d.readMsgSize) + d.readPos + 12 +func (d *diskQueue) moveToNextReadFile(readFileSize int64) { oldReadFileNum := d.readFileNum d.readFileNum = d.nextReadFileNum d.readPos = d.nextReadPos - d.depth -= 1 - - if d.diskLimitFeatIsOn { - d.readMessages += 1 - } // see if we need to clean up the old file if oldReadFileNum != d.nextReadFileNum { @@ -751,9 +751,21 @@ func (d *diskQueue) moveForward() { if d.diskLimitFeatIsOn { d.readMessages = 0 - d.writeBytes -= readFileLen + d.writeBytes -= readFileSize } } +} + +func (d *diskQueue) moveForward() { + // add bytes for the number of messages and the size of the message + readFileSize := int64(d.readMsgSize) + d.readPos + 12 + d.depth -= 1 + + if d.diskLimitFeatIsOn { + d.readMessages += 1 + } + + d.moveToNextReadFile(readFileSize) d.checkTailCorruption(d.depth) } From 43806396b5a3d9efb41212fa4b9b70de6993034a Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 4 Jun 2021 15:30:53 +0000 Subject: [PATCH 08/67] Rename func to make it more readable. --- diskqueue.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 1aa8cf5..f563181 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -424,7 +424,7 @@ func (d *diskQueue) metaDataFileSize() int64 { return metaDataFileSize } -func (d *diskQueue) makeSpace() error { +func (d *diskQueue) freeUpDiskSpace() error { var err error // check a .bad file exists if it does, delete that first @@ -516,7 +516,7 @@ func (d *diskQueue) writeOne(data []byte) error { // check if we have enough space to write this message for d.diskLimitFeatIsOn && d.writeBytes+d.metaDataFileSize() > d.maxBytesDiskSpace { - err = d.makeSpace() + err = d.freeUpDiskSpace() } // add all data to writeBuf before writing to file From aaba003e0fa56e58fbfc7ba82aa313fd987c0b6e Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 4 Jun 2021 18:02:43 +0000 Subject: [PATCH 09/67] Make space when the new writeMessage will surpass the disk size limit. Test that DiskQueue never surpasses the disk size limit. --- diskqueue.go | 9 ++++++-- diskqueue_test.go | 57 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index f563181..c4649a9 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -442,7 +442,7 @@ func (d *diskQueue) freeUpDiskSpace() error { } // read total messages number at the end of the file - _, err = d.readFile.Seek(numFileMsgBytes, 2) + _, err = d.readFile.Seek(-numFileMsgBytes, 2) if err != nil { d.readFile.Close() d.readFile = nil @@ -515,8 +515,13 @@ func (d *diskQueue) writeOne(data []byte) error { } // check if we have enough space to write this message - for d.diskLimitFeatIsOn && d.writeBytes+d.metaDataFileSize() > d.maxBytesDiskSpace { + if d.diskLimitFeatIsOn && d.writeBytes+d.metaDataFileSize()+int64(4+dataLen)+numFileMsgBytes > d.maxBytesDiskSpace { err = d.freeUpDiskSpace() + if err != nil { + d.logf(ERROR, "Not able to free up space: %s", err) + } + + err = nil } // add all data to writeBuf before writing to file diff --git a/diskqueue_test.go b/diskqueue_test.go index 4155768..b213ef8 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -350,7 +350,7 @@ func TestDiskQueueSyncAfterReadWithDiskSizeImplementation(t *testing.T) { panic(err) } defer os.RemoveAll(tmpDir) - dq := NewWithDiskSpace(dqName, tmpDir, 6000, 1<<11, 0, 1<<10, 2500, 50*time.Millisecond, l) + dq := NewWithDiskSpace(dqName, tmpDir, 7000, 1<<11, 0, 1<<10, 2500, 50*time.Millisecond, l) defer dq.Close() msgSize := 1000 @@ -504,6 +504,61 @@ completeReadFileAgain: d.readPos == 0 && d.writePos == 0 { // success + goto meetDiskSizeLimit + } + time.Sleep(100 * time.Millisecond) + } + panic("fail") + +meetDiskSizeLimit: + // write a complete file + dq.Put(msg) + dq.Put(msg) + dq.Put(msg) + + // meet the file size limit exactly (2048 bytes) when writeFileNum + // is ahead of readFileNum + dq.Put(msg) + dq.Put(msg) + dq.Put(msg) + + for i := 0; i < 10; i++ { + // test that read position and messages reset when a file is completely read + // test the readFileNum correctly increments + d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) + if d.depth == 6 && + d.writeBytes == 6040 && + d.readFileNum == 3 && + d.writeFileNum == 5 && + d.readMessages == 0 && + d.writeMessages == 0 && + d.readPos == 0 && + d.writePos == 0 { + // success + goto surpassDiskSizeLimit + } + time.Sleep(100 * time.Millisecond) + } + panic("fail") + +surpassDiskSizeLimit: + t.Log("Start") + dq.Put(msg) + t.Log("Msg put") + + for i := 0; i < 10; i++ { + // test that read position and messages reset when a file is completely read + // test the readFileNum correctly increments + d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) + if d.depth == 4 && + d.writeBytes == 4024 && + d.readFileNum == 4 && + d.writeFileNum == 5 && + d.readMessages == 0 && + d.writeMessages == 1 && + d.readPos == 0 && + d.writePos == 1004 { + // success goto done } time.Sleep(100 * time.Millisecond) From 9f86e2869355e166bd54230ea469f100bfe6296c Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 4 Jun 2021 18:24:09 +0000 Subject: [PATCH 10/67] Make space until there is enough space to write the message --- diskqueue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diskqueue.go b/diskqueue.go index c4649a9..a6dcea3 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -515,7 +515,7 @@ func (d *diskQueue) writeOne(data []byte) error { } // check if we have enough space to write this message - if d.diskLimitFeatIsOn && d.writeBytes+d.metaDataFileSize()+int64(4+dataLen)+numFileMsgBytes > d.maxBytesDiskSpace { + for d.diskLimitFeatIsOn && d.writeBytes+d.metaDataFileSize()+int64(4+dataLen)+numFileMsgBytes > d.maxBytesDiskSpace { err = d.freeUpDiskSpace() if err != nil { d.logf(ERROR, "Not able to free up space: %s", err) From 24a5f6801703e22ee2e3eb084cb1202ea854b9b1 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 4 Jun 2021 20:02:30 +0000 Subject: [PATCH 11/67] Abstract code and make a reachFileSizeLimit flag. --- diskqueue.go | 134 +++++++++++++++++++++++++++++---------------------- 1 file changed, 77 insertions(+), 57 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index a6dcea3..f581eae 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -424,6 +424,56 @@ func (d *diskQueue) metaDataFileSize() int64 { return metaDataFileSize } +func (d *diskQueue) removeReadFile() error { + var err error + + if d.readFile == nil { + curFileName := d.fileName(d.readFileNum) + d.readFile, err = os.OpenFile(curFileName, os.O_RDONLY, 0600) + if err != nil { + return err + } + } + + closeReadFile := func() { + d.readFile.Close() + d.readFile = nil + } + defer closeReadFile() + + // read total messages number at the end of the file + _, err = d.readFile.Seek(-numFileMsgBytes, 2) + if err != nil { + return err + } + + var totalMessages int64 + err = binary.Read(d.reader, binary.BigEndian, &totalMessages) + if err != nil { + return err + } + + // update depth with the remaining number of messages + d.depth -= totalMessages - d.readMessages + + // get the size of the file + stat, err := d.readFile.Stat() + if err != nil { + return err + } + readFileSize := stat.Size() + + // we have not finished reading this file + if d.readFileNum == d.nextReadFileNum { + d.nextReadFileNum++ + d.nextReadPos = 0 + } + + d.moveToNextReadFile(readFileSize) + + return nil +} + func (d *diskQueue) freeUpDiskSpace() error { var err error @@ -433,52 +483,13 @@ func (d *diskQueue) freeUpDiskSpace() error { if d.readFileNum == d.writeFileNum { d.skipToNextRWFile() } else { - if d.readFile == nil { - curFileName := d.fileName(d.readFileNum) - d.readFile, err = os.OpenFile(curFileName, os.O_RDONLY, 0600) - if err != nil { - return err - } - } - - // read total messages number at the end of the file - _, err = d.readFile.Seek(-numFileMsgBytes, 2) - if err != nil { - d.readFile.Close() - d.readFile = nil - return err - } - - var totalMessages int64 - err = binary.Read(d.reader, binary.BigEndian, &totalMessages) + err = d.removeReadFile() if err != nil { - d.readFile.Close() - d.readFile = nil - return err - } - - // update depth with the remaining number of messages - d.depth -= totalMessages - d.readMessages - - // get the size of the file - stat, err := d.readFile.Stat() - if err != nil { - d.readFile.Close() - d.readFile = nil + d.logf(ERROR, "DISKQUEUE(%s) not able to delete file(%s) - %s", + d.name, d.fileName(d.readFileNum), err) + d.handleReadError() return err } - readFileSize := stat.Size() - - d.readFile.Close() - d.readFile = nil - - // we have not finished reading this file - if d.readFileNum == d.nextReadFileNum { - d.nextReadFileNum++ - d.nextReadPos = 0 - } - - d.moveToNextReadFile(readFileSize) } return nil @@ -514,14 +525,29 @@ func (d *diskQueue) writeOne(data []byte) error { return fmt.Errorf("invalid message write size (%d) minMsgSize=%d maxMsgSize=%d", dataLen, d.minMsgSize, d.maxMsgSize) } - // check if we have enough space to write this message - for d.diskLimitFeatIsOn && d.writeBytes+d.metaDataFileSize()+int64(4+dataLen)+numFileMsgBytes > d.maxBytesDiskSpace { - err = d.freeUpDiskSpace() - if err != nil { - d.logf(ERROR, "Not able to free up space: %s", err) + totalBytes := int64(4 + dataLen) + reachedFileSizeLimit := false + + if d.diskLimitFeatIsOn { + // check if we have enough space to write this message + metaDataFileSize := d.metaDataFileSize() + + // check if we will reach or surpass file size limit + if d.writePos+totalBytes+numFileMsgBytes >= d.maxBytesPerFile { + reachedFileSizeLimit = true + } + + expectedBytesIncrease := totalBytes + if reachedFileSizeLimit { + expectedBytesIncrease += numFileMsgBytes } - err = nil + // keep freeing up disk space until we have enough space to write this message + for metaDataFileSize+d.writeBytes+expectedBytesIncrease > d.maxBytesDiskSpace { + d.freeUpDiskSpace() + } + } else if d.writePos+totalBytes >= d.maxBytesPerFile { + reachedFileSizeLimit = true } // add all data to writeBuf before writing to file @@ -537,10 +563,8 @@ func (d *diskQueue) writeOne(data []byte) error { return err } - totalBytes := int64(4 + dataLen) - // check if we reached the file size limit with this message - if d.diskLimitFeatIsOn && d.writePos+totalBytes+numFileMsgBytes >= d.maxBytesPerFile { + if d.diskLimitFeatIsOn && reachedFileSizeLimit { // write number of messages in binary to file err = binary.Write(&d.writeBuf, binary.BigEndian, d.writeMessages+1) if err != nil { @@ -559,16 +583,12 @@ func (d *diskQueue) writeOne(data []byte) error { d.writePos += totalBytes d.depth += 1 - fileSize := d.writePos - if d.diskLimitFeatIsOn { - // save space for the number of messages in this file - fileSize += numFileMsgBytes d.writeBytes += totalBytes d.writeMessages += 1 } - if fileSize >= d.maxBytesPerFile { + if reachedFileSizeLimit { if d.readFileNum == d.writeFileNum { d.maxBytesPerFileRead = d.writePos } From 60415d70f68df9127f633b40d360b39042b9161d Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 4 Jun 2021 21:08:49 +0000 Subject: [PATCH 12/67] Update testing to account for metadata file size. --- diskqueue_test.go | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/diskqueue_test.go b/diskqueue_test.go index b213ef8..6f27e34 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -350,7 +350,7 @@ func TestDiskQueueSyncAfterReadWithDiskSizeImplementation(t *testing.T) { panic(err) } defer os.RemoveAll(tmpDir) - dq := NewWithDiskSpace(dqName, tmpDir, 7000, 1<<11, 0, 1<<10, 2500, 50*time.Millisecond, l) + dq := NewWithDiskSpace(dqName, tmpDir, 6040, 1<<11, 0, 1<<10, 2500, 50*time.Millisecond, l) defer dq.Close() msgSize := 1000 @@ -434,7 +434,6 @@ completeReadFile: // test that read position and messages reset when a file is completely read // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) - t.Logf("Write bytes: %d", d.writeBytes) if d.depth == 1 && d.writeBytes == 1004 && d.readFileNum == 1 && @@ -516,18 +515,34 @@ meetDiskSizeLimit: dq.Put(msg) dq.Put(msg) - // meet the file size limit exactly (2048 bytes) when writeFileNum + // meet the disk size limit exactly (6040 bytes) when writeFileNum // is ahead of readFileNum dq.Put(msg) dq.Put(msg) - dq.Put(msg) + + totalDiskBytes := int64(5*(msgSize+4) + 8) + + metaDataFile, err := os.OpenFile(dq.(*diskQueue).metaDataFileName(), os.O_RDONLY, 0600) + + var metaDataFileSize int64 + if err == nil { + var stat os.FileInfo + + stat, err = metaDataFile.Stat() + if err == nil { + metaDataFileSize = stat.Size() + } + } + + diskBytesRemaining := 6040 - metaDataFileSize - (totalDiskBytes + 12) + dq.Put(make([]byte, diskBytesRemaining)) for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 6 && - d.writeBytes == 6040 && + d.writeBytes == 6040-metaDataFileSize && d.readFileNum == 3 && d.writeFileNum == 5 && d.readMessages == 0 && @@ -542,22 +557,20 @@ meetDiskSizeLimit: panic("fail") surpassDiskSizeLimit: - t.Log("Start") - dq.Put(msg) - t.Log("Msg put") + dq.Put(make([]byte, 1)) for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 4 && - d.writeBytes == 4024 && + d.writeBytes == 3025-metaDataFileSize && d.readFileNum == 4 && d.writeFileNum == 5 && d.readMessages == 0 && d.writeMessages == 1 && d.readPos == 0 && - d.writePos == 1004 { + d.writePos == 5 { // success goto done } From c39849fed709c5133e2b18733acc0de87df5ffbe Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Mon, 7 Jun 2021 20:36:26 +0000 Subject: [PATCH 13/67] Handle the deletion of .bad files if it exists. --- diskqueue.go | 68 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index f581eae..562cd03 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -7,9 +7,11 @@ import ( "errors" "fmt" "io" + "io/fs" "math/rand" "os" "path" + "path/filepath" "sync" "time" ) @@ -474,21 +476,67 @@ func (d *diskQueue) removeReadFile() error { return nil } +func (d *diskQueue) getOldestBadFileInfo() (fs.FileInfo, error) { + var oldestBadFileInfo fs.FileInfo + + getFirstBadFile := func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + // if the entry is a directory, skip it + return fs.SkipDir + } + + if err != nil { + return err + } + + var e error + + if filepath.Ext(d.Name()) == ".bad" { + oldestBadFileInfo, e = d.Info() + if e != nil { + oldestBadFileInfo = nil + } + } + + return nil + } + + err := filepath.WalkDir(d.dataPath, getFirstBadFile) + if err != nil { + return nil, err + } + + return oldestBadFileInfo, nil +} + func (d *diskQueue) freeUpDiskSpace() error { var err error + var oldestBadFileInfo fs.FileInfo - // check a .bad file exists if it does, delete that first + oldestBadFileInfo, err = d.getOldestBadFileInfo() - // else delete the read file (makeSpace) - if d.readFileNum == d.writeFileNum { - d.skipToNextRWFile() + // check if a .bad file exists and no error occurred. if it does, delete that first + if err == nil && oldestBadFileInfo != nil { + badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) + + err = os.Remove(badFileFilePath) + if err == nil { + d.writeBytes -= oldestBadFileInfo.Size() + } else { + d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) + } } else { - err = d.removeReadFile() - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) not able to delete file(%s) - %s", - d.name, d.fileName(d.readFileNum), err) - d.handleReadError() - return err + // delete the read file (makeSpace) + if d.readFileNum == d.writeFileNum { + d.skipToNextRWFile() + } else { + err = d.removeReadFile() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) not able to delete file(%s) - %s", + d.name, d.fileName(d.readFileNum), err) + d.handleReadError() + return err + } } } From c00921ed862f7924582d26274238add4dd109d36 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Mon, 7 Jun 2021 20:42:19 +0000 Subject: [PATCH 14/67] Get oldest bad file info does not need to return an error. It either finds a bad file or does not. --- diskqueue.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 562cd03..3c02024 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -476,7 +476,7 @@ func (d *diskQueue) removeReadFile() error { return nil } -func (d *diskQueue) getOldestBadFileInfo() (fs.FileInfo, error) { +func (d *diskQueue) getOldestBadFileInfo() fs.FileInfo { var oldestBadFileInfo fs.FileInfo getFirstBadFile := func(path string, d fs.DirEntry, err error) error { @@ -503,20 +503,19 @@ func (d *diskQueue) getOldestBadFileInfo() (fs.FileInfo, error) { err := filepath.WalkDir(d.dataPath, getFirstBadFile) if err != nil { - return nil, err + return nil } - return oldestBadFileInfo, nil + return oldestBadFileInfo } func (d *diskQueue) freeUpDiskSpace() error { var err error - var oldestBadFileInfo fs.FileInfo - oldestBadFileInfo, err = d.getOldestBadFileInfo() + oldestBadFileInfo := d.getOldestBadFileInfo() - // check if a .bad file exists and no error occurred. if it does, delete that first - if err == nil && oldestBadFileInfo != nil { + // check if a .bad file exists. If it does, delete that first + if oldestBadFileInfo != nil { badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) err = os.Remove(badFileFilePath) From 0709e65c8089129c69fa098822366f7452d286bd Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Mon, 7 Jun 2021 21:08:30 +0000 Subject: [PATCH 15/67] Track size of bad files. --- diskqueue.go | 55 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 3c02024..642a698 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -69,6 +69,7 @@ type diskQueue struct { writeMessages int64 writeBytes int64 depth int64 + badBytes int64 sync.RWMutex @@ -308,6 +309,7 @@ func (d *diskQueue) skipToNextRWFile() error { d.depth = 0 if d.diskLimitFeatIsOn { + d.badBytes = 0 d.writeBytes = 0 d.readMessages = 0 d.writeMessages = 0 @@ -419,8 +421,8 @@ func (d *diskQueue) metaDataFileSize() int64 { } } if err != nil { - // use max file size (8 int64 fields) - metaDataFileSize = 64 + // use max file size (9 int64 fields) + metaDataFileSize = 72 } return metaDataFileSize @@ -511,21 +513,27 @@ func (d *diskQueue) getOldestBadFileInfo() fs.FileInfo { func (d *diskQueue) freeUpDiskSpace() error { var err error + badFileExists := false - oldestBadFileInfo := d.getOldestBadFileInfo() + if d.badBytes > 0 { + oldestBadFileInfo := d.getOldestBadFileInfo() - // check if a .bad file exists. If it does, delete that first - if oldestBadFileInfo != nil { - badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) + // check if a .bad file exists. If it does, delete that first + if oldestBadFileInfo != nil { + badFileExists = true + badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) - err = os.Remove(badFileFilePath) - if err == nil { - d.writeBytes -= oldestBadFileInfo.Size() - } else { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) + err = os.Remove(badFileFilePath) + if err == nil { + d.writeBytes -= oldestBadFileInfo.Size() + } else { + d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) + } } - } else { - // delete the read file (makeSpace) + } + + if !badFileExists { + // delete the read file (make space) if d.readFileNum == d.writeFileNum { d.skipToNextRWFile() } else { @@ -580,7 +588,7 @@ func (d *diskQueue) writeOne(data []byte) error { metaDataFileSize := d.metaDataFileSize() // check if we will reach or surpass file size limit - if d.writePos+totalBytes+numFileMsgBytes >= d.maxBytesPerFile { + if d.badBytes+d.writePos+totalBytes+numFileMsgBytes >= d.maxBytesPerFile { reachedFileSizeLimit = true } @@ -698,10 +706,11 @@ func (d *diskQueue) retrieveMetaData() error { // if user is using disk space limit feature if d.diskLimitFeatIsOn { - _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", + _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n%d", &d.depth, &d.readFileNum, &d.readMessages, &d.readPos, - &d.writeBytes, &d.writeFileNum, &d.writeMessages, &d.writePos) + &d.writeBytes, &d.writeFileNum, &d.writeMessages, &d.writePos, + &d.badBytes) } else { _, err = fmt.Fscanf(f, "%d\n%d,%d\n%d,%d\n", &d.depth, @@ -735,10 +744,11 @@ func (d *diskQueue) persistMetaData() error { // if user is using disk space limit feature if d.diskLimitFeatIsOn { - _, err = fmt.Fprintf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", + _, err = fmt.Fprintf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n%d", d.depth, d.readFileNum, d.readMessages, d.readPos, - d.writeBytes, d.writeFileNum, d.writeMessages, d.writePos) + d.writeBytes, d.writeFileNum, d.writeMessages, d.writePos, + d.badBytes) } else { _, err = fmt.Fprintf(f, "%d\n%d,%d\n%d,%d\n", d.depth, @@ -869,6 +879,15 @@ func (d *diskQueue) handleReadError() { d.name, badFn, badRenameFn) } + if d.diskLimitFeatIsOn { + d.badBytes += d.maxBytesPerFileRead + if d.maxBytesPerFileRead == d.maxBytesPerFile { + // this could mean that we were not able to get the + // correct file size + d.badBytes += int64(d.maxMsgSize) + 4 + } + } + d.readFileNum++ d.readPos = 0 d.nextReadFileNum = d.readFileNum From 65854ca303e9310719a784cfef973a38185af0dc Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Tue, 8 Jun 2021 14:02:11 +0000 Subject: [PATCH 16/67] Decrease badBytes when we delete a bad file and move badBytes to check if we need to make disk space. --- diskqueue.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 642a698..7f5e676 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -526,6 +526,7 @@ func (d *diskQueue) freeUpDiskSpace() error { err = os.Remove(badFileFilePath) if err == nil { d.writeBytes -= oldestBadFileInfo.Size() + d.badBytes -= oldestBadFileInfo.Size() } else { d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) } @@ -588,7 +589,7 @@ func (d *diskQueue) writeOne(data []byte) error { metaDataFileSize := d.metaDataFileSize() // check if we will reach or surpass file size limit - if d.badBytes+d.writePos+totalBytes+numFileMsgBytes >= d.maxBytesPerFile { + if d.writePos+totalBytes+numFileMsgBytes >= d.maxBytesPerFile { reachedFileSizeLimit = true } @@ -598,7 +599,7 @@ func (d *diskQueue) writeOne(data []byte) error { } // keep freeing up disk space until we have enough space to write this message - for metaDataFileSize+d.writeBytes+expectedBytesIncrease > d.maxBytesDiskSpace { + for d.badBytes+metaDataFileSize+d.writeBytes+expectedBytesIncrease > d.maxBytesDiskSpace { d.freeUpDiskSpace() } } else if d.writePos+totalBytes >= d.maxBytesPerFile { From 2c879c8f178c0ac9292d30acc426d37435e9ad21 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Tue, 8 Jun 2021 15:48:45 +0000 Subject: [PATCH 17/67] Remove badBytes from metadata and make it an atomic field. --- diskqueue.go | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 7f5e676..8c8a720 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -175,6 +175,12 @@ func (d *diskQueue) start() { d.logf(ERROR, "DISKQUEUE(%s) failed to retrieveMetaData - %s", d.name, err) } + // get the size of all the bad files + badFileInfos := d.getAllBadFileInfo() + for _, badFileInfo := range badFileInfos { + d.badBytes += badFileInfo.Size() + } + go d.ioLoop() } @@ -309,7 +315,6 @@ func (d *diskQueue) skipToNextRWFile() error { d.depth = 0 if d.diskLimitFeatIsOn { - d.badBytes = 0 d.writeBytes = 0 d.readMessages = 0 d.writeMessages = 0 @@ -478,8 +483,8 @@ func (d *diskQueue) removeReadFile() error { return nil } -func (d *diskQueue) getOldestBadFileInfo() fs.FileInfo { - var oldestBadFileInfo fs.FileInfo +func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { + var badFileInfos []fs.FileInfo getFirstBadFile := func(path string, d fs.DirEntry, err error) error { if d.IsDir() { @@ -491,12 +496,10 @@ func (d *diskQueue) getOldestBadFileInfo() fs.FileInfo { return err } - var e error - if filepath.Ext(d.Name()) == ".bad" { - oldestBadFileInfo, e = d.Info() - if e != nil { - oldestBadFileInfo = nil + badFileInfo, e := d.Info() + if e != nil && badFileInfo != nil { + badFileInfos = append(badFileInfos, badFileInfo) } } @@ -508,7 +511,7 @@ func (d *diskQueue) getOldestBadFileInfo() fs.FileInfo { return nil } - return oldestBadFileInfo + return badFileInfos } func (d *diskQueue) freeUpDiskSpace() error { @@ -516,10 +519,11 @@ func (d *diskQueue) freeUpDiskSpace() error { badFileExists := false if d.badBytes > 0 { - oldestBadFileInfo := d.getOldestBadFileInfo() + badFileInfos := d.getAllBadFileInfo() // check if a .bad file exists. If it does, delete that first - if oldestBadFileInfo != nil { + if badFileInfos != nil { + oldestBadFileInfo := badFileInfos[0] badFileExists = true badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) @@ -707,11 +711,10 @@ func (d *diskQueue) retrieveMetaData() error { // if user is using disk space limit feature if d.diskLimitFeatIsOn { - _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n%d", + _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", &d.depth, &d.readFileNum, &d.readMessages, &d.readPos, - &d.writeBytes, &d.writeFileNum, &d.writeMessages, &d.writePos, - &d.badBytes) + &d.writeBytes, &d.writeFileNum, &d.writeMessages, &d.writePos) } else { _, err = fmt.Fscanf(f, "%d\n%d,%d\n%d,%d\n", &d.depth, @@ -745,11 +748,10 @@ func (d *diskQueue) persistMetaData() error { // if user is using disk space limit feature if d.diskLimitFeatIsOn { - _, err = fmt.Fprintf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n%d", + _, err = fmt.Fprintf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", d.depth, d.readFileNum, d.readMessages, d.readPos, - d.writeBytes, d.writeFileNum, d.writeMessages, d.writePos, - d.badBytes) + d.writeBytes, d.writeFileNum, d.writeMessages, d.writePos) } else { _, err = fmt.Fprintf(f, "%d\n%d,%d\n%d,%d\n", d.depth, From aedd5038e4f7da280e41f0b11d08c7f38cd9efe3 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Tue, 8 Jun 2021 15:57:48 +0000 Subject: [PATCH 18/67] Recalculate badBytes if it is a negative number. --- diskqueue.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 8c8a720..104c4f7 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -518,13 +518,22 @@ func (d *diskQueue) freeUpDiskSpace() error { var err error badFileExists := false - if d.badBytes > 0 { + if d.badBytes != 0 { badFileInfos := d.getAllBadFileInfo() + // if badBytes is negative, something went wrong, so recalculate total bad files size + if d.badBytes < 0 { + d.badBytes = 0 + for _, badFileInfo := range badFileInfos { + d.badBytes += badFileInfo.Size() + } + } + // check if a .bad file exists. If it does, delete that first if badFileInfos != nil { - oldestBadFileInfo := badFileInfos[0] badFileExists = true + + oldestBadFileInfo := badFileInfos[0] badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) err = os.Remove(badFileFilePath) @@ -534,6 +543,9 @@ func (d *diskQueue) freeUpDiskSpace() error { } else { d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) } + } else { + // there are no bad files + d.badBytes = 0 } } From 621b1bc16be4ebee5cf2cb28470c9be444f4b675 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Tue, 8 Jun 2021 16:08:33 +0000 Subject: [PATCH 19/67] Add comments and rename varaibles to improve readability. --- diskqueue.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 104c4f7..ba08b85 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -175,7 +175,7 @@ func (d *diskQueue) start() { d.logf(ERROR, "DISKQUEUE(%s) failed to retrieveMetaData - %s", d.name, err) } - // get the size of all the bad files + // get the size of all the .bad files badFileInfos := d.getAllBadFileInfo() for _, badFileInfo := range badFileInfos { d.badBytes += badFileInfo.Size() @@ -486,7 +486,7 @@ func (d *diskQueue) removeReadFile() error { func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { var badFileInfos []fs.FileInfo - getFirstBadFile := func(path string, d fs.DirEntry, err error) error { + getBadFileInfos := func(path string, d fs.DirEntry, err error) error { if d.IsDir() { // if the entry is a directory, skip it return fs.SkipDir @@ -506,7 +506,7 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { return nil } - err := filepath.WalkDir(d.dataPath, getFirstBadFile) + err := filepath.WalkDir(d.dataPath, getBadFileInfos) if err != nil { return nil } @@ -518,10 +518,11 @@ func (d *diskQueue) freeUpDiskSpace() error { var err error badFileExists := false + // delete .bad files before deleting non-corrupted files (i.e. readFile) if d.badBytes != 0 { badFileInfos := d.getAllBadFileInfo() - // if badBytes is negative, something went wrong, so recalculate total bad files size + // if badBytes is negative, something went wrong, so recalculate total .bad files size if d.badBytes < 0 { d.badBytes = 0 for _, badFileInfo := range badFileInfos { @@ -538,7 +539,6 @@ func (d *diskQueue) freeUpDiskSpace() error { err = os.Remove(badFileFilePath) if err == nil { - d.writeBytes -= oldestBadFileInfo.Size() d.badBytes -= oldestBadFileInfo.Size() } else { d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) From 9cde8b0270022b690d49b6ce99160ddd33b1b2c7 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Tue, 8 Jun 2021 17:19:17 +0000 Subject: [PATCH 20/67] Update metaData file size now that badBytes is removed from metaData file. --- diskqueue.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index ba08b85..3c7d4a5 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -69,7 +69,6 @@ type diskQueue struct { writeMessages int64 writeBytes int64 depth int64 - badBytes int64 sync.RWMutex @@ -85,6 +84,7 @@ type diskQueue struct { syncTimeout time.Duration // duration of time per fsync exitFlag int32 needSync bool + badBytes int64 // keeps track of the position where we have read // (but not yet sent over readChan) @@ -426,8 +426,8 @@ func (d *diskQueue) metaDataFileSize() int64 { } } if err != nil { - // use max file size (9 int64 fields) - metaDataFileSize = 72 + // use max file size (8 int64 fields) + metaDataFileSize = 64 } return metaDataFileSize From b215b85cfd2fb688a29f5928cebb4a77b9971f7c Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Tue, 8 Jun 2021 17:31:29 +0000 Subject: [PATCH 21/67] Prevent user from writing data that is bigger than disk size limit. --- diskqueue.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/diskqueue.go b/diskqueue.go index 3c7d4a5..16efe22 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -601,6 +601,11 @@ func (d *diskQueue) writeOne(data []byte) error { reachedFileSizeLimit := false if d.diskLimitFeatIsOn { + // If there the data to be written is bigger than the disk size limit, do not write + if totalBytes+8 > d.maxBytesDiskSpace { + return errors.New("Not enough disk space to write message") + } + // check if we have enough space to write this message metaDataFileSize := d.metaDataFileSize() From e2f76a8f348f21c1bfc2d24ed6f2df292c1b7d69 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Tue, 8 Jun 2021 17:49:23 +0000 Subject: [PATCH 22/67] Make the code more readable by changing the order of checks in writeOne(). --- diskqueue.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 16efe22..5633692 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -601,14 +601,6 @@ func (d *diskQueue) writeOne(data []byte) error { reachedFileSizeLimit := false if d.diskLimitFeatIsOn { - // If there the data to be written is bigger than the disk size limit, do not write - if totalBytes+8 > d.maxBytesDiskSpace { - return errors.New("Not enough disk space to write message") - } - - // check if we have enough space to write this message - metaDataFileSize := d.metaDataFileSize() - // check if we will reach or surpass file size limit if d.writePos+totalBytes+numFileMsgBytes >= d.maxBytesPerFile { reachedFileSizeLimit = true @@ -619,6 +611,14 @@ func (d *diskQueue) writeOne(data []byte) error { expectedBytesIncrease += numFileMsgBytes } + // If the data to be written is bigger than the disk size limit, do not write + if expectedBytesIncrease > d.maxBytesDiskSpace { + return errors.New("message size surpasses disk size limit") + } + + // check if we have enough space to write this message + metaDataFileSize := d.metaDataFileSize() + // keep freeing up disk space until we have enough space to write this message for d.badBytes+metaDataFileSize+d.writeBytes+expectedBytesIncrease > d.maxBytesDiskSpace { d.freeUpDiskSpace() From f76ab78d1d692bb1db557c157c189eddc0298aae Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Tue, 8 Jun 2021 20:58:31 +0000 Subject: [PATCH 23/67] Test the scenario when msgSize needs the deletion of several files. --- diskqueue.go | 21 +++++++++----- diskqueue_test.go | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 5633692..02f0ba7 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -438,10 +438,13 @@ func (d *diskQueue) removeReadFile() error { if d.readFile == nil { curFileName := d.fileName(d.readFileNum) + d.readFile, err = os.OpenFile(curFileName, os.O_RDONLY, 0600) if err != nil { return err } + + d.reader = bufio.NewReader(d.readFile) } closeReadFile := func() { @@ -556,8 +559,7 @@ func (d *diskQueue) freeUpDiskSpace() error { } else { err = d.removeReadFile() if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) not able to delete file(%s) - %s", - d.name, d.fileName(d.readFileNum), err) + d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(d.readFileNum), err) d.handleReadError() return err } @@ -900,12 +902,17 @@ func (d *diskQueue) handleReadError() { } if d.diskLimitFeatIsOn { - d.badBytes += d.maxBytesPerFileRead - if d.maxBytesPerFileRead == d.maxBytesPerFile { - // this could mean that we were not able to get the - // correct file size - d.badBytes += int64(d.maxMsgSize) + 4 + var stat os.FileInfo + stat, err = os.Stat(badRenameFn) + var badFileSize int64 + if err == nil { + badFileSize = stat.Size() + } else { + badFileSize = int64(d.maxMsgSize+d.maxMsgSize) + 4 } + + d.badBytes += badFileSize + d.writeBytes -= badFileSize } d.readFileNum++ diff --git a/diskqueue_test.go b/diskqueue_test.go index 6f27e34..57e32c3 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -581,6 +581,76 @@ surpassDiskSizeLimit: done: } +func TestDiskSizeImplementationMsgSizeGreaterThanFileSize(t *testing.T) { + // write three files + + l := NewTestLogger(t) + dqName := "test_disk_queue_read_after_sync" + strconv.Itoa(int(time.Now().Unix())) + tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano())) + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpDir) + dq := NewWithDiskSpace(dqName, tmpDir, 1<<12, 1<<10, 0, 1<<12, 2500, 50*time.Millisecond, l) + defer dq.Close() + + msgSize := 1000 + msg := make([]byte, msgSize) + + // file size: 1533 + dq.Put(msg) + dq.Put(make([]byte, 517)) + + // file size: 1032 + dq.Put(msg) + dq.Put(make([]byte, 16)) + + // file size: 1512 + dq.Put(make([]byte, 1500)) + + for i := 0; i < 10; i++ { + d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) + if d.depth == 5 && + d.writeBytes == 4077 && + d.readFileNum == 0 && + d.writeFileNum == 3 && + d.readMessages == 0 && + d.writeMessages == 0 && + d.readPos == 0 && + d.writePos == 0 { + // success + goto writeLargeMsg + } + time.Sleep(100 * time.Millisecond) + } + panic("fail") + +writeLargeMsg: + // Write a large message that causes the deletion of three files + dq.Put(make([]byte, 3000)) + + for i := 0; i < 10; i++ { + // test that read position and messages reset when a file is completely read + // test the readFileNum correctly increments + d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) + if d.depth == 1 && + d.writeBytes == 3012 && + d.readFileNum == 3 && + d.writeFileNum == 4 && + d.readMessages == 0 && + d.writeMessages == 0 && + d.readPos == 0 && + d.writePos == 0 { + // success + goto done + } + time.Sleep(100 * time.Millisecond) + } + panic("fail") + +done: +} + func TestDiskQueueTorture(t *testing.T) { var wg sync.WaitGroup From 02ebb4b5d66a3fc1704ba3de45be7bf7a8f0c18f Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 9 Jun 2021 14:31:13 +0000 Subject: [PATCH 24/67] Use regex when getting diskqueue files with .bad extension. --- diskqueue.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 02f0ba7..fcbfb21 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -12,6 +12,8 @@ import ( "os" "path" "path/filepath" + "regexp" + "strings" "sync" "time" ) @@ -489,7 +491,21 @@ func (d *diskQueue) removeReadFile() error { func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { var badFileInfos []fs.FileInfo + // the directory containing DiskQueue files + var mainDir string + if d.dataPath == "/" { + mainDir = d.dataPath + } else { + pathArray := strings.Split(d.dataPath, "/") + mainDir = pathArray[len(pathArray)-1] + } + getBadFileInfos := func(path string, d fs.DirEntry, err error) error { + if d.Name() == mainDir { + // we want to see the contents of this directory + return nil + } + if d.IsDir() { // if the entry is a directory, skip it return fs.SkipDir @@ -499,9 +515,10 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { return err } - if filepath.Ext(d.Name()) == ".bad" { + matched, _ := regexp.Match(`test_dq.diskqueue.\d\d\d\d\d\d.dat.bad`, []byte(d.Name())) + if matched { badFileInfo, e := d.Info() - if e != nil && badFileInfo != nil { + if e == nil && badFileInfo != nil { badFileInfos = append(badFileInfos, badFileInfo) } } From e7f85221af2e8dab2e70af26f5d5bca484f428e6 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 9 Jun 2021 16:23:56 +0000 Subject: [PATCH 25/67] Update the tracking of .bad files to use regex. --- diskqueue.go | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index fcbfb21..e15ef98 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -182,6 +182,7 @@ func (d *diskQueue) start() { for _, badFileInfo := range badFileInfos { d.badBytes += badFileInfo.Size() } + d.logf(DEBUG, "BadBytes: %d", d.badBytes) go d.ioLoop() } @@ -499,15 +500,18 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { pathArray := strings.Split(d.dataPath, "/") mainDir = pathArray[len(pathArray)-1] } + d.logf(DEBUG, "mainDir: %s", mainDir) - getBadFileInfos := func(path string, d fs.DirEntry, err error) error { - if d.Name() == mainDir { + getBadFileInfos := func(pathStr string, dirEntry fs.DirEntry, err error) error { + if dirEntry.Name() == mainDir { // we want to see the contents of this directory + d.logf(DEBUG, "We found main directory") return nil } - if d.IsDir() { + if dirEntry.IsDir() { // if the entry is a directory, skip it + d.logf(DEBUG, "Skipping dir: %s", dirEntry.Name()) return fs.SkipDir } @@ -515,9 +519,18 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { return err } - matched, _ := regexp.Match(`test_dq.diskqueue.\d\d\d\d\d\d.dat.bad`, []byte(d.Name())) + var matched bool + + regExp, err := regexp.Compile(d.name + `.diskqueue.\d\d\d\d\d\d.dat.bad`) + if err == nil { + d.logf(DEBUG, "Compile successful") + matched = regExp.MatchString(dirEntry.Name()) + } else { + matched, _ = regexp.Match(`.diskqueue.\d\d\d\d\d\d.dat.bad`, []byte(dirEntry.Name())) + } + d.logf(DEBUG, "%s, matched: %s. Name: %s", dirEntry.Name(), matched, d.name) if matched { - badFileInfo, e := d.Info() + badFileInfo, e := dirEntry.Info() if e == nil && badFileInfo != nil { badFileInfos = append(badFileInfos, badFileInfo) } @@ -531,6 +544,8 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { return nil } + d.logf(DEBUG, "%s", badFileInfos) + return badFileInfos } From 9792c990b142ce68cf538b6172ac5b35bd5794d4 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 9 Jun 2021 16:32:02 +0000 Subject: [PATCH 26/67] Remove log messages. --- diskqueue.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index e15ef98..ff57569 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -182,7 +182,6 @@ func (d *diskQueue) start() { for _, badFileInfo := range badFileInfos { d.badBytes += badFileInfo.Size() } - d.logf(DEBUG, "BadBytes: %d", d.badBytes) go d.ioLoop() } @@ -500,18 +499,15 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { pathArray := strings.Split(d.dataPath, "/") mainDir = pathArray[len(pathArray)-1] } - d.logf(DEBUG, "mainDir: %s", mainDir) getBadFileInfos := func(pathStr string, dirEntry fs.DirEntry, err error) error { if dirEntry.Name() == mainDir { // we want to see the contents of this directory - d.logf(DEBUG, "We found main directory") return nil } if dirEntry.IsDir() { // if the entry is a directory, skip it - d.logf(DEBUG, "Skipping dir: %s", dirEntry.Name()) return fs.SkipDir } @@ -523,12 +519,11 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { regExp, err := regexp.Compile(d.name + `.diskqueue.\d\d\d\d\d\d.dat.bad`) if err == nil { - d.logf(DEBUG, "Compile successful") matched = regExp.MatchString(dirEntry.Name()) } else { matched, _ = regexp.Match(`.diskqueue.\d\d\d\d\d\d.dat.bad`, []byte(dirEntry.Name())) } - d.logf(DEBUG, "%s, matched: %s. Name: %s", dirEntry.Name(), matched, d.name) + if matched { badFileInfo, e := dirEntry.Info() if e == nil && badFileInfo != nil { @@ -544,8 +539,6 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { return nil } - d.logf(DEBUG, "%s", badFileInfos) - return badFileInfos } From 2438eaf2ad6ff213209cc23719f6a6baaef48be0 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 9 Jun 2021 16:43:50 +0000 Subject: [PATCH 27/67] Start writing a test to check if .bad files are deleted or accounted for correctly. --- diskqueue_test.go | 74 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/diskqueue_test.go b/diskqueue_test.go index 57e32c3..09210de 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -651,6 +651,80 @@ writeLargeMsg: done: } +func createBadFile(dqName string, filePath string, fileNum int64, numBytes int) error { + fn := fmt.Sprintf(path.Join(filePath, "%s.diskqueue.%06d.dat.bad"), dqName, fileNum) + + badFile, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return err + } + + defer badFile.Close() + + _, err = badFile.Write(make([]byte, numBytes)) + + return err +} + +func TestDiskSizeImplementationWithBadFiles(t *testing.T) { + // write three files + + l := NewTestLogger(t) + dqName := "test_disk_queue_read_after_sync" + strconv.Itoa(int(time.Now().Unix())) + tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano())) + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpDir) + + // make 2 bad files + createBadFile(dqName, tmpDir, 0, 1533) + createBadFile(dqName, tmpDir, 1, 1032) + + dq := NewWithDiskSpace(dqName, tmpDir, 1<<12, 1<<10, 0, 1<<12, 2500, 50*time.Millisecond, l) + defer dq.Close() + + msgSize := 1000 + msg := make([]byte, msgSize) + + // file size: 1533 + dq.Put(msg) + dq.Put(make([]byte, 517)) + + // file size: 1032 + dq.Put(msg) + dq.Put(make([]byte, 16)) + + // file size: 1512 + dq.Put(make([]byte, 1500)) + + // read two files + <-dq.ReadChan() + <-dq.ReadChan() + + <-dq.ReadChan() + <-dq.ReadChan() + + for i := 0; i < 10; i++ { + d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) + if d.depth == 1 && + d.writeBytes == 1512 && + d.readFileNum == 2 && + d.writeFileNum == 3 && + d.readMessages == 0 && + d.writeMessages == 0 && + d.readPos == 0 && + d.writePos == 0 { + // success + goto done + } + time.Sleep(100 * time.Millisecond) + } + panic("fail") + +done: +} + func TestDiskQueueTorture(t *testing.T) { var wg sync.WaitGroup From 1fab6952aed9279eca0db28571198d345de4ddef Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 9 Jun 2021 18:44:47 +0000 Subject: [PATCH 28/67] Test that DiskQueue deleted .bad files first in order to make Disk Space. --- diskqueue_test.go | 104 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 92 insertions(+), 12 deletions(-) diff --git a/diskqueue_test.go b/diskqueue_test.go index 09210de..7ccb7b8 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -4,13 +4,16 @@ import ( "bufio" "bytes" "fmt" + "io/fs" "io/ioutil" "os" "path" "path/filepath" "reflect" + "regexp" "runtime" "strconv" + "strings" "sync" "sync/atomic" "testing" @@ -666,6 +669,60 @@ func createBadFile(dqName string, filePath string, fileNum int64, numBytes int) return err } +func totalBadFileDiskSize(diskQueueName string, dataPath string) int64 { + var badFileDiskSize int64 + + // the directory containing DiskQueue files + var mainDir string + if dataPath == "/" { + mainDir = dataPath + } else { + pathArray := strings.Split(dataPath, "/") + mainDir = pathArray[len(pathArray)-1] + } + + getBadFileInfos := func(pathStr string, dirEntry fs.DirEntry, err error) error { + if dirEntry.Name() == mainDir { + // we want to see the contents of this directory + return nil + } + + if dirEntry.IsDir() { + // if the entry is a directory, skip it + return fs.SkipDir + } + + if err != nil { + return err + } + + var matched bool + + regExp, err := regexp.Compile(diskQueueName + `.diskqueue.\d\d\d\d\d\d.dat.bad`) + if err == nil { + matched = regExp.MatchString(dirEntry.Name()) + } else { + matched, _ = regexp.Match(`.diskqueue.\d\d\d\d\d\d.dat.bad`, []byte(dirEntry.Name())) + } + + if matched { + badFileInfo, e := dirEntry.Info() + if e == nil && badFileInfo != nil { + badFileDiskSize += badFileInfo.Size() + } + } + + return nil + } + + err := filepath.WalkDir(dataPath, getBadFileInfos) + if err != nil { + return 0 + } + + return badFileDiskSize +} + func TestDiskSizeImplementationWithBadFiles(t *testing.T) { // write three files @@ -677,39 +734,62 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { } defer os.RemoveAll(tmpDir) + // there should be no .bad files + var badFileDiskSize int64 + badFileDiskSize = totalBadFileDiskSize(dqName, tmpDir) + if badFileDiskSize != 0 { + panic("fail") + } + // make 2 bad files - createBadFile(dqName, tmpDir, 0, 1533) + createBadFile(dqName, tmpDir, 0, 1503) createBadFile(dqName, tmpDir, 1, 1032) + badFileDiskSize = totalBadFileDiskSize(dqName, tmpDir) + if badFileDiskSize != 2535 { + panic("fail") + } + dq := NewWithDiskSpace(dqName, tmpDir, 1<<12, 1<<10, 0, 1<<12, 2500, 50*time.Millisecond, l) defer dq.Close() msgSize := 1000 msg := make([]byte, msgSize) - // file size: 1533 + // file size: 1497 dq.Put(msg) - dq.Put(make([]byte, 517)) + dq.Put(make([]byte, 481)) + + // no bad files should have been deleted + badFileDiskSize = totalBadFileDiskSize(dqName, tmpDir) + if badFileDiskSize != 2535 { + panic("fail") + } // file size: 1032 dq.Put(msg) dq.Put(make([]byte, 16)) + // one .bad file should be deleted in order to make space + badFileDiskSize = totalBadFileDiskSize(dqName, tmpDir) + if badFileDiskSize != 1032 { + panic("fail") + } + // file size: 1512 dq.Put(make([]byte, 1500)) - // read two files - <-dq.ReadChan() - <-dq.ReadChan() - - <-dq.ReadChan() - <-dq.ReadChan() + // check if the .bad files were deleted + badFileDiskSize = totalBadFileDiskSize(dqName, tmpDir) + if badFileDiskSize != 0 { + panic("fail") + } for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) - if d.depth == 1 && - d.writeBytes == 1512 && - d.readFileNum == 2 && + if d.depth == 5 && + d.writeBytes == 4041 && + d.readFileNum == 0 && d.writeFileNum == 3 && d.readMessages == 0 && d.writeMessages == 0 && From 3ec5f40cce78423e85a9e31e571405fc8081074f Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 9 Jun 2021 21:33:29 +0000 Subject: [PATCH 29/67] Update with to other branches - variable name change and adjust writeBytes, writeMsgs, and readMsgs in handReadError. --- diskqueue.go | 79 +++++++++++++++++++++++++---------------------- diskqueue_test.go | 6 ++-- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index ff57569..9973302 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -117,7 +117,7 @@ type diskQueue struct { logf AppLogFunc // disk limit implementation flag - diskLimitFeatIsOn bool + enableDiskLimitation bool } // New instantiates an instance of diskQueue, retrieving metadata @@ -139,30 +139,30 @@ func NewWithDiskSpace(name string, dataPath string, maxBytesDiskSpace int64, maxBytesPerFile int64, minMsgSize int32, maxMsgSize int32, syncEvery int64, syncTimeout time.Duration, logf AppLogFunc) Interface { - diskLimitFeatIsOn := true + enableDiskLimitation := true if maxBytesDiskSpace <= 0 { maxBytesDiskSpace = 0 - diskLimitFeatIsOn = false + enableDiskLimitation = false } d := diskQueue{ - name: name, - dataPath: dataPath, - maxBytesDiskSpace: maxBytesDiskSpace, - maxBytesPerFile: maxBytesPerFile, - minMsgSize: minMsgSize, - maxMsgSize: maxMsgSize, - readChan: make(chan []byte), - depthChan: make(chan int64), - writeChan: make(chan []byte), - writeResponseChan: make(chan error), - emptyChan: make(chan int), - emptyResponseChan: make(chan error), - exitChan: make(chan int), - exitSyncChan: make(chan int), - syncEvery: syncEvery, - syncTimeout: syncTimeout, - logf: logf, - diskLimitFeatIsOn: diskLimitFeatIsOn, + name: name, + dataPath: dataPath, + maxBytesDiskSpace: maxBytesDiskSpace, + maxBytesPerFile: maxBytesPerFile, + minMsgSize: minMsgSize, + maxMsgSize: maxMsgSize, + readChan: make(chan []byte), + depthChan: make(chan int64), + writeChan: make(chan []byte), + writeResponseChan: make(chan error), + emptyChan: make(chan int), + emptyResponseChan: make(chan error), + exitChan: make(chan int), + exitSyncChan: make(chan int), + syncEvery: syncEvery, + syncTimeout: syncTimeout, + logf: logf, + enableDiskLimitation: enableDiskLimitation, } d.start() @@ -316,7 +316,7 @@ func (d *diskQueue) skipToNextRWFile() error { d.nextReadPos = 0 d.depth = 0 - if d.diskLimitFeatIsOn { + if d.enableDiskLimitation { d.writeBytes = 0 d.readMessages = 0 d.writeMessages = 0 @@ -355,7 +355,7 @@ func (d *diskQueue) readOne() ([]byte, error) { stat, err := d.readFile.Stat() if err == nil { d.maxBytesPerFileRead = stat.Size() - if d.diskLimitFeatIsOn { + if d.enableDiskLimitation { // last 8 bytes are reserved for the number of messages in this file d.maxBytesPerFileRead -= numFileMsgBytes } @@ -627,7 +627,7 @@ func (d *diskQueue) writeOne(data []byte) error { totalBytes := int64(4 + dataLen) reachedFileSizeLimit := false - if d.diskLimitFeatIsOn { + if d.enableDiskLimitation { // check if we will reach or surpass file size limit if d.writePos+totalBytes+numFileMsgBytes >= d.maxBytesPerFile { reachedFileSizeLimit = true @@ -668,7 +668,7 @@ func (d *diskQueue) writeOne(data []byte) error { } // check if we reached the file size limit with this message - if d.diskLimitFeatIsOn && reachedFileSizeLimit { + if d.enableDiskLimitation && reachedFileSizeLimit { // write number of messages in binary to file err = binary.Write(&d.writeBuf, binary.BigEndian, d.writeMessages+1) if err != nil { @@ -687,7 +687,7 @@ func (d *diskQueue) writeOne(data []byte) error { d.writePos += totalBytes d.depth += 1 - if d.diskLimitFeatIsOn { + if d.enableDiskLimitation { d.writeBytes += totalBytes d.writeMessages += 1 } @@ -700,7 +700,7 @@ func (d *diskQueue) writeOne(data []byte) error { d.writeFileNum++ d.writePos = 0 - if d.diskLimitFeatIsOn { + if d.enableDiskLimitation { // add bytes for the number of messages in the file d.writeBytes += numFileMsgBytes d.writeMessages = 0 @@ -754,7 +754,7 @@ func (d *diskQueue) retrieveMetaData() error { defer f.Close() // if user is using disk space limit feature - if d.diskLimitFeatIsOn { + if d.enableDiskLimitation { _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", &d.depth, &d.readFileNum, &d.readMessages, &d.readPos, @@ -791,7 +791,7 @@ func (d *diskQueue) persistMetaData() error { } // if user is using disk space limit feature - if d.diskLimitFeatIsOn { + if d.enableDiskLimitation { _, err = fmt.Fprintf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", d.depth, d.readFileNum, d.readMessages, d.readPos, @@ -878,7 +878,7 @@ func (d *diskQueue) moveToNextReadFile(readFileSize int64) { d.logf(ERROR, "DISKQUEUE(%s) failed to Remove(%s) - %s", d.name, fn, err) } - if d.diskLimitFeatIsOn { + if d.enableDiskLimitation { d.readMessages = 0 d.writeBytes -= readFileSize } @@ -890,7 +890,7 @@ func (d *diskQueue) moveForward() { readFileSize := int64(d.readMsgSize) + d.readPos + 12 d.depth -= 1 - if d.diskLimitFeatIsOn { + if d.enableDiskLimitation { d.readMessages += 1 } @@ -926,14 +926,19 @@ func (d *diskQueue) handleReadError() { d.name, badFn, badRenameFn) } - if d.diskLimitFeatIsOn { - var stat os.FileInfo - stat, err = os.Stat(badRenameFn) + if d.enableDiskLimitation { var badFileSize int64 - if err == nil { - badFileSize = stat.Size() + if d.readFileNum == d.writeFileNum { + badFileSize = d.writeBytes } else { - badFileSize = int64(d.maxMsgSize+d.maxMsgSize) + 4 + var stat os.FileInfo + stat, err = os.Stat(badRenameFn) + if err == nil { + badFileSize = stat.Size() + } else { + // max file size + badFileSize = int64(d.maxMsgSize) + d.maxBytesPerFile + 4 + } } d.badBytes += badFileSize diff --git a/diskqueue_test.go b/diskqueue_test.go index 7ccb7b8..0f9656f 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -263,7 +263,7 @@ type md struct { writePos int64 } -func readMetaDataFile(fileName string, retried int, diskLimitFeatIsOn bool) md { +func readMetaDataFile(fileName string, retried int, enableDiskLimitation bool) md { f, err := os.OpenFile(fileName, os.O_RDONLY, 0600) if err != nil { // provide a simple retry that results in up to @@ -271,7 +271,7 @@ func readMetaDataFile(fileName string, retried int, diskLimitFeatIsOn bool) md { if retried < 9 { retried++ time.Sleep(50 * time.Millisecond) - return readMetaDataFile(fileName, retried, diskLimitFeatIsOn) + return readMetaDataFile(fileName, retried, enableDiskLimitation) } panic(err) } @@ -279,7 +279,7 @@ func readMetaDataFile(fileName string, retried int, diskLimitFeatIsOn bool) md { var ret md - if diskLimitFeatIsOn { + if enableDiskLimitation { _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", &ret.depth, &ret.readFileNum, &ret.readMessages, &ret.readPos, From bfed51c3c2bcf3af9a6544cecc45f4fd1d549662 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 10 Jun 2021 14:46:46 +0000 Subject: [PATCH 30/67] Adjust writeBytes as a result of the initial overestimation of the size of a bad file when we cannot get its accurate size. --- diskqueue.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 9973302..25464d7 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -547,17 +547,9 @@ func (d *diskQueue) freeUpDiskSpace() error { badFileExists := false // delete .bad files before deleting non-corrupted files (i.e. readFile) - if d.badBytes != 0 { + if d.badBytes > 0 { badFileInfos := d.getAllBadFileInfo() - // if badBytes is negative, something went wrong, so recalculate total .bad files size - if d.badBytes < 0 { - d.badBytes = 0 - for _, badFileInfo := range badFileInfos { - d.badBytes += badFileInfo.Size() - } - } - // check if a .bad file exists. If it does, delete that first if badFileInfos != nil { badFileExists = true @@ -572,7 +564,8 @@ func (d *diskQueue) freeUpDiskSpace() error { d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) } } else { - // there are no bad files + // we overestimated the size of some .bad files and removed too much from writeBytes + d.writeBytes += d.badBytes d.badBytes = 0 } } From d0a48bb109c9af3b98fac457241de3f789a2f25a Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 10 Jun 2021 14:56:36 +0000 Subject: [PATCH 31/67] Reset messages in handleReadError --- diskqueue.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/diskqueue.go b/diskqueue.go index 25464d7..cc79047 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -923,6 +923,9 @@ func (d *diskQueue) handleReadError() { var badFileSize int64 if d.readFileNum == d.writeFileNum { badFileSize = d.writeBytes + + // we moved on to the next writeFile + d.writeMessages = 0 } else { var stat os.FileInfo stat, err = os.Stat(badRenameFn) @@ -936,6 +939,8 @@ func (d *diskQueue) handleReadError() { d.badBytes += badFileSize d.writeBytes -= badFileSize + + d.readMessages = 0 } d.readFileNum++ From 872148f408973176dc0fca7dea3aa8bbf1e467e5 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 10 Jun 2021 15:07:17 +0000 Subject: [PATCH 32/67] Use MatchString as opposed to Match. --- diskqueue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diskqueue.go b/diskqueue.go index cc79047..572658d 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -521,7 +521,7 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { if err == nil { matched = regExp.MatchString(dirEntry.Name()) } else { - matched, _ = regexp.Match(`.diskqueue.\d\d\d\d\d\d.dat.bad`, []byte(dirEntry.Name())) + matched, _ = regexp.MatchString(`.diskqueue.\d\d\d\d\d\d.dat.bad`, dirEntry.Name()) } if matched { From a13c888f02cf795dc4b0436a8825892a0104beeb Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 10 Jun 2021 18:18:32 +0000 Subject: [PATCH 33/67] Throw error. --- diskqueue.go | 1 + 1 file changed, 1 insertion(+) diff --git a/diskqueue.go b/diskqueue.go index 572658d..78f8beb 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -562,6 +562,7 @@ func (d *diskQueue) freeUpDiskSpace() error { d.badBytes -= oldestBadFileInfo.Size() } else { d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) + return err } } else { // we overestimated the size of some .bad files and removed too much from writeBytes From c84524fac35fefb4626833a04c5f53b0232d656a Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 11 Jun 2021 17:09:31 +0000 Subject: [PATCH 34/67] Instead of overestimating bad file size, underestime it. This ensures we will never go over disk limit. --- diskqueue.go | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 78f8beb..61c573d 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -179,9 +179,7 @@ func (d *diskQueue) start() { // get the size of all the .bad files badFileInfos := d.getAllBadFileInfo() - for _, badFileInfo := range badFileInfos { - d.badBytes += badFileInfo.Size() - } + d.badBytes = int64(len(badFileInfos)) * d.maxBytesPerFile go d.ioLoop() } @@ -559,14 +557,17 @@ func (d *diskQueue) freeUpDiskSpace() error { err = os.Remove(badFileFilePath) if err == nil { - d.badBytes -= oldestBadFileInfo.Size() + if d.badBytes == d.maxBytesPerFile { + // recalculate estimate + d.badBytes = int64(len(badFileInfos)-1) * d.maxBytesPerFile + } else { + d.badBytes -= d.maxBytesPerFile + } } else { d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) return err } } else { - // we overestimated the size of some .bad files and removed too much from writeBytes - d.writeBytes += d.badBytes d.badBytes = 0 } } @@ -921,25 +922,14 @@ func (d *diskQueue) handleReadError() { } if d.enableDiskLimitation { - var badFileSize int64 if d.readFileNum == d.writeFileNum { - badFileSize = d.writeBytes - // we moved on to the next writeFile d.writeMessages = 0 - } else { - var stat os.FileInfo - stat, err = os.Stat(badRenameFn) - if err == nil { - badFileSize = stat.Size() - } else { - // max file size - badFileSize = int64(d.maxMsgSize) + d.maxBytesPerFile + 4 - } } - d.badBytes += badFileSize - d.writeBytes -= badFileSize + // use lower estimate of file size + d.badBytes += d.maxBytesPerFile + d.writeBytes -= d.maxBytesPerFile d.readMessages = 0 } From e9cd28a23d8217388c043e13ea0b20e9cf732153 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 11 Jun 2021 18:40:52 +0000 Subject: [PATCH 35/67] Add a check to see if readFile is corrupted. --- diskqueue.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 61c573d..3872d22 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -453,6 +453,19 @@ func (d *diskQueue) removeReadFile() error { } defer closeReadFile() + // get the size of the file + stat, err := d.readFile.Stat() + if err != nil { + return err + } + readFileSize := stat.Size() + + if readFileSize < d.maxBytesPerFile { + // if this file is corrupted we want to decrement + // at least d.maxBytesPerFile from d.writeBytes + return errors.New("file(" + d.fileName(d.readFileNum) + ") is corrupted") + } + // read total messages number at the end of the file _, err = d.readFile.Seek(-numFileMsgBytes, 2) if err != nil { @@ -465,16 +478,11 @@ func (d *diskQueue) removeReadFile() error { return err } + d.logf(DEBUG, "messages in file: %d", totalMessages) + // update depth with the remaining number of messages d.depth -= totalMessages - d.readMessages - // get the size of the file - stat, err := d.readFile.Stat() - if err != nil { - return err - } - readFileSize := stat.Size() - // we have not finished reading this file if d.readFileNum == d.nextReadFileNum { d.nextReadFileNum++ @@ -926,6 +934,7 @@ func (d *diskQueue) handleReadError() { // we moved on to the next writeFile d.writeMessages = 0 } + d.logf(DEBUG, "estimated file size: %d", d.maxBytesPerFile) // use lower estimate of file size d.badBytes += d.maxBytesPerFile From 3f2ff0554010ba2603f884096a6380c1c5e3b472 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 11 Jun 2021 20:05:58 +0000 Subject: [PATCH 36/67] Remove DEBUG logs. --- diskqueue.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 3872d22..18ba87d 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -478,8 +478,6 @@ func (d *diskQueue) removeReadFile() error { return err } - d.logf(DEBUG, "messages in file: %d", totalMessages) - // update depth with the remaining number of messages d.depth -= totalMessages - d.readMessages @@ -934,7 +932,6 @@ func (d *diskQueue) handleReadError() { // we moved on to the next writeFile d.writeMessages = 0 } - d.logf(DEBUG, "estimated file size: %d", d.maxBytesPerFile) // use lower estimate of file size d.badBytes += d.maxBytesPerFile From f9260275793004493442f2d419bb5636aded0074 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 11 Jun 2021 20:38:53 +0000 Subject: [PATCH 37/67] test .bad files. --- diskqueue_test.go | 89 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 67 insertions(+), 22 deletions(-) diff --git a/diskqueue_test.go b/diskqueue_test.go index 0f9656f..0309ed5 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -234,6 +234,11 @@ func TestDiskQueueCorruption(t *testing.T) { Equal(t, msg, <-dq.ReadChan()) } + badFilesCount := numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 1 { + panic("fail") + } + // corrupt the 4th (current) file dqFn = dq.(*diskQueue).fileName(3) os.Truncate(dqFn, 100) @@ -241,6 +246,10 @@ func TestDiskQueueCorruption(t *testing.T) { dq.Put(msg) // in 5th file Equal(t, msg, <-dq.ReadChan()) + badFilesCount = numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 2 { + panic("fail") + } // write a corrupt (len 0) message at the 5th (current) file dq.(*diskQueue).writeFile.Write([]byte{0, 0, 0, 0}) @@ -250,6 +259,10 @@ func TestDiskQueueCorruption(t *testing.T) { dq.Put(msg) Equal(t, msg, <-dq.ReadChan()) + badFilesCount = numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 3 { + panic("fail") + } } type md struct { @@ -669,8 +682,8 @@ func createBadFile(dqName string, filePath string, fileNum int64, numBytes int) return err } -func totalBadFileDiskSize(diskQueueName string, dataPath string) int64 { - var badFileDiskSize int64 +func numberOfBadFiles(diskQueueName string, dataPath string) int64 { + var badFilesCount int64 // the directory containing DiskQueue files var mainDir string @@ -706,10 +719,7 @@ func totalBadFileDiskSize(diskQueueName string, dataPath string) int64 { } if matched { - badFileInfo, e := dirEntry.Info() - if e == nil && badFileInfo != nil { - badFileDiskSize += badFileInfo.Size() - } + badFilesCount++ } return nil @@ -720,7 +730,7 @@ func totalBadFileDiskSize(diskQueueName string, dataPath string) int64 { return 0 } - return badFileDiskSize + return badFilesCount } func TestDiskSizeImplementationWithBadFiles(t *testing.T) { @@ -735,9 +745,9 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { defer os.RemoveAll(tmpDir) // there should be no .bad files - var badFileDiskSize int64 - badFileDiskSize = totalBadFileDiskSize(dqName, tmpDir) - if badFileDiskSize != 0 { + var badFilesCount int64 + badFilesCount = numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 0 { panic("fail") } @@ -745,43 +755,43 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { createBadFile(dqName, tmpDir, 0, 1503) createBadFile(dqName, tmpDir, 1, 1032) - badFileDiskSize = totalBadFileDiskSize(dqName, tmpDir) - if badFileDiskSize != 2535 { + badFilesCount = numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 2 { panic("fail") } - dq := NewWithDiskSpace(dqName, tmpDir, 1<<12, 1<<10, 0, 1<<12, 2500, 50*time.Millisecond, l) + dq := NewWithDiskSpace(dqName, tmpDir, 1<<12, 1<<10, 10, 1600, 2500, 50*time.Millisecond, l) defer dq.Close() msgSize := 1000 msg := make([]byte, msgSize) - // file size: 1497 + // file 0 size: 1497 dq.Put(msg) dq.Put(make([]byte, 481)) // no bad files should have been deleted - badFileDiskSize = totalBadFileDiskSize(dqName, tmpDir) - if badFileDiskSize != 2535 { + badFilesCount = numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 2 { panic("fail") } - // file size: 1032 + // file 1 size: 1032 dq.Put(msg) dq.Put(make([]byte, 16)) // one .bad file should be deleted in order to make space - badFileDiskSize = totalBadFileDiskSize(dqName, tmpDir) - if badFileDiskSize != 1032 { + badFilesCount = numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 1 { panic("fail") } - // file size: 1512 + // file 2 size: 1512 dq.Put(make([]byte, 1500)) // check if the .bad files were deleted - badFileDiskSize = totalBadFileDiskSize(dqName, tmpDir) - if badFileDiskSize != 0 { + badFilesCount = numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 0 { panic("fail") } @@ -796,6 +806,41 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { d.readPos == 0 && d.writePos == 0 { // success + goto corruptFiles + } + time.Sleep(100 * time.Millisecond) + } + panic("fail") + +corruptFiles: + // test removeReadFile when file is corrupted + // create bad files see if writebytes is updated properly + // check that after corrupting files, we make space appropriately + + // corrupt file 0 + dqFn := dq.(*diskQueue).fileName(0) + os.Truncate(dqFn, 1017) // 1 valid message, 1 corrupted message + + // when making space check that writeBytes is what it should be and that there are no .bad files + // this checks that disk limit check turns it into a .bad file + dq.Put(make([]byte, 100)) + + // check if the .bad files were deleted + badFilesCount = numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 0 { + panic("fail") + } + + for i := 0; i < 10; i++ { + d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) + if d.writeBytes == 3121 && + d.readFileNum == 1 && + d.writeFileNum == 3 && + d.readMessages == 0 && + d.writeMessages == 1 && + d.readPos == 0 && + d.writePos == 104 { + // success goto done } time.Sleep(100 * time.Millisecond) From eef61fbc67ac29aa32475fb3b578012e97334435 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 11 Jun 2021 20:42:11 +0000 Subject: [PATCH 38/67] In progress... corrupt file and then have readOne() deem it corrupted. --- diskqueue_test.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/diskqueue_test.go b/diskqueue_test.go index 0309ed5..45e8da7 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -841,12 +841,60 @@ corruptFiles: d.readPos == 0 && d.writePos == 104 { // success - goto done + goto readCorruptedFile } time.Sleep(100 * time.Millisecond) } panic("fail") +readCorruptedFile: + + // test handleReadError + // corrupt file 2 + // have readOne turn it into a bad file and then try to make space + dqFn = dq.(*diskQueue).fileName(2) + t.Logf("Filename: %s", dqFn) + os.Truncate(dqFn, 50) + + <-dq.ReadChan() + <-dq.ReadChan() + + <-dq.ReadChan() + // check if the file was converted into a .bad file + badFilesCount = numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 1 { + t.Logf("Bad files count: %d", badFilesCount) + panic("fail") + } + + dq.Put(msg) + // check if the corrupted file was deleted to make space + badFilesCount = numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 0 { + t.Logf("Bad files count: %d", badFilesCount) + panic("fail") + } + // t.Logf("%d, %d, %d, %d, %d, %d, %d, %d", d.depth, d.writeBytes, d.readFileNum, d.writeFileNum, d.readMessages, d.writeMessages, d.readPos, d.writePos) + + t.Log("Already READ") + // check if the .bad files were deleted + // badFilesCount = numberOfBadFiles(dqName, tmpDir) + // if badFilesCount != 0 { + // panic("fail") + // } + + // check writeBytes? break up into several sections? + + // check that as DiskQueue encounters more corrupt files, the size of the corrupt files never surpass + // the disk size limit + + // check writeBytes + goto done + + // add a message + + // + done: } From 6bcee725e82dd08a925649264229228ea83061d9 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 11 Jun 2021 20:42:49 +0000 Subject: [PATCH 39/67] Make code more readable. --- diskqueue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diskqueue.go b/diskqueue.go index 18ba87d..be2ae67 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -888,7 +888,7 @@ func (d *diskQueue) moveToNextReadFile(readFileSize int64) { func (d *diskQueue) moveForward() { // add bytes for the number of messages and the size of the message - readFileSize := int64(d.readMsgSize) + d.readPos + 12 + readFileSize := int64(d.readMsgSize) + d.readPos + numFileMsgBytes + 4 d.depth -= 1 if d.enableDiskLimitation { From 6e63d55c04e9617c0fc958311348b94891dc2b3d Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Mon, 14 Jun 2021 15:09:58 +0000 Subject: [PATCH 40/67] Ensure that diskqueue handles corrupted files correctly. --- diskqueue_test.go | 55 ++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/diskqueue_test.go b/diskqueue_test.go index 45e8da7..8d27c02 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -848,52 +848,57 @@ corruptFiles: panic("fail") readCorruptedFile: - // test handleReadError + + // there should be no "bad" files at this point + badFilesCount = numberOfBadFiles(dqName, tmpDir) + if badFilesCount != 0 { + panic("fail") + } + // corrupt file 2 // have readOne turn it into a bad file and then try to make space dqFn = dq.(*diskQueue).fileName(2) - t.Logf("Filename: %s", dqFn) - os.Truncate(dqFn, 50) + os.Truncate(dqFn, 1020) + // read file 1 <-dq.ReadChan() <-dq.ReadChan() - <-dq.ReadChan() + // wait for DiskQueue to notice that file 2 is corrupted + time.Sleep(20 * time.Millisecond) + // check if the file was converted into a .bad file badFilesCount = numberOfBadFiles(dqName, tmpDir) if badFilesCount != 1 { - t.Logf("Bad files count: %d", badFilesCount) panic("fail") } + // go over the disk limit + dq.Put(msg) dq.Put(msg) + // check if the corrupted file was deleted to make space badFilesCount = numberOfBadFiles(dqName, tmpDir) if badFilesCount != 0 { - t.Logf("Bad files count: %d", badFilesCount) panic("fail") } - // t.Logf("%d, %d, %d, %d, %d, %d, %d, %d", d.depth, d.writeBytes, d.readFileNum, d.writeFileNum, d.readMessages, d.writeMessages, d.readPos, d.writePos) - - t.Log("Already READ") - // check if the .bad files were deleted - // badFilesCount = numberOfBadFiles(dqName, tmpDir) - // if badFilesCount != 0 { - // panic("fail") - // } - - // check writeBytes? break up into several sections? - - // check that as DiskQueue encounters more corrupt files, the size of the corrupt files never surpass - // the disk size limit - - // check writeBytes - goto done - // add a message - - // + for i := 0; i < 10; i++ { + d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) + if d.writeBytes == 3081 && + d.readFileNum == 3 && + d.writeFileNum == 4 && + d.readMessages == 0 && + d.writeMessages == 1 && + d.readPos == 0 && + d.writePos == 1004 { + // success + goto done + } + time.Sleep(100 * time.Millisecond) + } + panic("fail") done: } From 86ade8f352f9e104215c43f6a45e148a235634e2 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Mon, 14 Jun 2021 20:30:27 +0000 Subject: [PATCH 41/67] Remove badBytes and get the accurate writeBytes data every time a writeFile is deleted or a readFile turns into a bad file. --- diskqueue.go | 176 +++++++++++++++++++++++----------------------- diskqueue_test.go | 28 +++++--- 2 files changed, 105 insertions(+), 99 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index be2ae67..bf14b1b 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -86,7 +86,6 @@ type diskQueue struct { syncTimeout time.Duration // duration of time per fsync exitFlag int32 needSync bool - badBytes int64 // keeps track of the position where we have read // (but not yet sent over readChan) @@ -177,10 +176,6 @@ func (d *diskQueue) start() { d.logf(ERROR, "DISKQUEUE(%s) failed to retrieveMetaData - %s", d.name, err) } - // get the size of all the .bad files - badFileInfos := d.getAllBadFileInfo() - d.badBytes = int64(len(badFileInfos)) * d.maxBytesPerFile - go d.ioLoop() } @@ -453,19 +448,6 @@ func (d *diskQueue) removeReadFile() error { } defer closeReadFile() - // get the size of the file - stat, err := d.readFile.Stat() - if err != nil { - return err - } - readFileSize := stat.Size() - - if readFileSize < d.maxBytesPerFile { - // if this file is corrupted we want to decrement - // at least d.maxBytesPerFile from d.writeBytes - return errors.New("file(" + d.fileName(d.readFileNum) + ") is corrupted") - } - // read total messages number at the end of the file _, err = d.readFile.Seek(-numFileMsgBytes, 2) if err != nil { @@ -487,7 +469,7 @@ func (d *diskQueue) removeReadFile() error { d.nextReadPos = 0 } - d.moveToNextReadFile(readFileSize) + d.moveToNextReadFile() return nil } @@ -519,16 +501,9 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { return err } - var matched bool - - regExp, err := regexp.Compile(d.name + `.diskqueue.\d\d\d\d\d\d.dat.bad`) - if err == nil { - matched = regExp.MatchString(dirEntry.Name()) - } else { - matched, _ = regexp.MatchString(`.diskqueue.\d\d\d\d\d\d.dat.bad`, dirEntry.Name()) - } + regExp, _ := regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat.bad$`) - if matched { + if regExp.MatchString(dirEntry.Name()) { badFileInfo, e := dirEntry.Info() if e == nil && badFileInfo != nil { badFileInfos = append(badFileInfos, badFileInfo) @@ -546,55 +521,6 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { return badFileInfos } -func (d *diskQueue) freeUpDiskSpace() error { - var err error - badFileExists := false - - // delete .bad files before deleting non-corrupted files (i.e. readFile) - if d.badBytes > 0 { - badFileInfos := d.getAllBadFileInfo() - - // check if a .bad file exists. If it does, delete that first - if badFileInfos != nil { - badFileExists = true - - oldestBadFileInfo := badFileInfos[0] - badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) - - err = os.Remove(badFileFilePath) - if err == nil { - if d.badBytes == d.maxBytesPerFile { - // recalculate estimate - d.badBytes = int64(len(badFileInfos)-1) * d.maxBytesPerFile - } else { - d.badBytes -= d.maxBytesPerFile - } - } else { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) - return err - } - } else { - d.badBytes = 0 - } - } - - if !badFileExists { - // delete the read file (make space) - if d.readFileNum == d.writeFileNum { - d.skipToNextRWFile() - } else { - err = d.removeReadFile() - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(d.readFileNum), err) - d.handleReadError() - return err - } - } - } - - return nil -} - // writeOne performs a low level filesystem write for a single []byte // while advancing write positions and rolling files, if necessary func (d *diskQueue) writeOne(data []byte) error { @@ -647,9 +573,45 @@ func (d *diskQueue) writeOne(data []byte) error { // check if we have enough space to write this message metaDataFileSize := d.metaDataFileSize() + // get total disk space of bad files + var badFilesSize int64 + badFileInfos := d.getAllBadFileInfo() + for _, badFileInfo := range badFileInfos { + badFilesSize += badFileInfo.Size() + } + // keep freeing up disk space until we have enough space to write this message - for d.badBytes+metaDataFileSize+d.writeBytes+expectedBytesIncrease > d.maxBytesDiskSpace { - d.freeUpDiskSpace() + for badFilesSize+metaDataFileSize+d.writeBytes+expectedBytesIncrease > d.maxBytesDiskSpace { + if badFilesSize > 0 { + // check if a .bad file exists. If it does, delete that first + oldestBadFileInfo := badFileInfos[0] + badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) + + err = os.Remove(badFileFilePath) + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) + return err + } else { + // recaclulate total bad files disk size + badFileInfos = d.getAllBadFileInfo() + badFilesSize = 0 + for _, badFileInfo := range badFileInfos { + badFilesSize += badFileInfo.Size() + } + } + } else { + // delete the read file (make space) + if d.readFileNum == d.writeFileNum { + d.skipToNextRWFile() + } else { + err = d.removeReadFile() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(d.readFileNum), err) + d.handleReadError() + return err + } + } + } } } else if d.writePos+totalBytes >= d.maxBytesPerFile { reachedFileSizeLimit = true @@ -862,7 +824,48 @@ func (d *diskQueue) checkTailCorruption(depth int64) { } } -func (d *diskQueue) moveToNextReadFile(readFileSize int64) { +func (d *diskQueue) updateWriteBytes() { + d.writeBytes = 0 + + // the directory containing DiskQueue files + var mainDir string + if d.dataPath == "/" { + mainDir = d.dataPath + } else { + pathArray := strings.Split(d.dataPath, "/") + mainDir = pathArray[len(pathArray)-1] + } + + getWriteBytes := func(pathStr string, dirEntry fs.DirEntry, err error) error { + if dirEntry.Name() == mainDir { + // we want to see the contents of this directory + return nil + } + + if dirEntry.IsDir() { + // if the entry is a directory, skip it + return fs.SkipDir + } + + if err != nil { + return err + } + + regExp, _ := regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat$`) + if regExp.MatchString(dirEntry.Name()) { + fileInfo, e := dirEntry.Info() + if e == nil && fileInfo != nil { + d.writeBytes += fileInfo.Size() + } + } + + return nil + } + + filepath.WalkDir(d.dataPath, getWriteBytes) +} + +func (d *diskQueue) moveToNextReadFile() { oldReadFileNum := d.readFileNum d.readFileNum = d.nextReadFileNum d.readPos = d.nextReadPos @@ -881,21 +884,19 @@ func (d *diskQueue) moveToNextReadFile(readFileSize int64) { if d.enableDiskLimitation { d.readMessages = 0 - d.writeBytes -= readFileSize + d.updateWriteBytes() } } } func (d *diskQueue) moveForward() { - // add bytes for the number of messages and the size of the message - readFileSize := int64(d.readMsgSize) + d.readPos + numFileMsgBytes + 4 d.depth -= 1 if d.enableDiskLimitation { d.readMessages += 1 } - d.moveToNextReadFile(readFileSize) + d.moveToNextReadFile() d.checkTailCorruption(d.depth) } @@ -930,13 +931,12 @@ func (d *diskQueue) handleReadError() { if d.enableDiskLimitation { if d.readFileNum == d.writeFileNum { // we moved on to the next writeFile + d.writeBytes = 0 d.writeMessages = 0 + } else { + d.updateWriteBytes() } - // use lower estimate of file size - d.badBytes += d.maxBytesPerFile - d.writeBytes -= d.maxBytesPerFile - d.readMessages = 0 } diff --git a/diskqueue_test.go b/diskqueue_test.go index 8d27c02..50c41c1 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -711,11 +711,11 @@ func numberOfBadFiles(diskQueueName string, dataPath string) int64 { var matched bool - regExp, err := regexp.Compile(diskQueueName + `.diskqueue.\d\d\d\d\d\d.dat.bad`) + regExp, _ := regexp.Compile(`^` + diskQueueName + `.diskqueue.\d\d\d\d\d\d.dat.bad$`) if err == nil { matched = regExp.MatchString(dirEntry.Name()) } else { - matched, _ = regexp.Match(`.diskqueue.\d\d\d\d\d\d.dat.bad`, []byte(dirEntry.Name())) + fmt.Println("Error:", err) } if matched { @@ -813,6 +813,7 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { panic("fail") corruptFiles: + t.Log("CORRUPT") // test removeReadFile when file is corrupted // create bad files see if writebytes is updated properly // check that after corrupting files, we make space appropriately @@ -833,7 +834,8 @@ corruptFiles: for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) - if d.writeBytes == 3121 && + t.Log(d.writeBytes, d.readFileNum, d.writeFileNum, d.readMessages, d.writeMessages) + if d.writeBytes == 2648 && d.readFileNum == 1 && d.writeFileNum == 3 && d.readMessages == 0 && @@ -845,7 +847,7 @@ corruptFiles: } time.Sleep(100 * time.Millisecond) } - panic("fail") + panic("fail3") readCorruptedFile: // test handleReadError @@ -853,7 +855,7 @@ readCorruptedFile: // there should be no "bad" files at this point badFilesCount = numberOfBadFiles(dqName, tmpDir) if badFilesCount != 0 { - panic("fail") + panic("fail1-") } // corrupt file 2 @@ -871,28 +873,32 @@ readCorruptedFile: // check if the file was converted into a .bad file badFilesCount = numberOfBadFiles(dqName, tmpDir) if badFilesCount != 1 { - panic("fail") + panic("fail2-") } // go over the disk limit + dq.Put(msg) + dq.Put(msg) dq.Put(msg) // check if the corrupted file was deleted to make space badFilesCount = numberOfBadFiles(dqName, tmpDir) if badFilesCount != 0 { - panic("fail") + t.Log("BAD FILE COUNT:", badFilesCount) + panic("fail3-") } for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) - if d.writeBytes == 3081 && + t.Log(d.writeBytes, d.readFileNum, d.writeFileNum, d.readMessages, d.writeMessages) + if d.writeBytes == 3132 && d.readFileNum == 3 && - d.writeFileNum == 4 && + d.writeFileNum == 5 && d.readMessages == 0 && - d.writeMessages == 1 && + d.writeMessages == 0 && d.readPos == 0 && - d.writePos == 1004 { + d.writePos == 0 { // success goto done } From 0ab256e046e8d62729ac2522260a1354b1d1a4ed Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Mon, 14 Jun 2021 20:49:23 +0000 Subject: [PATCH 42/67] Abstract general function to walk through all of the files in the directory DiskQueue writes and reads in. --- diskqueue.go | 93 ++++++++++++++++++++++++++-------------------------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index bf14b1b..98c3344 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -474,9 +474,8 @@ func (d *diskQueue) removeReadFile() error { return nil } -func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { - var badFileInfos []fs.FileInfo - +// walk through all of the files in the DiskQueue directory +func (d *diskQueue) walkDiskQueueDir(fn func(fs.DirEntry) error) error { // the directory containing DiskQueue files var mainDir string if d.dataPath == "/" { @@ -486,9 +485,9 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { mainDir = pathArray[len(pathArray)-1] } - getBadFileInfos := func(pathStr string, dirEntry fs.DirEntry, err error) error { + walkDiskQueueDir := func(pathStr string, dirEntry fs.DirEntry, err error) error { if dirEntry.Name() == mainDir { - // we want to see the contents of this directory + // we want to see the contents of the directory DiskQueue writes and reads in return nil } @@ -501,6 +500,27 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { return err } + e := fn(dirEntry) + if e != nil { + return e + } + + return nil + } + + err := filepath.WalkDir(d.dataPath, walkDiskQueueDir) + if err != nil { + return err + } + + return nil +} + +func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { + var badFileInfos []fs.FileInfo + + getAllBadFileInfo := func(dirEntry fs.DirEntry) error { + // only accept "bad" files created by this DiskQueue object regExp, _ := regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat.bad$`) if regExp.MatchString(dirEntry.Name()) { @@ -513,7 +533,7 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { return nil } - err := filepath.WalkDir(d.dataPath, getBadFileInfos) + err := d.walkDiskQueueDir(getAllBadFileInfo) if err != nil { return nil } @@ -521,6 +541,26 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { return badFileInfos } +// get the accurate total non-"bad" file size +func (d *diskQueue) updateWriteBytes() { + d.writeBytes = 0 + + updateWriteBytes := func(dirEntry fs.DirEntry) error { + // only accept files created by this DiskQueue object + regExp, _ := regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat$`) + if regExp.MatchString(dirEntry.Name()) { + fileInfo, e := dirEntry.Info() + if e == nil && fileInfo != nil { + d.writeBytes += fileInfo.Size() + } + } + + return nil + } + + d.walkDiskQueueDir(updateWriteBytes) +} + // writeOne performs a low level filesystem write for a single []byte // while advancing write positions and rolling files, if necessary func (d *diskQueue) writeOne(data []byte) error { @@ -824,47 +864,6 @@ func (d *diskQueue) checkTailCorruption(depth int64) { } } -func (d *diskQueue) updateWriteBytes() { - d.writeBytes = 0 - - // the directory containing DiskQueue files - var mainDir string - if d.dataPath == "/" { - mainDir = d.dataPath - } else { - pathArray := strings.Split(d.dataPath, "/") - mainDir = pathArray[len(pathArray)-1] - } - - getWriteBytes := func(pathStr string, dirEntry fs.DirEntry, err error) error { - if dirEntry.Name() == mainDir { - // we want to see the contents of this directory - return nil - } - - if dirEntry.IsDir() { - // if the entry is a directory, skip it - return fs.SkipDir - } - - if err != nil { - return err - } - - regExp, _ := regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat$`) - if regExp.MatchString(dirEntry.Name()) { - fileInfo, e := dirEntry.Info() - if e == nil && fileInfo != nil { - d.writeBytes += fileInfo.Size() - } - } - - return nil - } - - filepath.WalkDir(d.dataPath, getWriteBytes) -} - func (d *diskQueue) moveToNextReadFile() { oldReadFileNum := d.readFileNum d.readFileNum = d.nextReadFileNum From 23f936966c94983442f6a75d9ad47a62036b1043 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Mon, 14 Jun 2021 21:08:18 +0000 Subject: [PATCH 43/67] Abstract code to its own function. --- diskqueue.go | 116 ++++++++++++++++++++++++++++----------------------- 1 file changed, 63 insertions(+), 53 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 98c3344..1796ffc 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -431,6 +431,11 @@ func (d *diskQueue) metaDataFileSize() int64 { func (d *diskQueue) removeReadFile() error { var err error + if d.readFileNum == d.writeFileNum { + d.skipToNextRWFile() + return nil + } + if d.readFile == nil { curFileName := d.fileName(d.readFileNum) @@ -561,6 +566,62 @@ func (d *diskQueue) updateWriteBytes() { d.walkDiskQueueDir(updateWriteBytes) } +func (d *diskQueue) freeUpDiskSpace(totalBytes int64, reachedFileSizeLimit bool) error { + var err error + + expectedBytesIncrease := totalBytes + if reachedFileSizeLimit { + expectedBytesIncrease += numFileMsgBytes + } + + // If the data to be written is bigger than the disk size limit, do not write + if expectedBytesIncrease > d.maxBytesDiskSpace { + return errors.New("message size surpasses disk size limit") + } + + // check if we have enough space to write this message + metaDataFileSize := d.metaDataFileSize() + + // get total disk space of bad files + var badFilesSize int64 + badFileInfos := d.getAllBadFileInfo() + for _, badFileInfo := range badFileInfos { + badFilesSize += badFileInfo.Size() + } + + // keep freeing up disk space until we have enough space to write this message + for badFilesSize+metaDataFileSize+d.writeBytes+expectedBytesIncrease > d.maxBytesDiskSpace { + if badFilesSize > 0 { + // check if a .bad file exists. If it does, delete that first + oldestBadFileInfo := badFileInfos[0] + badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) + + err = os.Remove(badFileFilePath) + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) + return err + } else { + // recaclulate total bad files disk size + badFileInfos = d.getAllBadFileInfo() + badFilesSize = 0 + for _, badFileInfo := range badFileInfos { + badFilesSize += badFileInfo.Size() + } + } + } else { + // delete the read file (make space) + err = d.removeReadFile() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(d.readFileNum), err) + d.handleReadError() + return err + + } + } + } + return nil +} + // writeOne performs a low level filesystem write for a single []byte // while advancing write positions and rolling files, if necessary func (d *diskQueue) writeOne(data []byte) error { @@ -600,59 +661,8 @@ func (d *diskQueue) writeOne(data []byte) error { reachedFileSizeLimit = true } - expectedBytesIncrease := totalBytes - if reachedFileSizeLimit { - expectedBytesIncrease += numFileMsgBytes - } - - // If the data to be written is bigger than the disk size limit, do not write - if expectedBytesIncrease > d.maxBytesDiskSpace { - return errors.New("message size surpasses disk size limit") - } - - // check if we have enough space to write this message - metaDataFileSize := d.metaDataFileSize() - - // get total disk space of bad files - var badFilesSize int64 - badFileInfos := d.getAllBadFileInfo() - for _, badFileInfo := range badFileInfos { - badFilesSize += badFileInfo.Size() - } - - // keep freeing up disk space until we have enough space to write this message - for badFilesSize+metaDataFileSize+d.writeBytes+expectedBytesIncrease > d.maxBytesDiskSpace { - if badFilesSize > 0 { - // check if a .bad file exists. If it does, delete that first - oldestBadFileInfo := badFileInfos[0] - badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) - - err = os.Remove(badFileFilePath) - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) - return err - } else { - // recaclulate total bad files disk size - badFileInfos = d.getAllBadFileInfo() - badFilesSize = 0 - for _, badFileInfo := range badFileInfos { - badFilesSize += badFileInfo.Size() - } - } - } else { - // delete the read file (make space) - if d.readFileNum == d.writeFileNum { - d.skipToNextRWFile() - } else { - err = d.removeReadFile() - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(d.readFileNum), err) - d.handleReadError() - return err - } - } - } - } + // free disk space if needed + d.freeUpDiskSpace(totalBytes, reachedFileSizeLimit) } else if d.writePos+totalBytes >= d.maxBytesPerFile { reachedFileSizeLimit = true } From 2c66983ed693c39c58805b683c79776df9af454d Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Tue, 15 Jun 2021 13:50:58 +0000 Subject: [PATCH 44/67] Remove readMsgSize and make code more readable. --- diskqueue.go | 56 +++++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 1796ffc..e1bc49a 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -92,10 +92,6 @@ type diskQueue struct { nextReadPos int64 nextReadFileNum int64 - // keep track of the msg size we have read - // (but not yet sent over readChan) - readMsgSize int32 - readFile *os.File writeFile *os.File reader *bufio.Reader @@ -322,6 +318,7 @@ func (d *diskQueue) skipToNextRWFile() error { // while advancing read positions and rolling files, if necessary func (d *diskQueue) readOne() ([]byte, error) { var err error + var msgSize int32 if d.readFile == nil { curFileName := d.fileName(d.readFileNum) @@ -358,22 +355,22 @@ func (d *diskQueue) readOne() ([]byte, error) { d.reader = bufio.NewReader(d.readFile) } - err = binary.Read(d.reader, binary.BigEndian, &d.readMsgSize) + err = binary.Read(d.reader, binary.BigEndian, &msgSize) if err != nil { d.readFile.Close() d.readFile = nil return nil, err } - if d.readMsgSize < d.minMsgSize || d.readMsgSize > d.maxMsgSize { + if msgSize < d.minMsgSize || msgSize > d.maxMsgSize { // this file is corrupt and we have no reasonable guarantee on // where a new message should begin d.readFile.Close() d.readFile = nil - return nil, fmt.Errorf("invalid message read size (%d)", d.readMsgSize) + return nil, fmt.Errorf("invalid message read size (%d)", msgSize) } - readBuf := make([]byte, d.readMsgSize) + readBuf := make([]byte, msgSize) _, err = io.ReadFull(d.reader, readBuf) if err != nil { d.readFile.Close() @@ -381,7 +378,7 @@ func (d *diskQueue) readOne() ([]byte, error) { return nil, err } - totalBytes := int64(4 + d.readMsgSize) + totalBytes := int64(4 + msgSize) // we only advance next* because we have not yet sent this to consumers // (where readFileNum, readPos will actually be advanced) @@ -505,12 +502,7 @@ func (d *diskQueue) walkDiskQueueDir(fn func(fs.DirEntry) error) error { return err } - e := fn(dirEntry) - if e != nil { - return e - } - - return nil + return fn(dirEntry) } err := filepath.WalkDir(d.dataPath, walkDiskQueueDir) @@ -521,7 +513,7 @@ func (d *diskQueue) walkDiskQueueDir(fn func(fs.DirEntry) error) error { return nil } -func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { +func (d *diskQueue) getAllBadFileInfo() ([]fs.FileInfo, error) { var badFileInfos []fs.FileInfo getAllBadFileInfo := func(dirEntry fs.DirEntry) error { @@ -539,15 +531,12 @@ func (d *diskQueue) getAllBadFileInfo() []fs.FileInfo { } err := d.walkDiskQueueDir(getAllBadFileInfo) - if err != nil { - return nil - } - return badFileInfos + return badFileInfos, err } // get the accurate total non-"bad" file size -func (d *diskQueue) updateWriteBytes() { +func (d *diskQueue) updateWriteBytes() error { d.writeBytes = 0 updateWriteBytes := func(dirEntry fs.DirEntry) error { @@ -563,9 +552,10 @@ func (d *diskQueue) updateWriteBytes() { return nil } - d.walkDiskQueueDir(updateWriteBytes) + return d.walkDiskQueueDir(updateWriteBytes) } +// free up disk space if needed to write new data to file func (d *diskQueue) freeUpDiskSpace(totalBytes int64, reachedFileSizeLimit bool) error { var err error @@ -584,7 +574,10 @@ func (d *diskQueue) freeUpDiskSpace(totalBytes int64, reachedFileSizeLimit bool) // get total disk space of bad files var badFilesSize int64 - badFileInfos := d.getAllBadFileInfo() + badFileInfos, err := d.getAllBadFileInfo() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to retrieve all .bad file info - %s", d.name, err) + } for _, badFileInfo := range badFileInfos { badFilesSize += badFileInfo.Size() } @@ -601,8 +594,11 @@ func (d *diskQueue) freeUpDiskSpace(totalBytes int64, reachedFileSizeLimit bool) d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) return err } else { - // recaclulate total bad files disk size - badFileInfos = d.getAllBadFileInfo() + // recaclulate total bad files disk size to get most accurate info + badFileInfos, err = d.getAllBadFileInfo() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to retrieve all .bad file info - %s", d.name, err) + } badFilesSize = 0 for _, badFileInfo := range badFileInfos { badFilesSize += badFileInfo.Size() @@ -893,7 +889,10 @@ func (d *diskQueue) moveToNextReadFile() { if d.enableDiskLimitation { d.readMessages = 0 - d.updateWriteBytes() + err = d.updateWriteBytes() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to update write bytes - %s", d.name, err) + } } } } @@ -943,7 +942,10 @@ func (d *diskQueue) handleReadError() { d.writeBytes = 0 d.writeMessages = 0 } else { - d.updateWriteBytes() + err = d.updateWriteBytes() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to update write bytes - %s", d.name, err) + } } d.readMessages = 0 From 779f347660ec741582c1cc68abdfeb61c1069cf2 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Tue, 15 Jun 2021 14:05:26 +0000 Subject: [PATCH 45/67] Make test code more readable. --- diskqueue_test.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/diskqueue_test.go b/diskqueue_test.go index 50c41c1..e4d1532 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -550,6 +550,7 @@ meetDiskSizeLimit: } } + // save space for msg len and number of msgs in file diskBytesRemaining := 6040 - metaDataFileSize - (totalDiskBytes + 12) dq.Put(make([]byte, diskBytesRemaining)) @@ -709,16 +710,8 @@ func numberOfBadFiles(diskQueueName string, dataPath string) int64 { return err } - var matched bool - regExp, _ := regexp.Compile(`^` + diskQueueName + `.diskqueue.\d\d\d\d\d\d.dat.bad$`) - if err == nil { - matched = regExp.MatchString(dirEntry.Name()) - } else { - fmt.Println("Error:", err) - } - - if matched { + if regExp.MatchString(dirEntry.Name()) { badFilesCount++ } @@ -789,7 +782,7 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { // file 2 size: 1512 dq.Put(make([]byte, 1500)) - // check if the .bad files were deleted + // check if all the .bad files were deleted badFilesCount = numberOfBadFiles(dqName, tmpDir) if badFilesCount != 0 { panic("fail") @@ -822,8 +815,6 @@ corruptFiles: dqFn := dq.(*diskQueue).fileName(0) os.Truncate(dqFn, 1017) // 1 valid message, 1 corrupted message - // when making space check that writeBytes is what it should be and that there are no .bad files - // this checks that disk limit check turns it into a .bad file dq.Put(make([]byte, 100)) // check if the .bad files were deleted @@ -879,6 +870,7 @@ readCorruptedFile: // go over the disk limit dq.Put(msg) + // write a complete file dq.Put(msg) dq.Put(msg) From 78610d83300d7d1b2f83bbfd39283b52119454fb Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 16 Jun 2021 11:53:56 -0400 Subject: [PATCH 46/67] Update go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 30a1fc9..54c31b6 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ -module github.com/nsqio/go-diskqueue +module github.com/kev1n80/go-diskqueue go 1.13 From 878cc6119ec517e4739f4e5f0ae7122cda5f6269 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 16 Jun 2021 18:24:15 +0000 Subject: [PATCH 47/67] Make regexp constants and close metadatafile after getting its size. --- diskqueue.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index e1bc49a..5926a5a 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -31,6 +31,8 @@ const ( numFileMsgBytes = 8 ) +var badFileNameRegexp, fileNameRegexp *regexp.Regexp + type AppLogFunc func(lvl LogLevel, f string, args ...interface{}) func (l LogLevel) String() string { @@ -172,6 +174,9 @@ func (d *diskQueue) start() { d.logf(ERROR, "DISKQUEUE(%s) failed to retrieveMetaData - %s", d.name, err) } + fileNameRegexp, _ = regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat$`) + badFileNameRegexp, _ = regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat.bad$`) + go d.ioLoop() } @@ -422,6 +427,8 @@ func (d *diskQueue) metaDataFileSize() int64 { metaDataFileSize = 64 } + metaDataFile.Close() + return metaDataFileSize } @@ -518,9 +525,7 @@ func (d *diskQueue) getAllBadFileInfo() ([]fs.FileInfo, error) { getAllBadFileInfo := func(dirEntry fs.DirEntry) error { // only accept "bad" files created by this DiskQueue object - regExp, _ := regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat.bad$`) - - if regExp.MatchString(dirEntry.Name()) { + if badFileNameRegexp.MatchString(dirEntry.Name()) { badFileInfo, e := dirEntry.Info() if e == nil && badFileInfo != nil { badFileInfos = append(badFileInfos, badFileInfo) @@ -541,8 +546,7 @@ func (d *diskQueue) updateWriteBytes() error { updateWriteBytes := func(dirEntry fs.DirEntry) error { // only accept files created by this DiskQueue object - regExp, _ := regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat$`) - if regExp.MatchString(dirEntry.Name()) { + if fileNameRegexp.MatchString(dirEntry.Name()) { fileInfo, e := dirEntry.Info() if e == nil && fileInfo != nil { d.writeBytes += fileInfo.Size() From 3b8bd418d708317d6e376900270e5d33879980ae Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 16 Jun 2021 19:31:48 +0000 Subject: [PATCH 48/67] Change writeBytes to totalDiskSpaceUsed and have it track writeBytes and badBytes for now. --- diskqueue.go | 109 ++++++++++++++++++++++++--------------------------- 1 file changed, 52 insertions(+), 57 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 5926a5a..49b3450 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -65,14 +65,14 @@ type diskQueue struct { // 64bit atomic vars need to be first for proper alignment on 32bit platforms // run-time state (also persisted to disk) - readPos int64 - writePos int64 - readFileNum int64 - writeFileNum int64 - readMessages int64 - writeMessages int64 - writeBytes int64 - depth int64 + readPos int64 + writePos int64 + readFileNum int64 + writeFileNum int64 + readMessages int64 + writeMessages int64 + totalDiskSpaceUsed int64 + depth int64 sync.RWMutex @@ -177,6 +177,8 @@ func (d *diskQueue) start() { fileNameRegexp, _ = regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat$`) badFileNameRegexp, _ = regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat.bad$`) + d.updateTotalDiskSpaceUsed() + go d.ioLoop() } @@ -311,7 +313,7 @@ func (d *diskQueue) skipToNextRWFile() error { d.depth = 0 if d.enableDiskLimitation { - d.writeBytes = 0 + d.totalDiskSpaceUsed = 0 d.readMessages = 0 d.writeMessages = 0 } @@ -541,27 +543,28 @@ func (d *diskQueue) getAllBadFileInfo() ([]fs.FileInfo, error) { } // get the accurate total non-"bad" file size -func (d *diskQueue) updateWriteBytes() error { - d.writeBytes = 0 +func (d *diskQueue) updateTotalDiskSpaceUsed() error { + d.totalDiskSpaceUsed = 0 - updateWriteBytes := func(dirEntry fs.DirEntry) error { + updateTotalDiskSpaceUsed := func(dirEntry fs.DirEntry) error { // only accept files created by this DiskQueue object - if fileNameRegexp.MatchString(dirEntry.Name()) { + if fileNameRegexp.MatchString(dirEntry.Name()) || badFileNameRegexp.MatchString(dirEntry.Name()) { fileInfo, e := dirEntry.Info() if e == nil && fileInfo != nil { - d.writeBytes += fileInfo.Size() + d.totalDiskSpaceUsed += fileInfo.Size() } } return nil } - return d.walkDiskQueueDir(updateWriteBytes) + return d.walkDiskQueueDir(updateTotalDiskSpaceUsed) } // free up disk space if needed to write new data to file func (d *diskQueue) freeUpDiskSpace(totalBytes int64, reachedFileSizeLimit bool) error { var err error + var badFileInfos []fs.FileInfo expectedBytesIncrease := totalBytes if reachedFileSizeLimit { @@ -576,46 +579,38 @@ func (d *diskQueue) freeUpDiskSpace(totalBytes int64, reachedFileSizeLimit bool) // check if we have enough space to write this message metaDataFileSize := d.metaDataFileSize() - // get total disk space of bad files - var badFilesSize int64 - badFileInfos, err := d.getAllBadFileInfo() - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to retrieve all .bad file info - %s", d.name, err) - } - for _, badFileInfo := range badFileInfos { - badFilesSize += badFileInfo.Size() - } - // keep freeing up disk space until we have enough space to write this message - for badFilesSize+metaDataFileSize+d.writeBytes+expectedBytesIncrease > d.maxBytesDiskSpace { - if badFilesSize > 0 { - // check if a .bad file exists. If it does, delete that first - oldestBadFileInfo := badFileInfos[0] - badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) + if metaDataFileSize+d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { + badFileInfos, err = d.getAllBadFileInfo() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to retrieve all .bad file info - %s", d.name, err) + } + for metaDataFileSize+d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { + if badFileInfos != nil { + // check if a .bad file exists. If it does, delete that first + oldestBadFileInfo := badFileInfos[0] + badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) - err = os.Remove(badFileFilePath) - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) - return err - } else { - // recaclulate total bad files disk size to get most accurate info - badFileInfos, err = d.getAllBadFileInfo() + err = os.Remove(badFileFilePath) if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to retrieve all .bad file info - %s", d.name, err) + d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) + d.updateTotalDiskSpaceUsed() + return err + } else { + // recaclulate total bad files disk size to get most accurate info + d.totalDiskSpaceUsed -= oldestBadFileInfo.Size() } - badFilesSize = 0 - for _, badFileInfo := range badFileInfos { - badFilesSize += badFileInfo.Size() - } - } - } else { - // delete the read file (make space) - err = d.removeReadFile() - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(d.readFileNum), err) - d.handleReadError() - return err + badFileInfos = badFileInfos[1:] + } else { + // delete the read file (make space) + err = d.removeReadFile() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(d.readFileNum), err) + d.handleReadError() + return err + + } } } } @@ -701,7 +696,7 @@ func (d *diskQueue) writeOne(data []byte) error { d.depth += 1 if d.enableDiskLimitation { - d.writeBytes += totalBytes + d.totalDiskSpaceUsed += totalBytes d.writeMessages += 1 } @@ -715,7 +710,7 @@ func (d *diskQueue) writeOne(data []byte) error { if d.enableDiskLimitation { // add bytes for the number of messages in the file - d.writeBytes += numFileMsgBytes + d.totalDiskSpaceUsed += numFileMsgBytes d.writeMessages = 0 } @@ -771,7 +766,7 @@ func (d *diskQueue) retrieveMetaData() error { _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", &d.depth, &d.readFileNum, &d.readMessages, &d.readPos, - &d.writeBytes, &d.writeFileNum, &d.writeMessages, &d.writePos) + &d.totalDiskSpaceUsed, &d.writeFileNum, &d.writeMessages, &d.writePos) } else { _, err = fmt.Fscanf(f, "%d\n%d,%d\n%d,%d\n", &d.depth, @@ -808,7 +803,7 @@ func (d *diskQueue) persistMetaData() error { _, err = fmt.Fprintf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", d.depth, d.readFileNum, d.readMessages, d.readPos, - d.writeBytes, d.writeFileNum, d.writeMessages, d.writePos) + d.totalDiskSpaceUsed, d.writeFileNum, d.writeMessages, d.writePos) } else { _, err = fmt.Fprintf(f, "%d\n%d,%d\n%d,%d\n", d.depth, @@ -893,7 +888,7 @@ func (d *diskQueue) moveToNextReadFile() { if d.enableDiskLimitation { d.readMessages = 0 - err = d.updateWriteBytes() + err = d.updateTotalDiskSpaceUsed() if err != nil { d.logf(ERROR, "DISKQUEUE(%s) failed to update write bytes - %s", d.name, err) } @@ -943,10 +938,10 @@ func (d *diskQueue) handleReadError() { if d.enableDiskLimitation { if d.readFileNum == d.writeFileNum { // we moved on to the next writeFile - d.writeBytes = 0 + d.totalDiskSpaceUsed = 0 d.writeMessages = 0 } else { - err = d.updateWriteBytes() + err = d.updateTotalDiskSpaceUsed() if err != nil { d.logf(ERROR, "DISKQUEUE(%s) failed to update write bytes - %s", d.name, err) } From d453ed4bcfeda826500fce97b5def6f92151da54 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 16 Jun 2021 20:55:25 +0000 Subject: [PATCH 49/67] Update panic messages. --- diskqueue_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/diskqueue_test.go b/diskqueue_test.go index e4d1532..7e216d4 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -838,7 +838,7 @@ corruptFiles: } time.Sleep(100 * time.Millisecond) } - panic("fail3") + panic("fail") readCorruptedFile: // test handleReadError @@ -846,7 +846,7 @@ readCorruptedFile: // there should be no "bad" files at this point badFilesCount = numberOfBadFiles(dqName, tmpDir) if badFilesCount != 0 { - panic("fail1-") + panic("fail") } // corrupt file 2 @@ -864,7 +864,7 @@ readCorruptedFile: // check if the file was converted into a .bad file badFilesCount = numberOfBadFiles(dqName, tmpDir) if badFilesCount != 1 { - panic("fail2-") + panic("fail") } // go over the disk limit @@ -878,7 +878,7 @@ readCorruptedFile: badFilesCount = numberOfBadFiles(dqName, tmpDir) if badFilesCount != 0 { t.Log("BAD FILE COUNT:", badFilesCount) - panic("fail3-") + panic("fail") } for i := 0; i < 10; i++ { From 87638f0f8bd263369d2dae9e0adfe9ade4238ac3 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 17 Jun 2021 15:39:51 +0000 Subject: [PATCH 50/67] Track the total disk size with one variable rather than several smaller ones. --- diskqueue.go | 34 +++++++------- diskqueue_test.go | 117 ++++++++++++++++++++++++++-------------------- 2 files changed, 85 insertions(+), 66 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 49b3450..50269ec 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -425,8 +425,8 @@ func (d *diskQueue) metaDataFileSize() int64 { } } if err != nil { - // use max file size (8 int64 fields) - metaDataFileSize = 64 + // use max file size (7 int64 fields) + metaDataFileSize = 56 } metaDataFile.Close() @@ -544,7 +544,7 @@ func (d *diskQueue) getAllBadFileInfo() ([]fs.FileInfo, error) { // get the accurate total non-"bad" file size func (d *diskQueue) updateTotalDiskSpaceUsed() error { - d.totalDiskSpaceUsed = 0 + d.totalDiskSpaceUsed = d.metaDataFileSize() updateTotalDiskSpaceUsed := func(dirEntry fs.DirEntry) error { // only accept files created by this DiskQueue object @@ -577,15 +577,14 @@ func (d *diskQueue) freeUpDiskSpace(totalBytes int64, reachedFileSizeLimit bool) } // check if we have enough space to write this message - metaDataFileSize := d.metaDataFileSize() - - // keep freeing up disk space until we have enough space to write this message - if metaDataFileSize+d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { + if d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { badFileInfos, err = d.getAllBadFileInfo() if err != nil { d.logf(ERROR, "DISKQUEUE(%s) failed to retrieve all .bad file info - %s", d.name, err) } - for metaDataFileSize+d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { + + // keep freeing up disk space until we have enough space to write this message + for d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { if badFileInfos != nil { // check if a .bad file exists. If it does, delete that first oldestBadFileInfo := badFileInfos[0] @@ -611,6 +610,7 @@ func (d *diskQueue) freeUpDiskSpace(totalBytes int64, reachedFileSizeLimit bool) return err } + d.updateTotalDiskSpaceUsed() } } } @@ -709,8 +709,6 @@ func (d *diskQueue) writeOne(data []byte) error { d.writePos = 0 if d.enableDiskLimitation { - // add bytes for the number of messages in the file - d.totalDiskSpaceUsed += numFileMsgBytes d.writeMessages = 0 } @@ -740,6 +738,10 @@ func (d *diskQueue) sync() error { } } + if d.enableDiskLimitation { + d.updateTotalDiskSpaceUsed() + } + err := d.persistMetaData() if err != nil { return err @@ -763,10 +765,10 @@ func (d *diskQueue) retrieveMetaData() error { // if user is using disk space limit feature if d.enableDiskLimitation { - _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", + _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d\n", &d.depth, &d.readFileNum, &d.readMessages, &d.readPos, - &d.totalDiskSpaceUsed, &d.writeFileNum, &d.writeMessages, &d.writePos) + &d.writeFileNum, &d.writeMessages, &d.writePos) } else { _, err = fmt.Fscanf(f, "%d\n%d,%d\n%d,%d\n", &d.depth, @@ -800,10 +802,10 @@ func (d *diskQueue) persistMetaData() error { // if user is using disk space limit feature if d.enableDiskLimitation { - _, err = fmt.Fprintf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", + _, err = fmt.Fprintf(f, "%d\n%d,%d,%d\n%d,%d,%d\n", d.depth, d.readFileNum, d.readMessages, d.readPos, - d.totalDiskSpaceUsed, d.writeFileNum, d.writeMessages, d.writePos) + d.writeFileNum, d.writeMessages, d.writePos) } else { _, err = fmt.Fprintf(f, "%d\n%d,%d\n%d,%d\n", d.depth, @@ -888,7 +890,7 @@ func (d *diskQueue) moveToNextReadFile() { if d.enableDiskLimitation { d.readMessages = 0 - err = d.updateTotalDiskSpaceUsed() + // err = d.updateTotalDiskSpaceUsed() if err != nil { d.logf(ERROR, "DISKQUEUE(%s) failed to update write bytes - %s", d.name, err) } @@ -941,7 +943,7 @@ func (d *diskQueue) handleReadError() { d.totalDiskSpaceUsed = 0 d.writeMessages = 0 } else { - err = d.updateTotalDiskSpaceUsed() + // err = d.updateTotalDiskSpaceUsed() if err != nil { d.logf(ERROR, "DISKQUEUE(%s) failed to update write bytes - %s", d.name, err) } diff --git a/diskqueue_test.go b/diskqueue_test.go index 7e216d4..5351f16 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -267,7 +267,6 @@ func TestDiskQueueCorruption(t *testing.T) { type md struct { depth int64 - writeBytes int64 readFileNum int64 writeFileNum int64 readMessages int64 @@ -293,10 +292,10 @@ func readMetaDataFile(fileName string, retried int, enableDiskLimitation bool) m var ret md if enableDiskLimitation { - _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d,%d\n", + _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d\n", &ret.depth, &ret.readFileNum, &ret.readMessages, &ret.readPos, - &ret.writeBytes, &ret.writeFileNum, &ret.writeMessages, &ret.writePos) + &ret.writeFileNum, &ret.writeMessages, &ret.writePos) } else { _, err = fmt.Fscanf(f, "%d\n%d,%d\n%d,%d\n", &ret.depth, @@ -358,6 +357,29 @@ next: done: } +func metaDataFileSize(metaDataFileName string) int64 { + metaDataFile, err := os.OpenFile(metaDataFileName, os.O_RDONLY, 0600) + + var metaDataFileSize int64 + if err == nil { + var stat os.FileInfo + + stat, err = metaDataFile.Stat() + if err == nil { + metaDataFileSize = stat.Size() + } + } + if err != nil { + // use max file size (7 int64 fields) + metaDataFileSize = 56 + } + + metaDataFile.Close() + + // use max file size (7 int64 fields) + return metaDataFileSize +} + func TestDiskQueueSyncAfterReadWithDiskSizeImplementation(t *testing.T) { l := NewTestLogger(t) dqName := "test_disk_queue_read_after_sync" + strconv.Itoa(int(time.Now().Unix())) @@ -373,16 +395,17 @@ func TestDiskQueueSyncAfterReadWithDiskSizeImplementation(t *testing.T) { msg := make([]byte, msgSize) dq.Put(msg) + metaDataSize := metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 1 && - d.writeBytes == 1004 && d.readFileNum == 0 && d.writeFileNum == 0 && d.readMessages == 0 && d.writeMessages == 1 && d.readPos == 0 && - d.writePos == 1004 { + d.writePos == 1004 && + dq.(*diskQueue).totalDiskSpaceUsed == 1004+metaDataSize { // success goto next } @@ -394,16 +417,17 @@ next: dq.Put(msg) <-dq.ReadChan() + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 1 && - d.writeBytes == 2008 && d.readFileNum == 0 && d.writeFileNum == 0 && d.readMessages == 1 && d.writeMessages == 2 && d.readPos == 1004 && - d.writePos == 2008 { + d.writePos == 2008 && + dq.(*diskQueue).totalDiskSpaceUsed == 2008+metaDataSize { // success goto completeWriteFile } @@ -420,18 +444,19 @@ completeWriteFile: dq.Put(make([]byte, bytesRemaining-4-oneByteMsgSizeIncrease)) dq.Put(make([]byte, 1)) + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that write position and messages reset when a new file is created // test the writeFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 3 && - d.writeBytes == 2048 && d.readFileNum == 0 && d.writeFileNum == 1 && d.readMessages == 1 && d.writeMessages == 0 && d.readPos == 1004 && - d.writePos == 0 { + d.writePos == 0 && + dq.(*diskQueue).totalDiskSpaceUsed == 2048+metaDataSize { // success goto completeReadFile } @@ -445,19 +470,20 @@ completeReadFile: <-dq.ReadChan() <-dq.ReadChan() <-dq.ReadChan() + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 1 && - d.writeBytes == 1004 && d.readFileNum == 1 && d.writeFileNum == 1 && d.readMessages == 0 && d.writeMessages == 1 && d.readPos == 0 && - d.writePos == 1004 { + d.writePos == 1004 && + dq.(*diskQueue).totalDiskSpaceUsed == 1004+metaDataSize { // success goto completeWriteFileAgain } @@ -477,18 +503,19 @@ completeWriteFileAgain: dq.Put(make([]byte, bytesRemaining-4-oneByteMsgSizeIncrease)) dq.Put(make([]byte, 1)) + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that write position and messages reset when a file is completely read // test the writeFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 7 && - d.writeBytes == 5068 && d.readFileNum == 1 && d.writeFileNum == 3 && d.readMessages == 0 && d.writeMessages == 0 && d.readPos == 0 && - d.writePos == 0 { + d.writePos == 0 && + dq.(*diskQueue).totalDiskSpaceUsed == 5068+metaDataSize { // success goto completeReadFileAgain } @@ -506,18 +533,19 @@ completeReadFileAgain: <-dq.ReadChan() <-dq.ReadChan() + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 0 && - d.writeBytes == 0 && d.readFileNum == 3 && d.writeFileNum == 3 && d.readMessages == 0 && d.writeMessages == 0 && d.readPos == 0 && - d.writePos == 0 { + d.writePos == 0 && + dq.(*diskQueue).totalDiskSpaceUsed == metaDataSize { // success goto meetDiskSizeLimit } @@ -538,20 +566,8 @@ meetDiskSizeLimit: totalDiskBytes := int64(5*(msgSize+4) + 8) - metaDataFile, err := os.OpenFile(dq.(*diskQueue).metaDataFileName(), os.O_RDONLY, 0600) - - var metaDataFileSize int64 - if err == nil { - var stat os.FileInfo - - stat, err = metaDataFile.Stat() - if err == nil { - metaDataFileSize = stat.Size() - } - } - // save space for msg len and number of msgs in file - diskBytesRemaining := 6040 - metaDataFileSize - (totalDiskBytes + 12) + diskBytesRemaining := 6040 - metaDataFileSize(dq.(*diskQueue).metaDataFileName()) - (totalDiskBytes + 12) dq.Put(make([]byte, diskBytesRemaining)) for i := 0; i < 10; i++ { @@ -559,13 +575,13 @@ meetDiskSizeLimit: // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 6 && - d.writeBytes == 6040-metaDataFileSize && d.readFileNum == 3 && d.writeFileNum == 5 && d.readMessages == 0 && d.writeMessages == 0 && d.readPos == 0 && - d.writePos == 0 { + d.writePos == 0 && + dq.(*diskQueue).totalDiskSpaceUsed == 6040 { // success goto surpassDiskSizeLimit } @@ -581,13 +597,13 @@ surpassDiskSizeLimit: // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 4 && - d.writeBytes == 3025-metaDataFileSize && d.readFileNum == 4 && d.writeFileNum == 5 && d.readMessages == 0 && d.writeMessages == 1 && d.readPos == 0 && - d.writePos == 5 { + d.writePos == 5 && + dq.(*diskQueue).totalDiskSpaceUsed == 3025 { // success goto done } @@ -625,16 +641,17 @@ func TestDiskSizeImplementationMsgSizeGreaterThanFileSize(t *testing.T) { // file size: 1512 dq.Put(make([]byte, 1500)) + metaDataSize := metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 5 && - d.writeBytes == 4077 && d.readFileNum == 0 && d.writeFileNum == 3 && d.readMessages == 0 && d.writeMessages == 0 && d.readPos == 0 && - d.writePos == 0 { + d.writePos == 0 && + dq.(*diskQueue).totalDiskSpaceUsed == 4077+metaDataSize { // success goto writeLargeMsg } @@ -646,18 +663,19 @@ writeLargeMsg: // Write a large message that causes the deletion of three files dq.Put(make([]byte, 3000)) + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 1 && - d.writeBytes == 3012 && d.readFileNum == 3 && d.writeFileNum == 4 && d.readMessages == 0 && d.writeMessages == 0 && d.readPos == 0 && - d.writePos == 0 { + d.writePos == 0 && + dq.(*diskQueue).totalDiskSpaceUsed == 3012+metaDataSize { // success goto done } @@ -788,16 +806,17 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { panic("fail") } + metaDataSize := metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 5 && - d.writeBytes == 4041 && d.readFileNum == 0 && d.writeFileNum == 3 && d.readMessages == 0 && d.writeMessages == 0 && d.readPos == 0 && - d.writePos == 0 { + d.writePos == 0 && + dq.(*diskQueue).totalDiskSpaceUsed == 4041+metaDataSize { // success goto corruptFiles } @@ -806,9 +825,8 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { panic("fail") corruptFiles: - t.Log("CORRUPT") // test removeReadFile when file is corrupted - // create bad files see if writebytes is updated properly + // create bad files see if totalDiskSpaceUsed is updated properly // check that after corrupting files, we make space appropriately // corrupt file 0 @@ -823,22 +841,22 @@ corruptFiles: panic("fail") } + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) - t.Log(d.writeBytes, d.readFileNum, d.writeFileNum, d.readMessages, d.writeMessages) - if d.writeBytes == 2648 && - d.readFileNum == 1 && + if d.readFileNum == 1 && d.writeFileNum == 3 && d.readMessages == 0 && d.writeMessages == 1 && d.readPos == 0 && - d.writePos == 104 { + d.writePos == 104 && + dq.(*diskQueue).totalDiskSpaceUsed == 2648+metaDataSize { // success goto readCorruptedFile } time.Sleep(100 * time.Millisecond) } - panic("fail") + panic("fail1") readCorruptedFile: // test handleReadError @@ -877,20 +895,19 @@ readCorruptedFile: // check if the corrupted file was deleted to make space badFilesCount = numberOfBadFiles(dqName, tmpDir) if badFilesCount != 0 { - t.Log("BAD FILE COUNT:", badFilesCount) panic("fail") } + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) - t.Log(d.writeBytes, d.readFileNum, d.writeFileNum, d.readMessages, d.writeMessages) - if d.writeBytes == 3132 && - d.readFileNum == 3 && + if d.readFileNum == 3 && d.writeFileNum == 5 && d.readMessages == 0 && d.writeMessages == 0 && d.readPos == 0 && - d.writePos == 0 { + d.writePos == 0 && + dq.(*diskQueue).totalDiskSpaceUsed == 3132+metaDataSize { // success goto done } From 4f017a3a7e9b1604dcb9a418e09c42d483968d32 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 17 Jun 2021 15:45:25 +0000 Subject: [PATCH 51/67] Remove unnecessary comments. --- diskqueue.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 50269ec..13db54c 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -890,7 +890,6 @@ func (d *diskQueue) moveToNextReadFile() { if d.enableDiskLimitation { d.readMessages = 0 - // err = d.updateTotalDiskSpaceUsed() if err != nil { d.logf(ERROR, "DISKQUEUE(%s) failed to update write bytes - %s", d.name, err) } @@ -943,7 +942,6 @@ func (d *diskQueue) handleReadError() { d.totalDiskSpaceUsed = 0 d.writeMessages = 0 } else { - // err = d.updateTotalDiskSpaceUsed() if err != nil { d.logf(ERROR, "DISKQUEUE(%s) failed to update write bytes - %s", d.name, err) } From 3d62b73e792eb0b47936f15cd119f650072a5b87 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 17 Jun 2021 15:53:45 +0000 Subject: [PATCH 52/67] Break up huge function into two smaller functions. --- diskqueue.go | 82 ++++++++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 13db54c..2cfe50d 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -561,11 +561,51 @@ func (d *diskQueue) updateTotalDiskSpaceUsed() error { return d.walkDiskQueueDir(updateTotalDiskSpaceUsed) } -// free up disk space if needed to write new data to file -func (d *diskQueue) freeUpDiskSpace(totalBytes int64, reachedFileSizeLimit bool) error { +func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { var err error var badFileInfos []fs.FileInfo + badFileInfos, err = d.getAllBadFileInfo() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to retrieve all .bad file info - %s", d.name, err) + } + + // keep freeing up disk space until we have enough space to write this message + for d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { + if badFileInfos != nil { + // check if a .bad file exists. If it does, delete that first + oldestBadFileInfo := badFileInfos[0] + badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) + + err = os.Remove(badFileFilePath) + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) + d.updateTotalDiskSpaceUsed() + return err + } else { + // recaclulate total bad files disk size to get most accurate info + d.totalDiskSpaceUsed -= oldestBadFileInfo.Size() + } + + badFileInfos = badFileInfos[1:] + } else { + // delete the read file (make space) + err = d.removeReadFile() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(d.readFileNum), err) + d.handleReadError() + return err + + } + d.updateTotalDiskSpaceUsed() + } + } + + return nil +} + +// check if there is enough available disk space to write new data to file +func (d *diskQueue) checkDiskSpace(totalBytes int64, reachedFileSizeLimit bool) error { expectedBytesIncrease := totalBytes if reachedFileSizeLimit { expectedBytesIncrease += numFileMsgBytes @@ -578,41 +618,7 @@ func (d *diskQueue) freeUpDiskSpace(totalBytes int64, reachedFileSizeLimit bool) // check if we have enough space to write this message if d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { - badFileInfos, err = d.getAllBadFileInfo() - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to retrieve all .bad file info - %s", d.name, err) - } - - // keep freeing up disk space until we have enough space to write this message - for d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { - if badFileInfos != nil { - // check if a .bad file exists. If it does, delete that first - oldestBadFileInfo := badFileInfos[0] - badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) - - err = os.Remove(badFileFilePath) - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) - d.updateTotalDiskSpaceUsed() - return err - } else { - // recaclulate total bad files disk size to get most accurate info - d.totalDiskSpaceUsed -= oldestBadFileInfo.Size() - } - - badFileInfos = badFileInfos[1:] - } else { - // delete the read file (make space) - err = d.removeReadFile() - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(d.readFileNum), err) - d.handleReadError() - return err - - } - d.updateTotalDiskSpaceUsed() - } - } + return d.freeDiskSpace(expectedBytesIncrease) } return nil } @@ -657,7 +663,7 @@ func (d *diskQueue) writeOne(data []byte) error { } // free disk space if needed - d.freeUpDiskSpace(totalBytes, reachedFileSizeLimit) + d.checkDiskSpace(totalBytes, reachedFileSizeLimit) } else if d.writePos+totalBytes >= d.maxBytesPerFile { reachedFileSizeLimit = true } From 069c7f8e3e08730e1802c3150373bc31286a9945 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 17 Jun 2021 18:07:33 +0000 Subject: [PATCH 53/67] Ensure that global regExp are created on start. --- diskqueue.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 2cfe50d..9b4d7d7 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -174,8 +174,8 @@ func (d *diskQueue) start() { d.logf(ERROR, "DISKQUEUE(%s) failed to retrieveMetaData - %s", d.name, err) } - fileNameRegexp, _ = regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat$`) - badFileNameRegexp, _ = regexp.Compile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat.bad$`) + fileNameRegexp = regexp.MustCompile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat$`) + badFileNameRegexp = regexp.MustCompile(`^` + d.name + `.diskqueue.\d\d\d\d\d\d.dat.bad$`) d.updateTotalDiskSpaceUsed() From 2e688130db8921af66af87dda9d4c6bb921a9780 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 17 Jun 2021 21:08:18 +0000 Subject: [PATCH 54/67] Add testing of depth, add info logs, and test when disk size limit is too small (not enough space for meta data file). --- diskqueue.go | 27 +++++++++++++++++++-------- diskqueue_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 9b4d7d7..1ac411b 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -23,12 +23,13 @@ import ( type LogLevel int const ( - DEBUG = LogLevel(1) - INFO = LogLevel(2) - WARN = LogLevel(3) - ERROR = LogLevel(4) - FATAL = LogLevel(5) - numFileMsgBytes = 8 + DEBUG = LogLevel(1) + INFO = LogLevel(2) + WARN = LogLevel(3) + ERROR = LogLevel(4) + FATAL = LogLevel(5) + numFileMsgBytes = 8 + maxMetaDataFileSize = 56 ) var badFileNameRegexp, fileNameRegexp *regexp.Regexp @@ -426,7 +427,7 @@ func (d *diskQueue) metaDataFileSize() int64 { } if err != nil { // use max file size (7 int64 fields) - metaDataFileSize = 56 + metaDataFileSize = maxMetaDataFileSize } metaDataFile.Close() @@ -585,6 +586,7 @@ func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { } else { // recaclulate total bad files disk size to get most accurate info d.totalDiskSpaceUsed -= oldestBadFileInfo.Size() + d.logf(INFO, "DISKQUEUE(%s) removed .bad file(%s) to free up disk space", d.name, oldestBadFileInfo.Name()) } badFileInfos = badFileInfos[1:] @@ -596,6 +598,8 @@ func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { d.handleReadError() return err + } else { + d.logf(INFO, "DISKQUEUE(%s) removed file(%s) to free up disk space", d.name, d.fileName(d.readFileNum)) } d.updateTotalDiskSpaceUsed() } @@ -616,6 +620,10 @@ func (d *diskQueue) checkDiskSpace(totalBytes int64, reachedFileSizeLimit bool) return errors.New("message size surpasses disk size limit") } + if d.maxBytesDiskSpace <= maxMetaDataFileSize { + return errors.New("disk size limit too small: not enough space for MetaData file size") + } + // check if we have enough space to write this message if d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { return d.freeDiskSpace(expectedBytesIncrease) @@ -663,7 +671,10 @@ func (d *diskQueue) writeOne(data []byte) error { } // free disk space if needed - d.checkDiskSpace(totalBytes, reachedFileSizeLimit) + err = d.checkDiskSpace(totalBytes, reachedFileSizeLimit) + if err != nil { + return err + } } else if d.writePos+totalBytes >= d.maxBytesPerFile { reachedFileSizeLimit = true } diff --git a/diskqueue_test.go b/diskqueue_test.go index 5351f16..6b1c64c 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -395,6 +395,10 @@ func TestDiskQueueSyncAfterReadWithDiskSizeImplementation(t *testing.T) { msg := make([]byte, msgSize) dq.Put(msg) + if dq.Depth() != 1 { + panic("fail") + } + metaDataSize := metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) @@ -417,6 +421,10 @@ next: dq.Put(msg) <-dq.ReadChan() + if dq.Depth() != 1 { + panic("fail") + } + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) @@ -444,6 +452,10 @@ completeWriteFile: dq.Put(make([]byte, bytesRemaining-4-oneByteMsgSizeIncrease)) dq.Put(make([]byte, 1)) + if dq.Depth() != 3 { + panic("fail") + } + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that write position and messages reset when a new file is created @@ -472,6 +484,10 @@ completeReadFile: <-dq.ReadChan() metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) + if dq.Depth() != 1 { + panic("fail") + } + for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read // test the readFileNum correctly increments @@ -503,6 +519,10 @@ completeWriteFileAgain: dq.Put(make([]byte, bytesRemaining-4-oneByteMsgSizeIncrease)) dq.Put(make([]byte, 1)) + if dq.Depth() != 7 { + panic("fail") + } + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that write position and messages reset when a file is completely read @@ -533,6 +553,10 @@ completeReadFileAgain: <-dq.ReadChan() <-dq.ReadChan() + if dq.Depth() != 0 { + panic("fail") + } + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read @@ -570,6 +594,10 @@ meetDiskSizeLimit: diskBytesRemaining := 6040 - metaDataFileSize(dq.(*diskQueue).metaDataFileName()) - (totalDiskBytes + 12) dq.Put(make([]byte, diskBytesRemaining)) + if dq.Depth() != 6 { + panic("fail") + } + for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read // test the readFileNum correctly increments @@ -592,6 +620,10 @@ meetDiskSizeLimit: surpassDiskSizeLimit: dq.Put(make([]byte, 1)) + if dq.Depth() != 4 { + panic("fail") + } + for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read // test the readFileNum correctly increments @@ -641,6 +673,10 @@ func TestDiskSizeImplementationMsgSizeGreaterThanFileSize(t *testing.T) { // file size: 1512 dq.Put(make([]byte, 1500)) + if dq.Depth() != 5 { + panic("fail") + } + metaDataSize := metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) @@ -663,6 +699,10 @@ writeLargeMsg: // Write a large message that causes the deletion of three files dq.Put(make([]byte, 3000)) + if dq.Depth() != 1 { + panic("fail") + } + metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read @@ -806,6 +846,10 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { panic("fail") } + if dq.Depth() != 5 { + panic("fail") + } + metaDataSize := metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) From be18a8965606b109aba7b84cf4a3014ea4da4d52 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Fri, 18 Jun 2021 18:34:52 +0000 Subject: [PATCH 55/67] Update name of dq objects to match function name. --- diskqueue_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/diskqueue_test.go b/diskqueue_test.go index 6b1c64c..952eb63 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -382,7 +382,7 @@ func metaDataFileSize(metaDataFileName string) int64 { func TestDiskQueueSyncAfterReadWithDiskSizeImplementation(t *testing.T) { l := NewTestLogger(t) - dqName := "test_disk_queue_read_after_sync" + strconv.Itoa(int(time.Now().Unix())) + dqName := "test_disk_queue_read_with_disk_size_implementation" + strconv.Itoa(int(time.Now().Unix())) tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano())) if err != nil { panic(err) @@ -650,7 +650,7 @@ func TestDiskSizeImplementationMsgSizeGreaterThanFileSize(t *testing.T) { // write three files l := NewTestLogger(t) - dqName := "test_disk_queue_read_after_sync" + strconv.Itoa(int(time.Now().Unix())) + dqName := "test_disk_queue_implementation_msg_size_greater_than_file_size" + strconv.Itoa(int(time.Now().Unix())) tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano())) if err != nil { panic(err) @@ -788,7 +788,7 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { // write three files l := NewTestLogger(t) - dqName := "test_disk_queue_read_after_sync" + strconv.Itoa(int(time.Now().Unix())) + dqName := "test_disk_queue_implementation_with_bad_files" + strconv.Itoa(int(time.Now().Unix())) tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano())) if err != nil { panic(err) From 26577b68dcdbfc774d49c1ea9e0bbed4079815c4 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 23 Jun 2021 18:29:58 +0000 Subject: [PATCH 56/67] Use ReadDir instead of WalkDir --- diskqueue.go | 46 ++++++++++++---------------------------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 1ac411b..e96bb9c 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -7,13 +7,10 @@ import ( "errors" "fmt" "io" - "io/fs" "math/rand" "os" "path" - "path/filepath" "regexp" - "strings" "sync" "time" ) @@ -487,46 +484,27 @@ func (d *diskQueue) removeReadFile() error { } // walk through all of the files in the DiskQueue directory -func (d *diskQueue) walkDiskQueueDir(fn func(fs.DirEntry) error) error { - // the directory containing DiskQueue files - var mainDir string - if d.dataPath == "/" { - mainDir = d.dataPath - } else { - pathArray := strings.Split(d.dataPath, "/") - mainDir = pathArray[len(pathArray)-1] - } - - walkDiskQueueDir := func(pathStr string, dirEntry fs.DirEntry, err error) error { - if dirEntry.Name() == mainDir { - // we want to see the contents of the directory DiskQueue writes and reads in - return nil - } +func (d *diskQueue) walkDiskQueueDir(fn func(os.DirEntry) error) error { + dirEntries, err := os.ReadDir(d.dataPath) - if dirEntry.IsDir() { - // if the entry is a directory, skip it - return fs.SkipDir - } + if err != nil { + return err + } + for _, dirEntry := range dirEntries { + err = fn(dirEntry) if err != nil { return err } - - return fn(dirEntry) - } - - err := filepath.WalkDir(d.dataPath, walkDiskQueueDir) - if err != nil { - return err } return nil } -func (d *diskQueue) getAllBadFileInfo() ([]fs.FileInfo, error) { - var badFileInfos []fs.FileInfo +func (d *diskQueue) getAllBadFileInfo() ([]os.FileInfo, error) { + var badFileInfos []os.FileInfo - getAllBadFileInfo := func(dirEntry fs.DirEntry) error { + getAllBadFileInfo := func(dirEntry os.DirEntry) error { // only accept "bad" files created by this DiskQueue object if badFileNameRegexp.MatchString(dirEntry.Name()) { badFileInfo, e := dirEntry.Info() @@ -547,7 +525,7 @@ func (d *diskQueue) getAllBadFileInfo() ([]fs.FileInfo, error) { func (d *diskQueue) updateTotalDiskSpaceUsed() error { d.totalDiskSpaceUsed = d.metaDataFileSize() - updateTotalDiskSpaceUsed := func(dirEntry fs.DirEntry) error { + updateTotalDiskSpaceUsed := func(dirEntry os.DirEntry) error { // only accept files created by this DiskQueue object if fileNameRegexp.MatchString(dirEntry.Name()) || badFileNameRegexp.MatchString(dirEntry.Name()) { fileInfo, e := dirEntry.Info() @@ -564,7 +542,7 @@ func (d *diskQueue) updateTotalDiskSpaceUsed() error { func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { var err error - var badFileInfos []fs.FileInfo + var badFileInfos []os.FileInfo badFileInfos, err = d.getAllBadFileInfo() if err != nil { From 3ffa67c4edcdbec1a7167f70ec414718228ed604 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 24 Jun 2021 15:35:45 +0000 Subject: [PATCH 57/67] Replace go v1.16 functions and objects with Date: Fri, 25 Jun 2021 19:41:29 +0000 Subject: [PATCH 58/67] Have metaData file size stay as 56 bytes and update testing. --- diskqueue.go | 42 ++++---------- diskqueue_test.go | 140 ++++++++++++++++------------------------------ 2 files changed, 59 insertions(+), 123 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 06bbd8c..5929a0f 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -407,32 +407,6 @@ func (d *diskQueue) readOne() ([]byte, error) { return readBuf, nil } -// get the size of the metaData file or its max possible size -func (d *diskQueue) metaDataFileSize() int64 { - var err error - var metaDataFile *os.File - - metaDataFile, err = os.OpenFile(d.metaDataFileName(), os.O_RDONLY, 0600) - - var metaDataFileSize int64 - if err == nil { - var stat os.FileInfo - - stat, err = metaDataFile.Stat() - if err == nil { - metaDataFileSize = stat.Size() - } - } - if err != nil { - // use max file size (7 int64 fields) - metaDataFileSize = maxMetaDataFileSize - } - - metaDataFile.Close() - - return metaDataFileSize -} - func (d *diskQueue) removeReadFile() error { var err error @@ -521,7 +495,7 @@ func (d *diskQueue) getAllBadFileInfo() ([]os.FileInfo, error) { // get the accurate total non-"bad" file size func (d *diskQueue) updateTotalDiskSpaceUsed() error { - d.totalDiskSpaceUsed = d.metaDataFileSize() + d.totalDiskSpaceUsed = maxMetaDataFileSize updateTotalDiskSpaceUsed := func(fileInfo os.FileInfo) error { // only accept files created by this DiskQueue object @@ -546,7 +520,7 @@ func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { // keep freeing up disk space until we have enough space to write this message for d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { - if badFileInfos != nil { + if len(badFileInfos) > 0 { // check if a .bad file exists. If it does, delete that first oldestBadFileInfo := badFileInfos[0] badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) @@ -590,11 +564,19 @@ func (d *diskQueue) checkDiskSpace(totalBytes int64, reachedFileSizeLimit bool) // If the data to be written is bigger than the disk size limit, do not write if expectedBytesIncrease > d.maxBytesDiskSpace { - return errors.New("message size surpasses disk size limit") + errorMsg := fmt.Sprintf( + "message size(%d) surpasses disk size limit(%d)", + expectedBytesIncrease, d.maxBytesDiskSpace) + d.logf(ERROR, "DISKQUEUE(%s) - %s", d.name, errorMsg) + return errors.New(errorMsg) } if d.maxBytesDiskSpace <= maxMetaDataFileSize { - return errors.New("disk size limit too small: not enough space for MetaData file size") + errorMsg := fmt.Sprintf( + "disk size limit too small(%d): not enough space for MetaData file size(%d)", + d.maxBytesDiskSpace, maxMetaDataFileSize) + d.logf(ERROR, "DISKQUEUE(%s) - %s", errorMsg) + return errors.New(errorMsg) } // check if we have enough space to write this message diff --git a/diskqueue_test.go b/diskqueue_test.go index 952eb63..4cd5239 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "fmt" - "io/fs" "io/ioutil" "os" "path" @@ -13,7 +12,6 @@ import ( "regexp" "runtime" "strconv" - "strings" "sync" "sync/atomic" "testing" @@ -357,29 +355,6 @@ next: done: } -func metaDataFileSize(metaDataFileName string) int64 { - metaDataFile, err := os.OpenFile(metaDataFileName, os.O_RDONLY, 0600) - - var metaDataFileSize int64 - if err == nil { - var stat os.FileInfo - - stat, err = metaDataFile.Stat() - if err == nil { - metaDataFileSize = stat.Size() - } - } - if err != nil { - // use max file size (7 int64 fields) - metaDataFileSize = 56 - } - - metaDataFile.Close() - - // use max file size (7 int64 fields) - return metaDataFileSize -} - func TestDiskQueueSyncAfterReadWithDiskSizeImplementation(t *testing.T) { l := NewTestLogger(t) dqName := "test_disk_queue_read_with_disk_size_implementation" + strconv.Itoa(int(time.Now().Unix())) @@ -399,7 +374,6 @@ func TestDiskQueueSyncAfterReadWithDiskSizeImplementation(t *testing.T) { panic("fail") } - metaDataSize := metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 1 && @@ -409,7 +383,7 @@ func TestDiskQueueSyncAfterReadWithDiskSizeImplementation(t *testing.T) { d.writeMessages == 1 && d.readPos == 0 && d.writePos == 1004 && - dq.(*diskQueue).totalDiskSpaceUsed == 1004+metaDataSize { + dq.(*diskQueue).totalDiskSpaceUsed == 1004+maxMetaDataFileSize { // success goto next } @@ -425,7 +399,6 @@ next: panic("fail") } - metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 1 && @@ -435,7 +408,7 @@ next: d.writeMessages == 2 && d.readPos == 1004 && d.writePos == 2008 && - dq.(*diskQueue).totalDiskSpaceUsed == 2008+metaDataSize { + dq.(*diskQueue).totalDiskSpaceUsed == 2008+maxMetaDataFileSize { // success goto completeWriteFile } @@ -456,7 +429,6 @@ completeWriteFile: panic("fail") } - metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that write position and messages reset when a new file is created // test the writeFileNum correctly increments @@ -468,7 +440,7 @@ completeWriteFile: d.writeMessages == 0 && d.readPos == 1004 && d.writePos == 0 && - dq.(*diskQueue).totalDiskSpaceUsed == 2048+metaDataSize { + dq.(*diskQueue).totalDiskSpaceUsed == 2048+maxMetaDataFileSize { // success goto completeReadFile } @@ -482,7 +454,6 @@ completeReadFile: <-dq.ReadChan() <-dq.ReadChan() <-dq.ReadChan() - metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) if dq.Depth() != 1 { panic("fail") @@ -499,7 +470,7 @@ completeReadFile: d.writeMessages == 1 && d.readPos == 0 && d.writePos == 1004 && - dq.(*diskQueue).totalDiskSpaceUsed == 1004+metaDataSize { + dq.(*diskQueue).totalDiskSpaceUsed == 1004+maxMetaDataFileSize { // success goto completeWriteFileAgain } @@ -523,7 +494,6 @@ completeWriteFileAgain: panic("fail") } - metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that write position and messages reset when a file is completely read // test the writeFileNum correctly increments @@ -535,7 +505,7 @@ completeWriteFileAgain: d.writeMessages == 0 && d.readPos == 0 && d.writePos == 0 && - dq.(*diskQueue).totalDiskSpaceUsed == 5068+metaDataSize { + dq.(*diskQueue).totalDiskSpaceUsed == 5068+maxMetaDataFileSize { // success goto completeReadFileAgain } @@ -557,7 +527,6 @@ completeReadFileAgain: panic("fail") } - metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read // test the readFileNum correctly increments @@ -569,15 +538,32 @@ completeReadFileAgain: d.writeMessages == 0 && d.readPos == 0 && d.writePos == 0 && - dq.(*diskQueue).totalDiskSpaceUsed == metaDataSize { + dq.(*diskQueue).totalDiskSpaceUsed == maxMetaDataFileSize { // success - goto meetDiskSizeLimit + goto done } time.Sleep(100 * time.Millisecond) } panic("fail") -meetDiskSizeLimit: +done: +} + +func TestDiskSizeImplementationDiskSizeLimit(t *testing.T) { + l := NewTestLogger(t) + dqName := "test_disk_queue_implementation_disk_size_limit" + strconv.Itoa(int(time.Now().Unix())) + tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano())) + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpDir) + dq := NewWithDiskSpace(dqName, tmpDir, 6040, 1<<11, 0, 1<<10, 2500, 50*time.Millisecond, l) + defer dq.Close() + + msgSize := 1000 + msg := make([]byte, msgSize) + + // meet disk size limit // write a complete file dq.Put(msg) dq.Put(msg) @@ -591,10 +577,11 @@ meetDiskSizeLimit: totalDiskBytes := int64(5*(msgSize+4) + 8) // save space for msg len and number of msgs in file - diskBytesRemaining := 6040 - metaDataFileSize(dq.(*diskQueue).metaDataFileName()) - (totalDiskBytes + 12) + diskBytesRemaining := 6040 - maxMetaDataFileSize - (totalDiskBytes + 12) dq.Put(make([]byte, diskBytesRemaining)) - if dq.Depth() != 6 { + depth := dq.Depth() + if depth != 6 { panic("fail") } @@ -603,8 +590,8 @@ meetDiskSizeLimit: // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 6 && - d.readFileNum == 3 && - d.writeFileNum == 5 && + d.readFileNum == 0 && + d.writeFileNum == 2 && d.readMessages == 0 && d.writeMessages == 0 && d.readPos == 0 && @@ -629,8 +616,8 @@ surpassDiskSizeLimit: // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 4 && - d.readFileNum == 4 && - d.writeFileNum == 5 && + d.readFileNum == 1 && + d.writeFileNum == 2 && d.readMessages == 0 && d.writeMessages == 1 && d.readPos == 0 && @@ -662,9 +649,9 @@ func TestDiskSizeImplementationMsgSizeGreaterThanFileSize(t *testing.T) { msgSize := 1000 msg := make([]byte, msgSize) - // file size: 1533 + // file size: 1496 dq.Put(msg) - dq.Put(make([]byte, 517)) + dq.Put(make([]byte, 480)) // file size: 1032 dq.Put(msg) @@ -677,7 +664,6 @@ func TestDiskSizeImplementationMsgSizeGreaterThanFileSize(t *testing.T) { panic("fail") } - metaDataSize := metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 5 && @@ -687,7 +673,7 @@ func TestDiskSizeImplementationMsgSizeGreaterThanFileSize(t *testing.T) { d.writeMessages == 0 && d.readPos == 0 && d.writePos == 0 && - dq.(*diskQueue).totalDiskSpaceUsed == 4077+metaDataSize { + dq.(*diskQueue).totalDiskSpaceUsed == 4040+maxMetaDataFileSize { // success goto writeLargeMsg } @@ -703,7 +689,6 @@ writeLargeMsg: panic("fail") } - metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { // test that read position and messages reset when a file is completely read // test the readFileNum correctly increments @@ -715,7 +700,7 @@ writeLargeMsg: d.writeMessages == 0 && d.readPos == 0 && d.writePos == 0 && - dq.(*diskQueue).totalDiskSpaceUsed == 3012+metaDataSize { + dq.(*diskQueue).totalDiskSpaceUsed == 3012+maxMetaDataFileSize { // success goto done } @@ -744,41 +729,12 @@ func createBadFile(dqName string, filePath string, fileNum int64, numBytes int) func numberOfBadFiles(diskQueueName string, dataPath string) int64 { var badFilesCount int64 - // the directory containing DiskQueue files - var mainDir string - if dataPath == "/" { - mainDir = dataPath - } else { - pathArray := strings.Split(dataPath, "/") - mainDir = pathArray[len(pathArray)-1] - } - - getBadFileInfos := func(pathStr string, dirEntry fs.DirEntry, err error) error { - if dirEntry.Name() == mainDir { - // we want to see the contents of this directory - return nil - } - - if dirEntry.IsDir() { - // if the entry is a directory, skip it - return fs.SkipDir - } - - if err != nil { - return err - } - + fileInfos, _ := ioutil.ReadDir(dataPath) + for _, fileInfo := range fileInfos { regExp, _ := regexp.Compile(`^` + diskQueueName + `.diskqueue.\d\d\d\d\d\d.dat.bad$`) - if regExp.MatchString(dirEntry.Name()) { + if regExp.MatchString(fileInfo.Name()) { badFilesCount++ } - - return nil - } - - err := filepath.WalkDir(dataPath, getBadFileInfos) - if err != nil { - return 0 } return badFilesCount @@ -837,8 +793,8 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { panic("fail") } - // file 2 size: 1512 - dq.Put(make([]byte, 1500)) + // file 2 size: 1503 + dq.Put(make([]byte, 1491)) // check if all the .bad files were deleted badFilesCount = numberOfBadFiles(dqName, tmpDir) @@ -846,11 +802,11 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { panic("fail") } - if dq.Depth() != 5 { + depth := dq.Depth() + if depth != 5 { panic("fail") } - metaDataSize := metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 5 && @@ -860,7 +816,7 @@ func TestDiskSizeImplementationWithBadFiles(t *testing.T) { d.writeMessages == 0 && d.readPos == 0 && d.writePos == 0 && - dq.(*diskQueue).totalDiskSpaceUsed == 4041+metaDataSize { + dq.(*diskQueue).totalDiskSpaceUsed == 4032+maxMetaDataFileSize { // success goto corruptFiles } @@ -885,7 +841,6 @@ corruptFiles: panic("fail") } - metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.readFileNum == 1 && @@ -894,13 +849,13 @@ corruptFiles: d.writeMessages == 1 && d.readPos == 0 && d.writePos == 104 && - dq.(*diskQueue).totalDiskSpaceUsed == 2648+metaDataSize { + dq.(*diskQueue).totalDiskSpaceUsed == 2639+maxMetaDataFileSize { // success goto readCorruptedFile } time.Sleep(100 * time.Millisecond) } - panic("fail1") + panic("fail") readCorruptedFile: // test handleReadError @@ -942,7 +897,6 @@ readCorruptedFile: panic("fail") } - metaDataSize = metaDataFileSize(dq.(*diskQueue).metaDataFileName()) for i := 0; i < 10; i++ { d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.readFileNum == 3 && @@ -951,7 +905,7 @@ readCorruptedFile: d.writeMessages == 0 && d.readPos == 0 && d.writePos == 0 && - dq.(*diskQueue).totalDiskSpaceUsed == 3132+metaDataSize { + dq.(*diskQueue).totalDiskSpaceUsed == 3132+maxMetaDataFileSize { // success goto done } From 89ba90bdc4b7651b584996ada2d053ce7e26c9de Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Mon, 28 Jun 2021 20:38:47 +0000 Subject: [PATCH 59/67] Add the number of bytes that were removed when a file is removed. --- diskqueue.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 5929a0f..b39dfe5 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -533,7 +533,7 @@ func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { } else { // recaclulate total bad files disk size to get most accurate info d.totalDiskSpaceUsed -= oldestBadFileInfo.Size() - d.logf(INFO, "DISKQUEUE(%s) removed .bad file(%s) to free up disk space", d.name, oldestBadFileInfo.Name()) + d.logf(INFO, "DISKQUEUE(%s) removed .bad file(%s) of size(%d bytes) to free up disk space", d.name, oldestBadFileInfo.Name(), oldestBadFileInfo.Size()) } badFileInfos = badFileInfos[1:] @@ -544,7 +544,6 @@ func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(d.readFileNum), err) d.handleReadError() return err - } else { d.logf(INFO, "DISKQUEUE(%s) removed file(%s) to free up disk space", d.name, d.fileName(d.readFileNum)) } @@ -855,9 +854,14 @@ func (d *diskQueue) moveToNextReadFile() { d.needSync = true fn := d.fileName(oldReadFileNum) + oldFileInfo, _ := os.Stat(fn) + oldFileSize := oldFileInfo.Size() + err := os.Remove(fn) if err != nil { d.logf(ERROR, "DISKQUEUE(%s) failed to Remove(%s) - %s", d.name, fn, err) + } else { + d.logf(INFO, "DISKQUEUE(%s) removed(%s) of size(%d bytes)", d.name, fn, oldFileSize) } if d.enableDiskLimitation { From 3077b4ff3c42365f550730979f969fd0e72f2f4c Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Mon, 28 Jun 2021 20:43:26 +0000 Subject: [PATCH 60/67] Only get the size if the file was able to be removed successfully. --- diskqueue.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index b39dfe5..d885f7e 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -855,13 +855,12 @@ func (d *diskQueue) moveToNextReadFile() { fn := d.fileName(oldReadFileNum) oldFileInfo, _ := os.Stat(fn) - oldFileSize := oldFileInfo.Size() err := os.Remove(fn) if err != nil { d.logf(ERROR, "DISKQUEUE(%s) failed to Remove(%s) - %s", d.name, fn, err) } else { - d.logf(INFO, "DISKQUEUE(%s) removed(%s) of size(%d bytes)", d.name, fn, oldFileSize) + d.logf(INFO, "DISKQUEUE(%s) removed(%s) of size(%d bytes)", d.name, fn, oldFileInfo.Size()) } if d.enableDiskLimitation { From 0834f63c34864fa0ed5a4afb870ce1caefa3e3c3 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Tue, 29 Jun 2021 16:13:31 +0000 Subject: [PATCH 61/67] Add function to get filder size and update log when removing read file to make space. --- diskqueue.go | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index d885f7e..f02d652 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -57,6 +57,7 @@ type Interface interface { Delete() error Depth() int64 Empty() error + TotalBytesFolderSize() int64 } // diskQueue implements a filesystem backed FIFO queue @@ -191,6 +192,20 @@ func (d *diskQueue) Depth() int64 { return depth } +// Returns to total size of the contents (files) in the directory located in the dataPath +func (d *diskQueue) TotalBytesFolderSize() int64 { + var totalFolderSizeBytes int64 + + getTotalFolderSizeBytes := func(fileInfo os.FileInfo) error { + totalFolderSizeBytes += fileInfo.Size() + return nil + } + + d.walkDiskQueueDir(getTotalFolderSizeBytes) + + return totalFolderSizeBytes +} + // ReadChan returns the receive-only []byte channel for reading data func (d *diskQueue) ReadChan() <-chan []byte { return d.readChan @@ -467,9 +482,12 @@ func (d *diskQueue) walkDiskQueueDir(fn func(os.FileInfo) error) error { } for _, fileInfo := range fileInfos { - err = fn(fileInfo) - if err != nil { - return err + // only go through files and skip directories + if !fileInfo.IsDir() { + err = fn(fileInfo) + if err != nil { + return err + } } } @@ -539,13 +557,14 @@ func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { badFileInfos = badFileInfos[1:] } else { // delete the read file (make space) + readFileToDeleteNum := d.readFileNum err = d.removeReadFile() if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(d.readFileNum), err) + d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(readFileToDeleteNum), err) d.handleReadError() return err } else { - d.logf(INFO, "DISKQUEUE(%s) removed file(%s) to free up disk space", d.name, d.fileName(d.readFileNum)) + d.logf(INFO, "DISKQUEUE(%s) removed file(%s) to free up disk space", d.name, d.fileName(readFileToDeleteNum)) } d.updateTotalDiskSpaceUsed() } From 368076980991b48793cc651c7037312683d117f1 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 7 Jul 2021 15:42:01 +0000 Subject: [PATCH 62/67] Make the code more readable. --- diskqueue.go | 80 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index f02d652..d91d050 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -69,8 +69,8 @@ type diskQueue struct { writePos int64 readFileNum int64 writeFileNum int64 - readMessages int64 - writeMessages int64 + readMessages int64 // Number of read messages. It's used to update depth. + writeMessages int64 // Number of write messages. It's used to update depth. totalDiskSpaceUsed int64 depth int64 @@ -138,7 +138,6 @@ func NewWithDiskSpace(name string, dataPath string, syncEvery int64, syncTimeout time.Duration, logf AppLogFunc) Interface { enableDiskLimitation := true if maxBytesDiskSpace <= 0 { - maxBytesDiskSpace = 0 enableDiskLimitation = false } d := diskQueue{ @@ -162,12 +161,25 @@ func NewWithDiskSpace(name string, dataPath string, enableDiskLimitation: enableDiskLimitation, } - d.start() + err := d.start() + if err != nil { + return nil + } + return &d } // Get the last known state of DiskQueue from metadata and start ioLoop -func (d *diskQueue) start() { +func (d *diskQueue) start() error { + // ensure that DiskQueue has enough space to write the metadata file + if d.enableDiskLimitation && d.maxBytesDiskSpace <= maxMetaDataFileSize { + errorMsg := fmt.Sprintf( + "disk size limit too small(%d): not enough space for MetaData file size(%d)", + d.maxBytesDiskSpace, maxMetaDataFileSize) + d.logf(ERROR, "DISKQUEUE(%s) - %s", errorMsg) + return errors.New(errorMsg) + } + // no need to lock here, nothing else could possibly be touching this instance err := d.retrieveMetaData() if err != nil && !os.IsNotExist(err) { @@ -180,6 +192,8 @@ func (d *diskQueue) start() { d.updateTotalDiskSpaceUsed() go d.ioLoop() + + return nil } // Depth returns the depth of the queue @@ -422,6 +436,25 @@ func (d *diskQueue) readOne() ([]byte, error) { return readBuf, nil } +func (d *diskQueue) removeBadFile(oldestBadFileInfo os.FileInfo) error { + var err error + badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) + + // remove file if it exists + err = os.Remove(badFileFilePath) + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) + d.updateTotalDiskSpaceUsed() + return err + } else { + // recaclulate total bad files disk size to get most accurate info + d.totalDiskSpaceUsed -= oldestBadFileInfo.Size() + d.logf(INFO, "DISKQUEUE(%s) removed .bad file(%s) of size(%d bytes) to free up disk space", d.name, oldestBadFileInfo.Name(), oldestBadFileInfo.Size()) + } + + return nil +} + func (d *diskQueue) removeReadFile() error { var err error @@ -540,20 +573,10 @@ func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { for d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { if len(badFileInfos) > 0 { // check if a .bad file exists. If it does, delete that first - oldestBadFileInfo := badFileInfos[0] - badFileFilePath := path.Join(d.dataPath, oldestBadFileInfo.Name()) - - err = os.Remove(badFileFilePath) + err = d.removeBadFile(badFileInfos[0]) if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove .bad file(%s) - %s", d.name, oldestBadFileInfo.Name(), err) - d.updateTotalDiskSpaceUsed() return err - } else { - // recaclulate total bad files disk size to get most accurate info - d.totalDiskSpaceUsed -= oldestBadFileInfo.Size() - d.logf(INFO, "DISKQUEUE(%s) removed .bad file(%s) of size(%d bytes) to free up disk space", d.name, oldestBadFileInfo.Name(), oldestBadFileInfo.Size()) } - badFileInfos = badFileInfos[1:] } else { // delete the read file (make space) @@ -574,12 +597,7 @@ func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { } // check if there is enough available disk space to write new data to file -func (d *diskQueue) checkDiskSpace(totalBytes int64, reachedFileSizeLimit bool) error { - expectedBytesIncrease := totalBytes - if reachedFileSizeLimit { - expectedBytesIncrease += numFileMsgBytes - } - +func (d *diskQueue) checkDiskSpace(expectedBytesIncrease int64) error { // If the data to be written is bigger than the disk size limit, do not write if expectedBytesIncrease > d.maxBytesDiskSpace { errorMsg := fmt.Sprintf( @@ -589,14 +607,6 @@ func (d *diskQueue) checkDiskSpace(totalBytes int64, reachedFileSizeLimit bool) return errors.New(errorMsg) } - if d.maxBytesDiskSpace <= maxMetaDataFileSize { - errorMsg := fmt.Sprintf( - "disk size limit too small(%d): not enough space for MetaData file size(%d)", - d.maxBytesDiskSpace, maxMetaDataFileSize) - d.logf(ERROR, "DISKQUEUE(%s) - %s", errorMsg) - return errors.New(errorMsg) - } - // check if we have enough space to write this message if d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { return d.freeDiskSpace(expectedBytesIncrease) @@ -638,13 +648,15 @@ func (d *diskQueue) writeOne(data []byte) error { reachedFileSizeLimit := false if d.enableDiskLimitation { + expectedBytesIncrease := totalBytes // check if we will reach or surpass file size limit if d.writePos+totalBytes+numFileMsgBytes >= d.maxBytesPerFile { reachedFileSizeLimit = true + expectedBytesIncrease += numFileMsgBytes } // free disk space if needed - err = d.checkDiskSpace(totalBytes, reachedFileSizeLimit) + err = d.checkDiskSpace(expectedBytesIncrease) if err != nil { return err } @@ -753,15 +765,16 @@ func (d *diskQueue) retrieveMetaData() error { } defer f.Close() + var depth int64 // if user is using disk space limit feature if d.enableDiskLimitation { _, err = fmt.Fscanf(f, "%d\n%d,%d,%d\n%d,%d,%d\n", - &d.depth, + &depth, &d.readFileNum, &d.readMessages, &d.readPos, &d.writeFileNum, &d.writeMessages, &d.writePos) } else { _, err = fmt.Fscanf(f, "%d\n%d,%d\n%d,%d\n", - &d.depth, + &depth, &d.readFileNum, &d.readPos, &d.writeFileNum, &d.writePos) } @@ -770,6 +783,7 @@ func (d *diskQueue) retrieveMetaData() error { return err } + d.depth = depth d.nextReadFileNum = d.readFileNum d.nextReadPos = d.readPos From e9fe0b0d0728a5cb707ad38fabb237292a21fc42 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 7 Jul 2021 17:34:56 +0000 Subject: [PATCH 63/67] Extract chunk of code into a new function. --- diskqueue.go | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index d91d050..9b53198 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -455,20 +455,13 @@ func (d *diskQueue) removeBadFile(oldestBadFileInfo os.FileInfo) error { return nil } -func (d *diskQueue) removeReadFile() error { +func (d *diskQueue) readNumOfMessages(fileName string) (int64, error) { var err error - if d.readFileNum == d.writeFileNum { - d.skipToNextRWFile() - return nil - } - if d.readFile == nil { - curFileName := d.fileName(d.readFileNum) - - d.readFile, err = os.OpenFile(curFileName, os.O_RDONLY, 0600) + d.readFile, err = os.OpenFile(fileName, os.O_RDONLY, 0600) if err != nil { - return err + return 0, err } d.reader = bufio.NewReader(d.readFile) @@ -483,11 +476,26 @@ func (d *diskQueue) removeReadFile() error { // read total messages number at the end of the file _, err = d.readFile.Seek(-numFileMsgBytes, 2) if err != nil { - return err + return 0, err } var totalMessages int64 err = binary.Read(d.reader, binary.BigEndian, &totalMessages) + if err != nil { + return 0, err + } + + return totalMessages, nil +} + +func (d *diskQueue) removeReadFile() error { + if d.readFileNum == d.writeFileNum { + d.skipToNextRWFile() + return nil + } + + curFileName := d.fileName(d.readFileNum) + totalMessages, err := d.readNumOfMessages(curFileName) if err != nil { return err } From 44b932a92c328f2af5e4d1c8b9e1a34b3faee7ac Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 7 Jul 2021 17:51:42 +0000 Subject: [PATCH 64/67] Improve readability of the code and update code from nsqio/go-diskqueue#29 issue 29 --- diskqueue.go | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 9b53198..dad2b95 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -553,7 +553,7 @@ func (d *diskQueue) getAllBadFileInfo() ([]os.FileInfo, error) { } // get the accurate total non-"bad" file size -func (d *diskQueue) updateTotalDiskSpaceUsed() error { +func (d *diskQueue) updateTotalDiskSpaceUsed() { d.totalDiskSpaceUsed = maxMetaDataFileSize updateTotalDiskSpaceUsed := func(fileInfo os.FileInfo) error { @@ -565,7 +565,10 @@ func (d *diskQueue) updateTotalDiskSpaceUsed() error { return nil } - return d.walkDiskQueueDir(updateTotalDiskSpaceUsed) + err := d.walkDiskQueueDir(updateTotalDiskSpaceUsed) + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to update write bytes - %s", d.name, err) + } } func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { @@ -936,6 +939,11 @@ func (d *diskQueue) handleReadError() { } d.writeFileNum++ d.writePos = 0 + + if d.enableDiskLimitation { + d.totalDiskSpaceUsed = 0 + d.writeMessages = 0 + } } badFn := d.fileName(d.readFileNum) @@ -952,27 +960,18 @@ func (d *diskQueue) handleReadError() { d.name, badFn, badRenameFn) } - if d.enableDiskLimitation { - if d.readFileNum == d.writeFileNum { - // we moved on to the next writeFile - d.totalDiskSpaceUsed = 0 - d.writeMessages = 0 - } else { - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to update write bytes - %s", d.name, err) - } - } - - d.readMessages = 0 - } - d.readFileNum++ d.readPos = 0 d.nextReadFileNum = d.readFileNum d.nextReadPos = 0 + if d.enableDiskLimitation { + d.readMessages = 0 + } // significant state change, schedule a sync on the next iteration d.needSync = true + + d.checkTailCorruption(d.depth) } // ioLoop provides the backend for exposing a go channel (via ReadChan()) From a4f9d67aefefea179b0004e1c3341696e21a8f1b Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 7 Jul 2021 18:24:06 +0000 Subject: [PATCH 65/67] totalDiskSpaceUsed does not change in handleReadError --- diskqueue.go | 1 - 1 file changed, 1 deletion(-) diff --git a/diskqueue.go b/diskqueue.go index dad2b95..d71a729 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -941,7 +941,6 @@ func (d *diskQueue) handleReadError() { d.writePos = 0 if d.enableDiskLimitation { - d.totalDiskSpaceUsed = 0 d.writeMessages = 0 } } From a821855461335029764927e2d929c178967a51b6 Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Wed, 7 Jul 2021 20:59:11 +0000 Subject: [PATCH 66/67] Iterate through each file rather than relying on the totalDiskSpaceUsed variable. --- diskqueue.go | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index d71a729..2e08a0e 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -580,28 +580,36 @@ func (d *diskQueue) freeDiskSpace(expectedBytesIncrease int64) error { d.logf(ERROR, "DISKQUEUE(%s) failed to retrieve all .bad file info - %s", d.name, err) } + if expectedBytesIncrease > d.maxBytesDiskSpace { + return fmt.Errorf("could not make space for expectedBytesIncrease = %d, with maxBytesDiskSpace = %d ", expectedBytesIncrease, d.maxBytesDiskSpace) + } + // keep freeing up disk space until we have enough space to write this message - for d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { - if len(badFileInfos) > 0 { - // check if a .bad file exists. If it does, delete that first - err = d.removeBadFile(badFileInfos[0]) - if err != nil { - return err - } - badFileInfos = badFileInfos[1:] + for _, badFileInfo := range badFileInfos { + if d.totalDiskSpaceUsed+expectedBytesIncrease <= d.maxBytesDiskSpace { + return nil + } + d.removeBadFile(badFileInfo) + } + for d.readFileNum <= d.writeFileNum { + if d.totalDiskSpaceUsed+expectedBytesIncrease <= d.maxBytesDiskSpace { + return nil + } + // delete the read file (make space) + readFileToDeleteNum := d.readFileNum + err = d.removeReadFile() + if err != nil { + d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(readFileToDeleteNum), err) + d.handleReadError() + return err } else { - // delete the read file (make space) - readFileToDeleteNum := d.readFileNum - err = d.removeReadFile() - if err != nil { - d.logf(ERROR, "DISKQUEUE(%s) failed to remove file(%s) - %s", d.name, d.fileName(readFileToDeleteNum), err) - d.handleReadError() - return err - } else { - d.logf(INFO, "DISKQUEUE(%s) removed file(%s) to free up disk space", d.name, d.fileName(readFileToDeleteNum)) - } - d.updateTotalDiskSpaceUsed() + d.logf(INFO, "DISKQUEUE(%s) removed file(%s) to free up disk space", d.name, d.fileName(readFileToDeleteNum)) } + d.updateTotalDiskSpaceUsed() + } + + if d.totalDiskSpaceUsed+expectedBytesIncrease > d.maxBytesDiskSpace { + return fmt.Errorf("could not make space for totalDiskSpaceUsed = %d, expectedBytesIncrease = %d, with maxBytesDiskSpace = %d ", d.totalDiskSpaceUsed, expectedBytesIncrease, d.maxBytesDiskSpace) } return nil From 6a640f323cdded79e22c5bc9bc5cbc738a92321d Mon Sep 17 00:00:00 2001 From: Kevin Cam Date: Thu, 8 Jul 2021 15:11:55 +0000 Subject: [PATCH 67/67] Revert "Merge branch 'master' into DiskSizeLimit" This reverts commit 0c456b49e0f550e540caa46599c59e167e4d4158, reversing changes made to a821855461335029764927e2d929c178967a51b6. --- diskqueue.go | 34 ++++++---------------------------- diskqueue_test.go | 2 -- 2 files changed, 6 insertions(+), 30 deletions(-) diff --git a/diskqueue.go b/diskqueue.go index 7fbe333..2e08a0e 100644 --- a/diskqueue.go +++ b/diskqueue.go @@ -79,7 +79,7 @@ type diskQueue struct { // instantiation time metadata name string dataPath string - maxBytesDiskSize int64 + maxBytesDiskSpace int64 maxBytesPerFile int64 // cannot change once created maxBytesPerFileRead int64 minMsgSize int32 @@ -94,10 +94,6 @@ type diskQueue struct { nextReadPos int64 nextReadFileNum int64 - // keep track of the msg size we have read - // (but not yet sent over readChan) - readMsgSize int32 - readFile *os.File writeFile *os.File reader *bufio.Reader @@ -127,17 +123,17 @@ func New(name string, dataPath string, maxBytesPerFile int64, minMsgSize int32, maxMsgSize int32, syncEvery int64, syncTimeout time.Duration, logf AppLogFunc) Interface { - return NewWithDiskSize(name, dataPath, + return NewWithDiskSpace(name, dataPath, 0, maxBytesPerFile, minMsgSize, maxMsgSize, syncEvery, syncTimeout, logf) } -// Another constructor that allows users to use Disk Size Limit feature -// If user is not using Disk Size Limit feature, maxBytesDiskSize will +// Another constructor that allows users to use Disk Space Limit feature +// If user is not using Disk Space Limit feature, maxBytesDiskSpace will // be 0 -func NewWithDiskSize(name string, dataPath string, - maxBytesDiskSize int64, maxBytesPerFile int64, +func NewWithDiskSpace(name string, dataPath string, + maxBytesDiskSpace int64, maxBytesPerFile int64, minMsgSize int32, maxMsgSize int32, syncEvery int64, syncTimeout time.Duration, logf AppLogFunc) Interface { enableDiskLimitation := true @@ -971,24 +967,6 @@ func (d *diskQueue) handleReadError() { d.name, badFn, badRenameFn) } - if d.enableDiskLimitation { - var badFileSize int64 - if d.readFileNum == d.writeFileNum { - badFileSize = d.writeBytes - } else { - var stat os.FileInfo - stat, err = os.Stat(badRenameFn) - if err == nil { - badFileSize = stat.Size() - } else { - // max file size - badFileSize = int64(d.maxMsgSize) + d.maxBytesPerFile + 4 + numFileMsgsBytes - } - } - - d.writeBytes -= badFileSize - } - d.readFileNum++ d.readPos = 0 d.nextReadFileNum = d.readFileNum diff --git a/diskqueue_test.go b/diskqueue_test.go index 70020e9..4cd5239 100644 --- a/diskqueue_test.go +++ b/diskqueue_test.go @@ -499,7 +499,6 @@ completeWriteFileAgain: // test the writeFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 7 && - d.writeBytes == 5068 && d.readFileNum == 1 && d.writeFileNum == 3 && d.readMessages == 0 && @@ -533,7 +532,6 @@ completeReadFileAgain: // test the readFileNum correctly increments d := readMetaDataFile(dq.(*diskQueue).metaDataFileName(), 0, true) if d.depth == 0 && - d.writeBytes == 0 && d.readFileNum == 3 && d.writeFileNum == 3 && d.readMessages == 0 &&