diff --git a/examples/Anomaly_Detection/Anomaly_Detection.ino b/examples/Anomaly_Detection/Anomaly_Detection.ino
new file mode 100644
index 00000000..c5aafe1f
--- /dev/null
+++ b/examples/Anomaly_Detection/Anomaly_Detection.ino
@@ -0,0 +1,79 @@
+/**
+ * Anomaly detection
+ * Detect when the frame changes by a reasonable amount
+ *
+ * BE SURE TO SET "TOOLS > CORE DEBUG LEVEL = DEBUG"
+ * to turn on debug messages
+ */
+#include <eloquent_esp32cam.h>
+#include <eloquent_esp32cam/anomaly/detection.h>
+
+using eloq::camera;
+using eloq::anomaly::detection;
+
+/**
+ *
+ */
+void setup() {
+ delay(3000);
+ Serial.begin(115200);
+ Serial.println("___ANOMALY DETECTION___");
+
+ // camera settings
+ // replace with your own model!
+ camera.pinout.xiao();
+ camera.brownout.disable();
+ camera.resolution.vga();
+ camera.quality.high();
+
+ // configure anomaly detection
+ detection.skip(4);
+ // the higher the stride, the faster the detection
+ // the higher the stride, the less the granularity
+ detection.stride(1);
+ // the higher the threshold, the less the sensitivity
+ // (at pixel level)
+ detection.threshold(5);
+ // the higher the detectionRatio, the less the sensitivity
+ // (at image level, from 0 to 1)
+ detection.detectionRatio(0.5);
+ // the higher the referenceRatio, the more the reference image can change over time
+ // (at image level, from 0 to 1)
+ detection.referenceRatio(0.2);
+ // optionally, you can enable rate limiting (aka debounce)
+ // anomaly won't trigger more often than the specified frequency
+ //detection.rate.atMostOnceEvery(5).seconds();
+
+ // init camera
+ while (!camera.begin().isOk())
+ Serial.println(camera.exception.toString());
+
+ Serial.println("Camera OK");
+ Serial.println("Awaiting anomaly...");
+}
+
+/**
+ *
+ */
+void loop() {
+
+ // Don't run more often than the time for an anomaly to come into view as the reference image can 'drift' away from 'normal'
+ delay(1000);
+ // capture picture
+ if (!camera.capture().isOk()) {
+ Serial.println(camera.exception.toString());
+ return;
+ }
+
+ // run anomaly detection
+ if (!detection.run().isOk()) {
+ Serial.println(detection.exception.toString());
+ return;
+ }
+
+ // on anomaly, perform action
+ if (detection.triggered()) {
+ Serial.print("Anomaly detected: "); Serial.println(detection.movingRatio);
+ }
+}
+
diff --git a/examples/Anomaly_Detection_Higher_Resolution/Anomaly_Detection_Higher_Resolution.ino b/examples/Anomaly_Detection_Higher_Resolution/Anomaly_Detection_Higher_Resolution.ino
new file mode 100644
index 00000000..9afa0853
--- /dev/null
+++ b/examples/Anomaly_Detection_Higher_Resolution/Anomaly_Detection_Higher_Resolution.ino
@@ -0,0 +1,100 @@
+/**
+ * Run anomaly detection at low resolution.
+ * On anomaly, capture frame at higher resolution
+ * for SD storage.
+ *
+ * BE SURE TO SET "TOOLS > CORE DEBUG LEVEL = INFO"
+ * to turn on debug messages
+ */
+#include <eloquent_esp32cam.h>
+#include <eloquent_esp32cam/anomaly/detection.h>
+
+using eloq::camera;
+using eloq::anomaly::detection;
+
+
+/**
+ *
+ */
+void setup() {
+ delay(3000);
+ Serial.begin(115200);
+ Serial.println("___ANOMALY DETECTION + SWITCH RESOLUTION___");
+
+ // camera settings
+ // replace with your own model!
+ camera.pinout.freenove_s3();
+ camera.brownout.disable();
+ camera.resolution.vga();
+ camera.quality.high();
+
+ // see example of anomaly detection for config values
+ detection.skip(5);
+ detection.stride(1);
+ detection.threshold(5);
+ // the higher the detectionRatio, the less the sensitivity
+ // (at image level, from 0 to 1)
+ detection.detectionRatio(0.5);
+ // the higher the referenceRatio, the more the reference image can change over time
+ // (at image level, from 0 to 1)
+ detection.referenceRatio(0.2);
+
+ // init camera
+ while (!camera.begin().isOk())
+ Serial.println(camera.exception.toString());
+
+ Serial.println("Camera OK");
+ Serial.println("Awaiting for anomaly...");
+}
+
+/**
+ *
+ */
+void loop() {
+ // Don't run more often than the time for an anomaly to come into view as the reference image can 'drift' away from 'normal'
+ delay(1000);
+ // capture picture
+ if (!camera.capture().isOk()) {
+ Serial.println(camera.exception.toString());
+ return;
+ }
+
+ // run anomaly detection
+ if (!detection.run().isOk()) {
+ Serial.println(detection.exception.toString());
+ return;
+ }
+
+ // on anomaly, perform action
+ if (detection.triggered()) {
+ Serial.printf(
+ "Anomaly of %.2f detected on frame of size %dx%d (%d bytes)\n",
+ detection.movingRatio,
+ camera.resolution.getWidth(),
+ camera.resolution.getHeight(),
+ camera.getSizeInBytes()
+ );
+
+ Serial.println("Taking photo of anomaly at higher resolution");
+
+ camera.resolution.at(FRAMESIZE_UXGA, []() {
+ Serial.printf(
+ "Switched to higher resolution: %dx%d. It took %d ms to switch\n",
+ camera.resolution.getWidth(),
+ camera.resolution.getHeight(),
+ camera.resolution.benchmark.millis()
+ );
+
+ camera.capture();
+
+ Serial.printf(
+ "Frame size is now %d bytes\n",
+ camera.getSizeInBytes()
+ );
+
+ // save to SD...
+ });
+
+ Serial.println("Resolution switched back to VGA");
+ }
+}
\ No newline at end of file
diff --git a/src/eloquent_esp32cam/anomaly/daemon.h b/src/eloquent_esp32cam/anomaly/daemon.h
new file mode 100644
index 00000000..afbb8cb8
--- /dev/null
+++ b/src/eloquent_esp32cam/anomaly/daemon.h
@@ -0,0 +1,91 @@
+#ifndef ELOQUENT_ESP32CAM_ANOMALY_DAEMON_H
+#define ELOQUENT_ESP32CAM_ANOMALY_DAEMON_H
+
+#include <functional>
+#include "../camera/camera.h"
+#include "../extra/esp32/multiprocessing/thread.h"
+
+using eloq::camera;
+using Eloquent::Extra::Esp32::Multiprocessing::Thread;
+using OnAnomalyCallback = std::function<void(void)>;
+
+
+namespace Eloquent {
+ namespace Esp32cam {
+ namespace Anomaly {
+ /**
+ * Run anomaly detection in a task
+ *
+ * @class Daemon
+ * @author jksemple
+ * @date 11/07/2024
+ * @file daemon.h
+ * @brief
+ */
+ template<typename T>
+ class Daemon {
+ public:
+ Thread thread;
+
+ /**
+ * Constructor
+ *
+ * @brief
+ */
+ Daemon(T* detection) :
+ thread("AnomalyDetection"),
+ _detection(detection) {
+
+ }
+
+ /**
+ * Run function when a difference from 'normal' is detected
+ *
+ * @brief
+ * @param callback
+ */
+ void onAnomaly(OnAnomalyCallback callback) {
+ _onAnomaly = callback;
+ }
+
+ /**
+ * Start anomaly detection in background
+ *
+ * @brief
+ */
+ void start() {
+ thread
+ .withArgs((void*) this)
+ .withStackSize(5000)
+ .run([](void *args) {
+ Daemon *self = (Daemon*) args;
+
+ delay(3000);
+
+ while (true) {
+ yield();
+ delay(1);
+
+ if (!camera.capture().isOk())
+ continue;
+
+ if (!self->_detection->run().isOk())
+ continue;
+
+ if (!self->_detection->triggered())
+ continue;
+
+ self->_onAnomaly();
+ }
+ });
+ }
+
+ protected:
+ T *_detection;
+ OnAnomalyCallback _onAnomaly;
+ };
+ }
+ }
+}
+
+#endif
diff --git a/src/eloquent_esp32cam/anomaly/detection.h b/src/eloquent_esp32cam/anomaly/detection.h
new file mode 100644
index 00000000..c6971297
--- /dev/null
+++ b/src/eloquent_esp32cam/anomaly/detection.h
@@ -0,0 +1,214 @@
+#ifndef ELOQUENT_ESP32CAM_ANOMALY_DETECTION
+#define ELOQUENT_ESP32CAM_ANOMALY_DETECTION
+
+#include <dl_image.hpp>
+#include "../extra/exception.h"
+#include "../extra/time/benchmark.h"
+#include "../extra/time/rate_limit.h"
+#include "../extra/pubsub.h"
+#include "./daemon.h"
+
+using eloq::camera;
+using Eloquent::Error::Exception;
+using Eloquent::Extra::Time::Benchmark;
+using Eloquent::Extra::Time::RateLimit;
+#if defined(ELOQUENT_EXTRA_PUBSUB_H)
+using Eloquent::Extra::PubSub;
+#endif
+
+
+namespace Eloquent {
+ namespace Esp32cam {
+ namespace Anomaly {
+ /**
+ * Detect anomaly using "fast" algorithm
+ */
+ class Detection {
+ public:
+ float movingRatio;
+ Exception exception;
+ Benchmark benchmark;
+ RateLimit rate;
+ Daemon<Detection> daemon;
+ #if defined(ELOQUENT_EXTRA_PUBSUB_H)
+ PubSub<Detection> mqtt;
+ #endif
+
+ /**
+ *
+ */
+ Detection() :
+ _stride(4),
+ _threshold(5),
+ _lowerDetectionRatio(0.2),
+ _upperDetectionRatio(0.5),
+ _referenceRatio(0.05),
+ _reference(NULL),
+ _skip(5),
+ movingRatio(0),
+ daemon(this),
+ #if defined(ELOQUENT_EXTRA_PUBSUB_H)
+ mqtt(this),
+ #endif
+ exception("AnomalyDetection") {
+
+ }
+
+ /**
+ * Set detection stride.
+ * The greater the value, the less accurate.
+ * The greater the value, the faster.
+ */
+ void stride(uint8_t stride) {
+ _stride = stride;
+ }
+
+ /**
+ * Set detection sensitivity (pixel level).
+ * The greater the value, the less sensitive the detection.
+ */
+ void threshold(uint8_t threshold) {
+ _threshold = threshold;
+ }
+
+ /**
+ * @brief Skip first frames (to avoid false detection)
+ * @param skip
+ */
+ void skip(uint8_t skip) {
+ _skip = skip;
+ }
+
+ /**
+ * Set reference image sensitivity (image level).
+ * The greater the value, the more the reference image can vary over time.
+ */
+ void referenceRatio(float ratio) {
+ _referenceRatio = ratio;
+ }
+
+ /**
+ * Set detection sensitivity (image level).
+ * The greater the value, the less sensitive the detection.
+ */
+ void lowerDetectionRatio(float ratio) {
+ _lowerDetectionRatio = ratio;
+ }
+
+ /**
+ * Set maximum detection sensitivity (image level).
+ * This protects against false detections when the light level across the whole image changes
+ * Useful when you know objects being detected will never fill more than a fraction of the image
+ */
+ void upperDetectionRatio(float ratio) {
+ _upperDetectionRatio = ratio;
+ }
+ /**
+ * Test if anomaly triggered
+ */
+ inline bool triggered() {
+ return movingRatio >= _lowerDetectionRatio && movingRatio <= _upperDetectionRatio;
+ }
+
+ Exception& setReference() {
+ // convert JPEG to RGB565
+ if (!camera.rgb565.convert().isOk())
+ return camera.rgb565.exception;
+
+ if (_reference == NULL) {
+ _reference = (uint16_t*) ps_malloc(camera.rgb565.length * sizeof(uint16_t));
+ }
+ copy(camera.rgb565);
+
+ return exception.clear();
+ }
+ /**
+ *
+ */
+ Exception& run() {
+ // skip fre first frames
+ if (_skip > 0 && _skip-- > 0)
+ return exception.set(String("Still ") + _skip + " frames to skip...");
+
+ // convert JPEG to RGB565
+ // this reduces the frame to 1/8th
+ if (!camera.rgb565.convert().isOk())
+ return camera.rgb565.exception;
+
+ // first frame, only copy frame to prev
+ if (_reference == NULL) {
+ _reference = (uint16_t*) malloc(camera.rgb565.length * sizeof(uint16_t));
+ copy(camera.rgb565);
+
+ return exception.set("First frame, can't detect anomaly").soft();
+ }
+
+ benchmark.timeit([this]() {
+ int movingPoints = dl::image::get_moving_point_number(
+ camera.rgb565.data,
+ _reference,
+ camera.rgb565.height,
+ camera.rgb565.width,
+ _stride,
+ _threshold
+ );
+
+ movingRatio = ((float) movingPoints) / camera.rgb565.length * _stride * _stride;
+ if (movingRatio < _referenceRatio) {
+ ESP_LOGD("AnomalyDetection", "Replacing reference frame - referenceRatio = %.2f", movingRatio);
+ copy(camera.rgb565);
+ }
+ });
+ ESP_LOGD("AnomalyDetection", "moving points ratio: %.2f", movingRatio);
+
+ // rate limit
+ if (triggered() && !rate)
+ return exception.set(rate.getRetryInMessage()).soft();
+
+ if (triggered())
+ rate.touch();
+
+ return exception.clear();
+ }
+ /**
+ * @brief Convert to JSON
+ */
+ String toJSON() {
+ return String("{\"anomaly\":") + (triggered() ? "true" : "false") + "}";
+ }
+
+ /**
+ * @brief Test if an MQTT message should be published
+ */
+ bool shouldPub() {
+ return triggered();
+ }
+
+ protected:
+ uint8_t _skip;
+ uint16_t *_reference;
+ uint8_t _stride;
+ uint8_t _threshold;
+ float _lowerDetectionRatio;
+ float _upperDetectionRatio;
+ float _referenceRatio;
+
+ /**
+ *
+ */
+ template<typename Frame>
+ void copy(Frame frame) {
+ memcpy((uint8_t*) _reference, (uint8_t*) frame.data, frame.length * sizeof(uint16_t));
+ }
+ };
+ }
+ }
+}
+
+namespace eloq {
+ namespace anomaly {
+ static Eloquent::Esp32cam::Anomaly::Detection detection;
+ }
+}
+
+#endif
diff --git a/src/eloquent_esp32cam/anomaly/roi_detection.h b/src/eloquent_esp32cam/anomaly/roi_detection.h
new file mode 100644
index 00000000..412d9b45
--- /dev/null
+++ b/src/eloquent_esp32cam/anomaly/roi_detection.h
@@ -0,0 +1,158 @@
+#ifndef ELOQUENT_ESP32CAM_ANOMALY_ROI_DETECTION
+#define ELOQUENT_ESP32CAM_ANOMALY_ROI_DETECTION
+
+#include "./detection.h"
+
+namespace Eloquent {
+ namespace Esp32cam {
+ namespace Anomaly {
+ /**
+ * Perform anomaly detection on Region of Interest
+ */
+ class RoI : public Detection {
+ public:
+ struct {
+ uint16_t x;
+ uint16_t y;
+ uint16_t width;
+ uint16_t height;
+ uint16_t x1;
+ uint16_t x2;
+ uint16_t y1;
+ uint16_t y2;
+ } coords;
+
+ /**
+ *
+ */
+ RoI() :
+ _x(0),
+ _y(0),
+ _w(0),
+ _h(0) {
+
+ }
+
+ /**
+ * Set x coordinate (top-left corner)
+ */
+ void x(float x) {
+ _x = x;
+ }
+
+ /**
+ * Set y coordinate (top-left corner)
+ */
+ void y(float y) {
+ _y = y;
+ }
+
+ /**
+ * Set width of RoI
+ */
+ void width(float width) {
+ _w = width;
+ }
+
+ /**
+ * Set height of RoI
+ */
+ void height(float height) {
+ _h = height;
+ }
+
+ /**
+ *
+ */
+ void updateCoords(uint16_t width, uint16_t height) {
+ coords.x = max<int>(0, _x < 1 ? _x * width : _x);
+ coords.y = max<int>(0, _y < 1 ? _y * height : _y);
+ coords.width = min<int>(width - coords.x, _w < 1 ? _w * width : _w);
+ coords.height = min<int>(height - coords.y, _h < 1 ? _h * height : _h);
+ coords.x1 = coords.x;
+ coords.y1 = coords.y;
+ coords.x2 = coords.x + coords.width;
+ coords.y2 = coords.y + coords.height;
+ }
+
+ /**
+ * Detect anomaly
+ */
+ template<typename Frame>
+ Exception& update(Frame& frame) {
+ if (!_w || !_h)
+ return exception.set("You MUST set a width and height for the RoI");
+
+ if (_reference == NULL) {
+ _reference = (uint8_t*) ps_malloc(_w * _h * sizeof(uint16_t));
+ _roi = (uint8_t*) ps_malloc(_w * _h * sizeof(uint16_t));
+ copy(frame, _reference);
+
+ return exception.set("First frame, can't detect anomaly").soft();
+ }
+
+ updateCoords(frame.width, frame.height);
+
+ benchmark.timeit([this, &frame]() {
+ copy(frame, _roi);
+
+ int movingPoints = dl::image::get_moving_point_number((uint16_t *) _roi, (uint16_t*) _reference, coords.height, coords.width, _stride, _threshold);
+ movingRatio = ((float) movingPoints) / sizeof(_roi) * _stride * _stride;
+ memcpy(_reference, _roi, sizeof(_reference));
+ });
+ if (movingRatio < _referenceRatio) {
+ // update reference
+ copy(frame, _reference);
+ }
+
+ ESP_LOGD(
+ "RoI AnomalyDetection",
+ "roi: (x=%d, y=%d, width=%d, height=%d). moving points: %.2f%%",
+ coords.x,
+ coords.y,
+ coords.width,
+ coords.height,
+ moving_ratio
+ );
+ if (triggered() && !rate_limiter)
+ return exception.set(rate.getRetryInMessage()).soft();
+
+ if (triggered())
+ rate_limiter.touch();
+
+ return exception.clear();
+ }
+
+
+ protected:
+ float _x;
+ float _y;
+ float _w;
+ float _h;
+ uint8_t *_roi;
+
+ /**
+ * Copy RoI of frame into buffer
+ */
+ template<typename Frame>
+ void copy(Frame& frame, uint8_t *dest) {
+ for (int i = coords.y1; i < coords.y2; i++)
+ memcpy(
+ dest + coords.width * (i - coords.y1) * sizeof(uint16_t),
+ frame.data + (frame.width * i + coords.x) * sizeof(uint16_t),
+ coords.width * sizeof(uint16_t)
+ );
+ }
+ };
+ }
+ }
+}
+
+namespace eloq {
+ namespace anomaly {
+ // create class alias
+ class RoI : public Eloquent::Esp32cam::Anomaly::RoI {};
+ }
+}
+
+#endif
\ No newline at end of file
diff --git a/src/eloquent_esp32cam/camera/rgb_565.h b/src/eloquent_esp32cam/camera/rgb_565.h
index 68dd0e11..c11cee44 100644
--- a/src/eloquent_esp32cam/camera/rgb_565.h
+++ b/src/eloquent_esp32cam/camera/rgb_565.h
@@ -21,6 +21,7 @@ namespace Eloquent {
size_t length;
size_t width;
size_t height;
+ jpg_scale_t scaling;
/**
*
@@ -30,7 +31,8 @@ namespace Eloquent {
camera(cam),
length(0),
width(0),
- height(0) {
+ height(0),
+ scaling(JPG_SCALE_8X) {
}
/**
@@ -94,8 +96,8 @@ namespace Eloquent {
return exception.set("Can't convert empty frame to RGB565");
if (!width) {
- width = camera->resolution.getWidth() / 8;
- height = camera->resolution.getHeight() / 8;
+ width = camera->resolution.getWidth() >> scaling;
+ height = camera->resolution.getHeight() >> scaling;
length = width * height;
ESP_LOGI("Camera", "Allocating %d bytes for %dx%d RGB565 image", length * 2, width, height);
@@ -106,7 +108,7 @@ namespace Eloquent {
return exception.set("Cannot allocate memory");
camera->mutex.threadsafe([this]() {
- if (!jpg2rgb565(camera->frame->buf, camera->frame->len, (uint8_t*) data, JPG_SCALE_8X))
+ if (!jpg2rgb565(camera->frame->buf, camera->frame->len, (uint8_t*) data, scaling))
exception.set("Error converting frame from JPEG to RGB565");
});
diff --git a/src/eloquent_esp32cam/motion/detection.h b/src/eloquent_esp32cam/motion/detection.h
index a62f9398..f345c894 100644
--- a/src/eloquent_esp32cam/motion/detection.h
+++ b/src/eloquent_esp32cam/motion/detection.h
@@ -131,7 +131,6 @@ namespace Eloquent {
movingRatio = ((float) movingPoints) / camera.rgb565.length * _stride * _stride;
copy(camera.rgb565);
});
-
ESP_LOGD("MotionDetection", "moving points ratio: %.2f", movingRatio);
// rate limit
@@ -143,7 +142,6 @@ namespace Eloquent {
return exception.clear();
}
-
/**
* @brief Convert to JSON
*/
diff --git a/src/eloquent_esp32cam/transform/crop.h b/src/eloquent_esp32cam/transform/crop.h
index b45fd9b5..d018a683 100644
--- a/src/eloquent_esp32cam/transform/crop.h
+++ b/src/eloquent_esp32cam/transform/crop.h
@@ -42,8 +42,8 @@ namespace Eloquent {
_src.height = height;
_src.x1 = 0;
_src.y1 = 0;
- _src.x2 = width;
- _src.y2 = height;
+ _src.x2 = width - 1;
+ _src.y2 = height - 1;
return *this;
}
@@ -64,8 +64,8 @@ namespace Eloquent {
_out.height = height;
_out.x1 = 0;
_out.y1 = 0;
- _out.x2 = width;
- _out.y2 = height;
+ _out.x2 = width - 1;
+ _out.y2 = height - 1;
return *this;
}
@@ -93,13 +93,13 @@ namespace Eloquent {
*/
Crop& squash() {
_src.x1 = 0;
- _src.x2 = _src.width;
+ _src.x2 = _src.width - 1;
_src.y1 = 0;
- _src.y2 = _src.height;
+ _src.y2 = _src.height - 1;
_out.x1 = 0;
- _out.x2 = _out.width;
+ _out.x2 = _out.width - 1;
_out.y1 = 0;
- _out.y2 = _out.height;
+ _out.y2 = _out.height - 1;
return *this;
}
@@ -114,12 +114,14 @@ namespace Eloquent {
_src.x1 = dx;
_src.y1 = dy;
- _src.x2 = _src.width - dx;
- _src.y2 = _src.height - dy;
+ _src.x2 = _src.width - dx - 1;
+ _src.y2 = _src.height - dy - 1;
_out.x1 = 0;
_out.y1 = 0;
- _out.x2 = _out.width;
- _out.y2 = _out.height;
+ _out.x2 = _out.width - 1;
+ _out.y2 = _out.height - 1;
+
+ return *this;
}
else if (_out.width > _src.width) {
uint16_t dx = (_out.width - _src.width) / 2;
@@ -127,17 +129,35 @@ namespace Eloquent {
_out.x1 = dx;
_out.y1 = dy;
- _out.x2 = _out.width - dx;
- _out.y2 = _out.height - dy;
+ _out.x2 = _out.width - dx - 1;
+ _out.y2 = _out.height - dy - 1;
_src.x1 = 0;
_src.y1 = 0;
- _src.x2 = _src.width;
- _src.y2 = _src.height;
+ _src.x2 = _src.width - 1;
+ _src.y2 = _src.height - 1;
}
return *this;
}
+ /**
+ * Manually set crop area origin
+ * @param x
+ * @param y
+ * @return
+ */
+ Crop& offset(int16_t x, int16_t y) {
+ if (x < 0) x += _src.width;
+ if (y < 0) y += _src.height;
+
+ _src.x1 = x;
+ _src.x2 = x + _out.width - 1;
+ _src.y1 = y;
+ _src.y2 = y + _out.height - 1;
+
+ return *this;
+ }
+
/**
* No interpolation
*/