Add price comparisons tables

ross-spencer · ross-spencer · commit bfb9d553afb9 · 2025-05-20T17:17:55.000+02:00
This is very very rough code to try and extract information about
price deviations among participants. It may be better presented
visually than it is as a table. The data should be there to do that
but not the UI skills! Let's test this first and see how it goes.
diff --git a/htmx/index.htm b/htmx/index.htm
@@ -33,58 +33,68 @@
       <h1>ITN diagnostics</h1>
       <section id="license-holders">
          <div>
-         <h2>License holders</h2>
-         <div
-            hx-get="{baseURL}/participants"
-            hx-trigger="load"
-            hx-swap="innerHTML"
-            />
-         </div>
+            <h2>License holders</h2>
+            <div
+               hx-get="{baseURL}/participants"
+               hx-trigger="load"
+               hx-swap="innerHTML"
+               />
+            </div>
       </section>
       <hr>
       <section id="node-counts">
-        <h2>Node counts</h2>
-        <div
-           hx-get="{baseURL}/count_active_participants"
-           hx-trigger="load"
-           hx-swap="innerHTML"
-           />
-        </div>
-     </section>
-     <hr>
+      <h2>Node counts</h2>
+      <div
+         hx-get="{baseURL}/count_active_participants"
+         hx-trigger="load"
+         hx-swap="innerHTML"
+         />
+      </div>
+      </section>
+      <hr>
       <section id="collector-counts">
-         <h2>Active collector counts</h2>
-         <div
-            hx-get="{baseURL}/online_collectors"
-            hx-trigger="load, every 90s"
-            hx-swap="innerHTML"
-            />
-         </div>
+      <h2>Active collector counts</h2>
+      <div
+         hx-get="{baseURL}/online_collectors"
+         hx-trigger="load, every 90s"
+         hx-swap="innerHTML"
+         />
+      </div>
+      </section>
+      <hr>
+      <section id="price comparisons">
+      <h2>Price comparisons</h2>
+      <div
+         hx-get="{baseURL}/validator_price_stats_hx"
+         hx-trigger="load, every 300s"
+         hx-swap="innerHTML"
+         />
+      </div>
       </section>
       <hr>
       <section id="locations map">
-        <center><h2>Collector map</h2></center>
-        <div
-           hx-get="{baseURL}/locations_map"
-           hx-trigger="load"
-           hx-swap="innerHTML"
-           />
-        </div>
-     </section>
-     <hr>
+      <center><h2>Collector map</h2></center>
+      <div
+         hx-get="{baseURL}/locations_map"
+         hx-trigger="load"
+         hx-swap="innerHTML"
+         />
+      </div>
+      </section>
+      <hr>
       <section id="locations">
-         <h2>Collector locations</h2>
-         <div
-            hx-get="{baseURL}/locations"
-            hx-trigger="load"
-            hx-swap="innerHTML"
-            />
-         </div>
+      <h2>Collector locations</h2>
+      <div
+         hx-get="{baseURL}/locations"
+         hx-trigger="load"
+         hx-swap="innerHTML"
+         />
+      </div>
       </section>
       <hr>
       <div class="footer__bottom text--center">
-         <div class="footer__copyright">ITN | Diagnostics</div>
+      <div class="footer__copyright">ITN | Diagnostics</div>
+      </div>
       </div>
-    </div>
    </body>
 </html>
diff --git a/src/itn_api/api.py b/src/itn_api/api.py
@@ -220,6 +220,50 @@ async def get_locations():
     return await reports.get_locations(app)
 
 
+@app.get("/validator_price_stats", tags=[TAG_STATISTICS])
+async def validator_price_stats() -> dict:
+    """Count active participants."""
+    cursor = app.state.connection.cursor()
+
+    try:
+        cursor.execute(
+            """SELECT distinct feed_id
+            from data_points
+            where date_time >= (SELECT DATE_SUB(NOW(), INTERVAL 1 DAY));
+            """
+        )
+    except mariadb.Error:
+        return "zero collectors online"
+    feeds = list(cursor)
+    try:
+        cursor.execute(
+            """select address, feed_id, min(source_price), max(source_price)
+            from data_points
+            where date_time < date_sub(Now(), interval 1 hour)
+            group by address, feed_id;
+            """
+        )
+    except mariadb.Error as err:
+        return {"error": f"{err}"}
+    hour_data = list(cursor)
+    try:
+        cursor.execute(
+            """select address, feed_id, min(source_price), max(source_price)
+            from data_points
+            where date_time < date_sub(Now(), interval 1 day)
+            group by address, feed_id;
+            """
+        )
+    except mariadb.Error as err:
+        return {"error": f"{err}"}
+    day_data = list(cursor)
+
+    out = await reports._analyze_price_stats(feeds, hour_data, day_data)
+
+    cursor.close()
+    return out
+
+
 # HTMX #################################################################
 # HTMX #################################################################
 # HTMX #################################################################
@@ -229,7 +273,7 @@ async def get_locations():
 async def get_itn_participants() -> str:
     """Return ITN aliases and licenses."""
     all_holders = reports.get_all_license_holders(app, 0, None)
-    htmx = htm_helpers.aliases_to_html(all_holders)
+    htmx = await htm_helpers.aliases_to_html(all_holders)
     return htmx.strip()
 
 
@@ -295,7 +339,7 @@ async def get_online_collectors() -> str:
             participant_count_1h_feed_average[address] = 0
             participant_count_1m_feed_average[address] = 0
 
-    htmx = htm_helpers.participants_count_table(
+    htmx = await htm_helpers.participants_count_table(
         participants_count_total,
         participants_count_24hr,
         participant_count_24h_feed_average,
@@ -309,14 +353,14 @@ async def get_online_collectors() -> str:
 async def get_locations_hx():
     """Return countries participating in the ITN."""
     locations = await reports.get_locations_stake_key(app)
-    return htm_helpers.locations_table(locations)
+    return await htm_helpers.locations_table(locations)
 
 
 @app.get("/locations_map", tags=[TAG_HTMX], response_class=HTMLResponse)
 async def get_locations_map_hx():
     """Return countries participating in the ITN."""
     locations = await reports.get_locations(app)
-    return htm_helpers.locations_map(locations)
+    return await htm_helpers.locations_map(locations)
 
 
 @app.get("/count_active_participants", tags=[TAG_HTMX], response_class=HTMLResponse)
@@ -332,6 +376,12 @@ async def count_active_participants():
     return f"{data[0][0]}"
 
 
+@app.get("/validator_price_stats_hx", tags=[TAG_HTMX], response_class=HTMLResponse)
+async def htmx_validator_price_stats() -> str:
+    price_data = await validator_price_stats()
+    return await htm_helpers.price_comparisons_section(price_data)
+
+
 def main():
     """Primary entry point for this script."""
 
diff --git a/src/itn_api/htm_helpers.py b/src/itn_api/htm_helpers.py
@@ -8,7 +8,7 @@
 logger = logging.getLogger(__name__)
 
 
-def aliases_to_html(alias_report: dict) -> str:
+async def aliases_to_html(alias_report: dict) -> str:
     """Take the alias report and convert it to HTML.
 
     e.g.
@@ -65,7 +65,7 @@ def aliases_to_html(alias_report: dict) -> str:
     return f"{head}\n{rows}\n{count_row}</table>\n"
 
 
-def participants_count_table(
+async def participants_count_table(
     participants_count_total,
     participants_count_24hr,
     participant_count_24h_feed_average,
@@ -74,7 +74,7 @@ def participants_count_table(
 ):
     """Return a table with active participant counts."""
 
-    logging.info("formatting participants table")
+    logging.info("formatting participants count table")
 
     if not participants_count_total:
         return "zero collectors online"
@@ -113,10 +113,10 @@ def participants_count_table(
     return f"{head}\n{rows}</table>\n"
 
 
-def locations_table(locations):
+async def locations_table(locations):
     """Create a table for participant locations."""
 
-    logging.info("formatting participants table")
+    logging.info("formatting participants location table")
 
     if not locations:
         return "no locations available"
@@ -157,7 +157,7 @@ def locations_table(locations):
     return f"{head}\n{rows}\n{country_count}</table>\n"
 
 
-def locations_map(locations):
+async def locations_map(locations):
     """Create a map for participant locations."""
 
     logging.info("formatting participants map")
@@ -183,6 +183,8 @@ def locations_map(locations):
 
     collectors_map.get_root().html.add_child(folium.Element(collector_count_html))
 
+    if not locations:
+        return "problem gathering collectors, please try again shortly"
     for locale in locations:
         region = locale["region"]
         country = locale["country"]
@@ -208,3 +210,64 @@ def locations_map(locations):
     )
 
     return collectors_map_html
+
+
+async def price_comparisons_section(price_data: dict) -> str:
+    """Output the ITN price comparison table so that we can start
+    to understand deviation better.
+    """
+
+    feeds = [item for item in price_data.keys()]
+
+    htm = ""
+
+    threshold = 1
+
+    for feed in feeds:
+        htm = f"{htm}<div><h3>{feed}</h3>\n"
+        summary = "<h4>summary</h4>\n"
+        htm = f"{htm}{summary}\n"
+
+        o1 = f"min values in last hour greater than {threshold}%: {price_data[feed]["breached_hourly_min"]} | range diff (%): {price_data[feed]["min_min_hourly_diff"]}"
+        o2 = f"max values in last hour greater than {threshold}%: {price_data[feed]["breached_hourly_max"]} | range diff (%): {price_data[feed]["max_max_hourly_diff"]}"
+        o3 = f"min values in last day greater than {threshold}%: {price_data[feed]["breached_daily_min"]} | range diff (%): {price_data[feed]["min_min_daily_diff"]}"
+        o4 = f"max values in last day greater than {threshold}%: {price_data[feed]["breached_daily_max"]} | range diff (%): {price_data[feed]["max_max_daily_diff"]}"
+
+        htm = f"{htm}\n<pre>{o1}\n{o2}\n{o3}\n{o4}</pre>\n</div>\n"
+
+    for feed in feeds:
+        htm = f"{htm}<div>\n"
+        summary = f"<h4>detailed: {feed}</h4>\n"
+        htm = f"{htm}{summary}\n"
+        htm = f"{htm}<table><th>price (min hour)</th><th>key</th>\n"
+        for item in price_data[feed]["min_price_hour"]:
+            price = item[0]
+            key = item[1]
+            tr = f"<tr><td>{price:1.7f}</td><td>{key}</td></tr>"
+            htm = f"{htm}{tr}\n"
+        htm = f"{htm}</table>\n<br>\n"
+        htm = f"{htm}<table><th>prices (max hour)</th><th>key</th>\n"
+        for item in price_data[feed]["max_price_hour"]:
+            price = item[0]
+            key = item[1]
+            tr = f"<tr><td>{price:1.7f}</td><td>{key}</td></tr>"
+            htm = f"{htm}{tr}\n"
+        htm = f"{htm}</table>\n<br>\n"
+        """
+        htm = f"{htm}<table><th>prices (min day)</th><th>key</th>\n"
+        for item in price_data[feed]["min_price_day"]:
+            price = item[0]
+            key = item[1]
+            tr = f"<tr><td>{price:1.7f}</td><td>{key}</td></tr>"
+            htm = f"{htm}{tr}\n"
+        htm = f"{htm}</table>\n<br>\n"
+        htm = f"{htm}<table><th>prices (max day)</th><th>key</th>\n"
+        for item in price_data[feed]["max_price_day"]:
+            price = item[0]
+            key = item[1]
+            tr = f"<tr><td>{price:1.7f}</td><td>{key}</td></tr>"
+            htm = f"{htm}{tr}\n"
+        htm = f"{htm}</table>\n<br>\n"
+        """
+
+    return htm
diff --git a/src/itn_api/reports.py b/src/itn_api/reports.py
