Add /health endpoint for comprehensive service readiness checking

.github/workflows/build-testing.yml

@@ -41,7 +41,7 @@ jobs:
       friendbot_ref: horizon-v23.0.0
       lab_ref: main
       test_matrix: |
-        {j
+        {
           "network": ["testnet", "pubnet", "local"],
           "core": ["core", null],
           "horizon": ["horizon", null],

.github/workflows/build.yml

@@ -482,6 +482,18 @@ jobs:
         echo "supervisorctl tail -f horizon" | docker exec -i stellar sh &
         go run tests/test_horizon_ingesting.go
         curl http://localhost:8000
+    # Test the /health endpoint through nginx
+    - name: Run health endpoint test
+      if: ${{ matrix.horizon }}
+      run: |
+        docker logs stellar -f &
+        echo "supervisorctl tail -f horizon" | docker exec -i stellar sh &
+        echo "supervisorctl tail -f readiness" | docker exec -i stellar sh &
+        # Ensure readiness service is running
+        docker exec stellar supervisorctl status readiness || docker exec stellar supervisorctl start readiness
+        # Copy the test file into the container and run it there
+        docker cp tests/test_health_endpoint.go stellar:/tmp/
+        docker exec stellar sh -c "cd /tmp && go run test_health_endpoint.go"
     - name: Run friendbot test
       if: ${{ matrix.horizon && matrix.network == 'local' }}
       run: |

.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+*.pyc
+test_health_endpoint

Dockerfile

@@ -22,6 +22,7 @@ EXPOSE 6060
 EXPOSE 6061
 EXPOSE 8000
 EXPOSE 8002
+EXPOSE 8004
 EXPOSE 8100
 EXPOSE 11625
 EXPOSE 11626

README.md

@@ -152,6 +152,37 @@ $ curl http://localhost:8000/friendbot?addr=G...
 
 _Note: In local mode a local friendbot is running. In testnet and futurenet modes requests to the local `:8000/friendbot` endpoint will be proxied to the friendbot deployments for the respective network._
 
+### Readiness Endpoint
+
+The quickstart image provides a `/health` endpoint that indicates when all services are fully ready for use. This endpoint reports HTTP 200 when the image is ready and HTTP 503 when services are still starting up or experiencing issues.
+
+Example usage:
+
+```bash
+$ curl http://localhost:8000/health
+```
+
+Example response when ready:
+```json
+{
+  "status": "ready",
+  "services": {
+    "stellar-core": "ready",
+    "horizon": "ready",
+    "horizon_health": {
+      "database_connected": true,
+      "core_up": true,
+      "core_synced": true
+    },
+    "stellar-rpc": "ready"
+  }
+}
+```
+
+The endpoint automatically detects which services are running and only reports "ready" when all detected services are functioning properly. This eliminates the need to write custom scripts to test multiple service endpoints individually.
+
+_Note: The `/health` endpoint provides comprehensive readiness status for all detected services, replacing Horizon's built-in `/health` endpoint with expanded functionality._
+
 ### Using in GitHub Actions
 
 The quickstart image can be run in GitHub Actions workflows using the provided action. This is useful for testing smart contracts, running integration tests, or any other CI/CD workflows that need a Stellar network.
@@ -307,9 +338,9 @@ Managing UIDs between a docker container and a host volume can be complicated. A
 
 The image exposes one main port through which services provide their APIs:
 
-| Port | Service                         | Description    |
-| ---- | ------------------------------- | -------------- |
-| 8000 | lab, horizon, stellar-rpc, friendbot | main http port |
+| Port | Service                                    | Description    |
+| ---- | ------------------------------------------ | -------------- |
+| 8000 | lab, horizon, stellar-rpc, friendbot, health | main http port |
 
 The image also exposes a few other ports that most developers do not need, but area available:
 
@@ -318,6 +349,7 @@ The image also exposes a few other ports that most developers do not need, but a
 | 5432  | postgresql                 | database access port |
 | 6060  | horizon                    | admin port           |
 | 6061  | stellar-rpc                | admin port           |
+| 8004  | health service             | internal port        |
 | 11625 | stellar-core               | peer node port       |
 | 11626 | stellar-core               | main http port       |
 | 11725 | stellar-core (horizon)     | peer node port       |

common/nginx/etc/conf.d/health.conf

@@ -0,0 +1,6 @@
+location /health {
+        rewrite /health / break;
+        proxy_set_header Host $http_host;
+        proxy_pass http://127.0.0.1:8004;
+        proxy_redirect off;
+}

common/readiness/bin/readiness-service.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+
+import json
+import logging
+import os
+import sys
+import time
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import urllib.request
+import urllib.error
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+class HealthCheckHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == '/' or self.path == '/health':
+            self.handle_readiness_check()
+        else:
+            self.send_error(404)
+
+    def handle_readiness_check(self):
+        """Handle readiness check requests"""
+
+        # Detect enabled services by checking if they're running
+        # rather than relying on environment variables which may not be passed to supervisord
+        enable_core = self.is_service_intended_to_run('stellar-core')
+        enable_horizon = self.is_service_intended_to_run('horizon')
+        enable_rpc = self.is_service_intended_to_run('stellar-rpc')
+
+        response = {
+            'status': 'ready',
+            'services': {}
+        }
+
+        all_healthy = True
+
+        # Check stellar-core if enabled
+        if enable_core:
+            if self.check_stellar_core():
+                response['services']['stellar-core'] = 'ready'
+            else:
+                response['services']['stellar-core'] = 'not ready'
+                all_healthy = False
+
+        # Check horizon if enabled
+        if enable_horizon:
+            horizon_status = self.check_horizon()
+            if horizon_status['ready']:
+                response['services']['horizon'] = 'ready'
+                # Include Horizon's detailed health info
+                response['services']['horizon_health'] = horizon_status['health']
+            else:
+                response['services']['horizon'] = 'not ready'
+                all_healthy = False
+
+        # Check stellar-rpc if enabled
+        if enable_rpc:
+            if self.check_stellar_rpc():
+                response['services']['stellar-rpc'] = 'ready'
+            else:
+                response['services']['stellar-rpc'] = 'not ready'
+                all_healthy = False
+
+        if not all_healthy:
+            response['status'] = 'not ready'
+            status_code = 503
+        else:
+            status_code = 200
+
+        # Send response
+        self.send_response(status_code)
+        self.send_header('Content-Type', 'application/json')
+        self.end_headers()
+
+        response_json = json.dumps(response)
+        self.wfile.write(response_json.encode('utf-8'))
+
+        logger.info(f"Readiness check - Status: {response['status']}, Services: {response['services']}")
+
+    def is_service_intended_to_run(self, service_name):
+        """Check if a service is intended to run by testing if it's reachable"""
+        if service_name == 'stellar-core':
+            # Check if stellar-core is running on its default port
+            try:
+                with urllib.request.urlopen('http://localhost:11626/info', timeout=2) as resp:
+                    return True
+            except:
+                return False
+        elif service_name == 'horizon':
+            # Check if horizon is running on its default port  
+            try:
+                with urllib.request.urlopen('http://localhost:8001', timeout=2) as resp:
+                    return True
+            except:
+                return False
+        elif service_name == 'stellar-rpc':
+            # Check if stellar-rpc is running by calling its health method
+            try:
+                request_data = {
+                    'jsonrpc': '2.0',
+                    'id': 10235,
+                    'method': 'getHealth'
+                }
+
+                req = urllib.request.Request(
+                    'http://localhost:8003',
+                    data=json.dumps(request_data).encode('utf-8'),
+                    headers={'Content-Type': 'application/json'}
+                )
+
+                with urllib.request.urlopen(req, timeout=2) as resp:
+                    return True
+            except:
+                return False
+        return False
+
+    def check_stellar_core(self):
+        """Check if stellar-core is healthy"""
+        try:
+            with urllib.request.urlopen('http://localhost:11626/info', timeout=5) as resp:
+                return resp.status == 200
+        except Exception as e:
+            logger.debug(f"stellar-core check failed: {e}")
+            return False
+
+    def check_horizon(self):
+        """Check if horizon is ready and get its health status"""
+        try:
+            # First check the root endpoint
+            with urllib.request.urlopen('http://localhost:8001', timeout=5) as resp:
+                if resp.status != 200:
+                    return {'ready': False, 'health': None}
+
+                data = json.load(resp)
+                protocol_version = data.get('supported_protocol_version', 0)
+                core_ledger = data.get('core_latest_ledger', 0)
+                history_ledger = data.get('history_latest_ledger', 0)
+
+                # Basic readiness check
+                basic_ready = protocol_version > 0 and core_ledger > 0 and history_ledger > 0
+
+            # Try to get Horizon's own health endpoint
+            horizon_health = None
+            try:
+                with urllib.request.urlopen('http://localhost:8001/health', timeout=5) as health_resp:
+                    if health_resp.status == 200:
+                        horizon_health = json.load(health_resp)
+            except Exception:
+                # Health endpoint might not be available, that's ok
+                pass
+
+            return {
+                'ready': basic_ready,
+                'health': horizon_health
+            }
+
+        except Exception as e:
+            logger.debug(f"horizon check failed: {e}")
+            return {'ready': False, 'health': None}
+
+    def check_stellar_rpc(self):
+        """Check if stellar-rpc is healthy"""
+        try:
+            request_data = {
+                'jsonrpc': '2.0',
+                'id': 10235,
+                'method': 'getHealth'
+            }
+
+            req = urllib.request.Request(
+                'http://localhost:8003',
+                data=json.dumps(request_data).encode('utf-8'),
+                headers={'Content-Type': 'application/json'}
+            )
+
+            with urllib.request.urlopen(req, timeout=5) as resp:
+                if resp.status != 200:
+                    return False
+
+                data = json.load(resp)
+                return data.get('result', {}).get('status') == 'healthy'
+        except Exception as e:
+            logger.debug(f"stellar-rpc check failed: {e}")
+            return False
+
+    def log_message(self, format, *args):
+        """Override to use our logger"""
+        logger.info(format % args)
+
+def main():
+    port = 8004
+    server = HTTPServer(('0.0.0.0', port), HealthCheckHandler)
+    logger.info(f"Readiness service starting on port {port}")
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info("Readiness service shutting down")
+        server.shutdown()
+
+if __name__ == '__main__':
+    main()

common/readiness/bin/start

@@ -0,0 +1,9 @@
+#! /bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+echo "starting readiness service..."
+set -e
+
+# Use the Python-based readiness service
+exec python3 "$DIR/readiness-service.py"

common/supervisor/etc/supervisord.conf.d/readiness.conf

@@ -0,0 +1,8 @@
+[program:readiness]
+user=stellar
+directory=/opt/stellar/readiness
+command=/opt/stellar/readiness/bin/start
+autostart=true
+autorestart=true
+priority=60
+redirect_stderr=true

start

@@ -16,6 +16,7 @@ export FBHOME="$STELLAR_HOME/friendbot"
 export LABHOME="$STELLAR_HOME/lab"
 export NXHOME="$STELLAR_HOME/nginx"
 export STELLAR_RPC_HOME="$STELLAR_HOME/stellar-rpc"
+export READINESS_HOME="$STELLAR_HOME/readiness"
 
 export CORELOG="/var/log/stellar-core"
 
@@ -363,6 +364,12 @@ function copy_defaults() {
       $CP /opt/stellar-default/$NETWORK/nginx/ $NXHOME
     fi
   fi
+
+  if [ -d $READINESS_HOME/etc ]; then
+    echo "readiness: config directory exists, skipping copy"
+  else
+    $CP /opt/stellar-default/common/readiness/ $READINESS_HOME
+  fi
 }
 
 function copy_pgpass() {