v1.0.2 changes

1. Support for contextmanager in SQLiteClient
2. PyPy3 platform in build matrix
3. Unittest cases for backup
4. Deleted travis.yml. Going forward only github actions will be used for building and testing
5. Readme updates

Author: Abhishek Singh

.dockerignore

@@ -16,4 +16,5 @@ example.db
 venv/
 start_server.py
 run_client.py
+.github
 

.github/workflows/sqlite_build.yaml

@@ -9,7 +9,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos-latest, windows-latest, ubuntu-18.04, ubuntu-20.04]
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.6, 3.7, 3.8, 3.9, '3.10.0-beta.3', 'pypy3']
 
     steps:
     - name: Checkout

.travis.yml

@@ -1,22 +1,26 @@
-# sqlite_rx [![Downloads](https://pepy.tech/badge/sqlite-rx)](https://pepy.tech/project/sqlite-rx) [![Travis](https://travis-ci.org/aosingh/sqlite_rx.svg?branch=master)](https://travis-ci.org/aosingh/sqlite_rx)  [![PyPI version](https://badge.fury.io/py/sqlite-rx.svg)](https://pypi.python.org/pypi/sqlite-rx) [![Coverage Status](https://coveralls.io/repos/github/aosingh/sqlite_rx/badge.svg?branch=master)](https://coveralls.io/github/aosingh/sqlite_rx?branch=master)
+# sqlite_rx [![Downloads](https://pepy.tech/badge/sqlite-rx)](https://pepy.tech/project/sqlite-rx) [![sqlite-rx](https://github.com/aosingh/sqlite-rx/actions/workflows/sqlite_build.yaml/badge.svg)]
+
+[![PyPI version](https://badge.fury.io/py/sqlite-rx.svg)](https://pypi.python.org/pypi/sqlite-rx) 
+
 [![Python 3.6](https://img.shields.io/badge/python-3.6-blue.svg)]((https://www.python.org/downloads/release/python-370/)) [![Python 3.7](https://img.shields.io/badge/python-3.7-blue.svg)](https://www.python.org/downloads/release/python-370/) [![Python 3.8](https://img.shields.io/badge/python-3.8-blue.svg)](https://www.python.org/downloads/release/python-380/)
 [![Python 3.9](https://img.shields.io/badge/python-3.9-blue.svg)]((https://www.python.org/downloads/release/python-390/))
 [![PyPy3.6](https://img.shields.io/badge/python-PyPy3.6-blue.svg)](https://www.pypy.org/index.html)
+
 ## Background
 
 [SQLite](https://www.sqlite.org/index.html) is a lightweight database written in C. 
-The Python programming language has in-built support to interact with the database(locally) which is either stored on disk or in memory.
+The Python programming language has in-built support to interact with the database (locally) which is either stored on disk or in memory.
 
-## Introducing sqlite_rx - SQLite remote query execution
+## sqlite_rx
 With `sqlite_rx`, clients should be able to communicate with an `SQLiteServer` in a fast, simple and secure manner and execute queries remotely.
 
 Key Features
 
 - Python Client and Server for [SQLite](https://www.sqlite.org/index.html) database built using [ZeroMQ](http://zguide.zeromq.org/page:all) as the transport layer and [msgpack](https://msgpack.org/index.html) for serialization/deserialization.
-- Supports authentication using [ZeroMQ Authentication Protocol (ZAP)](https://rfc.zeromq.org/spec:27/ZAP/)
-- Supports encryption using [CurveZMQ](http://curvezmq.org/)
-- Allows the users to define a generic authorization policy during server startup
-
+- Authentication using [ZeroMQ Authentication Protocol (ZAP)](https://rfc.zeromq.org/spec:27/ZAP/)
+- Encryption using [CurveZMQ](http://curvezmq.org/)
+- Generic authorization policy during server startup
+- Schedule regular backups
 
 
 # Install
@@ -58,6 +62,7 @@ def main():
     server = SQLiteServer(database=":memory:",
                           bind_address="tcp://127.0.0.1:5000")
     server.start()
+    server.join()
 
 if __name__ == '__main__':
     main()

README.md

@@ -3,7 +3,7 @@ services:
 
   sqlite_server:
     image: aosingh/sqlite_rx
-    command: sqlite-server --log-level DEBUG --database /data/database.db --backup-database /data/backup.db
+    command: sqlite-server --log-level DEBUG --database /data/main.db --backup-database /data/backup.db
     ports:
       - 5001:5000
     volumes:

docker-compose.yaml

@@ -3,7 +3,6 @@
 import socket
 import threading
 import zlib
-from pprint import pformat
 
 import msgpack
 import zmq
@@ -161,7 +160,6 @@ def execute(self,
                 socks = dict(self._poller.poll(request_timeout))
                 if socks.get(self._client) == zmq.POLLIN:
                     response = self._recv_response()
-                    LOG.debug("Response %s", pformat(response))
                     return response
                 else:
                     LOG.warning("No response from server, retrying...")
@@ -175,6 +173,12 @@ def execute(self,
                     self._send_request(request)
 
         raise SQLiteRxConnectionError("No response after retrying. Abandoning Request")
+    
+    def __enter__(self):
+        return self
+    
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.shutdown()
 
     def shutdown(self):
         try:
@@ -183,10 +187,10 @@ def shutdown(self):
             self._poller.unregister(self._client)
         except zmq.ZMQError as e:
             if e.errno in (zmq.EINVAL,
-                          zmq.EPROTONOSUPPORT,
-                          zmq.ENOCOMPATPROTO,
-                          zmq.EADDRINUSE,
-                          zmq.EADDRNOTAVAIL,):
+                           zmq.EPROTONOSUPPORT,
+                           zmq.ENOCOMPATPROTO,
+                           zmq.EADDRINUSE,
+                           zmq.EADDRNOTAVAIL,):
                 LOG.error("ZeroMQ Transportation endpoint was not setup")
 
             elif e.errno in (zmq.ENODEV, zmq.ENOTSOCK,):

sqlite_rx/client.py

@@ -6,12 +6,11 @@
 import sys
 import traceback
 import zlib
-from pprint import pformat
 from signal import SIGTERM, SIGINT, signal
 
 from typing import List, Union, Callable
 
-import billiard as multiprocess
+import billiard as multiprocessing
 import msgpack
 import zmq
 from sqlite_rx import get_version
@@ -31,7 +30,7 @@
 __all__ = ['SQLiteServer']
 
 
-class SQLiteZMQProcess(multiprocess.Process):
+class SQLiteZMQProcess(multiprocessing.Process):
 
     def __init__(self, *args, **kwargs):
         """The :class: ``sqlite_rx.server.SQLiteServer`` is intended to run as an isolated process.
@@ -181,18 +180,12 @@ def handle_signal(self, signum, frame):
         self.socket.close()
         self.loop.stop()
 
-        # self.terminate()
-
-        # If backup process was started then wait for the backup thread to finish. 
         if self.back_up_recurring_thread:
-            # self.back_up_recurring_thread.join()
-            # pthread_kill(self.back_up_recurring_thread.get_ident())
             self.back_up_recurring_thread.cancel()
 
         os._exit(os.EX_OK)
 
 
-
     def run(self):
         LOG.info("Setting up signal handlers")
 
@@ -254,7 +247,6 @@ def __call__(self, message: List):
             self._rep_stream.send(zlib.compress(msgpack.dumps(result)))
 
     def execute(self, message: dict, *args, **kwargs):
-        LOG.debug("Request received is %s", pformat(message))
         execute_many = message['execute_many']
         execute_script = message['execute_script']
         error = None

sqlite_rx/server.py

@@ -0,0 +1,52 @@
+import os
+import platform
+import signal
+import tempfile
+import pytest
+
+import sqlite3
+import logging.config
+
+from collections import namedtuple
+from sqlite_rx import backup, get_default_logger_settings
+from sqlite_rx.client import SQLiteClient
+from sqlite_rx.server import SQLiteServer
+
+logging.config.dictConfig(get_default_logger_settings(level="DEBUG"))
+
+LOG = logging.getLogger(__file__)
+
+backup_event = namedtuple('backup_event', ('client', 'backup_database'))
+
+
+@pytest.fixture(scope="module")
+def plain_client():
+    auth_config = {
+        sqlite3.SQLITE_OK: {
+            sqlite3.SQLITE_DROP_TABLE
+        }
+    }
+    with tempfile.NamedTemporaryFile() as back_db_file, tempfile.NamedTemporaryFile() as main_db_file:
+        server = SQLiteServer(bind_address="tcp://127.0.0.1:5003",
+                              database=main_db_file.name,
+                              auth_config=auth_config,
+                              backup_database=back_db_file.name,
+                              backup_interval=1)
+        
+        client = SQLiteClient(connect_address="tcp://127.0.0.1:5003")
+
+        event = backup_event(client=client, backup_database=back_db_file.name)
+
+        server.start()
+
+        LOG.info("Started Test SQLiteServer")
+
+        yield event
+
+        if platform.system().lower() == 'windows':
+            os.system("taskkill  /F /pid "+str(server.pid))
+        else:
+            os.kill(server.pid, signal.SIGINT)
+
+        server.join()
+        client.shutdown()

sqlite_rx/tests/backup/__init__.py

@@ -0,0 +1,77 @@
+import sys
+import platform
+import sqlite3
+import os
+import time
+
+from sqlite_rx import backup
+
+sqlite_error_prefix = "sqlite3.OperationalError"
+
+if platform.python_implementation() == "PyPy":
+    sqlite_error_prefix = "_sqlite3.OperationalError"
+
+
+def test_not_present(plain_client):
+    result = plain_client.client.execute('SELECT * FROM IDOLS')
+    expected_result = {
+        'items': [],
+        'error': {
+            'message': '{0}: no such table: IDOLS'.format(sqlite_error_prefix),
+            'type': '{0}'.format(sqlite_error_prefix)}}
+    assert type(result) == dict
+    assert result == expected_result
+    
+
+
+def test_table_creation(plain_client):
+    result = plain_client.client.execute('CREATE TABLE stocks (date text, trans text, symbol text, qty real, price real)')
+    expected_result = {"error": None, 'items': []}
+    assert result == expected_result
+
+
+def test_table_rows_insertion(plain_client):
+    purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+                 ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+                 ('2006-04-06', 'SELL', 'XOM', 500, 53.00),
+                 ]
+
+    result = plain_client.client.execute('INSERT INTO stocks VALUES (?,?,?,?,?)', *purchases, execute_many=True)
+    expected_result = {'error': None, 'items': [], 'rowcount': 27}
+    assert result == expected_result
+
+    if (sys.version_info.major == 3 and sys.version_info.minor >= 7):
+        time.sleep(1) # wait for the backup thread to finish backing up.
+
+        backup_database = plain_client.backup_database
+        backup_connection = sqlite3.connect(database=backup_database,
+                                            isolation_level=None,
+                                            check_same_thread=False)                      
+        assert os.path.exists(backup_database) == True
+        result = backup_connection.execute("SELECT * FROM stocks").fetchall()
+        assert len(result) == 27
+    

sqlite_rx/tests/backup/conftest.py

@@ -0,0 +1,12 @@
+import sys
+import pytest
+
+from sqlite_rx.exception import SQLiteRxBackUpError
+from sqlite_rx.server import SQLiteServer
+
+
+def test_backup_exception():
+
+    if not (sys.version_info.major == 3 and sys.version_info.minor >= 7):
+        with pytest.raises(SQLiteRxBackUpError):
+            server = SQLiteServer(bind_address="tcp://127.0.0.1:5002", database=":memory:", backup_database='backup.db')