initiating work on strict typing, need sqlalchemy-stubs

attempting to use sqlalchemy stubs

fleshed out mypy.ini - verified works with vscode; updated mypy

improved support for mypy-sqlalchemy

questionable_model_change

fixed issues except python/mypy#4049

adding some mypy workarounds for mypy #4049 and #4291

Author: bbarker

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "sqlalchemy-stubs"]
+	path = sqlalchemy-stubs
+	url = https://github.com/JelleZijlstra/sqlalchemy-stubs.git

browse/controllers.py

@@ -1,52 +1,26 @@
 """Browse controllers."""
-import re
+
+from typing import Dict, Optional, Tuple
+
 from flask import request
-from arxiv import status
-from typing import Tuple, Dict, Any, Optional
-from browse.services.document import metadata
-from browse.services.document.metadata import AbsNotFoundException,\
-    AbsVersionNotFoundException
-from browse.domain.identifier import IdentifierException
-from browse.services.database.models import get_institution
-from werkzeug.exceptions import InternalServerError
+from flask_api import status
+from browse.services.database import get_institution
 
-Response = Tuple[Dict[str, Any], int, Dict[str, Any]]
 
+InstitutionResp = Tuple[Dict[str, str], int]
 
-def get_abs_page(arxiv_id: str) -> Response:
-    """Get the data that constitutes an /abs page."""
-    response_data = {}  # type: Dict[str, Any]
 
-    try:
-        abs_meta = metadata.get_abs(arxiv_id)
-        response_data['abs_meta'] = abs_meta
-    except AbsNotFoundException as e:
-        return {'not_found': True, 'arxiv_id': arxiv_id}, \
-            status.HTTP_404_NOT_FOUND, {}
-    except AbsVersionNotFoundException as e:
-        arxiv_id_latest = re.sub(r'(v[\d]+)$', '', arxiv_id)
-        return {'version_not_found': True,
-                'arxiv_id': arxiv_id,
-                'arxiv_id_latest': arxiv_id_latest},\
-            status.HTTP_404_NOT_FOUND, {}
-    except IdentifierException as e:
-        print(f'Got IdentifierException {e}')
-        return {'arxiv_id': arxiv_id}, status.HTTP_404_NOT_FOUND, {}
-    except IOError as e:
-        # TODO: handle differently?
-        raise InternalServerError(
-            "There was a problem. If this problem "
-            "persists, please contact [email protected]."
-        )
-
-    return response_data, status.HTTP_200_OK, {}
-
-
-def get_institution_from_request() -> Optional[str]:
+def get_institution_from_request() -> InstitutionResp:
     """Get the institution name from the request context."""
     institution_str = None
     try:
-        institution_str = get_institution(request.remote_addr)
+        institution_opt: Optional[str] = get_institution(request.remote_addr)
+        response: InstitutionResp = ({
+            'institution': institution_opt
+        }, status.HTTP_200_OK) if institution_opt is not None else ({
+            'explanation': 'Institution not found.'
+        }, status.HTTP_400_BAD_REQUEST)
+        return response
 
     except IOError:
         # TODO: log this

browse/factory.py

@@ -1,20 +1,22 @@
 """Application factory for browse service components."""
 
 from flask import Flask
-from browse.services.database import models
-from browse import views
+from browse.services.database.models import dbx as db
 
 
-def create_web_app() -> Flask:
+def create_web_app(config_filename: str) -> Flask:
     """Initialize an instance of the browse web application."""
+    from browse.views import blueprint
 
-    app = Flask('browse', static_folder='static', template_folder='templates')
-    app.config.from_pyfile('config.py')
+    app: Flask = Flask('browse', static_folder='static',
+                       template_folder='templates')
+    app.config.from_pyfile(config_filename)
 
-    models.init_app(app)
+    from browse.url_converter import ArXivConverter
+    app.url_map.converters['arxiv'] = ArXivConverter
 
-    # from browse.url_converter import ArXivConverter
-    # app.url_map.converters['arxiv'] = ArXivConverter
-    app.register_blueprint(views.blueprint)
+    db.init_app(app)
+
+    app.register_blueprint(blueprint)
 
     return app

browse/services/database/__init__.py

@@ -1,37 +1,46 @@
 """Import db instance and define utility functions."""
 
 import ipaddress
-from browse.services.database.models import db
+from browse.services.database.models import dbx
 from browse.services.database.models import MemberInstitution, \
     MemberInstitutionIP
 from sqlalchemy.sql import func
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm.exc import NoResultFound
+from flask_sqlalchemy import SQLAlchemy
 
+from typing import Optional
 
-# def get_institution(ip: str):
-#     """Get institution label from IP address."""
-#     decimal_ip = int(ipaddress.ip_address(ip))
-#     try:
-#         stmt = (
-#             db.session.query(
-#                 MemberInstitution.label,
-#                 func.sum(MemberInstitutionIP.exclude).label("exclusions")
-#             ).
-#             join(MemberInstitutionIP).
-#             filter(
-#                 MemberInstitutionIP.start <= decimal_ip,
-#                 MemberInstitutionIP.end >= decimal_ip
-#             ).
-#             group_by(MemberInstitution.label).
-#             subquery()
-#         )
-#
-#         return (
-#                     db.session.query(stmt.c.label).
-#                     filter(stmt.c.exclusions == 0).one().label
-#                )
-#     except NoResultFound:
-#         return None
-#     except SQLAlchemyError as e:
-#         raise IOError('Database error: %s' % e) from e
+# Temporary fix for https://github.com/python/mypy/issues/4049 :
+db: SQLAlchemy = dbx
+
+
+def get_institution(ip: str) -> Optional[str]:
+    """Get institution label from IP address."""
+    decimal_ip = int(ipaddress.ip_address(ip))
+    try:
+        stmt = (
+            db.session.query(
+                MemberInstitution.label,
+                func.sum(MemberInstitutionIP.exclude).label("exclusions")
+            ).
+            join(MemberInstitutionIP).
+            filter(
+                MemberInstitutionIP.start <= decimal_ip,
+                MemberInstitutionIP.end >= decimal_ip
+            ).
+            group_by(MemberInstitution.label).
+            subquery()
+        )
+
+        institution: str = db.session.query(stmt.c.label) \
+            .filter(stmt.c.exclusions == 0).one().label
+
+        return institution
+    except NoResultFound:
+        return None
+    except SQLAlchemyError as e:
+        raise IOError('Database error: %s' % e) from e
+
+
+__all__ = ["get_institution", "db"]

browse/services/database/models.py

@@ -4,16 +4,14 @@
 from sqlalchemy import BigInteger, Column, DateTime, Enum, \
     ForeignKey, Index, Integer, String, text
 from sqlalchemy.orm import relationship
-from sqlalchemy.sql import func
-from sqlalchemy.exc import SQLAlchemyError
-from sqlalchemy.orm.exc import NoResultFound
-from typing import Any, Optional
-from werkzeug.local import LocalProxy
+from flask_sqlalchemy import SQLAlchemy, Model
+from typing import Any, NewType
+#SQLAlchemyType = NewType('SQLAlchemyType', Any)
 
-db: Any = SQLAlchemy()
+dbx: SQLAlchemy = SQLAlchemy()
 
 
-class MemberInstitution(db.Model):
+class MemberInstitution(dbx.Model):
     """Primary model for arXiv member insitution data."""
 
     __tablename__ = 'Subscription_UniversalInstitution'
@@ -27,7 +25,7 @@ class MemberInstitution(db.Model):
     note = Column(String(255))
 
 
-class MemberInstitutionContact(db.Model):
+class MemberInstitutionContact(dbx.Model):
     """Model for arXiv member institution contact information."""
 
     __tablename__ = 'Subscription_UniversalInstitutionContact'
@@ -44,7 +42,7 @@ class MemberInstitutionContact(db.Model):
     Subscription_UniversalInstitution = relationship('MemberInstitution')
 
 
-class MemberInstitutionIP(db.Model):
+class MemberInstitutionIP(dbx.Model):
     """Model for arXiv member insitution IP address ranges and exclusions."""
 
     __tablename__ = 'Subscription_UniversalInstitutionIP'
@@ -62,7 +60,7 @@ class MemberInstitutionIP(db.Model):
     Subscription_UniversalInstitution = relationship('MemberInstitution')
 
 
-class SciencewisePing(db.Model):
+class SciencewisePing(dbx.Model):
     """Model for ScienceWISE (trackback) pings."""
 
     __tablename__ = 'arXiv_sciencewise_pings'
@@ -71,7 +69,7 @@ class SciencewisePing(db.Model):
     updated = Column(DateTime)
 
 
-class TrackbackPing(db.Model):
+class TrackbackPing(dbx.Model):
     """Primary model for arXiv trackback data."""
 
     __tablename__ = 'arXiv_trackback_pings'
@@ -99,7 +97,7 @@ class TrackbackPing(db.Model):
     site_id = Column(Integer)
 
 
-class TrackbackSite(db.Model):
+class TrackbackSite(dbx.Model):
     """Model for sites that submit trackbacks to arXiv."""
 
     __tablename__ = 'arXiv_trackback_sites'

browse/views.py

@@ -1,34 +1,14 @@
-"""Provides the user intefaces for browse."""
-from typing import Union
+"""Views."""
+from typing import Any
 
-from browse.controllers import get_abs_page, get_institution_from_request
-from flask import Blueprint, render_template, redirect, Response, session
-from arxiv import status
-from arxiv.base import exceptions
+from browse.controllers import get_institution_from_request
+from flask import Blueprint, render_template
 
 blueprint = Blueprint('browse', __name__, url_prefix='')
 
 
-@blueprint.before_request
-def before_request() -> None:
-    if 'institution' not in session:
-        institution = get_institution_from_request()
-        session['institution'] = institution
-
-
-@blueprint.after_request
-def apply_response_headers(response: Response) -> Response:
-    """Hook for applying response headers to all responses."""
-    """Prevent UI redress attacks"""
-    response.headers["Content-Security-Policy"] = "frame-ancestors 'none'"
-    response.headers["X-Frame-Options"] = "SAMEORIGIN"
-    return response
-
-
-# @blueprint.route('/abs/<arxiv:document_id>', methods=['GET'])
-@blueprint.route('/abs/', methods=['GET'], defaults={'arxiv_id': ''})
-@blueprint.route('/abs/<path:arxiv_id>', methods=['GET'])
-def abstract(arxiv_id: str) -> Union[str, Response]:
+@blueprint.route('/abs/<arxiv:document_id>', methods=['GET'])
+def abstract(document_id: str) -> Any:
     """Abstract (abs) page view."""
     response, code, headers = get_abs_page(arxiv_id)
     if code == status.HTTP_200_OK:

mypy.ini

@@ -1,10 +1,12 @@
 [mypy]
 mypy_path = $MYPYPATH:./sqlalchemy-stubs
+#mypy_path = $MYPYPATH
+
 
 #
 # Covered by --strict, with some turned off:
 #
-#disallow_untyped_calls=True
+disallow_untyped_calls=True
 disallow_untyped_defs=True
 check_untyped_defs=True
 # currently an issue with sql alchemy
@@ -13,15 +15,14 @@ disallow_subclassing_any=false
 disallow_any_decorated=false
 warn_redundant_casts=True
 warn_return_any=True
-#warn_unused_ignores=True
-# this seems to be at least somewhat non-functioning:
-#warn_unused_configs=True
+warn_unused_ignores=True
+warn_unused_configs=True
 #may be worth reconsidering this one:
 no_implicit_optional=True
 strict_optional=True
 
 #
-# Other:
+# Other: 
 #
 ignore_missing_imports=True
 

requirements/dev.txt

@@ -12,9 +12,11 @@ Jinja2==2.9.6
 lazy-object-proxy==1.3.1
 MarkupSafe==1.0
 mccabe==0.6.1
-mypy==0.540
+mypy==0.550
 mysqlclient==1.3.10
 nose2==0.6.5
+pep8==1.7.1
+psutil==5.4.1
 pylint==1.7.4
 six==1.11.0
 sqlacodegen==1.1.6

sqlalchemy-stubs

@@ -0,0 +1,21 @@
+------------------------------------------------------------------------------
+\\
+arXiv:acc-phys/9411001
+From: Salman Habib <[email protected]>
+Date: Tue, 1 Nov 1994 18:57:48 GMT   (199kb)
+
+Title: Symplectic Computation of Lyapunov Exponents
+Authors: Salman Habib and Robert D. Ryne
+Categories: acc-phys physics.acc-ph
+Comments: 12 pages, uuencoded PostScript (figures included)
+Report-no: LA-UR-94-3641
+\\
+  A recently developed method for the calculation of Lyapunov exponents of
+dynamical systems is described. The method is applicable whenever the
+linearized dynamics is Hamiltonian. By utilizing the exponential representation
+of symplectic matrices, this approach avoids the renormalization and
+reorthogonalization procedures necessary in usual techniques. It is also easily
+extendible to damped systems. The method is illustrated by considering two
+examples of physical interest: a model system that describes the beam halo in
+charged particle beams and the driven van der Pol oscillator.
+\\

tests/data/abs_files/ftp/acc-phys/papers/9411/9411001.abs

@@ -0,0 +1,21 @@
+------------------------------------------------------------------------------
+\\
+arXiv:acc-phys/9411002
+From: Salman Habib <[email protected]>
+Date: Sat, 19 Nov 1994 21:03:56 GMT   (12kb)
+
+Title: Order $\hbar$ Corrections to the Classical Dynamics of a Particle with
+  Intrinsic Spin Moving in a Constant Magnetic Field
+Authors: Patrick L. Nash (University of Texas at San Antonio)
+Categories: acc-phys physics.acc-ph
+\\
+  $O(\hbar)$ effects that modify the classical orbit of a charged particle are
+described for the case of a classical spin-1/2 particle moving in a constant
+magnetic field, using a manifestly covariant formalism reported previously. It
+is found that the coupling between the momentum and spin gives rise to a shift
+in the cyclotron frequency, which is explicitly calculated. In addition the
+orbit is found to exhibit $O(\hbar)$ oscillations along the axis of the uniform
+static magnetic field whenever the gyromagnetic ratio $g$ of the particle is
+not 2. This oscillation is found to occur at a frequency $\propto$ $\frac{g}{2}
+- 1$, and is an observable source of electric dipole radiation.
+\\

tests/data/abs_files/ftp/acc-phys/papers/9411/9411002.abs

@@ -0,0 +1,17 @@
+------------------------------------------------------------------------------
+\\
+arXiv:acc-phys/9411003
+From: Peter V. Vorob'ev. <[email protected]>
+Date: Tue, 22 Nov 1994 12:45:16 GMT   (69kb)
+
+Title: Candela Photo-Injector Experimental Results
+Authors: C. Travier, L. Boy, J.N. Cayla, B. Leblond, P. Georges, P. Thomas
+Categories: acc-phys physics.acc-ph
+Comments: 8 pages, 6 figures, LATEX
+Report-no: LAL/RT/94-15
+\\
+  The CANDELA photo-injector is a two cell S-band photo-injector. The copper
+cathode is illuminated by a 500 fs Ti:sapphire laser. This paper presents
+energy spectrum measurements of the dark current and intense electron emission
+that occurs when the laser power density is very high.
+\\