Disallow missing, star and unused imports (#518)

- Resolves #516
- Resolves #517

Author: zero323

mypy.ini

@@ -1 +1,19 @@
 [mypy]
+
+[mypy-pyspark.cloudpickle.*]
+ignore_errors = True
+
+[mypy-py4j.*]
+ignore_missing_imports = True
+
+[mypy-numpy.*]
+ignore_missing_imports = True
+
+[mypy-scipy.*]
+ignore_missing_imports = True
+
+[mypy-pandas.*]
+ignore_missing_imports = True
+
+[mypy-pyarrow]
+ignore_missing_imports = True

third_party/3/pyspark/__init__.pyi

@@ -16,36 +16,49 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Stubs for pyspark (Python 3)
-#
-
 from typing import Callable, Optional, TypeVar
 
-from pyspark.status import *
-from pyspark.accumulators import (
+from pyspark.accumulators import (  # noqa: F401
     Accumulator as Accumulator,
     AccumulatorParam as AccumulatorParam,
 )
-from pyspark.broadcast import Broadcast as Broadcast
-from pyspark.conf import SparkConf as SparkConf
-from pyspark.context import SparkContext as SparkContext
-from pyspark.files import SparkFiles as SparkFiles
-from pyspark.profiler import BasicProfiler as BasicProfiler, Profiler as Profiler
-from pyspark.rdd import RDD as RDD, RDDBarrier as RDDBarrier
-from pyspark.serializers import (
+from pyspark.broadcast import Broadcast as Broadcast  # noqa: F401
+from pyspark.conf import SparkConf as SparkConf  # noqa: F401
+from pyspark.context import SparkContext as SparkContext  # noqa: F401
+from pyspark.files import SparkFiles as SparkFiles  # noqa: F401
+from pyspark.status import (
+    StatusTracker as StatusTracker,
+    SparkJobInfo as SparkJobInfo,
+    SparkStageInfo as SparkStageInfo,
+)  # noqa: F401
+from pyspark.profiler import (  # noqa: F401
+    BasicProfiler as BasicProfiler,
+    Profiler as Profiler,
+)
+from pyspark.rdd import RDD as RDD, RDDBarrier as RDDBarrier  # noqa: F401
+from pyspark.serializers import (  # noqa: F401
     MarshalSerializer as MarshalSerializer,
     PickleSerializer as PickleSerializer,
 )
-from pyspark.storagelevel import StorageLevel as StorageLevel
-from pyspark.taskcontext import (
+from pyspark.status import (  # noqa: F401
+    SparkJobInfo as SparkJobInfo,
+    SparkStageInfo as SparkStageInfo,
+    StatusTracker as StatusTracker,
+)
+from pyspark.storagelevel import StorageLevel as StorageLevel  # noqa: F401
+from pyspark.taskcontext import (  # noqa: F401
     BarrierTaskContext as BarrierTaskContext,
     BarrierTaskInfo as BarrierTaskInfo,
     TaskContext as TaskContext,
 )
-from pyspark.util import InheritableThread as InheritableThread
+from pyspark.util import InheritableThread as InheritableThread  # noqa: F401
 
 # Compatiblity imports
-from pyspark.sql import SQLContext as SQLContext, HiveContext as HiveContext, Row as Row
+from pyspark.sql import (  # noqa: F401
+    SQLContext as SQLContext,
+    HiveContext as HiveContext,
+    Row as Row,
+)
 
 T = TypeVar("T")
 F = TypeVar("F", bound=Callable)
@@ -58,8 +71,3 @@ def copy_func(
     doc: Optional[str] = ...,
 ) -> F: ...
 def keyword_only(func: F) -> F: ...
-
-# Names in __all__ with no definition:
-#   SparkJobInfo
-#   SparkStageInfo
-#   StatusTracker

third_party/3/pyspark/_typing.pyi

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from typing import Any, Generic, Iterable, List, Optional, Sized, TypeVar, Union
+from typing import Iterable, Sized, TypeVar, Union
 from typing_extensions import Protocol
 
 T = TypeVar("T", covariant=True)

third_party/3/pyspark/accumulators.pyi

@@ -29,7 +29,6 @@ T = TypeVar("T")
 U = TypeVar("U", bound=SupportsIAdd)
 
 import socketserver as SocketServer
-from typing import Any
 
 class Accumulator(Generic[T]):
     aid: int

third_party/3/pyspark/cloudpickle.pyi

@@ -20,7 +20,7 @@
 #
 
 from typing import overload
-from typing import Any, List, Optional, Tuple
+from typing import List, Optional, Tuple
 
 from py4j.java_gateway import JVMView, JavaObject  # type: ignore[import]
 

third_party/3/pyspark/conf.pyi

@@ -26,7 +26,7 @@ from py4j.java_gateway import JavaGateway, JavaObject  # type: ignore[import]
 from pyspark.accumulators import Accumulator, AccumulatorParam
 from pyspark.broadcast import Broadcast
 from pyspark.conf import SparkConf
-from pyspark.profiler import Profiler
+from pyspark.profiler import Profiler  # noqa: F401
 from pyspark.resource.information import ResourceInformation
 from pyspark.rdd import RDD
 from pyspark.serializers import Serializer

third_party/3/pyspark/context.pyi

@@ -16,7 +16,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyspark.serializers import UTF8Deserializer as UTF8Deserializer, read_int as read_int, write_int as write_int, write_with_length as write_with_length  # type: ignore[attr-defined]
+from pyspark.serializers import (  # noqa: F401
+    UTF8Deserializer as UTF8Deserializer,
+    read_int as read_int,
+    write_int as write_int,
+    write_with_length as write_with_length,
+)
 from typing import Any
 
 def compute_real_exit_code(exit_code: Any): ...

third_party/3/pyspark/daemon.pyi

@@ -16,10 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Stubs for pyspark.join (Python 3)
-#
-
-from typing import Hashable, Iterable, List, Optional, Tuple, TypeVar
+from typing import Hashable, Iterable, Optional, Tuple, TypeVar
 
 from pyspark.resultiterable import ResultIterable
 import pyspark.rdd

third_party/3/pyspark/join.pyi

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from pyspark.ml import (
+from pyspark.ml import (  # noqa: F401
     classification as classification,
     clustering as clustering,
     evaluation as evaluation,
@@ -31,12 +31,15 @@ from pyspark.ml import (
     tuning as tuning,
     util as util,
 )
-from pyspark.ml.base import (
+from pyspark.ml.base import (  # noqa: F401
     Estimator as Estimator,
     Model as Model,
     PredictionModel as PredictionModel,
     Predictor as Predictor,
     Transformer as Transformer,
     UnaryTransformer as UnaryTransformer,
 )
-from pyspark.ml.pipeline import Pipeline as Pipeline, PipelineModel as PipelineModel
+from pyspark.ml.pipeline import (  # noqa: F401
+    Pipeline as Pipeline,
+    PipelineModel as PipelineModel,
+)

third_party/3/pyspark/ml/__init__.pyi

@@ -18,35 +18,39 @@
 
 from typing import overload
 from typing import (
-    Any,
     Callable,
-    Dict,
     Generic,
     Iterable,
     List,
     Optional,
     Tuple,
     Type,
-    TypeVar,
 )
 from pyspark.ml._typing import M, P, T, ParamMap
 
 import _thread
 
 import abc
 from abc import abstractmethod
-from pyspark.ml.common import inherit_doc as inherit_doc
-from pyspark.ml.param import Params, Param
-from pyspark.ml.param.shared import *
-from pyspark.sql.column import Column
-from pyspark.sql.dataframe import DataFrame
-from pyspark.sql.functions import udf as udf
-from pyspark.sql.types import (
+from pyspark import since as since  # noqa: F401
+from pyspark.ml.common import inherit_doc as inherit_doc  # noqa: F401
+from pyspark.ml.param.shared import (
+    HasFeaturesCol as HasFeaturesCol,
+    HasInputCol as HasInputCol,
+    HasLabelCol as HasLabelCol,
+    HasOutputCol as HasOutputCol,
+    HasPredictionCol as HasPredictionCol,
+    Params as Params,
+)
+from pyspark.sql.functions import udf as udf  # noqa: F401
+from pyspark.sql.types import (  # noqa: F401
     DataType,
     StructField as StructField,
     StructType as StructType,
 )
 
+from pyspark.sql.dataframe import DataFrame
+
 class _FitMultipleIterator:
     fitSingleModel: Callable[[int], Transformer]
     numModel: int

third_party/3/pyspark/ml/base.pyi

@@ -16,36 +16,48 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from typing import Any, Dict, List, Optional, Type, TypeVar
+from typing import Any, List, Optional, Type
 from pyspark.ml._typing import JM, M, P, T, ParamMap
 
 import abc
 from abc import abstractmethod
-from pyspark.ml.base import Estimator, Model, Transformer, PredictionModel, Predictor
+from pyspark.ml import Estimator, Model, PredictionModel, Predictor, Transformer
 from pyspark.ml.base import _PredictorParams
-from pyspark.ml.linalg import Matrix, Vector
-from pyspark.ml.param.shared import *
+from pyspark.ml.param.shared import (
+    HasAggregationDepth,
+    HasBlockSize,
+    HasElasticNetParam,
+    HasFitIntercept,
+    HasMaxIter,
+    HasParallelism,
+    HasProbabilityCol,
+    HasRawPredictionCol,
+    HasRegParam,
+    HasSeed,
+    HasSolver,
+    HasStandardization,
+    HasStepSize,
+    HasThreshold,
+    HasThresholds,
+    HasTol,
+    HasWeightCol,
+)
+from pyspark.ml.regression import _FactorizationMachinesParams
 from pyspark.ml.tree import (
     _DecisionTreeModel,
     _DecisionTreeParams,
-    _TreeEnsembleModel,
-    _RandomForestParams,
     _GBTParams,
     _HasVarianceImpurity,
+    _RandomForestParams,
     _TreeClassifierParams,
-    _TreeEnsembleParams,
-)
-from pyspark.ml.regression import (
-    _FactorizationMachinesParams,
-    DecisionTreeRegressionModel,
-)
-from pyspark.ml.util import *
-from pyspark.ml.wrapper import (
-    JavaPredictionModel,
-    JavaPredictor,
-    JavaWrapper,
-    JavaTransformer,
+    _TreeEnsembleModel,
 )
+from pyspark.ml.util import HasTrainingSummary, JavaMLReadable, JavaMLWritable
+from pyspark.ml.wrapper import JavaPredictionModel, JavaPredictor, JavaWrapper
+
+from pyspark.ml.linalg import Matrix, Vector
+from pyspark.ml.param import Param
+from pyspark.ml.regression import DecisionTreeRegressionModel
 from pyspark.sql.dataframe import DataFrame
 
 class _ClassifierParams(HasRawPredictionCol, _PredictorParams): ...