Skip to content

Commit b63b85d

Browse files
JavierMontonJavierMonton
authored
Core/Scala3 (#54)
Core module migrated to Scala3.
1 parent 56c74b7 commit b63b85d

File tree

35 files changed

+643
-58
lines changed

35 files changed

+643
-58
lines changed

‎.github/workflows/bigquery-integration.yml

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,12 @@ jobs:
2626
service_account_key: ${{ secrets.BQ_SERVICE_ACCOUNT_64 }}
2727
export_default_credentials: true
2828
- name: Integration Tests
29-
run: sbt -v -Dfile.encoding=UTF-8 coverage +it:test
30-
- name: Coverage Report
31-
run: sbt -v -Dfile.encoding=UTF-8 coverageReport
32-
- name: Codecov
33-
uses: codecov/codecov-action@v1
34-
with:
35-
token: ${{ secrets.CODECOV_TOKEN }}
36-
verbose: false # optional (default = false)
37-
flags: bq-integration
29+
run: sbt -v -Dfile.encoding=UTF-8 +it:test
30+
#- name: Coverage Report
31+
# run: sbt -v -Dfile.encoding=UTF-8 coverageReport
32+
#- name: Codecov
33+
# uses: codecov/codecov-action@v1
34+
# with:
35+
# token: ${{ secrets.CODECOV_TOKEN }}
36+
# verbose: false # optional (default = false)
37+
# flags: bq-integration

‎.github/workflows/ci-tests.yml

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,12 @@ jobs:
1818
with:
1919
java-version: "[email protected]"
2020
- name: Build and Unit Tests
21-
run: sbt -v -Dfile.encoding=UTF-8 +clean coverage +test
22-
- name: Coverage Report
23-
run: sbt -v -Dfile.encoding=UTF-8 coverageReport
24-
- name: Codecov
25-
uses: codecov/codecov-action@v1
26-
with:
27-
token: ${{ secrets.CODECOV_TOKEN }}
28-
verbose: false # optional (default = false)
29-
flags: unittests
21+
run: sbt -v -Dfile.encoding=UTF-8 +clean +test
22+
#- name: Coverage Report
23+
# run: sbt -v -Dfile.encoding=UTF-8 coverageReport
24+
#- name: Codecov
25+
# uses: codecov/codecov-action@v1
26+
# with:
27+
# token: ${{ secrets.CODECOV_TOKEN }}
28+
# verbose: false # optional (default = false)
29+
# flags: unittests

‎bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/SqlInstanceToBigQuery.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
package org.datatools.bigdatatypes.bigquery
22

33
import com.google.cloud.bigquery.Field
4+
import org.datatools.bigdatatypes.basictypes.SqlType
45
import org.datatools.bigdatatypes.conversions.SqlInstanceConversion
56
import org.datatools.bigdatatypes.formats.Formats
6-
import org.datatools.bigdatatypes.types.basic.SqlType
77

88
/** Type class to convert generic SqlTypes received as instance into BigQuery specific fields
99
* This uses [[SqlTypeToBigQuery]] to create BigQuery Fields

‎bigquery/src/main/scala/org/datatools/bigdatatypes/bigquery/SqlTypeToBigQuery.scala

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,9 @@ package org.datatools.bigdatatypes.bigquery
22

33
import com.google.cloud.bigquery.Field.Mode
44
import com.google.cloud.bigquery.{Field, StandardSQLTypeName}
5+
import org.datatools.bigdatatypes.basictypes._
56
import org.datatools.bigdatatypes.conversions._
67
import org.datatools.bigdatatypes.formats.Formats
7-
import org.datatools.bigdatatypes.types.basic
8-
import org.datatools.bigdatatypes.types.basic._
98

109
/** Type class to convert generic SqlTypes into BigQuery specific fields
1110
* In BigQuery, a table is made with a List of fields so as an example:
@@ -41,7 +40,7 @@ object SqlTypeToBigQuery {
4140
def getSchema(sqlType: SqlType)(implicit f: Formats): List[Field] = sqlType match {
4241
case SqlStruct(Nil, _) => Nil
4342
case SqlStruct((name, sqlType) :: records, mode) =>
44-
getSchemaWithName(f.transformKey(name, sqlType), sqlType) :: getSchema(basic.SqlStruct(records, mode))
43+
getSchemaWithName(f.transformKey(name, sqlType), sqlType) :: getSchema(SqlStruct(records, mode))
4544
}
4645

4746
/** Basic SqlTypes conversions to BigQuery Fields

‎bigquery/src/test/scala/org/datatools/bigdatatypes/bigquery/SqlTypeToBigQueryInstanceSpec.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ package org.datatools.bigdatatypes.bigquery
22

33
import com.google.cloud.bigquery.Field
44
import org.datatools.bigdatatypes.TestTypes._
5+
import org.datatools.bigdatatypes.basictypes.SqlType
56
import org.datatools.bigdatatypes.bigquery.SqlInstanceToBigQuery.InstanceSyntax
67
import org.datatools.bigdatatypes.conversions.SqlTypeConversion
78
import org.datatools.bigdatatypes.conversions.SqlTypeConversion._
89
import org.datatools.bigdatatypes.formats.Formats.implicitDefaultFormats
9-
import org.datatools.bigdatatypes.types.basic.SqlType
1010
import org.datatools.bigdatatypes.{BigQueryTestTypes, UnitSpec}
1111

1212
/** These tests defines how to convert an SqlType instance into BigQueryFields

‎build.sbt

Lines changed: 44 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
//used to build Sonatype releases
2-
lazy val versionNumber = "0.3.1"
2+
lazy val versionNumber = "0.3.2"
33
lazy val projectName = "big-data-types"
44
version := versionNumber
55
name := projectName
66

77
lazy val scala213 = "2.13.5"
88
lazy val scala212 = "2.12.12"
9-
lazy val scala211 = "2.11.12"
10-
lazy val supportedScalaVersions = List(scala213, scala212)
11-
scalaVersion := scala212
9+
lazy val scala3 = "3.0.0-RC2"
10+
lazy val supportedScalaVersions = List(scala3, scala213, scala212)
11+
scalaVersion := scala213
1212

13-
assemblyMergeStrategy in assembly := {
13+
assembly / assemblyMergeStrategy := {
1414
case PathList("META-INF", xs @ _*) => MergeStrategy.discard
1515
case x => MergeStrategy.first
1616
}
@@ -29,19 +29,27 @@ lazy val publishSettings = Seq(
2929
licenses := Seq("APL2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")),
3030
publishMavenStyle := true
3131
)
32-
lazy val noPublishSettings =
33-
skip in publish := true
32+
33+
lazy val noPublishSettings = {
34+
publish / skip := true
35+
}
3436

3537
publishSettings
3638

3739
//Dependencies
38-
lazy val coreDependencies = Seq(
40+
lazy val coreDependencies2 = Seq(
3941
"ch.qos.logback" % "logback-classic" % "1.2.3",
4042
"org.clapper" %% "grizzled-slf4j" % "1.3.4",
4143
"com.chuusai" %% "shapeless" % "2.3.3",
4244
scalatest % Test
4345
)
4446

47+
lazy val coreDependencies3 = Seq(
48+
"ch.qos.logback" % "logback-classic" % "1.2.3",
49+
"org.clapper" % "grizzled-slf4j_2.13" % "1.3.4",
50+
scalatest % Test
51+
)
52+
4553
lazy val bigqueryDependencies = Seq(
4654
"com.google.cloud" % "google-cloud-bigquery" % "1.127.11",
4755
scalatest % "it,test"
@@ -57,7 +65,7 @@ lazy val scalatest = "org.scalatest" %% "scalatest" % "3.2.7"
5765
//Project settings
5866
lazy val root = (project in file("."))
5967
.configs(IntegrationTest)
60-
.settings(noPublishSettings)
68+
.settings(noPublishSettings, crossScalaVersions := Nil)
6169
.aggregate(
6270
core,
6371
bigquery,
@@ -69,8 +77,15 @@ lazy val core = (project in file("core")).settings(
6977
name := projectName + "-core",
7078
publishSettings,
7179
crossScalaVersions := supportedScalaVersions,
72-
crossVersionSharedSources,
73-
libraryDependencies ++= coreDependencies
80+
crossVersionSharedSourcesScala3, //different one for Scala 2 or 3
81+
//for Scala 2 or 3
82+
libraryDependencies ++= {
83+
CrossVersion.partialVersion(scalaVersion.value) match {
84+
case Some((2, _)) => coreDependencies2
85+
case Some((3, _)) => coreDependencies3
86+
case _ => Nil
87+
}
88+
}
7489
)
7590

7691
lazy val bigquery = (project in file("bigquery"))
@@ -79,7 +94,7 @@ lazy val bigquery = (project in file("bigquery"))
7994
name := projectName + "-bigquery",
8095
publishSettings,
8196
Defaults.itSettings,
82-
crossScalaVersions := supportedScalaVersions,
97+
crossScalaVersions := List(scala212, scala213),
8398
crossVersionSharedSources,
8499
libraryDependencies ++= bigqueryDependencies
85100
)
@@ -112,18 +127,31 @@ lazy val examples = (project in file("examples"))
112127
)
113128
.dependsOn(spark % "test->test;compile->compile")
114129

115-
116-
117130
lazy val crossVersionSharedSources: Seq[Setting[_]] =
118131
Seq(Compile, Test).map { sc =>
119-
(unmanagedSourceDirectories in sc) ++= {
120-
(unmanagedSourceDirectories in sc).value.flatMap { dir: File =>
132+
(sc / unmanagedSourceDirectories) ++= {
133+
(sc / unmanagedSourceDirectories).value.flatMap { dir: File =>
121134
if (dir.getName != "scala") Seq(dir)
122135
else
123136
CrossVersion.partialVersion(scalaVersion.value) match {
137+
case Some((3, _)) => Seq(new File(dir.getPath + "_3"))
124138
case Some((2, y)) if y >= 13 => Seq(new File(dir.getPath + "_2.13+"))
125139
case Some((2, y)) if y >= 11 => Seq(new File(dir.getPath + "_2.13-"))
126140
}
127141
}
128142
}
129143
}
144+
145+
lazy val crossVersionSharedSourcesScala3: Seq[Setting[_]] =
146+
Seq(Compile, Test).map { sc =>
147+
(sc / unmanagedSourceDirectories) ++= {
148+
(sc / unmanagedSourceDirectories).value.flatMap { dir: File =>
149+
if (dir.getName != "scala") Seq(dir)
150+
else
151+
CrossVersion.partialVersion(scalaVersion.value) match {
152+
case Some((3, _)) => Seq(new File(dir.getPath + "_3"))
153+
case Some((2, _)) => Seq(new File(dir.getPath + "_2"))
154+
}
155+
}
156+
}
157+
}

‎core/src/main/scala/org/datatools/bigdatatypes/types/basic/SqlType.scala renamed to ‎core/src/main/scala_2/org/datatools/bigdatatypes/basictypes/SqlType.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package org.datatools.bigdatatypes.types.basic
1+
package org.datatools.bigdatatypes.basictypes
22

33
/** Abstract representation of the type of a generic SQL database */
44
sealed trait SqlType {

‎core/src/main/scala/org/datatools/bigdatatypes/types/basic/SqlTypeMode.scala renamed to ‎core/src/main/scala_2/org/datatools/bigdatatypes/basictypes/SqlTypeMode.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package org.datatools.bigdatatypes.types.basic
1+
package org.datatools.bigdatatypes.basictypes
22

33
/** The mode of a sql type. e.g: Required, Nullable, Repeated.
44
*/

‎core/src/main/scala/org/datatools/bigdatatypes/conversions/SqlInstanceConversion.scala renamed to ‎core/src/main/scala_2/org/datatools/bigdatatypes/conversions/SqlInstanceConversion.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package org.datatools.bigdatatypes.conversions
22

3-
import org.datatools.bigdatatypes.types.basic._
3+
import org.datatools.bigdatatypes.basictypes.SqlType
44

55
/** Type class to convert instances into [[SqlType]]
66
*

‎core/src/main/scala/org/datatools/bigdatatypes/conversions/SqlTypeConversion.scala renamed to ‎core/src/main/scala_2/org/datatools/bigdatatypes/conversions/SqlTypeConversion.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
package org.datatools.bigdatatypes.conversions
22

3-
import java.sql.{Date, Timestamp}
3+
import org.datatools.bigdatatypes.basictypes._
44

5-
import org.datatools.bigdatatypes.types.basic._
5+
import java.sql.{Date, Timestamp}
66
import shapeless._
77
import shapeless.labelled.FieldType
88

‎core/src/main/scala/org/datatools/bigdatatypes/formats/Formats.scala renamed to ‎core/src/main/scala_2/org/datatools/bigdatatypes/formats/Formats.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package org.datatools.bigdatatypes.formats
22

3-
import org.datatools.bigdatatypes.types.basic.SqlType
3+
import org.datatools.bigdatatypes.basictypes.SqlType
44

55
trait Formats {
66

‎core/src/main/scala/org/datatools/bigdatatypes/formats/KeyTypeExampleFormats.scala renamed to ‎core/src/main/scala_2/org/datatools/bigdatatypes/formats/KeyTypeExampleFormats.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package org.datatools.bigdatatypes.formats
22

3-
import org.datatools.bigdatatypes.types.basic.{SqlBool, SqlDate, SqlTimestamp, SqlType}
3+
import org.datatools.bigdatatypes.basictypes._
44

55
/** Converts CamelCase field names to snake_case
66
*/

‎core/src/main/scala/org/datatools/bigdatatypes/formats/SnakifyFormats.scala renamed to ‎core/src/main/scala_2/org/datatools/bigdatatypes/formats/SnakifyFormats.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package org.datatools.bigdatatypes.formats
22

3-
import org.datatools.bigdatatypes.types.basic.SqlType
3+
import org.datatools.bigdatatypes.basictypes.SqlType
44

55
/** Converts CamelCase field names to snake_case
66
*/

‎core/src/main/scala_3/org/datatools/bigdatatypes/basictypes/SqlType.scala

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package org.datatools.bigdatatypes.basictypes
2+
3+
import org.datatools.bigdatatypes.basictypes.SqlType.{SqlBool, SqlDate, SqlDecimal, SqlDouble, SqlFloat, SqlInt, SqlLong, SqlString, SqlStruct, SqlTimestamp}
4+
import org.datatools.bigdatatypes.basictypes.SqlTypeMode._
5+
6+
/** Abstract representation of the type of a generic SQL database */
7+
enum SqlType {
8+
9+
/** @return the [[SqlTypeMode]] of this SqlType
10+
*/
11+
def mode: SqlTypeMode
12+
13+
/** Promotes the type to a new mode if the conversion makes sense. e.g:
14+
* [[List[Option[String] ] ]] and [[Option[List[String] ] ]] should be SqlString(Repeated)
15+
*
16+
* @param mode the mode we want to convert to
17+
* @return a new [[SqlType]] with the mode
18+
*/
19+
def changeMode(mode: SqlTypeMode): SqlType =
20+
if (this.mode.isValidConversion(mode))
21+
this match {
22+
case SqlInt(_) => SqlInt(mode)
23+
case SqlLong(_) => SqlLong(mode)
24+
case SqlFloat(_) => SqlFloat(mode)
25+
case SqlDouble(_) => SqlDouble(mode)
26+
case SqlDecimal(_) => SqlDecimal(mode)
27+
case SqlBool(_) => SqlBool(mode)
28+
case SqlString(_) => SqlString(mode)
29+
case SqlTimestamp(_) => SqlTimestamp(mode)
30+
case SqlDate(_) => SqlDate(mode)
31+
case SqlStruct(records, _) => SqlStruct(records, mode)
32+
}
33+
else this
34+
35+
case SqlInt(mode: SqlTypeMode = Required)
36+
case SqlLong(mode: SqlTypeMode = Required)
37+
case SqlFloat(mode: SqlTypeMode = Required)
38+
case SqlDouble(mode: SqlTypeMode = Required)
39+
case SqlDecimal(mode: SqlTypeMode = Required)
40+
case SqlBool(mode: SqlTypeMode = Required)
41+
case SqlString(mode: SqlTypeMode = Required)
42+
case SqlTimestamp(mode: SqlTypeMode = Required)
43+
case SqlDate(mode: SqlTypeMode = Required)
44+
case SqlStruct(records: List[(String, SqlType)], mode: SqlTypeMode = Required)
45+
}
46+
47+

‎core/src/main/scala_3/org/datatools/bigdatatypes/basictypes/SqlTypeMode.scala

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package org.datatools.bigdatatypes.basictypes
2+
3+
import org.datatools.bigdatatypes.basictypes.SqlTypeMode.{Nullable, Repeated, Required}
4+
5+
/** The mode of a sql type. e.g: Required, Nullable, Repeated.
6+
*/
7+
enum SqlTypeMode {
8+
9+
/** Tells you if you can change the mode for another.
10+
*
11+
* e.g:
12+
* [[[Option[String] ]] should be SqlString(Nullable)
13+
* [[List[Option[String] ] ]] should be SqlString(Repeated).
14+
*
15+
* @param newMode the mode we want to convert to
16+
* @return true if this conversion makes sense, false if not
17+
*/
18+
def isValidConversion(newMode: SqlTypeMode): Boolean = (this, newMode) match {
19+
case (Repeated, _) => false
20+
case (Nullable, Required) => false
21+
case (_, _) => true
22+
}
23+
24+
/** Nullable field */
25+
case Nullable
26+
27+
/** Repeated or array field */
28+
case Repeated
29+
30+
/** Mandatory field */
31+
case Required
32+
}
33+

‎core/src/main/scala_3/org/datatools/bigdatatypes/conversions/SqlInstanceConversion.scala

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package org.datatools.bigdatatypes.conversions
2+
3+
import org.datatools.bigdatatypes.basictypes.SqlType
4+
5+
/** Type class to convert instances into [[SqlType]]
6+
*
7+
* @tparam A is a Scala type
8+
*/
9+
trait SqlInstanceConversion[-A] {
10+
11+
/**
12+
* @param value an instance that implements SqlInstanceConversion
13+
* @return the [[SqlType]] representation of [[A]]
14+
*/
15+
def getType(value: A): SqlType
16+
}
17+
18+
object SqlInstanceConversion {
19+
20+
/** Summoner method
21+
*/
22+
def apply[A](using a: SqlInstanceConversion[A]): SqlInstanceConversion[A] = a
23+
24+
}
25+

0 commit comments

Comments
 (0)