diff --git a/Jenkinsfile b/Jenkinsfile index 99c6255..3bf37ff 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,7 @@ def isMasterBranch = 'master'.equals(env.BRANCH_NAME); pipeline { agent { node { - label 'alpine-jdk8-mvn-3.5' + label 'alpine-jdk8-mvn3.6' } } parameters { diff --git a/README.md b/README.md index e996f8d..20e0b6d 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # strongbox-db ## The goal of this project -This project has been created in order speed up the building of the OrientDB database snapshots during the initial [Strongbox](https://github.com/strongbox/strongbox) startup. Before this project was born, it took some time to do all of the database changes from the beginning of time up until the most recent version, so that the OrientDB database structure would be populated properly. +This project has been created in order speed up the building of the JanusGraph database snapshots during the initial [Strongbox](https://github.com/strongbox/strongbox) startup. Before this project was born, it took some time to do all of the database changes from the beginning of time up until the most recent version, so that the JanusGraph database structure would be populated properly. ## Architecture This application consists of the following modules: -* `strongbox-db-liquibase` which creates a `jar` file consiting of the liquibase changesets that construct the [Strongbox](https://github.com/strongbox/strongbox) OrientDB database schema +* `strongbox-db-schema` which creates a `jar` file consiting of the changesets that construct the [Strongbox](https://github.com/strongbox/strongbox) JanusGraph database schema * `strongbox-db-import` which creates a `zip` file with packed built database snapshot built from the liquibase changesets ## How is this project used in [strongbox](https://github.com/strongbox/strongbox) ? -[strongbox](https://github.com/strongbox/strongbox) uses this project submodules as required dependencies. During the startup of [Strongbox](https://github.com/strongbox/strongbox), the application detects if an OrientDB database already exists. If not, then the application uses an extracted empty OrientDB database snapshot from the `strongbox-db-import` artifact. If the OrientDB database already exists, then the application applies all missing liquibase changesets from the `strongbox-db-liquibase` artifact. +[strongbox](https://github.com/strongbox/strongbox) uses this project submodules as required dependencies. During the startup of [Strongbox](https://github.com/strongbox/strongbox), the application detects if a JanusGraph database already exists. If one doesn't exist, it uses an extracted empty JanusGraph database snapshot from the `strongbox-db-import` artifact; if a database exists, then the application applies all missing changesets from the `strongbox-db-schema` artifact. ## How to build this project ? To build the code, simply execute: @@ -17,5 +17,5 @@ To build the code, simply execute: ## What's the result of the build process ? This project produces the following artifacts: -* `strongbox-db-liquibase-${version}.jar` : Located in `strongbox-db-liquibase/target`, which contains current database liquibase changesets +* `strongbox-db-schema-${version}.jar` : Located in `strongbox-db-schema/target`, which contains current database changesets * `strongbox-db-import-${version}.zip` : Located in `strongbox-db-import/target`, which contains zipped fresh database snapshot built from the above changesets diff --git a/pom.xml b/pom.xml index 7cad30f..54af1ac 100644 --- a/pom.xml +++ b/pom.xml @@ -2,17 +2,17 @@ 4.0.0 - + org.carlspring.strongbox strongbox-parent - 1.0-SNAPSHOT + 1.0-PR-62-SNAPSHOT - + strongbox-db pom - 1.0-SNAPSHOT - + 1.0-PR-19-SNAPSHOT + @@ -48,7 +48,7 @@ - strongbox-db-liquibase + strongbox-db-schema strongbox-db-import strongbox-db-server diff --git a/strongbox-db-import/pom.xml b/strongbox-db-import/pom.xml index 4d89a31..3ee1c95 100644 --- a/strongbox-db-import/pom.xml +++ b/strongbox-db-import/pom.xml @@ -8,37 +8,28 @@ org.carlspring.strongbox strongbox-db - 1.0-SNAPSHOT + 1.0-PR-19-SNAPSHOT ../pom.xml strongbox-db-import 2019 - - ${project.build.directory}/db - - ${project.groupId} - strongbox-db-liquibase + strongbox-db-server ${project.version} ${project.groupId} - strongbox-db-server + strongbox-db-schema ${project.version} org.springframework.boot - spring-boot-starter-jdbc - - - - com.orientechnologies - orientdb-jdbc + spring-boot-starter @@ -46,49 +37,9 @@ commons-beanutils - - - org.apache.ant - ant - 1.10.5 - - - org.liquibase - liquibase-core - - - org.apache.ant - ant - - - - - org.unbroken-dome.liquibase-orientdb - liquibase-orientdb - - - com.mattbertolini - liquibase-slf4j - - - org.hibernate.validator - hibernate-validator - - - - - org.apache.commons - commons-lang3 - - - - src/main/resources - true - - org.springframework.boot @@ -101,7 +52,30 @@ run - -Xmx1024m + + -Xmx1024m + -Djdk.attach.allowAttachSelf=true + --add-exports java.base/jdk.internal.misc=ALL-UNNAMED + --add-exports java.base/jdk.internal.ref=ALL-UNNAMED + --add-exports java.base/sun.nio.ch=ALL-UNNAMED + --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED + --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED + --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED + --add-exports java.sql/java.sql=ALL-UNNAMED + + --add-opens java.base/java.lang.module=ALL-UNNAMED + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED + --add-opens java.base/jdk.internal.ref=ALL-UNNAMED + --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED + --add-opens java.base/jdk.internal.math=ALL-UNNAMED + --add-opens java.base/jdk.internal.module=ALL-UNNAMED + --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED + --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED + + + --strongbox.db.janus-graph.storage-root=${project.build.directory}/db + --strongbox.dbimport.root=${project.build.directory} + @@ -120,11 +94,107 @@ src/main/assembly/db-schema.xml - + - + + + + jdk-default + + !11 + + + + + org.springframework.boot + spring-boot-maven-plugin + + + strongbox-export-db + prepare-package + + run + + + + -Xmx1024m + + + --strongbox.db.janus-graph.storage-root=${project.build.directory}/db + --strongbox.dbimport.root=${project.build.directory} + + + + + + + + + + jdk11 + + 11 + + + + + org.springframework.boot + spring-boot-maven-plugin + + + strongbox-export-db + prepare-package + + run + + + + -Xmx1024m + -Djdk.attach.allowAttachSelf=true + --add-exports java.base/jdk.internal.misc=ALL-UNNAMED + --add-exports java.base/jdk.internal.ref=ALL-UNNAMED + --add-exports java.base/sun.nio.ch=ALL-UNNAMED + --add-exports + java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED + --add-exports + java.rmi/sun.rmi.registry=ALL-UNNAMED + --add-exports + java.rmi/sun.rmi.server=ALL-UNNAMED + --add-exports + java.sql/java.sql=ALL-UNNAMED + + --add-opens + java.base/java.lang.module=ALL-UNNAMED + --add-opens + java.base/jdk.internal.loader=ALL-UNNAMED + --add-opens + java.base/jdk.internal.ref=ALL-UNNAMED + --add-opens + java.base/jdk.internal.reflect=ALL-UNNAMED + --add-opens + java.base/jdk.internal.math=ALL-UNNAMED + --add-opens + java.base/jdk.internal.module=ALL-UNNAMED + --add-opens + java.base/jdk.internal.util.jar=ALL-UNNAMED + --add-opens + jdk.management/com.sun.management.internal=ALL-UNNAMED + + + --strongbox.db.janus-graph.storage-root=${project.build.directory}/db + --strongbox.dbimport.root=${project.build.directory} + + + + + + + + + + diff --git a/strongbox-db-import/src/main/assembly/db-schema.xml b/strongbox-db-import/src/main/assembly/db-schema.xml index 794716a..e2f794f 100644 --- a/strongbox-db-import/src/main/assembly/db-schema.xml +++ b/strongbox-db-import/src/main/assembly/db-schema.xml @@ -9,8 +9,8 @@ - ${project.build.directory}/db/strongbox - META-INF/resources/strongbox/db/strongbox + ${project.build.directory}/db + META-INF/org/carlsparing/strongbox/db \ No newline at end of file diff --git a/strongbox-db-import/src/main/java/org/carlspring/strongbox/db/orient/OrientDbConfiguration.java b/strongbox-db-import/src/main/java/org/carlspring/strongbox/db/orient/OrientDbConfiguration.java deleted file mode 100644 index bff6c37..0000000 --- a/strongbox-db-import/src/main/java/org/carlspring/strongbox/db/orient/OrientDbConfiguration.java +++ /dev/null @@ -1,106 +0,0 @@ -package org.carlspring.strongbox.db.orient; - -import java.lang.reflect.Field; -import java.util.Properties; - -import javax.sql.DataSource; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.autoconfigure.liquibase.LiquibaseDataSource; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.DependsOn; -import org.springframework.util.ReflectionUtils; -import org.strongbox.db.server.EmbeddedOrientDbServer; -import org.strongbox.db.server.OrientDbServer; -import org.strongbox.db.server.OrientDbServerConfiguration; -import org.strongbox.db.server.OrientDbServerProperties; -import org.strongbox.db.server.OrientDbStudioConfiguration; -import org.strongbox.db.server.OrientDbStudioProperties; - -import com.orientechnologies.orient.core.db.ODatabasePool; -import com.orientechnologies.orient.core.db.ODatabaseType; -import com.orientechnologies.orient.core.db.OrientDB; -import com.orientechnologies.orient.core.db.OrientDBConfig; -import com.orientechnologies.orient.jdbc.OrientDataSource; - -/** - * @author Przemyslaw Fusik - */ -@Configuration -class OrientDbConfiguration -{ - - private static final Logger logger = LoggerFactory.getLogger(OrientDbConfiguration.class); - - @Bean - OrientDbServer orientDbServer(OrientDbServerConfiguration serverProperties, OrientDbStudioConfiguration studioProperties) { - return new EmbeddedOrientDbServer(studioProperties, serverProperties); - } - - @Bean - @ConfigurationProperties(prefix = "strongbox.orientdb.studio") - OrientDbStudioConfiguration orientDbStudioProperties() { - return new OrientDbStudioProperties(); - } - - @Bean - @ConfigurationProperties(prefix = "strongbox.orientdb.server") - OrientDbServerConfiguration orientDbServerProperties() { - return new OrientDbServerProperties(); - } - - @Bean(destroyMethod = "close") - @DependsOn("orientDbServer") - OrientDB orientDB(OrientDbServerConfiguration orientDbServerProperties) - { - OrientDB orientDB = new OrientDB(StringUtils.substringBeforeLast(orientDbServerProperties.getUrl(), "/"), - orientDbServerProperties.getUsername(), - orientDbServerProperties.getPassword(), - OrientDBConfig.defaultConfig()); - String database = orientDbServerProperties.getDatabase(); - - if (!orientDB.exists(database)) - { - logger.info(String.format("Creating database [%s]...", database)); - - orientDB.create(database, ODatabaseType.PLOCAL); - } - else - { - logger.info("Re-using existing database " + database + "."); - } - return orientDB; - } - - @Bean - @LiquibaseDataSource - DataSource dataSource(ODatabasePool pool, - OrientDB orientDB) - { - OrientDataSource ds = new OrientDataSource(orientDB); - - ds.setInfo(new Properties()); - - // DEV note: - // NPEx hotfix for OrientDataSource.java:134 :) - Field poolField = ReflectionUtils.findField(OrientDataSource.class, "pool"); - ReflectionUtils.makeAccessible(poolField); - ReflectionUtils.setField(poolField, ds, pool); - - return ds; - } - - @Bean(destroyMethod = "close") - ODatabasePool databasePool(OrientDB orientDB, OrientDbServerConfiguration orientDbServerProperties) - { - return new ODatabasePool(orientDB, - orientDbServerProperties.getDatabase(), - orientDbServerProperties.getUsername(), - orientDbServerProperties.getPassword()); - } - -} diff --git a/strongbox-db-import/src/main/java/org/carlspring/strongbox/db/Application.java b/strongbox-db-import/src/main/java/org/carlspring/strongbox/dbimport/Application.java similarity index 61% rename from strongbox-db-import/src/main/java/org/carlspring/strongbox/db/Application.java rename to strongbox-db-import/src/main/java/org/carlspring/strongbox/dbimport/Application.java index 2bdd916..6ea2137 100644 --- a/strongbox-db-import/src/main/java/org/carlspring/strongbox/db/Application.java +++ b/strongbox-db-import/src/main/java/org/carlspring/strongbox/dbimport/Application.java @@ -1,19 +1,23 @@ -package org.carlspring.strongbox.db; +package org.carlspring.strongbox.dbimport; +import org.janusgraph.core.JanusGraph; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.validation.ValidationAutoConfiguration; import org.springframework.context.ConfigurableApplicationContext; /** * @author Przemyslaw Fusik */ -@SpringBootApplication +@SpringBootApplication(exclude = ValidationAutoConfiguration.class) public class Application { public static void main(String[] args) { ConfigurableApplicationContext ctx = SpringApplication.run(Application.class, args); + ctx.getBean(JanusGraph.class); + SpringApplication.exit(ctx, () -> 0); } } diff --git a/strongbox-db-import/src/main/java/org/carlspring/strongbox/dbimport/EmbeddedDbServerConfiguration.java b/strongbox-db-import/src/main/java/org/carlspring/strongbox/dbimport/EmbeddedDbServerConfiguration.java new file mode 100644 index 0000000..1cb36d4 --- /dev/null +++ b/strongbox-db-import/src/main/java/org/carlspring/strongbox/dbimport/EmbeddedDbServerConfiguration.java @@ -0,0 +1,67 @@ +package org.carlspring.strongbox.dbimport; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.carlspring.strongbox.db.schema.StrongboxSchema; +import org.janusgraph.core.JanusGraph; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationPropertiesScan; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.strongbox.db.server.CassandraEmbeddedConfiguration; +import org.strongbox.db.server.CassandraEmbeddedProperties; +import org.strongbox.db.server.JanusGraphConfiguration; +import org.strongbox.db.server.JanusGraphProperties; +import org.strongbox.db.server.JanusGraphServer; +import org.strongbox.db.server.JanusGraphWithEmbeddedCassandra; +import org.strongbox.util.ConfigurationUtils; + +/** + * @author Przemyslaw Fusik + * @author sbespalov + */ +@Configuration +@ConfigurationPropertiesScan +class EmbeddedDbServerConfiguration +{ + + @Bean + JanusGraphServer embeddedDbServer(CassandraEmbeddedConfiguration cassandraConfiguration, + JanusGraphConfiguration janusGraphConfiguration) + throws IOException + { + return new JanusGraphWithEmbeddedCassandra(cassandraConfiguration, janusGraphConfiguration, () -> null); + } + + @Bean + JanusGraph janusGraph(JanusGraphServer server) + throws Exception + { + return new StrongboxSchema().createSchema(server.getJanusGraph()); + } + + @Bean + JanusGraphProperties dbImportJanusGraphProperties(@Value("${strongbox.dbimport.root}") String dbImportRoot) + throws IOException + { + ConfigurationUtils.extractConfigurationFile(dbImportRoot, "janusgraph-cassandra.properties"); + Path configFilePath = Paths.get(dbImportRoot).resolve("etc").resolve("conf").resolve("janusgraph-cassandra.properties"); + + return new JanusGraphProperties( String.format("file:%s", configFilePath.toAbsolutePath().toString())); + } + + @Bean + CassandraEmbeddedProperties dbImportCassandraEmbeddedProperties(@Value("${strongbox.dbimport.root}") String dbImportRoot) + throws IOException + { + ConfigurationUtils.extractConfigurationFile(dbImportRoot, "cassandra.yaml"); + Path rootPath = Paths.get(dbImportRoot).toAbsolutePath(); + Path configFilePath = rootPath.resolve("etc").resolve("conf").resolve("cassandra.yaml"); + Path storageRootPath = rootPath.resolve("db"); + + return new CassandraEmbeddedProperties(storageRootPath.toString(), String.format("file:%s", configFilePath.toString())); + } + +} diff --git a/strongbox-db-import/src/main/resources/application.yml b/strongbox-db-import/src/main/resources/application.yml index 620fd9d..2a5aef6 100644 --- a/strongbox-db-import/src/main/resources/application.yml +++ b/strongbox-db-import/src/main/resources/application.yml @@ -1,23 +1,6 @@ spring: main: web-application-type: NONE - liquibase: - change-log: classpath:/db/changelog/db.changelog-master.xml strongbox: - orientdb: - server: - protocol: remote - host: 127.0.0.1 - port: "2025" - database: strongbox - username: admin - password: password - path: @strongbox.orientdb.path@ - studio: - ipAddress: 127.0.0.1 - port: 2480 - enabled: true - path: @strongbox.orientdb.path@ -logging: - level: - liquibase: INFO \ No newline at end of file + dbimport: + root: ./target diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/db.changelog-master.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/db.changelog-master.xml deleted file mode 100644 index 6523632..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/db.changelog-master.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/db.changelog-1.0.0.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/db.changelog-1.0.0.xml deleted file mode 100644 index 5811154..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/db.changelog-1.0.0.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.10__ArtifactEntry.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.10__ArtifactEntry.xml deleted file mode 100644 index 3a29163..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.10__ArtifactEntry.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.11__ArtifactEntry_cleanup.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.11__ArtifactEntry_cleanup.xml deleted file mode 100644 index e08ec53..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.11__ArtifactEntry_cleanup.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - artifactCoordinates IS NULL - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.12__Configuration_cleanup.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.12__Configuration_cleanup.xml deleted file mode 100644 index 237a238..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.12__Configuration_cleanup.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.13__ArtifactEntry_path.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.13__ArtifactEntry_path.xml deleted file mode 100644 index 374ec08..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.13__ArtifactEntry_path.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.14__User_cleanup.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.14__User_cleanup.xml deleted file mode 100644 index 05e66fb..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.14__User_cleanup.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.15__Authorization_cleanup.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.15__Authorization_cleanup.xml deleted file mode 100644 index 656fd39..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.15__Authorization_cleanup.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.16__CronTaskConfiguration_cleanup.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.16__CronTaskConfiguration_cleanup.xml deleted file mode 100644 index 717a529..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.16__CronTaskConfiguration_cleanup.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.17__ArtifactArchiveListing.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.17__ArtifactArchiveListing.xml deleted file mode 100644 index 9dab10f..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.17__ArtifactArchiveListing.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.18__ArtifactGroup.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.18__ArtifactGroup.xml deleted file mode 100644 index 76ec48e..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.18__ArtifactGroup.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.19__ArtifactEntry.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.19__ArtifactEntry.xml deleted file mode 100644 index c8291f1..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.19__ArtifactEntry.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.1__GenericEntity.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.1__GenericEntity.xml deleted file mode 100644 index 9c5f443..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.1__GenericEntity.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.20__User.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.20__User.xml deleted file mode 100644 index 3a15116..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.20__User.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.21__ArtifactEntry.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.21__ArtifactEntry.xml deleted file mode 100644 index fde581f..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.21__ArtifactEntry.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.22__UserEntry.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.22__UserEntry.xml deleted file mode 100644 index 4329f5c..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.22__UserEntry.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.2__ArtifactEntry.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.2__ArtifactEntry.xml deleted file mode 100644 index 14aaf94..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.2__ArtifactEntry.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.3__AuthorizationConfig.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.3__AuthorizationConfig.xml deleted file mode 100644 index 97f69be..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.3__AuthorizationConfig.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.4__BinaryConfiguration.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.4__BinaryConfiguration.xml deleted file mode 100644 index 7351386..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.4__BinaryConfiguration.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.5__CronTaskConfiguration.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.5__CronTaskConfiguration.xml deleted file mode 100644 index e6ed941..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.5__CronTaskConfiguration.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.6__User.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.6__User.xml deleted file mode 100644 index fff2dd9..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.6__User.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.7__Configuration.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.7__Configuration.xml deleted file mode 100644 index bffbc8a..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.7__Configuration.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.8__AuthorizationConfig.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.8__AuthorizationConfig.xml deleted file mode 100644 index 43b3b9b..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.8__AuthorizationConfig.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.9__Repository.xml b/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.9__Repository.xml deleted file mode 100644 index 71e2de4..0000000 --- a/strongbox-db-liquibase/src/main/resources/db/changelog/v1.0.0/v1.0.0.9__Repository.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/strongbox-db-liquibase/pom.xml b/strongbox-db-schema/pom.xml similarity index 60% rename from strongbox-db-liquibase/pom.xml rename to strongbox-db-schema/pom.xml index 8a79119..447701e 100644 --- a/strongbox-db-liquibase/pom.xml +++ b/strongbox-db-schema/pom.xml @@ -8,18 +8,23 @@ org.carlspring.strongbox strongbox-db - 1.0-SNAPSHOT + 1.0-PR-19-SNAPSHOT ../pom.xml - strongbox-db-liquibase + strongbox-db-schema 2019 - com.google.guava - guava + ${project.groupId} + strongbox-db-server + ${project.version} + + + org.janusgraph + janusgraph-core diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/Edges.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/Edges.java new file mode 100644 index 0000000..e61bb1b --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/Edges.java @@ -0,0 +1,13 @@ +package org.carlspring.strongbox.db.schema; + +public interface Edges +{ + + String ARTIFACT_HAS_ARTIFACT_COORDINATES = "ArtifactHasArtifactCoordinates"; + String ARTIFACT_HAS_TAGS = "ArtifactHasTags"; + String ARTIFACT_GROUP_HAS_ARTIFACTS = "ArtifactGroupHasArtifacts"; + String ARTIFACT_GROUP_HAS_TAGGED_ARTIFACTS = "ArtifactGroupHasTaggedArtifacts"; + String EXTENDS = "Extends"; + String USER_HAS_SECURITY_ROLES = "UserHasSecurityRoles"; + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/Properties.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/Properties.java new file mode 100644 index 0000000..7ce4739 --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/Properties.java @@ -0,0 +1,48 @@ +package org.carlspring.strongbox.db.schema; + +/** + * @author ankit.tomar + */ +public interface Properties +{ + + String UUID = "uuid"; + String STORAGE_ID = "storageId"; + String REPOSITORY_ID = "repositoryId"; + String NAME = "name"; + String LAST_UPDATED = "lastUpdated"; + String SIZE_IN_BYTES = "sizeInBytes"; + String LAST_USED = "lastUsed"; + String CREATED = "created"; + String DOWNLOAD_COUNT = "downloadCount"; + String CHECKSUMS = "checksums"; + String ARTIFACT_FILE_EXISTS = "artifactFileExists"; + String VERSION = "version"; + String USERNAME = "username"; + String PASSWORD = "password"; + String ENABLED = "enabled"; + String SECURITY_TOKEN_KEY = "securityTokenKey"; + String SOURCE_ID = "sourceId"; + String FILE_NAMES = "filenames"; + String TAG_NAME = "tagName"; + String COORDINATES_EXTENSION = "coordinates.extension"; + String COORDINATES_NAME = "coordinates.name"; + String COORDINATES_GROUP_ID = "coordinates.groupId"; + String COORDINATES_ARTIFACT_ID = "coordinates.artifactId"; + String COORDINATES_CLASSIFIER = "coordinates.classifier"; + String COORDINATES_SCOPE = "coordinates.scope"; + String COORDINATES_ID = "coordinates.id"; + String COORDINATES_FILENAME = "coordinates.filename"; + String COORDINATES_BUILD = "coordinates.build"; + String COORDINATES_ABI = "coordinates.abi"; + String COORDINATES_PLATFORM = "coordinates.platform"; + String COORDINATES_PACKAGING = "coordinates.packaging"; + String COORDINATES_DISTRIBUTION = "coordinates.distribution"; + String COORDINATES_PATH = "coordinates.path"; + String COORDINATES_BASE_NAME = "coordinates.base_name"; + String COORDINATES_RELEASE = "coordinates.release"; + String COORDINATES_ARCHITECTURE = "coordinates.architecture"; + String COORDINATES_PACKAGE_TYPE = "coordinates.package_type"; + String COORDINATES_LANGUAGE_IMPLEMENTATION_VERSION = "coordinates.languageImplementationVersion"; + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/SchemaTemplate.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/SchemaTemplate.java new file mode 100644 index 0000000..ae96f08 --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/SchemaTemplate.java @@ -0,0 +1,95 @@ +package org.carlspring.strongbox.db.schema; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.carlspring.strongbox.db.schema.migration.Changeset; +import org.janusgraph.core.JanusGraph; +import org.janusgraph.core.schema.JanusGraphManagement; +import org.janusgraph.graphdb.database.management.ManagementSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author sbespalov + */ +public class SchemaTemplate +{ + private static final Logger logger = LoggerFactory.getLogger(SchemaTemplate.class); + + private final JanusGraph jg; + + public SchemaTemplate(JanusGraph jg) + { + this.jg = jg; + } + + public void applyChangeset(Changeset changeset) + { + logger.info(String.format("Apply changeset [%s]-[%s].", changeset.getVersion(), changeset.getName())); + + JanusGraphManagement jgm = jg.openManagement(); + Set compositeIndexes; + Map relationIndexes; + try + { + changeset.applySchemaChanges(jgm); + + logger.info(String.format("Create indexes [%s]-[%s].", changeset.getVersion(), changeset.getName())); + compositeIndexes = changeset.createVertexIndexes(jgm); + relationIndexes = changeset.createRelationIndexes(jgm); + + jgm.commit(); + } + catch (Exception e) + { + jgm.rollback(); + throw new RuntimeException(String.format("Failed to apply changeset [%s]-[%s].", + changeset.getVersion(), + changeset.getName()), + e); + } + + GraphTraversalSource g = jg.traversal(); + try + { + changeset.applyGraphChanges(g); + g.tx().commit(); + } + catch (Exception e) + { + g.tx().rollback(); + throw new RuntimeException(String.format("Failed to apply changeset [%s]-[%s].", + changeset.getVersion(), + changeset.getName()), + e); + } + + // Waiting index status + try + { + for (String janusGraphIndex : compositeIndexes) + { + logger.info(String.format("Wait index [%s] to be registered.", janusGraphIndex)); + ManagementSystem.awaitGraphIndexStatus(jg, janusGraphIndex).call(); + } + + for (Entry relationIndex : relationIndexes.entrySet()) + { + logger.info(String.format("Wait index [%s] to be registered.", relationIndex.getKey())); + ManagementSystem.awaitRelationIndexStatus(jg, relationIndex.getKey(), relationIndex.getValue()).call(); + } + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new RuntimeException(String.format("Failed to enable indexes for [%s]-[%s].", + changeset.getVersion(), + changeset.getName()), + e); + } + } + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/StrongboxSchema.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/StrongboxSchema.java new file mode 100644 index 0000000..5354500 --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/StrongboxSchema.java @@ -0,0 +1,182 @@ +package org.carlspring.strongbox.db.schema; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.carlspring.strongbox.db.schema.changelog.Changelog; +import org.carlspring.strongbox.db.schema.migration.Changeset; +import org.carlspring.strongbox.db.schema.migration.ChangesetVersion; +import org.carlspring.strongbox.db.schema.migration.JanusgraphChangelogStorage; +import org.janusgraph.core.JanusGraph; +import org.janusgraph.core.RelationType; +import org.janusgraph.core.schema.JanusGraphIndex; +import org.janusgraph.core.schema.JanusGraphManagement; +import org.janusgraph.core.schema.JanusGraphManagement.IndexJobFuture; +import org.janusgraph.core.schema.SchemaAction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author sbespalov + */ +public class StrongboxSchema +{ + + private static final Logger logger = LoggerFactory.getLogger(StrongboxSchema.class); + + private final SortedSet changeSets; + + public StrongboxSchema() + { + this(Changelog.changeSets); + } + + public StrongboxSchema(SortedSet changeSets) + { + this.changeSets = changeSets; + } + + public JanusGraph createSchema(JanusGraph jg) + throws InterruptedException + { + logger.info("Apply schema changes."); + applyChangesets(jg); + enableIndexes(jg); + printSchema(jg); + logger.info("Schema changes applied."); + + return jg; + } + + private void printSchema(JanusGraph jg) + { + JanusGraphManagement jgm = jg.openManagement(); + try + { + logger.info(String.format("Schema: %n%s", jgm.printSchema())); + } + finally + { + jgm.rollback(); + } + } + + private void enableIndexes(JanusGraph jg) + { + JanusGraphManagement jgm = jg.openManagement(); + try + { + Set vertexIndexes = fetchVertexIndexes(jgm); + Map relationIndexes = fetchRelationIndexes(jgm); + + enableVertexIndexes(jgm, vertexIndexes); + enableRelationIndexes(jgm, relationIndexes); + + jgm.commit(); + } + catch (Exception e) + { + logger.error("Failed to enable indexes.", e); + jgm.rollback(); + throw new RuntimeException(e); + } + } + + private void applyChangesets(JanusGraph jg) + { + JanusgraphChangelogStorage changelogStorage = new JanusgraphChangelogStorage(jg); + changelogStorage.init(); + + ChangesetVersion schemaVersion = changelogStorage.getSchemaVersion(); + logger.info("Current schema version {}", schemaVersion.getVersion()); + SchemaTemplate schemaTemplate = new SchemaTemplate(jg); + for (Changeset changeset : changeSets) + { + if (schemaVersion.compareTo(changeset) >= 0) + { + continue; + } + + Changeset storedChangeset = changelogStorage.prepare(changeset); + try + { + schemaTemplate.applyChangeset(storedChangeset); + } + catch (Exception e) + { + logger.error(String.format("Failed to apply changeset [%s]-[%s]", changeset.getVersion(), changeset.getName()), e); + throw new RuntimeException(e); + } + } + } + + private Map fetchRelationIndexes(JanusGraphManagement jgm) + { + Iterable relationTypes = jgm.getRelationTypes(RelationType.class); + + return StreamSupport.stream(relationTypes.spliterator(), false) + .flatMap(r -> StreamSupport.stream(jgm.getRelationIndexes(r).spliterator(), false)) + .collect(Collectors.toMap(e -> e.name(), e -> e.getType().name())); + } + + private Set fetchVertexIndexes(JanusGraphManagement jgm) + { + Iterable vertexIndexes = jgm.getGraphIndexes(Vertex.class); + return StreamSupport.stream(vertexIndexes.spliterator(), false) + .map(JanusGraphIndex::name) + .collect(Collectors.toSet()); + } + + protected void enableVertexIndexes(JanusGraphManagement jgm, + Set indexes) + throws InterruptedException, + ExecutionException + { + for (String janusGraphIndex : indexes) + { + logger.info(String.format("Enabling index [%s].", janusGraphIndex)); + Optional.ofNullable(jgm.updateIndex(jgm.getGraphIndex(janusGraphIndex), SchemaAction.ENABLE_INDEX)) + .ifPresent(this::waitIndexUpdate); + } + } + + private void waitIndexUpdate(IndexJobFuture future) + { + try + { + future.get(); + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + catch (ExecutionException e) + { + throw new RuntimeException(e); + } + } + + protected void enableRelationIndexes(JanusGraphManagement jgm, + Map indexes) + throws InterruptedException, + ExecutionException + { + Set> entrySet = indexes.entrySet(); + for (Entry e : entrySet) + { + logger.info(String.format("Enabling index [%s].", e.getKey())); + Optional.ofNullable(jgm.updateIndex(jgm.getRelationIndex(jgm.getRelationType(e.getValue()), e.getKey()), + SchemaAction.ENABLE_INDEX)) + .ifPresent(this::waitIndexUpdate); + } + } + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/Vertices.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/Vertices.java new file mode 100644 index 0000000..bd5000e --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/Vertices.java @@ -0,0 +1,18 @@ +package org.carlspring.strongbox.db.schema; + +public interface Vertices +{ + + String ARTIFACT = "Artifact"; + String GENERIC_ARTIFACT_COORDINATES = "GenericArtifactCoordinates"; + String RAW_ARTIFACT_COORDINATES = "RawArtifactCoordinates"; + String MAVEN_ARTIFACT_COORDINATES = "MavenArtifactCoordinates"; + String NPM_ARTIFACT_COORDINATES = "NpmArtifactCoordinates"; + String NUGET_ARTIFACT_COORDINATES = "NugetArtifactCoordinates"; + String PYPI_ARTIFACT_COORDINATES = "PypiArtifactCoordinates"; + String RPM_ARTIFACT_COORDINATES = "RpmArtifactCoordinates"; + String ARTIFACT_TAG = "ArtifactTag"; + String ARTIFACT_ID_GROUP = "ArtifactIdGroup"; + String USER = "User"; + String SECURITY_ROLE = "SecurityRole"; +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/changelog/Changelog.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/changelog/Changelog.java new file mode 100644 index 0000000..580568d --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/changelog/Changelog.java @@ -0,0 +1,18 @@ +package org.carlspring.strongbox.db.schema.changelog; + +import java.util.Arrays; +import java.util.Collections; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.carlspring.strongbox.db.schema.migration.Changeset; + +public interface Changelog +{ + + SortedSet changeSets = Collections.unmodifiableSortedSet(new TreeSet<>( + Arrays.asList(new Changeset[] { new V1_0_0_1_InitialSchema(), + new V1_0_0_2_InitialIndexes() + }))); + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/changelog/V1_0_0_1_InitialSchema.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/changelog/V1_0_0_1_InitialSchema.java new file mode 100644 index 0000000..8a4a28e --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/changelog/V1_0_0_1_InitialSchema.java @@ -0,0 +1,360 @@ +package org.carlspring.strongbox.db.schema.changelog; + +import static org.carlspring.strongbox.db.schema.Edges.ARTIFACT_GROUP_HAS_ARTIFACTS; +import static org.carlspring.strongbox.db.schema.Edges.ARTIFACT_GROUP_HAS_TAGGED_ARTIFACTS; +import static org.carlspring.strongbox.db.schema.Edges.ARTIFACT_HAS_ARTIFACT_COORDINATES; +import static org.carlspring.strongbox.db.schema.Edges.ARTIFACT_HAS_TAGS; +import static org.carlspring.strongbox.db.schema.Edges.EXTENDS; +import static org.carlspring.strongbox.db.schema.Edges.USER_HAS_SECURITY_ROLES; +import static org.carlspring.strongbox.db.schema.Properties.ARTIFACT_FILE_EXISTS; +import static org.carlspring.strongbox.db.schema.Properties.CHECKSUMS; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_ABI; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_ARCHITECTURE; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_ARTIFACT_ID; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_BASE_NAME; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_BUILD; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_CLASSIFIER; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_DISTRIBUTION; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_EXTENSION; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_FILENAME; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_GROUP_ID; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_ID; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_LANGUAGE_IMPLEMENTATION_VERSION; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_NAME; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_PACKAGE_TYPE; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_PACKAGING; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_PATH; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_PLATFORM; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_RELEASE; +import static org.carlspring.strongbox.db.schema.Properties.COORDINATES_SCOPE; +import static org.carlspring.strongbox.db.schema.Properties.CREATED; +import static org.carlspring.strongbox.db.schema.Properties.DOWNLOAD_COUNT; +import static org.carlspring.strongbox.db.schema.Properties.ENABLED; +import static org.carlspring.strongbox.db.schema.Properties.FILE_NAMES; +import static org.carlspring.strongbox.db.schema.Properties.LAST_UPDATED; +import static org.carlspring.strongbox.db.schema.Properties.LAST_USED; +import static org.carlspring.strongbox.db.schema.Properties.NAME; +import static org.carlspring.strongbox.db.schema.Properties.PASSWORD; +import static org.carlspring.strongbox.db.schema.Properties.REPOSITORY_ID; +import static org.carlspring.strongbox.db.schema.Properties.SECURITY_TOKEN_KEY; +import static org.carlspring.strongbox.db.schema.Properties.SIZE_IN_BYTES; +import static org.carlspring.strongbox.db.schema.Properties.SOURCE_ID; +import static org.carlspring.strongbox.db.schema.Properties.STORAGE_ID; +import static org.carlspring.strongbox.db.schema.Properties.TAG_NAME; +import static org.carlspring.strongbox.db.schema.Properties.UUID; +import static org.carlspring.strongbox.db.schema.Properties.VERSION; +import static org.carlspring.strongbox.db.schema.Vertices.ARTIFACT; +import static org.carlspring.strongbox.db.schema.Vertices.ARTIFACT_ID_GROUP; +import static org.carlspring.strongbox.db.schema.Vertices.ARTIFACT_TAG; +import static org.carlspring.strongbox.db.schema.Vertices.GENERIC_ARTIFACT_COORDINATES; +import static org.carlspring.strongbox.db.schema.Vertices.MAVEN_ARTIFACT_COORDINATES; +import static org.carlspring.strongbox.db.schema.Vertices.NPM_ARTIFACT_COORDINATES; +import static org.carlspring.strongbox.db.schema.Vertices.NUGET_ARTIFACT_COORDINATES; +import static org.carlspring.strongbox.db.schema.Vertices.PYPI_ARTIFACT_COORDINATES; +import static org.carlspring.strongbox.db.schema.Vertices.RAW_ARTIFACT_COORDINATES; +import static org.carlspring.strongbox.db.schema.Vertices.SECURITY_ROLE; +import static org.carlspring.strongbox.db.schema.Vertices.USER; +import static org.carlspring.strongbox.db.schema.util.SchemaUtils.addEdgePropertyConstraints; +import static org.carlspring.strongbox.db.schema.util.SchemaUtils.addVertexPropertyConstraints; +import static org.janusgraph.core.Multiplicity.MANY2ONE; +import static org.janusgraph.core.Multiplicity.MULTI; +import static org.janusgraph.core.Multiplicity.ONE2MANY; +import static org.janusgraph.core.Multiplicity.ONE2ONE; + +import java.util.Optional; + +import org.carlspring.strongbox.db.schema.migration.Changeset; +import org.janusgraph.core.Cardinality; +import org.janusgraph.core.schema.ConsistencyModifier; +import org.janusgraph.core.schema.JanusGraphManagement; + +public class V1_0_0_1_InitialSchema implements Changeset +{ + + @Override + public String getVersion() + { + return "1.0.0.1"; + } + + @Override + public String getAuthor() + { + return "serge.bespalov@gmail.com"; + } + + @Override + public void applySchemaChanges(JanusGraphManagement jgm) + { + // Properties + makeProperties(jgm); + + // Vertices + makeVertices(jgm); + + // Edges + makeEdges(jgm); + + // Add property constraints + makeVertexConstraints(jgm); + + // Add connection constraints + makeEdgeConstraints(jgm); + } + + private void makeProperties(JanusGraphManagement jgm) + { + Optional.of(jgm.makePropertyKey(UUID).dataType(String.class).make()) + .ifPresent(p -> jgm.setConsistency(p, ConsistencyModifier.LOCK)); + jgm.makePropertyKey(STORAGE_ID).dataType(String.class).make(); + jgm.makePropertyKey(REPOSITORY_ID).dataType(String.class).make(); + jgm.makePropertyKey(NAME).dataType(String.class).make(); + jgm.makePropertyKey(LAST_UPDATED).dataType(Long.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(TAG_NAME).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + + // Artifact + jgm.makePropertyKey(SIZE_IN_BYTES).dataType(Long.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(LAST_USED).dataType(Long.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(CREATED).dataType(Long.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(DOWNLOAD_COUNT).dataType(Integer.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(FILE_NAMES).dataType(String.class).cardinality(Cardinality.SET).make(); + jgm.makePropertyKey(CHECKSUMS).dataType(String.class).cardinality(Cardinality.SET).make(); + + // RemoteArtifact + jgm.makePropertyKey(ARTIFACT_FILE_EXISTS).dataType(Boolean.class).cardinality(Cardinality.SINGLE).make(); + + // Common coordinates + jgm.makePropertyKey(VERSION).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_EXTENSION).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_NAME).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + + // Maven + jgm.makePropertyKey(COORDINATES_GROUP_ID).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_ARTIFACT_ID).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_CLASSIFIER).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + + // Npm + jgm.makePropertyKey(COORDINATES_SCOPE).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + + // Nuget + jgm.makePropertyKey(COORDINATES_ID).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + + // P2 + jgm.makePropertyKey(COORDINATES_FILENAME).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + + // Pypi + jgm.makePropertyKey(COORDINATES_BUILD).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_LANGUAGE_IMPLEMENTATION_VERSION).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_ABI).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_PLATFORM).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_PACKAGING).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_DISTRIBUTION).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + + // Raw + jgm.makePropertyKey(COORDINATES_PATH).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + + // Rpm + jgm.makePropertyKey(COORDINATES_BASE_NAME).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_RELEASE).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_ARCHITECTURE).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(COORDINATES_PACKAGE_TYPE).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + + // User + jgm.makePropertyKey(PASSWORD).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(ENABLED).dataType(Boolean.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(SECURITY_TOKEN_KEY).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + jgm.makePropertyKey(SOURCE_ID).dataType(String.class).cardinality(Cardinality.SINGLE).make(); + } + + private void makeVertices(JanusGraphManagement jgm) + { + jgm.makeVertexLabel(ARTIFACT).make(); + jgm.makeVertexLabel(GENERIC_ARTIFACT_COORDINATES).make(); + jgm.makeVertexLabel(RAW_ARTIFACT_COORDINATES).make(); + jgm.makeVertexLabel(MAVEN_ARTIFACT_COORDINATES).make(); + jgm.makeVertexLabel(NPM_ARTIFACT_COORDINATES).make(); + jgm.makeVertexLabel(NUGET_ARTIFACT_COORDINATES).make(); + jgm.makeVertexLabel(PYPI_ARTIFACT_COORDINATES).make(); + jgm.makeVertexLabel(ARTIFACT_TAG).make(); + jgm.makeVertexLabel(ARTIFACT_ID_GROUP).make(); + jgm.makeVertexLabel(USER).make(); + jgm.makeVertexLabel(SECURITY_ROLE).make(); + } + + private void makeEdges(JanusGraphManagement jgm) + { + jgm.makeEdgeLabel(ARTIFACT_HAS_ARTIFACT_COORDINATES).multiplicity(MANY2ONE).make(); + jgm.makeEdgeLabel(ARTIFACT_HAS_TAGS).multiplicity(MULTI).make(); + jgm.makeEdgeLabel(EXTENDS).multiplicity(ONE2ONE).make(); + jgm.makeEdgeLabel(ARTIFACT_GROUP_HAS_ARTIFACTS).multiplicity(ONE2MANY).make(); + jgm.makeEdgeLabel(ARTIFACT_GROUP_HAS_TAGGED_ARTIFACTS).multiplicity(MULTI).make(); + jgm.makeEdgeLabel(USER_HAS_SECURITY_ROLES).multiplicity(MULTI).make(); + } + + private void makeVertexConstraints(JanusGraphManagement jgm) + { + // Vertex Property Constraints + addVertexPropertyConstraints(jgm, + ARTIFACT, + UUID, + STORAGE_ID, + REPOSITORY_ID, + CREATED, + LAST_UPDATED, + LAST_USED, + SIZE_IN_BYTES, + DOWNLOAD_COUNT, + FILE_NAMES, + CHECKSUMS, + ARTIFACT_FILE_EXISTS); + + addVertexPropertyConstraints(jgm, + GENERIC_ARTIFACT_COORDINATES, + UUID, + VERSION, + COORDINATES_ID, + COORDINATES_EXTENSION, + COORDINATES_NAME, + COORDINATES_PATH, + COORDINATES_SCOPE, + COORDINATES_GROUP_ID, + COORDINATES_ARTIFACT_ID, + COORDINATES_CLASSIFIER, + COORDINATES_DISTRIBUTION, + COORDINATES_BUILD, + COORDINATES_ABI, + COORDINATES_PLATFORM, + COORDINATES_PACKAGING, + COORDINATES_LANGUAGE_IMPLEMENTATION_VERSION, + CREATED); + + addVertexPropertyConstraints(jgm, + RAW_ARTIFACT_COORDINATES, + UUID, + VERSION, + COORDINATES_EXTENSION, + COORDINATES_NAME, + COORDINATES_PATH, + CREATED); + + addVertexPropertyConstraints(jgm, + MAVEN_ARTIFACT_COORDINATES, + UUID, + VERSION, + COORDINATES_EXTENSION, + COORDINATES_NAME, + COORDINATES_GROUP_ID, + COORDINATES_ARTIFACT_ID, + COORDINATES_CLASSIFIER, + CREATED); + + addVertexPropertyConstraints(jgm, + NPM_ARTIFACT_COORDINATES, + UUID, + VERSION, + COORDINATES_EXTENSION, + COORDINATES_NAME, + COORDINATES_SCOPE, + CREATED); + + addVertexPropertyConstraints(jgm, + NUGET_ARTIFACT_COORDINATES, + UUID, + VERSION, + COORDINATES_EXTENSION, + COORDINATES_NAME, + COORDINATES_ID, + CREATED); + + addVertexPropertyConstraints(jgm, + PYPI_ARTIFACT_COORDINATES, + UUID, + VERSION, + COORDINATES_EXTENSION, + COORDINATES_NAME, + COORDINATES_BUILD, + COORDINATES_ABI, + COORDINATES_PLATFORM, + COORDINATES_PACKAGING, + COORDINATES_DISTRIBUTION, + COORDINATES_LANGUAGE_IMPLEMENTATION_VERSION, + CREATED); + + addVertexPropertyConstraints(jgm, + ARTIFACT_TAG, + UUID, + CREATED); + + addVertexPropertyConstraints(jgm, + ARTIFACT_ID_GROUP, + UUID, + STORAGE_ID, + REPOSITORY_ID, + NAME, + CREATED); + + addVertexPropertyConstraints(jgm, + USER, + UUID, + PASSWORD, + ENABLED, + SECURITY_TOKEN_KEY, + SOURCE_ID, + CREATED, + LAST_UPDATED); + + addVertexPropertyConstraints(jgm, + SECURITY_ROLE, + UUID, + CREATED); + + addEdgePropertyConstraints(jgm, + ARTIFACT_GROUP_HAS_TAGGED_ARTIFACTS, + TAG_NAME); + } + + private void makeEdgeConstraints(JanusGraphManagement jgm) + { + jgm.addConnection(jgm.getEdgeLabel(ARTIFACT_HAS_ARTIFACT_COORDINATES), + jgm.getVertexLabel(ARTIFACT), + jgm.getVertexLabel(GENERIC_ARTIFACT_COORDINATES)); + + jgm.addConnection(jgm.getEdgeLabel(ARTIFACT_HAS_TAGS), + jgm.getVertexLabel(ARTIFACT), + jgm.getVertexLabel(ARTIFACT_TAG)); + + jgm.addConnection(jgm.getEdgeLabel(ARTIFACT_GROUP_HAS_ARTIFACTS), + jgm.getVertexLabel(ARTIFACT_ID_GROUP), + jgm.getVertexLabel(ARTIFACT)); + + jgm.addConnection(jgm.getEdgeLabel(ARTIFACT_GROUP_HAS_TAGGED_ARTIFACTS), + jgm.getVertexLabel(ARTIFACT_ID_GROUP), + jgm.getVertexLabel(ARTIFACT)); + + jgm.addConnection(jgm.getEdgeLabel(EXTENDS), + jgm.getVertexLabel(RAW_ARTIFACT_COORDINATES), + jgm.getVertexLabel(GENERIC_ARTIFACT_COORDINATES)); + + jgm.addConnection(jgm.getEdgeLabel(EXTENDS), + jgm.getVertexLabel(NUGET_ARTIFACT_COORDINATES), + jgm.getVertexLabel(GENERIC_ARTIFACT_COORDINATES)); + + jgm.addConnection(jgm.getEdgeLabel(EXTENDS), + jgm.getVertexLabel(NPM_ARTIFACT_COORDINATES), + jgm.getVertexLabel(GENERIC_ARTIFACT_COORDINATES)); + + jgm.addConnection(jgm.getEdgeLabel(EXTENDS), + jgm.getVertexLabel(MAVEN_ARTIFACT_COORDINATES), + jgm.getVertexLabel(GENERIC_ARTIFACT_COORDINATES)); + + jgm.addConnection(jgm.getEdgeLabel(EXTENDS), + jgm.getVertexLabel(PYPI_ARTIFACT_COORDINATES), + jgm.getVertexLabel(GENERIC_ARTIFACT_COORDINATES)); + + jgm.addConnection(jgm.getEdgeLabel(USER_HAS_SECURITY_ROLES), + jgm.getVertexLabel(USER), + jgm.getVertexLabel(SECURITY_ROLE)); + + } + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/changelog/V1_0_0_2_InitialIndexes.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/changelog/V1_0_0_2_InitialIndexes.java new file mode 100644 index 0000000..68c631e --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/changelog/V1_0_0_2_InitialIndexes.java @@ -0,0 +1,103 @@ +package org.carlspring.strongbox.db.schema.changelog; + +import static org.carlspring.strongbox.db.schema.Edges.ARTIFACT_GROUP_HAS_TAGGED_ARTIFACTS; +import static org.carlspring.strongbox.db.schema.Properties.REPOSITORY_ID; +import static org.carlspring.strongbox.db.schema.Properties.STORAGE_ID; +import static org.carlspring.strongbox.db.schema.Properties.TAG_NAME; +import static org.carlspring.strongbox.db.schema.Properties.UUID; +import static org.carlspring.strongbox.db.schema.Vertices.ARTIFACT; +import static org.carlspring.strongbox.db.schema.Vertices.ARTIFACT_ID_GROUP; +import static org.carlspring.strongbox.db.schema.Vertices.ARTIFACT_TAG; +import static org.carlspring.strongbox.db.schema.Vertices.GENERIC_ARTIFACT_COORDINATES; +import static org.carlspring.strongbox.db.schema.Vertices.SECURITY_ROLE; +import static org.carlspring.strongbox.db.schema.Vertices.USER; +import static org.carlspring.strongbox.db.schema.util.SchemaUtils.buildIndex; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.lang3.StringUtils; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.carlspring.strongbox.db.schema.migration.Changeset; +import org.janusgraph.core.schema.JanusGraphManagement; + +public class V1_0_0_2_InitialIndexes implements Changeset +{ + + @Override + public String getVersion() + { + return "1.0.0.2"; + } + + @Override + public String getAuthor() + { + return "serge.bespalov@gmail.com"; + } + + @Override + public Set createVertexIndexes(JanusGraphManagement jgm) + { + Set result = new HashSet<>(); + + result.add(buildIndex(jgm, + Vertex.class, + jgm.getVertexLabel(ARTIFACT), + true, + jgm.getPropertyKey(UUID))); + result.add(buildIndex(jgm, + Vertex.class, + jgm.getVertexLabel(GENERIC_ARTIFACT_COORDINATES), + true, + jgm.getPropertyKey(UUID))); + result.add(buildIndex(jgm, + Vertex.class, + jgm.getVertexLabel(ARTIFACT_TAG), + true, + true, + jgm.getPropertyKey(UUID))); + result.add(buildIndex(jgm, + Vertex.class, + jgm.getVertexLabel(ARTIFACT_ID_GROUP), + true, + jgm.getPropertyKey(UUID))); + result.add(buildIndex(jgm, + Vertex.class, + jgm.getVertexLabel(ARTIFACT_ID_GROUP), + false, + false, + jgm.getPropertyKey(STORAGE_ID), + jgm.getPropertyKey(REPOSITORY_ID))); + result.add(buildIndex(jgm, + Vertex.class, + jgm.getVertexLabel(USER), + true, + jgm.getPropertyKey(UUID))); + result.add(buildIndex(jgm, + Vertex.class, + jgm.getVertexLabel(SECURITY_ROLE), + true, + jgm.getPropertyKey(UUID))); + + return result; + } + + @Override + public Map createRelationIndexes(JanusGraphManagement jgm) + { + Map result = new HashMap<>(); + String name = ARTIFACT_GROUP_HAS_TAGGED_ARTIFACTS + "By" + StringUtils.capitalize(TAG_NAME); + jgm.buildEdgeIndex(jgm.getEdgeLabel(ARTIFACT_GROUP_HAS_TAGGED_ARTIFACTS), + name, + Direction.OUT, + jgm.getPropertyKey(TAG_NAME)); + result.put(name, ARTIFACT_GROUP_HAS_TAGGED_ARTIFACTS); + + return result; + } + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/ChangelogStorage.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/ChangelogStorage.java new file mode 100644 index 0000000..c0f5046 --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/ChangelogStorage.java @@ -0,0 +1,22 @@ +package org.carlspring.strongbox.db.schema.migration; + +/** + * @author sbespalov + */ +public interface ChangelogStorage +{ + + String EDGE_CHANGESET = "Changeset"; + String VERTEX_SCHEMA_VERSION = "SchemaVersion"; + String PROPERTY_VERSION_VALUE = "versionValue"; + String PROPERTY_APPLY_DATE = "applyDate"; + String PROPERTY_CHANGESET_NAME = "changesetName"; + String PROPERTY_AUTHOR = "changesetAuthor"; + + void init(); + + ChangesetVersion getSchemaVersion(); + + void upgrade(Changeset version); + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/Changeset.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/Changeset.java new file mode 100644 index 0000000..ef632ec --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/Changeset.java @@ -0,0 +1,42 @@ +package org.carlspring.strongbox.db.schema.migration; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.janusgraph.core.schema.JanusGraphManagement; + +/** + * @author sbespalov + */ +public interface Changeset extends ChangesetVersion +{ + default String getName() + { + return getClass().getSimpleName(); + } + + String getAuthor(); + + default void applySchemaChanges(JanusGraphManagement jgm) + { + + } + + default Set createVertexIndexes(JanusGraphManagement jgm) + { + return Collections.emptySet(); + } + + default Map createRelationIndexes(JanusGraphManagement jgm) + { + return Collections.emptyMap(); + } + + default void applyGraphChanges(GraphTraversalSource g) + { + + } + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/ChangesetVersion.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/ChangesetVersion.java new file mode 100644 index 0000000..e4bf3a3 --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/ChangesetVersion.java @@ -0,0 +1,35 @@ +package org.carlspring.strongbox.db.schema.migration; + +/** + * @author sbespalov + */ +public interface ChangesetVersion extends Comparable +{ + + String getVersion(); + + @Override + default int compareTo(ChangesetVersion other) + { + String version1 = this.getVersion(); + String version2 = other.getVersion(); + + String[] levels1 = version1.split("\\."); + String[] levels2 = version2.split("\\."); + + int length = Math.max(levels1.length, levels2.length); + for (int i = 0; i < length; i++) + { + Integer v1 = i < levels1.length ? Integer.parseInt(levels1[i]) : 0; + Integer v2 = i < levels2.length ? Integer.parseInt(levels2[i]) : 0; + int compare = v1.compareTo(v2); + if (compare != 0) + { + return compare; + } + } + + return 0; + } + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/ChangesetVersionValue.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/ChangesetVersionValue.java new file mode 100644 index 0000000..a52c76b --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/ChangesetVersionValue.java @@ -0,0 +1,24 @@ +package org.carlspring.strongbox.db.schema.migration; + +/** + * @author sbespalov + */ +public class ChangesetVersionValue implements ChangesetVersion +{ + + public static ChangesetVersion ZERO = new ChangesetVersionValue("0"); + + private final String version; + + public ChangesetVersionValue(String version) + { + this.version = version; + } + + @Override + public String getVersion() + { + return version; + } + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/JanusgraphChangelogStorage.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/JanusgraphChangelogStorage.java new file mode 100644 index 0000000..3b1509d --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/migration/JanusgraphChangelogStorage.java @@ -0,0 +1,228 @@ +package org.carlspring.strongbox.db.schema.migration; + +import java.util.Date; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.janusgraph.core.EdgeLabel; +import org.janusgraph.core.JanusGraph; +import org.janusgraph.core.PropertyKey; +import org.janusgraph.core.VertexLabel; +import org.janusgraph.core.schema.ConsistencyModifier; +import org.janusgraph.core.schema.JanusGraphManagement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author sbespalov + */ +public class JanusgraphChangelogStorage implements ChangelogStorage +{ + + private static final Logger logger = LoggerFactory.getLogger(JanusgraphChangelogStorage.class); + + private final JanusGraph jg; + + public JanusgraphChangelogStorage(JanusGraph jg) + { + this.jg = jg; + } + + @Override + public void init() + { + JanusGraphManagement jgm = jg.openManagement(); + try + { + initSchema(jgm); + jgm.commit(); + } + catch (Throwable e) + { + jgm.rollback(); + throw new RuntimeException("Failed to init schema storage.", e); + } + } + + private void initSchema(JanusGraphManagement jgm) + { + logger.info("Check the changelog storage schema to be initialized."); + VertexLabel schemaVersionLabel = Optional.ofNullable(jgm.getVertexLabel(VERTEX_SCHEMA_VERSION)) + .orElseGet(() -> jgm.makeVertexLabel(VERTEX_SCHEMA_VERSION).make()); + EdgeLabel changesetLabel = Optional.ofNullable(jgm.getEdgeLabel(EDGE_CHANGESET)) + .orElseGet(() -> jgm.makeEdgeLabel(EDGE_CHANGESET).make()); + if (schemaVersionLabel.mappedConnections().isEmpty()) + { + jgm.addConnection(changesetLabel, + schemaVersionLabel, + schemaVersionLabel); + } + + if (jgm.getPropertyKey(PROPERTY_VERSION_VALUE) == null) + { + PropertyKey property = jgm.makePropertyKey(PROPERTY_VERSION_VALUE).dataType(String.class).make(); + jgm.setConsistency(property, ConsistencyModifier.LOCK); + jgm.addProperties(schemaVersionLabel, property); + } + if (jgm.getPropertyKey(PROPERTY_APPLY_DATE) == null) + { + PropertyKey property = jgm.makePropertyKey(PROPERTY_APPLY_DATE).dataType(Date.class).make(); + jgm.addProperties(changesetLabel, property); + } + if (jgm.getPropertyKey(PROPERTY_AUTHOR) == null) + { + PropertyKey property = jgm.makePropertyKey(PROPERTY_AUTHOR).dataType(String.class).make(); + jgm.addProperties(changesetLabel, property); + } + if (jgm.getPropertyKey(PROPERTY_CHANGESET_NAME) == null) + { + PropertyKey property = jgm.makePropertyKey(PROPERTY_CHANGESET_NAME).dataType(String.class).make(); + jgm.addProperties(changesetLabel, property); + } + } + + @Override + public ChangesetVersion getSchemaVersion() + { + GraphTraversalSource g = jg.traversal(); + try + { + ChangesetVersion version = fetchSchemaVersion(g); + g.tx().commit(); + return version; + } + catch (Throwable e) + { + g.tx().rollback(); + throw new RuntimeException("Failed to get stored schema version.", e); + } + } + + private ChangesetVersion fetchSchemaVersion(GraphTraversalSource g) + { + GraphTraversal t = g.V() + .hasLabel(VERTEX_SCHEMA_VERSION) + .not(__.inE(EDGE_CHANGESET)) + .until(__.not(__.out(EDGE_CHANGESET))) + .repeat(__.out()); + if (t.hasNext()) + { + return new ChangesetVersionValue((String) t.next().property(PROPERTY_VERSION_VALUE).value()); + } + + return g.addV(VERTEX_SCHEMA_VERSION) + .property(PROPERTY_VERSION_VALUE, ChangesetVersionValue.ZERO.getVersion()) + .map(v -> new ChangesetVersionValue((String) v.get().property(PROPERTY_VERSION_VALUE).value())) + .next(); + } + + public Changeset prepare(Changeset changeset) + { + return new StoredChangeset(changeset); + } + + @Override + public void upgrade(Changeset version) + { + ChangesetVersion currentVersion = getSchemaVersion(); + GraphTraversalSource g = jg.traversal(); + try + { + upgrade(g, currentVersion, version); + g.tx().commit(); + } + catch (Exception e) + { + g.tx().rollback(); + throw new RuntimeException(String.format("Failed to upgrade schema version from [%s] to [%s]", currentVersion, version), + e); + } + } + + private void upgrade(GraphTraversalSource g, + ChangesetVersion from, + Changeset to) + { + GraphTraversal updateSchemaVersion = g.V() + .hasLabel(VERTEX_SCHEMA_VERSION) + .has(PROPERTY_VERSION_VALUE, from.getVersion()) + .addE(EDGE_CHANGESET) + .property(PROPERTY_APPLY_DATE, new Date()) + .property(PROPERTY_CHANGESET_NAME, to.getName()) + .property(PROPERTY_AUTHOR, to.getAuthor()) + .to(__.addV(VERTEX_SCHEMA_VERSION) + .property(PROPERTY_VERSION_VALUE, to.getVersion())); + if (updateSchemaVersion.hasNext()) + { + updateSchemaVersion.next(); + } + else + { + g.addV(VERTEX_SCHEMA_VERSION).property(PROPERTY_VERSION_VALUE, to.getVersion()).next(); + } + } + + /** + * @author sbespalov + */ + public class StoredChangeset implements Changeset + { + + private final Changeset target; + + public StoredChangeset(Changeset target) + { + this.target = target; + } + + public String getName() + { + return target.getName(); + } + + public String getAuthor() + { + return target.getAuthor(); + } + + public String getVersion() + { + return target.getVersion(); + } + + public void applySchemaChanges(JanusGraphManagement jgm) + { + target.applySchemaChanges(jgm); + } + + public int compareTo(ChangesetVersion other) + { + return target.compareTo(other); + } + + public Set createVertexIndexes(JanusGraphManagement jgm) + { + return target.createVertexIndexes(jgm); + } + + public Map createRelationIndexes(JanusGraphManagement jgm) + { + return target.createRelationIndexes(jgm); + } + + @Override + public void applyGraphChanges(GraphTraversalSource g) + { + target.applyGraphChanges(g); + upgrade(g, fetchSchemaVersion(g), target); + } + + } + +} diff --git a/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/util/SchemaUtils.java b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/util/SchemaUtils.java new file mode 100644 index 0000000..1405877 --- /dev/null +++ b/strongbox-db-schema/src/main/java/org/carlspring/strongbox/db/schema/util/SchemaUtils.java @@ -0,0 +1,81 @@ +package org.carlspring.strongbox.db.schema.util; + +import java.util.Arrays; +import java.util.Optional; + +import org.apache.commons.lang3.StringUtils; +import org.apache.tinkerpop.gremlin.structure.Element; +import org.janusgraph.core.EdgeLabel; +import org.janusgraph.core.PropertyKey; +import org.janusgraph.core.VertexLabel; +import org.janusgraph.core.schema.ConsistencyModifier; +import org.janusgraph.core.schema.JanusGraphIndex; +import org.janusgraph.core.schema.JanusGraphManagement; +import org.janusgraph.core.schema.JanusGraphSchemaType; +import org.janusgraph.core.schema.JanusGraphManagement.IndexBuilder; + +public class SchemaUtils +{ + + public static void addVertexPropertyConstraints(JanusGraphManagement jgm, + String vertex, + String... propertykeys) + { + VertexLabel vertexLabel = jgm.getVertexLabel(vertex); + for (String propertyKey : propertykeys) + { + jgm.addProperties(vertexLabel, jgm.getPropertyKey(propertyKey)); + } + } + + public static void addEdgePropertyConstraints(JanusGraphManagement jgm, + String label, + String... propertykeys) + { + EdgeLabel edge = jgm.getEdgeLabel(label); + for (String propertyKey : propertykeys) + { + jgm.addProperties(edge, jgm.getPropertyKey(propertyKey)); + } + } + + public static String buildIndex(final JanusGraphManagement jgm, + final Class elementType, + final JanusGraphSchemaType schemaType, + final boolean unique, + final PropertyKey... properties) + { + return buildIndex(jgm, elementType, schemaType, unique, true, properties); + } + + public static String buildIndex(final JanusGraphManagement jgm, + final Class elementType, + final JanusGraphSchemaType schemaType, + final boolean unique, + final boolean consistent, + final PropertyKey... properties) + { + final String name = schemaType.name() + "By" + Arrays.stream(properties) + .map(PropertyKey::name) + .map(StringUtils::capitalize) + .reduce((p1, + p2) -> p1 + "And" + p2) + .get(); + + IndexBuilder indexBuilder = jgm.buildIndex(name, elementType) + .indexOnly(schemaType); + Arrays.stream(properties).forEach(indexBuilder::addKey); + + if (unique) + { + indexBuilder = indexBuilder.unique(); + } + JanusGraphIndex intex = indexBuilder.buildCompositeIndex(); + if (consistent) + { + jgm.setConsistency(intex, ConsistencyModifier.LOCK); + } + + return intex.name(); + } +} diff --git a/strongbox-db-schema/src/test/java/org/carlspring/strongbox/db/schema/SchemaMigrationTest.java b/strongbox-db-schema/src/test/java/org/carlspring/strongbox/db/schema/SchemaMigrationTest.java new file mode 100644 index 0000000..a6b00f8 --- /dev/null +++ b/strongbox-db-schema/src/test/java/org/carlspring/strongbox/db/schema/SchemaMigrationTest.java @@ -0,0 +1,131 @@ +package org.carlspring.strongbox.db.schema; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.util.TreeSet; + +import org.carlspring.strongbox.db.schema.migration.Changeset; +import org.carlspring.strongbox.db.schema.migration.ChangesetVersionValue; +import org.carlspring.strongbox.db.schema.migration.JanusgraphChangelogStorage; +import org.janusgraph.core.JanusGraph; +import org.janusgraph.core.schema.JanusGraphManagement; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.strongbox.db.server.InMemoryJanusGraphServer; +import org.strongbox.db.server.JanusGraphConfiguration; +import org.strongbox.db.server.JanusGraphProperties; +import org.strongbox.db.server.JanusGraphServer; +import org.strongbox.util.ConfigurationUtils; + +public class SchemaMigrationTest +{ + + private static final Changeset V1 = new Changeset() + { + + @Override + public String getVersion() + { + return "1.0.0.1"; + } + + @Override + public String getAuthor() + { + return "sbespalov"; + } + + }; + + private static final Changeset V2 = new Changeset() + { + + @Override + public String getVersion() + { + return "1.0.0.2"; + } + + @Override + public String getAuthor() + { + return "sbespalov"; + } + + }; + + private static final Changeset V3 = new Changeset() + { + + @Override + public String getVersion() + { + return "1.0.0.3"; + } + + @Override + public void applySchemaChanges(JanusGraphManagement jgm) + { + throw new RuntimeException(getVersion()); + } + + @Override + public String getAuthor() + { + return "sbespalov"; + } + + }; + + private JanusGraphServer janusGraphServer; + + @BeforeAll + public static void init() + throws IOException + { + ConfigurationUtils.extractConfigurationFile("./target", "janusgraph-inmemory.properties"); + } + + @BeforeEach + public void setUp() + throws Exception + { + JanusGraphConfiguration janusGraphConfiguration = new JanusGraphProperties( + "file:./target/etc/conf/janusgraph-inmemory.properties"); + janusGraphServer = new InMemoryJanusGraphServer(janusGraphConfiguration, () -> null); + janusGraphServer.start(); + } + + @Test + public void testUpdateSchemaVersion() + throws InterruptedException + { + JanusGraph jg = janusGraphServer.getJanusGraph(); + JanusgraphChangelogStorage changelogStorage = new JanusgraphChangelogStorage(jg); + TreeSet changeSets = new TreeSet<>(); + StrongboxSchema strongboxSchema = new StrongboxSchema(changeSets); + + strongboxSchema.createSchema(jg); + assertThat(changelogStorage.getSchemaVersion().getVersion()).isEqualTo(ChangesetVersionValue.ZERO.getVersion()); + + changeSets.add(V2); + strongboxSchema.createSchema(jg); + assertThat(changelogStorage.getSchemaVersion().getVersion()).isEqualTo(V2.getVersion()); + + changeSets.add(V3); + assertThatThrownBy(() -> strongboxSchema.createSchema(jg)).withFailMessage("Failed to apply changeset [1.0.0.3]-[]"); + assertThat(changelogStorage.getSchemaVersion().getVersion()).isEqualTo(V2.getVersion()); + } + + @AfterEach + public void tearDwon() + throws Exception + { + janusGraphServer.stop(); + } + +} diff --git a/strongbox-db-server/pom.xml b/strongbox-db-server/pom.xml index 9a40a73..b08ea5a 100644 --- a/strongbox-db-server/pom.xml +++ b/strongbox-db-server/pom.xml @@ -1,15 +1,15 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" + xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> 4.0.0 org.carlspring.strongbox strongbox-db - 1.0-SNAPSHOT + 1.0-PR-19-SNAPSHOT ../pom.xml @@ -21,37 +21,80 @@ UTF-8 - - + + com.fasterxml.jackson.core jackson-databind - com.orientechnologies - orientdb-core + org.janusgraph + janusgraph-cql + + + org.janusgraph + janusgraph-inmemory - com.orientechnologies - orientdb-client + org.apache.cassandra + cassandra-all - com.orientechnologies - orientdb-server + io.dropwizard.metrics + metrics-core + - com.orientechnologies - orientdb-object + org.apache.tinkerpop + gremlin-core - com.orientechnologies - orientdb-graphdb + org.apache.tinkerpop + gremlin-driver - - - org.webjars - orientdb-studio + + org.apache.tinkerpop + gremlin-server + + + org.apache.tinkerpop + gremlin-groovy + + + org.opencypher.gremlin + cypher-gremlin-neo4j-driver + + + org.opencypher.gremlin + cypher-gremlin-server-plugin + + + + org.neo4j.driver + neo4j-java-driver + + + org.neo4j + neo4j-ogm-api + + + + + + commons-lang + commons-lang + 2.6 + + + + javax.annotation + javax.annotation-api + + diff --git a/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/driver/CypherGremlinStatementRunner.java b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/driver/CypherGremlinStatementRunner.java new file mode 100644 index 0000000..093dad1 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/driver/CypherGremlinStatementRunner.java @@ -0,0 +1,168 @@ +package org.opencypher.gremlin.neo4j.driver; + +import static org.opencypher.gremlin.translation.ReturnProperties.ID; +import static org.opencypher.gremlin.translation.ReturnProperties.INV; +import static org.opencypher.gremlin.translation.ReturnProperties.LABEL; +import static org.opencypher.gremlin.translation.ReturnProperties.OUTV; +import static org.opencypher.gremlin.translation.ReturnProperties.RELATIONSHIP_TYPE; +import static org.opencypher.gremlin.translation.ReturnProperties.TYPE; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.CompletionStage; + +import org.janusgraph.core.JanusGraphTransaction; +import org.janusgraph.core.JanusGraphVertex; +import org.janusgraph.graphdb.relations.RelationIdentifier; +import org.neo4j.driver.v1.Record; +import org.neo4j.driver.v1.Session; +import org.neo4j.driver.v1.Statement; +import org.neo4j.driver.v1.StatementResult; +import org.neo4j.driver.v1.StatementResultCursor; +import org.neo4j.driver.v1.StatementRunner; +import org.neo4j.driver.v1.Value; +import org.neo4j.driver.v1.types.TypeSystem; +import org.opencypher.gremlin.client.CypherGremlinClient; +import org.opencypher.gremlin.neo4j.ogm.transaction.GremlinTransaction; +import org.strongbox.util.Commons; + +/** + * @author sbespalov + */ +public class CypherGremlinStatementRunner implements StatementRunner +{ + + private final Session session; + private final GremlinTransaction gremlinTransaction; + + public CypherGremlinStatementRunner(GremlinTransaction gremlinTransaction) + { + this.gremlinTransaction = gremlinTransaction; + this.session = new GremlinServerSession(null, + CypherGremlinClient.inMemory(gremlinTransaction.getNativeTransaction().traversal()), + new JanusGraphValueConverter(false)); + } + + public StatementResult run(String statementTemplate, + Value parameters) + { + return session.run(statementTemplate, parameters); + } + + public StatementResult run(String statementTemplate, + Map statementParameters) + { + return session.run(statementTemplate, statementParameters); + } + + public StatementResult run(String statementTemplate, + Record statementParameters) + { + return session.run(statementTemplate, statementParameters); + } + + public StatementResult run(String statementTemplate) + { + return session.run(statementTemplate); + } + + public StatementResult run(Statement statement) + { + return session.run(statement); + } + + public TypeSystem typeSystem() + { + return session.typeSystem(); + } + + class JanusGraphValueConverter extends GremlinCypherValueConverter + { + + public JanusGraphValueConverter(boolean ignoreIds) + { + super(ignoreIds); + } + + @Override + Record toRecord(Map map) + { + map.entrySet().forEach(this::normalizeValue); + + return super.toRecord(map); + } + + private void normalizeValue(Entry e) + { + Object value = e.getValue(); + if (value instanceof Map) + { + Map nodeMap = (Map) value; + nodeMap.entrySet().forEach(this::normalizeValue); + } + else if (e.getValue() instanceof RelationIdentifier) + { + JanusGraphTransaction tx = (JanusGraphTransaction) gremlinTransaction.getNativeTransaction(); + RelationIdentifier relationIdentifier = (RelationIdentifier) e.getValue(); + if (ID.equals(e.getKey())) + { + e.setValue(relationIdentifier.getRelationId()); + + return; + } + + Map expectedValue = new HashMap<>(); + expectedValue.put(TYPE, RELATIONSHIP_TYPE); + expectedValue.put(ID, relationIdentifier.getRelationId()); + expectedValue.put(OUTV, relationIdentifier.getOutVertexId()); + expectedValue.put(INV, relationIdentifier.getInVertexId()); + + JanusGraphVertex typeVertex = tx.getVertex(relationIdentifier.getTypeId()); + expectedValue.put(LABEL, typeVertex.label()); + + e.setValue(expectedValue); + } + else if (value instanceof Date) + { + e.setValue(Commons.toLocalDateTime((Date) value)); + } + } + + } + + @Override + public CompletionStage runAsync(String statementTemplate, + Value parameters) + { + throw new UnsupportedOperationException(); + } + + @Override + public CompletionStage runAsync(String statementTemplate, + Map statementParameters) + { + throw new UnsupportedOperationException(); + } + + @Override + public CompletionStage runAsync(String statementTemplate, + Record statementParameters) + { + throw new UnsupportedOperationException(); + } + + @Override + public CompletionStage runAsync(String statementTemplate) + { + throw new UnsupportedOperationException(); + } + + @Override + public CompletionStage runAsync(Statement statement) + { + throw new UnsupportedOperationException(); + } + +} diff --git a/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/driver/Neo4jDriverEntityAdapter.java b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/driver/Neo4jDriverEntityAdapter.java new file mode 100644 index 0000000..8b17ed2 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/driver/Neo4jDriverEntityAdapter.java @@ -0,0 +1,117 @@ +package org.opencypher.gremlin.neo4j.driver; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.neo4j.driver.internal.value.ListValue; +import org.neo4j.driver.v1.Value; +import org.neo4j.driver.v1.types.Entity; +import org.neo4j.driver.v1.types.Node; +import org.neo4j.driver.v1.types.Path; +import org.neo4j.driver.v1.types.Relationship; +import org.neo4j.ogm.driver.TypeSystem.NoNativeTypes; + +/** + * @author sbespalov + */ +public class Neo4jDriverEntityAdapter +{ + + public boolean isPath(Object value) + { + return value instanceof Path; + } + + public boolean isNode(Object value) + { + return value instanceof Node; + } + + public boolean isRelationship(Object value) + { + return value instanceof Relationship; + } + + public long nodeId(Object node) + { + return ((Node) node).id(); + } + + public List labels(Object value) + { + Node node = (Node) value; + List labels = new ArrayList<>(); + for (String label : node.labels()) + { + labels.add(label); + } + return labels; + } + + public long relationshipId(Object relationship) + { + return ((Relationship) relationship).id(); + } + + public String relationshipType(Object relationship) + { + return ((Relationship) relationship).type(); + } + + public Long startNodeId(Object relationship) + { + return ((Relationship) relationship).startNodeId(); + } + + public Long endNodeId(Object relationship) + { + return ((Relationship) relationship).endNodeId(); + } + + public Map properties(Object container) + { + return ((Entity) container).asMap(this::toMapped); + } + + public List nodesInPath(Object pathValue) + { + Path path = (Path) pathValue; + List nodes = new ArrayList<>(path.length()); + for (Node node : path.nodes()) + { + nodes.add(node); + } + return nodes; + } + + public List relsInPath(Object pathValue) + { + Path path = (Path) pathValue; + List rels = new ArrayList<>(path.length()); + for (Relationship rel : path.relationships()) + { + rels.add(rel); + } + return rels; + } + + private Object toMapped(Value value) + { + + if (value == null) + { + return null; + } + + if (value instanceof ListValue) + { + return value.asList(this::toMapped); + } + + Object object = value.asObject(); + return NoNativeTypes.INSTANCE.getNativeToMappedTypeAdapter(object.getClass()) + .apply(object); + } + +} diff --git a/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/CypherQueryUtils.java b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/CypherQueryUtils.java new file mode 100644 index 0000000..f2b9c65 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/CypherQueryUtils.java @@ -0,0 +1,294 @@ +package org.opencypher.gremlin.neo4j.ogm; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.strongbox.util.Commons; + +public class CypherQueryUtils +{ + + private static final String UNWIND = "UNWIND"; + private static final String MERGE = "MERGE"; + private static final String MATCH = "MATCH"; + private static final String WHERE = "WHERE"; + private static final String WITH = "WITH"; + private static final String SET = "SET"; + private static final String RETURN = "RETURN"; + + private static final Logger logger = LoggerFactory.getLogger(CypherQueryUtils.class); + + public static String inlineParameters(String cypherStatement, + Map parameterMap) + { + String placeholderFormat; + Map params; + Collection> rows = (Collection>) parameterMap.get("rows"); + if (rows == null || rows.size() == 0) + { + // regular case + placeholderFormat = "$%s"; + params = parameterMap; + } + else + { + // the case with `UNWIND {rows} as row` + placeholderFormat = "row.%s"; + params = rows.iterator().next(); + } + + List> props = params.entrySet() + .stream() + .flatMap(CypherQueryUtils::flatten) + .collect(Collectors.toList()); + + for (Pair p : props) + { + cypherStatement = cypherStatement.replace(String.format(placeholderFormat, p.getKey()), + inlinedValue(p.getKey(), p.getValue())); + } + + return cypherStatement; + } + + private static String inlinedValue(String key, Object value) + { + if (value instanceof Number) + { + return value.toString(); + } + else if (value instanceof LocalDateTime) + { + return String.valueOf(Commons.toLong((LocalDateTime) value)); + } + else if (value instanceof Collection) { + return "[" + ((Collection)value).stream().map(e-> inlinedValue(key, e)).collect(Collectors.joining(",")) + "]"; + } + else if (value == null) + { + return "null"; + } + + return "'" + value.toString() + "'"; + } + + private static Stream> flatten(Map.Entry root) + { + if (root.getValue() instanceof Map) + { + return ((Map) root.getValue()).entrySet() + .stream() + .map(e -> Pair.of(root.getKey() + "." + e.getKey(), + e.getValue())) + .flatMap(CypherQueryUtils::flatten); + } + return Stream.of(Pair.of(root.getKey(), root.getValue())); + } + + public static String normalizeMergeByIdWithParams(String cypherStatement, + Map props) + { + if (!cypherStatement.startsWith(UNWIND)) + { + return cypherStatement; + } + + // cleanup multiple labels for DomainEntity inheritance + // TODO: make it generic + cypherStatement = cypherStatement.replace(String.format(":`%s`", "DomainEntity"), ""); + + int mergeIndex = cypherStatement.indexOf(MERGE); + int setIndex = cypherStatement.indexOf(SET); + int returnIndex = cypherStatement.indexOf(RETURN); + + if (mergeIndex < 0 || setIndex < 0 || returnIndex < 0) + { + return cypherStatement; + } + if (mergeIndex >= setIndex || setIndex >= returnIndex) + { + // not a MERGE statement + return cypherStatement; + } + + String mergeClause = cypherStatement.substring(mergeIndex, setIndex); + String setClause = cypherStatement.substring(setIndex, returnIndex); + + String alias; + if (mergeClause.contains("n:")) + { + alias = "n"; + } + else if (mergeClause.contains("rel:")) + { + alias = "rel"; + } + else + { + throw new IllegalArgumentException(String.format("Failet to parse node alias from [%s].", mergeClause)); + } + + // specify concrete properties to set + String propsClause = props.keySet() + .stream() + .map(p -> String.format("%s.%s = row.props.%s", alias, p, p)) + .reduce((p1, + p2) -> p1 + "," + p2) + .get(); + + return cypherStatement.replace(setClause, "SET " + propsClause + " "); + } + + public static String normalizeMatchByIdWithRelationResult(String cypherStatement, + String id) + { + // MATCH (n:`RepositoryArtifactIdGroup`) + // WHERE n.`uuid` = { id } + // WITH n + // RETURN n, + // _______[ + // _________[ + // ___________(n)-[r_r1:`RepositoryArtifactIdGroupEntity_ArtifactGroupEntity`]->(a1:`ArtifactGroup`) + // ___________| [ r_r1, a1 ] + // _________] + // _______] + if (!cypherStatement.startsWith(MATCH)) + { + return cypherStatement; + } + + int whereIndex = cypherStatement.indexOf(WHERE); + int withIndex = cypherStatement.indexOf(WITH); + int returnIndex = cypherStatement.indexOf(RETURN); + + if (whereIndex < 0 || withIndex < 0 || returnIndex < 0) + { + return cypherStatement; + } + + if (whereIndex >= withIndex || withIndex >= returnIndex) + { + return cypherStatement; + } + + String matchClause = cypherStatement.substring(0, whereIndex); + String whereClause = cypherStatement.substring(whereIndex, withIndex); + String withClause = cypherStatement.substring(withIndex, returnIndex); + String returnClause = cypherStatement.substring(returnIndex); + + logger.trace(String.format("Cyphter With: %s", withClause)); + logger.trace(String.format("Cyphter Return: %s", returnClause)); + + String returnResults = returnClause.replace(RETURN, ""); + List returnTokens = new ArrayList<>(); + for (int i = 0, j = returnResults.indexOf(","); j > 0; i = j + 1, j = returnResults.indexOf(",", i)) + { + String returnToken = returnResults.substring(i, j).trim(); + if (returnToken.startsWith("[")) + { + // this is relations query result token, which is commonly the + // last + returnTokens.add(returnResults.substring(i).trim()); + break; + } + else + { + // this is just a regular return token like some alias + returnTokens.add(returnToken); + } + } + + // return relations subquery should be last element + String returnRelationsClause = returnTokens.get(returnTokens.size() - 1); + returnTokens.remove(returnTokens.size() - 1); + if (!returnRelationsClause.startsWith("[") && !returnRelationsClause.endsWith("]")) + { + logger.trace("Return clause without relations."); + + return cypherStatement; + } + + // Remove list outer brackets + returnRelationsClause = returnRelationsClause.substring(1, returnRelationsClause.length() - 1).trim(); + // Remove frist bracket for first element and last bracket for last + // element + returnRelationsClause = returnRelationsClause.substring(1, returnRelationsClause.length() - 1).trim(); + + Map withRelations = new HashMap<>(); + for (String returnRelationClause : returnRelationsClause.split("\\]\\s*,\\s*\\[")) + { + if (!returnRelationClause.contains("|")) + { + throw new RuntimeException("Relation pattern not match."); + } + + String[] returnRelationTokens = returnRelationClause.split("\\|"); + if (returnRelationTokens.length != 2) + { + throw new RuntimeException("Return relation pattern not match."); + } + + String relationQueryResult = returnRelationTokens[1].trim(); + // remove brackets + relationQueryResult = relationQueryResult.substring(1, relationQueryResult.length() - 1).trim(); + // map relation results into relation query + withRelations.put(relationQueryResult.trim(), returnRelationTokens[0].trim()); + } + + withClause = withRelations.entrySet() + .stream() + .map(e -> { + String relationQueryResult = e.getKey(); + Arrays.stream(relationQueryResult.split(",")).forEach(r -> returnTokens.add(r)); + String localWith = returnTokens.stream() + .reduce((r1, + r2) -> r1.trim() + ", " + r2.trim()) + .get(); + return MATCH + " " + e.getValue() + " " + WITH + " " + localWith; + }) + .reduce((w1, + w2) -> w1 + " " + w2) + .get(); + + returnClause = RETURN + " " + returnTokens.stream() + .reduce((r1, + r2) -> r1.trim() + ", " + r2.trim()) + .get(); + + cypherStatement = matchClause + " " + whereClause + " " + WITH + " n " + withClause + " " + returnClause; + cypherStatement = cypherStatement.replace("{ id }", "'" + id + "'"); + return cypherStatement; + } + + public static void main(String[] args) + { + String query = "MATCH (n:`RepositoryArtifactIdGroup`) " + + "WHERE n.`uuid` = { id } " + + "WITH n " + + "RETURN n, " + + "[ [ (n)-[r_r1:`RepositoryArtifactIdGroupEntity_ArtifactGroupEntity`]->(a1:`ArtifactGroup`) | [ r_r1, a1 ] ], " + + + " [ (n)-[r_r2:`RepositoryArtifactIdGroupEntity_ArtifactGroupEntity`]->(a2:`ArtifactGroup`) | [ r_r2, a2 ] ] ]"; + + // String query = "MATCH (n:`RepositoryArtifactIdGroup`) WHERE n.`uuid` + // = { id } WITH n RETURN n,[ [ + // (n)-[r_r1:`RepositoryArtifactIdGroupEntity_ArtifactGroupEntity`]->(a1:`ArtifactGroup`) + // | [ r_r1, a1 ] ] ]"; + + System.out.println(normalizeMatchByIdWithRelationResult(query, "123")); + } + +} diff --git a/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/GremlinGraphDriver.java b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/GremlinGraphDriver.java new file mode 100644 index 0000000..67292a2 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/GremlinGraphDriver.java @@ -0,0 +1,79 @@ +package org.opencypher.gremlin.neo4j.ogm; + +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.neo4j.ogm.config.Configuration; +import org.neo4j.ogm.driver.AbstractConfigurableDriver; +import org.neo4j.ogm.request.Request; +import org.neo4j.ogm.transaction.Transaction; +import org.neo4j.ogm.transaction.Transaction.Type; +import org.neo4j.ogm.transaction.TransactionManager; +import org.opencypher.gremlin.neo4j.driver.CypherGremlinStatementRunner; +import org.opencypher.gremlin.neo4j.ogm.request.GremlinRequest; +import org.opencypher.gremlin.neo4j.ogm.transaction.GremlinTransaction; + +/** + * @author sbespalov + */ +public class GremlinGraphDriver extends AbstractConfigurableDriver +{ + + private org.apache.tinkerpop.gremlin.structure.Transaction transactionContext; + + public GremlinGraphDriver(org.apache.tinkerpop.gremlin.structure.Transaction transactionContext) + { + this.transactionContext = transactionContext; + } + + @Override + public void configure(Configuration config) + { + + } + + @Override + public Function, Transaction>> getTransactionFactorySupplier() + { + return transactionManager -> (type, + bookmarks) -> { + return createTransaction(transactionManager, type); + }; + + } + + private Transaction createTransaction(TransactionManager transactionManager, + Type type) + { + // TODO: Type.READ_ONLY + Graph tx = transactionContext.createThreadedTx(); + + return new GremlinTransaction(transactionManager, tx, type); + } + + @Override + public void close() + { + transactionContext.close(); + } + + @Override + public Request request(Transaction transaction) + { + return new GremlinRequest(new CypherGremlinStatementRunner((GremlinTransaction) transaction)); + } + + @Override + public Configuration getConfiguration() + { + return null; + } + + @Override + protected String getTypeSystemName() + { + return GremlinGraphDriver.class.getName() + ".types"; + } + +} diff --git a/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/request/GremlinRequest.java b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/request/GremlinRequest.java new file mode 100644 index 0000000..acf9adf --- /dev/null +++ b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/request/GremlinRequest.java @@ -0,0 +1,262 @@ +package org.opencypher.gremlin.neo4j.ogm.request; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; + +import org.apache.commons.lang3.tuple.Pair; +import org.neo4j.driver.v1.StatementResult; +import org.neo4j.driver.v1.StatementRunner; +import org.neo4j.driver.v1.exceptions.ClientException; +import org.neo4j.ogm.exception.CypherException; +import org.neo4j.ogm.model.GraphModel; +import org.neo4j.ogm.model.GraphRowListModel; +import org.neo4j.ogm.model.RestModel; +import org.neo4j.ogm.model.RowModel; +import org.neo4j.ogm.request.DefaultRequest; +import org.neo4j.ogm.request.GraphModelRequest; +import org.neo4j.ogm.request.GraphRowListModelRequest; +import org.neo4j.ogm.request.Request; +import org.neo4j.ogm.request.RestModelRequest; +import org.neo4j.ogm.request.RowModelRequest; +import org.neo4j.ogm.request.Statement; +import org.neo4j.ogm.response.EmptyResponse; +import org.neo4j.ogm.response.Response; +import org.opencypher.gremlin.neo4j.driver.Neo4jDriverEntityAdapter; +import org.opencypher.gremlin.neo4j.ogm.CypherQueryUtils; +import org.opencypher.gremlin.neo4j.ogm.response.GremlinGraphRowModelResponse; +import org.opencypher.gremlin.neo4j.ogm.response.GremlinModelResponse; +import org.opencypher.gremlin.neo4j.ogm.response.GremlinRestModelResponse; +import org.opencypher.gremlin.neo4j.ogm.response.GremlinRowModelResponse; +import org.opencypher.gremlin.translation.CypherAst; +import org.opencypher.gremlin.translation.groovy.GroovyPredicate; +import org.opencypher.gremlin.translation.translator.Translator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GremlinRequest implements Request +{ + + private static final Logger logger = LoggerFactory.getLogger(GremlinRequest.class); + + private final StatementRunner statementRunner; + private final Neo4jDriverEntityAdapter entityAdapter = new Neo4jDriverEntityAdapter(); + + public GremlinRequest(StatementRunner statementRunner) + { + this.statementRunner = statementRunner; + } + + @Override + public Response execute(GraphModelRequest query) + { + if (query.getStatement().length() == 0) + { + return new EmptyResponse<>(); + } + return new GremlinModelResponse(executeRequest(query), entityAdapter); + } + + @Override + public Response execute(RowModelRequest query) + { + if (query.getStatement().length() == 0) + { + return new EmptyResponse(); + } + return new GremlinRowModelResponse(executeRequest(query), entityAdapter); + } + + @Override + public Response execute(DefaultRequest query) + { + final List rowModels = new ArrayList<>(); + String[] columns = null; + for (Statement statement : query.getStatements()) + { + + StatementResult result = executeRequest(statement); + + if (columns == null) + { + try + { + List columnSet = result.keys(); + columns = columnSet.toArray(new String[columnSet.size()]); + } + catch (ClientException e) + { + throw new CypherException(e.code(), e.getMessage(), e); + } + } + try (GremlinRowModelResponse rowModelResponse = new GremlinRowModelResponse(result, entityAdapter)) + { + RowModel model; + while ((model = rowModelResponse.next()) != null) + { + rowModels.add(model); + } + result.consume(); + } + } + + return new MultiStatementBasedResponse(columns, rowModels); + } + + @Override + public Response execute(GraphRowListModelRequest query) + { + if (query.getStatement().length() == 0) + { + return new EmptyResponse(); + } + return new GremlinGraphRowModelResponse(executeRequest(query), entityAdapter); + } + + @Override + public Response execute(RestModelRequest query) + { + if (query.getStatement().length() == 0) + { + return new EmptyResponse(); + } + return new GremlinRestModelResponse(executeRequest(query), entityAdapter); + } + + private org.neo4j.driver.v1.StatementResult executeRequest(Statement query) + { + Map parameterMap = query.getParameters(); + String cypherStatement = query.getStatement(); + logger.debug("Cypher: {} with params {}", cypherStatement, parameterMap); + + Pair> cypherWithParams = Pair.of(cypherStatement, parameterMap); + cypherStatement = Optional.of(cypherWithParams) + .map(this::inlineParameters) + .map(this::normalizeMergeByIdWithParams) + .map(this::normalizeMatchByIdWithRelationResult) + .map(Pair::getLeft) + .get(); + + logger.trace("Cypher(normalized): {}", cypherStatement); + if (logger.isTraceEnabled()) + { + CypherAst ast = CypherAst.parse(cypherStatement, parameterMap); + Translator translator = Translator.builder() + .gremlinGroovy() + .enableCypherExtensions() + .build(); + logger.trace("Gremlin: {}", ast.buildTranslation(translator)); + } + + Class pageableClass; + try + { + pageableClass = Class.forName("org.springframework.data.domain.Pageable"); + } + catch (ClassNotFoundException e) + { + pageableClass = null; + } + if (pageableClass != null) + { + for (Iterator> iterator = parameterMap.entrySet().iterator(); iterator.hasNext();) + { + Entry next = iterator.next(); + if (pageableClass.isInstance(next.getValue())) + { + iterator.remove(); + } + } + } + + return statementRunner.run(cypherStatement, parameterMap); + } + + protected Pair> normalizeMatchByIdWithRelationResult(Pair> cyphterWithParams) + { + return Optional.of(cyphterWithParams) + .map(p -> p.getRight()) + .map(p -> p.get("id")) + .map(id -> id.toString()) + .map(id -> CypherQueryUtils.normalizeMatchByIdWithRelationResult(cyphterWithParams.getLeft(), + id)) + .map(s -> Pair.of(s, cyphterWithParams.getRight())) + .orElse(cyphterWithParams); + } + + protected Pair> normalizeMergeByIdWithParams(Pair> cyphterWithParams) + { + return Optional.of(cyphterWithParams) + .map(p -> p.getRight()) + .map(p -> p.get("rows")) + .filter(r -> r instanceof Collection) + .map(r -> (Collection) r) + .map(r -> r.iterator().next()) + .map(r -> (Map) r) + .map(r -> r.get("props")) + .filter(p -> p instanceof Map) + .map(p -> (Map) p) + .map(p -> CypherQueryUtils.normalizeMergeByIdWithParams(cyphterWithParams.getLeft(), p)) + .map(s -> Pair.of(s, cyphterWithParams.getRight())) + .orElse(cyphterWithParams); + } + + protected Pair> inlineParameters(Pair> cyphterWithParams) + { + return Optional.of(cyphterWithParams.getRight()) + .filter(p -> !p.isEmpty()) + .map(p -> CypherQueryUtils.inlineParameters(cyphterWithParams.getLeft(), p)) + .map(s -> Pair.of(s, cyphterWithParams.getRight())) + .orElse(cyphterWithParams); + } + + private static class MultiStatementBasedResponse implements Response + { + // This implementation is not good, but it preserved the current + // behaviour while fixing another bug. + // While the statements executed in + // org.neo4j.ogm.drivers.bolt.request.BoltRequest.execute(org.neo4j.ogm.request.DefaultRequest) + // might return different columns, only the ones of the first result are + // used. :( + private final String[] columns; + private final List rowModels; + + private int currentRow = 0; + + MultiStatementBasedResponse(String[] columns, + List rowModels) + { + this.columns = columns; + this.rowModels = rowModels; + } + + @Override + public RowModel next() + { + if (currentRow < rowModels.size()) + { + return rowModels.get(currentRow++); + } + return null; + } + + @Override + public void close() + { + } + + @Override + public String[] columns() + { + return this.columns; + } + } + +} diff --git a/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinGraphRowModelResponse.java b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinGraphRowModelResponse.java new file mode 100644 index 0000000..df447d1 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinGraphRowModelResponse.java @@ -0,0 +1,42 @@ +package org.opencypher.gremlin.neo4j.ogm.response; + +import java.util.Arrays; + +import org.neo4j.driver.v1.StatementResult; +import org.neo4j.ogm.model.GraphRowListModel; +import org.neo4j.ogm.response.model.DefaultGraphRowListModel; +import org.neo4j.ogm.result.adapter.GraphRowModelAdapter; +import org.opencypher.gremlin.neo4j.driver.Neo4jDriverEntityAdapter; + +public class GremlinGraphRowModelResponse extends GremlinResponse +{ + + private final GraphRowModelAdapter adapter; + + public GremlinGraphRowModelResponse(StatementResult result, + Neo4jDriverEntityAdapter entityAdapter) + { + + super(result); + + this.adapter = new GraphRowModelAdapter(new GremlinModelResponse.GremlinGraphModelAdapter(entityAdapter)); + this.adapter.setColumns(Arrays.asList(columns())); + } + + @Override + public GraphRowListModel fetchNext() + { + if (result.hasNext()) + { + DefaultGraphRowListModel model = new DefaultGraphRowListModel(); + model.add(adapter.adapt(result.next().asMap())); + + while (result.hasNext()) + { + model.add(adapter.adapt(result.next().asMap())); + } + return model; + } + return null; + } +} diff --git a/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinModelResponse.java b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinModelResponse.java new file mode 100644 index 0000000..18d9ab0 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinModelResponse.java @@ -0,0 +1,118 @@ +package org.opencypher.gremlin.neo4j.ogm.response; + +import java.util.List; +import java.util.Map; + +import org.neo4j.driver.v1.StatementResult; +import org.neo4j.ogm.model.GraphModel; +import org.neo4j.ogm.result.adapter.GraphModelAdapter; +import org.opencypher.gremlin.neo4j.driver.Neo4jDriverEntityAdapter; + +public class GremlinModelResponse extends GremlinResponse +{ + + private final GremlinGraphModelAdapter adapter; + + public GremlinModelResponse(StatementResult result, + Neo4jDriverEntityAdapter entityAdapter) + { + + super(result); + + this.adapter = new GremlinGraphModelAdapter(entityAdapter); + } + + @Override + public GraphModel fetchNext() + { + if (result.hasNext()) + { + return adapter.adapt(result.next().asMap()); + } + return null; + } + + static class GremlinGraphModelAdapter extends GraphModelAdapter + { + + private final Neo4jDriverEntityAdapter entityAdapter; + + GremlinGraphModelAdapter(Neo4jDriverEntityAdapter entityAdapter) + { + this.entityAdapter = entityAdapter; + } + + @Override + public boolean isPath(Object value) + { + return entityAdapter.isPath(value); + } + + @Override + public boolean isNode(Object value) + { + return entityAdapter.isNode(value); + } + + @Override + public boolean isRelationship(Object value) + { + return entityAdapter.isRelationship(value); + } + + @Override + public long nodeId(Object node) + { + return entityAdapter.nodeId(node); + } + + @Override + public List labels(Object entity) + { + return entityAdapter.labels(entity); + } + + @Override + public long relationshipId(Object relationship) + { + return entityAdapter.relationshipId(relationship); + } + + @Override + public String relationshipType(Object entity) + { + return entityAdapter.relationshipType(entity); + } + + @Override + public Long startNodeId(Object relationship) + { + return entityAdapter.startNodeId(relationship); + } + + @Override + public Long endNodeId(Object relationship) + { + return entityAdapter.endNodeId(relationship); + } + + @Override + public Map properties(Object container) + { + return entityAdapter.properties(container); + } + + @Override + public List nodesInPath(Object pathValue) + { + return entityAdapter.nodesInPath(pathValue); + } + + @Override + public List relsInPath(Object pathValue) + { + return entityAdapter.relsInPath(pathValue); + } + + } +} \ No newline at end of file diff --git a/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinResponse.java b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinResponse.java new file mode 100644 index 0000000..ec5b380 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinResponse.java @@ -0,0 +1,47 @@ +package org.opencypher.gremlin.neo4j.ogm.response; + +import java.util.Set; + +import org.neo4j.driver.v1.Record; +import org.neo4j.driver.v1.StatementResult; +import org.neo4j.ogm.response.Response; + +public abstract class GremlinResponse implements Response +{ + + final StatementResult result; + + GremlinResponse(StatementResult result) + { + this.result = result; + } + + @Override + public T next() + { + return fetchNext(); + } + + protected abstract T fetchNext(); + + @Override + public void close() + { + result.consume(); + } + + @Override + public String[] columns() + { + if (result.hasNext()) + { + Record record = result.peek(); + if (record != null) + { + Set columns = result.peek().asMap().keySet(); + return columns.toArray(new String[columns.size()]); + } + } + return new String[0]; + } +} \ No newline at end of file diff --git a/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinRestModelResponse.java b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinRestModelResponse.java new file mode 100644 index 0000000..d9b792d --- /dev/null +++ b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinRestModelResponse.java @@ -0,0 +1,155 @@ +package org.opencypher.gremlin.neo4j.ogm.response; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.neo4j.driver.v1.Record; +import org.neo4j.driver.v1.StatementResult; +import org.neo4j.ogm.config.ObjectMapperFactory; +import org.neo4j.ogm.model.QueryStatistics; +import org.neo4j.ogm.model.RestModel; +import org.neo4j.ogm.response.model.DefaultRestModel; +import org.neo4j.ogm.response.model.QueryStatisticsModel; +import org.neo4j.ogm.result.adapter.RestModelAdapter; +import org.neo4j.ogm.result.adapter.ResultAdapter; +import org.opencypher.gremlin.neo4j.driver.Neo4jDriverEntityAdapter; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class GremlinRestModelResponse extends GremlinResponse +{ + + private final RestModelAdapter restModelAdapter; + private final QueryStatisticsModel statisticsModel; + private final Iterator resultProjection; + + public GremlinRestModelResponse(StatementResult result, + Neo4jDriverEntityAdapter entityAdapter) + { + + super(result); + + this.restModelAdapter = new GremlinRestModelAdapter(entityAdapter); + this.resultProjection = result.list().iterator(); + this.statisticsModel = new StatisticsModelAdapter().adapt(result); + } + + @Override + public RestModel fetchNext() + { + return DefaultRestModel.basedOn(buildModel()) + .orElse(null); + } + + private Map buildModel() + { + Map row = new LinkedHashMap<>(); + if (resultProjection.hasNext()) + { + row = restModelAdapter.adapt(resultProjection.next().asMap()); + } + + return row; + } + + @Override + public Optional getStatistics() + { + return Optional.of(statisticsModel); + } + + static class GremlinRestModelAdapter extends RestModelAdapter + { + + private final Neo4jDriverEntityAdapter entityAdapter; + + public GremlinRestModelAdapter(Neo4jDriverEntityAdapter entityAdapter) + { + this.entityAdapter = entityAdapter; + } + + @Override + public boolean isNode(Object value) + { + return entityAdapter.isNode(value); + } + + @Override + public boolean isRelationship(Object value) + { + return entityAdapter.isRelationship(value); + } + + @Override + public long nodeId(Object node) + { + return entityAdapter.nodeId(node); + } + + @Override + public List labels(Object value) + { + return entityAdapter.labels(value); + } + + @Override + public long relationshipId(Object relationship) + { + return entityAdapter.relationshipId(relationship); + } + + @Override + public String relationshipType(Object relationship) + { + return entityAdapter.relationshipType(relationship); + } + + @Override + public Long startNodeId(Object relationship) + { + return entityAdapter.startNodeId(relationship); + } + + @Override + public Long endNodeId(Object relationship) + { + return entityAdapter.endNodeId(relationship); + } + + @Override + public Map properties(Object container) + { + return entityAdapter.properties(container); + } + } + + static class StatisticsModelAdapter + implements ResultAdapter + { + + protected static final ObjectMapper mapper = ObjectMapperFactory.objectMapper(); + + @Override + public QueryStatisticsModel adapt(org.neo4j.driver.v1.StatementResult result) + { + QueryStatisticsModel queryStatisticsModel = new QueryStatisticsModel(); + org.neo4j.driver.v1.summary.SummaryCounters stats = result.consume().counters(); + queryStatisticsModel.setContains_updates(stats.containsUpdates()); + queryStatisticsModel.setNodes_created(stats.nodesCreated()); + queryStatisticsModel.setNodes_deleted(stats.nodesDeleted()); + queryStatisticsModel.setProperties_set(stats.propertiesSet()); + queryStatisticsModel.setRelationships_created(stats.relationshipsCreated()); + queryStatisticsModel.setRelationship_deleted(stats.relationshipsDeleted()); + queryStatisticsModel.setLabels_added(stats.labelsAdded()); + queryStatisticsModel.setLabels_removed(stats.labelsRemoved()); + queryStatisticsModel.setIndexes_added(stats.indexesAdded()); + queryStatisticsModel.setIndexes_removed(stats.indexesRemoved()); + queryStatisticsModel.setConstraints_added(stats.constraintsAdded()); + queryStatisticsModel.setConstraints_removed(stats.constraintsRemoved()); + return queryStatisticsModel; + } + } +} diff --git a/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinRowModelResponse.java b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinRowModelResponse.java new file mode 100644 index 0000000..dc6b7e0 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/response/GremlinRowModelResponse.java @@ -0,0 +1,65 @@ +package org.opencypher.gremlin.neo4j.ogm.response; + +import java.util.Arrays; + +import org.neo4j.driver.v1.StatementResult; +import org.neo4j.ogm.model.RowModel; +import org.neo4j.ogm.result.adapter.RowModelAdapter; +import org.opencypher.gremlin.neo4j.driver.Neo4jDriverEntityAdapter; + +public class GremlinRowModelResponse extends GremlinResponse +{ + + private final GremlinRowModelAdapter adapter; + + public GremlinRowModelResponse(StatementResult result, + Neo4jDriverEntityAdapter entityAdapter) + { + + super(result); + + this.adapter = new GremlinRowModelAdapter(entityAdapter); + this.adapter.setColumns(Arrays.asList(columns())); + } + + @Override + public RowModel fetchNext() + { + if (result.hasNext()) + { + return adapter.adapt(result.next().asMap()); + } + return null; + } + + class GremlinRowModelAdapter extends RowModelAdapter + + { + + private final Neo4jDriverEntityAdapter entityAdapter; + + public GremlinRowModelAdapter(Neo4jDriverEntityAdapter entityAdapter) + { + this.entityAdapter = entityAdapter; + } + + @Override + public boolean isPath(Object value) + { + return entityAdapter.isPath(value); + } + + @Override + public boolean isNode(Object value) + { + return entityAdapter.isNode(value); + } + + @Override + public boolean isRelationship(Object value) + { + return entityAdapter.isRelationship(value); + } + + } +} diff --git a/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/transaction/GremlinTransaction.java b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/transaction/GremlinTransaction.java new file mode 100644 index 0000000..23ad431 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/opencypher/gremlin/neo4j/ogm/transaction/GremlinTransaction.java @@ -0,0 +1,120 @@ +package org.opencypher.gremlin.neo4j.ogm.transaction; + +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.neo4j.ogm.exception.TransactionException; +import org.neo4j.ogm.transaction.AbstractTransaction; +import org.neo4j.ogm.transaction.TransactionManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GremlinTransaction extends AbstractTransaction +{ + + private final Logger logger = LoggerFactory.getLogger(GremlinTransaction.class); + + private final Graph nativeTransaction; + + public GremlinTransaction(TransactionManager transactionManager, + Graph nativeTransaction, + Type type) + { + super(transactionManager); + this.type = type; + this.nativeTransaction = nativeTransaction; + } + + public Graph getNativeTransaction() + { + return nativeTransaction; + } + + @Override + public void rollback() + { + try + { + doRollback(); + } + catch (Exception e) + { + throw new TransactionException(e.getLocalizedMessage(), e); + } + finally + { + super.rollback(); + } + } + + protected void doRollback() throws Exception + { + if (!transactionManager.canRollback()) + { + logger.debug("Skip rolback."); + + return; + } + + logger.debug("Rolling back native transaction: {}", nativeTransaction); + + if (!nativeTransaction.tx().isOpen()) + { + logger.warn("Transaction is already closed"); + + return; + } + + try + { + nativeTransaction.tx().rollback(); + nativeTransaction.tx().close(); + } + finally + { + nativeTransaction.close(); + } + } + + @Override + public void commit() + { + try + { + doCommit(); + } + catch (Exception e) + { + super.rollback(); + throw new TransactionException(e.getLocalizedMessage(), e); + } + + super.commit(); + } + + protected void doCommit() throws Exception + { + if (!transactionManager.canCommit()) + { + logger.debug("Skip commit."); + + return; + } + + if (!nativeTransaction.tx().isOpen()) + { + throw new IllegalStateException("Transaction is already closed"); + } + + logger.debug("Committing native transaction: {}", nativeTransaction); + + try + { + nativeTransaction.tx().commit(); + nativeTransaction.tx().close(); + } + finally + { + nativeTransaction.close(); + } + } + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/CassandraEmbeddedConfiguration.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/CassandraEmbeddedConfiguration.java new file mode 100644 index 0000000..b41f566 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/CassandraEmbeddedConfiguration.java @@ -0,0 +1,10 @@ +package org.strongbox.db.server; + +public interface CassandraEmbeddedConfiguration +{ + + String getStorageRoot(); + + String getConfigLocation(); + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/CassandraEmbeddedProperties.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/CassandraEmbeddedProperties.java new file mode 100644 index 0000000..d4648fc --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/CassandraEmbeddedProperties.java @@ -0,0 +1,29 @@ +package org.strongbox.db.server; + +public class CassandraEmbeddedProperties + implements CassandraEmbeddedConfiguration +{ + + private final String storageRoot; + private final String configLocation; + + public CassandraEmbeddedProperties(String storageRoot, + String configLocation) + { + this.storageRoot = storageRoot; + this.configLocation = configLocation; + } + + @Override + public String getStorageRoot() + { + return storageRoot; + } + + @Override + public String getConfigLocation() + { + return configLocation; + } + +} \ No newline at end of file diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/EmbeddedDbServer.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/EmbeddedDbServer.java new file mode 100644 index 0000000..a720434 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/EmbeddedDbServer.java @@ -0,0 +1,10 @@ +package org.strongbox.db.server; + +public interface EmbeddedDbServer +{ + + void start() throws Exception; + + void stop() throws Exception; + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/EmbeddedOrientDbServer.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/EmbeddedOrientDbServer.java deleted file mode 100644 index 8247a1c..0000000 --- a/strongbox-db-server/src/main/java/org/strongbox/db/server/EmbeddedOrientDbServer.java +++ /dev/null @@ -1,283 +0,0 @@ -package org.strongbox.db.server; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.JarURLConnection; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Enumeration; -import java.util.LinkedList; -import java.util.List; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.orientechnologies.orient.core.config.OGlobalConfiguration; -import com.orientechnologies.orient.graph.server.command.OServerCommandGetGephi; -import com.orientechnologies.orient.server.OServer; -import com.orientechnologies.orient.server.OServerMain; -import com.orientechnologies.orient.server.config.OServerCommandConfiguration; -import com.orientechnologies.orient.server.config.OServerConfiguration; -import com.orientechnologies.orient.server.config.OServerEntryConfiguration; -import com.orientechnologies.orient.server.config.OServerNetworkConfiguration; -import com.orientechnologies.orient.server.config.OServerNetworkListenerConfiguration; -import com.orientechnologies.orient.server.config.OServerNetworkProtocolConfiguration; -import com.orientechnologies.orient.server.config.OServerParameterConfiguration; -import com.orientechnologies.orient.server.config.OServerUserConfiguration; -import com.orientechnologies.orient.server.network.protocol.http.command.get.OServerCommandGetStaticContent; - -/** - * An embedded configuration of OrientDb server. - * - * @author Alex Oreshkevich - */ -public class EmbeddedOrientDbServer - implements OrientDbServer -{ - - private static final Logger logger = LoggerFactory.getLogger(EmbeddedOrientDbServer.class); - - public static final String ORIENTDB_STUDIO_VERSION = "2.2.0"; - - private OServer server; - private OServerConfiguration serverConfiguration; - - private OrientDbStudioConfiguration studioProperties; - private OrientDbServerConfiguration serverProperties; - - public EmbeddedOrientDbServer(OrientDbStudioConfiguration studioProperties, - OrientDbServerConfiguration serverProperties) - { - super(); - - this.studioProperties = studioProperties; - this.serverProperties = serverProperties; - } - - @PostConstruct - public void start() - { - try - { - init(); - prepareStudio(); - activate(); - } - catch (Exception e) - { - throw new RuntimeException("Unable to start the embedded OrientDb server!", e); - } - } - - public JarFile getStudioClasspathLocation() - throws IOException - { - URL systemResource = OServer.class.getResource(String.format("/META-INF/resources/webjars/orientdb-studio/%s", - ORIENTDB_STUDIO_VERSION)); - JarURLConnection connection = (JarURLConnection) systemResource.openConnection(); - - return connection.getJarFile(); - } - - private void prepareStudio() - throws IOException - { - if (!studioProperties.isEnabled()) - { - logger.info("OrientDB Studio disabled with, skip initialization."); - - return; - } - - OServerNetworkListenerConfiguration httpListener = new OServerNetworkListenerConfiguration(); - httpListener.ipAddress = studioProperties.getIpAddress(); - httpListener.portRange = String.valueOf(studioProperties.getPort()); - httpListener.protocol = "http"; - httpListener.socket = "default"; - - OServerCommandConfiguration httpCommandConfiguration1 = new OServerCommandConfiguration(); - httpCommandConfiguration1.implementation = OServerCommandGetStaticContent.class.getCanonicalName(); - httpCommandConfiguration1.pattern = "GET|www GET|studio/ GET| GET|*.htm GET|*.html GET|*.xml GET|*.jpeg GET|*.jpg GET|*.png GET|*.gif GET|*.js GET|*.css GET|*.swf GET|*.ico GET|*.txt GET|*.otf GET|*.pjs GET|*.svg GET|*.json GET|*.woff GET|*.ttf GET|*.svgz"; - httpCommandConfiguration1.stateful = false; - httpCommandConfiguration1.parameters = new OServerEntryConfiguration[] { new OServerEntryConfiguration( - "http.cache:*.htm *.html", - "Cache-Control: no-cache, no-store, max-age=0, must-revalidate\\r\\nPragma: no-cache"), - new OServerEntryConfiguration( - "http.cache:default", - "Cache-Control: max-age=120") }; - - OServerCommandConfiguration httpCommandConfiguration2 = new OServerCommandConfiguration(); - httpCommandConfiguration2.implementation = OServerCommandGetGephi.class.getCanonicalName(); - - httpListener.commands = new OServerCommandConfiguration[] { httpCommandConfiguration1, - httpCommandConfiguration2 }; - httpListener.parameters = new OServerParameterConfiguration[] { new OServerParameterConfiguration("utf-8", - "network.http.charset") }; - - serverConfiguration.network.listeners.add(httpListener); - - OServerNetworkProtocolConfiguration httpProtocol = new OServerNetworkProtocolConfiguration(); - httpProtocol.name = "http"; - httpProtocol.implementation = "com.orientechnologies.orient.server.network.protocol.http.ONetworkProtocolHttpDb"; - - serverConfiguration.network.protocols.add(httpProtocol); - - Path studioPath = resolvePath(studioProperties.getPath()).resolve("studio"); - if (Files.exists(studioPath)) - { - logger.info(String.format("OrientDB Studio is already available at [%s], skipping initialization. %n" + - "If you want to force the initialization of OrientDB Studio, please remove it's " + - "folder shown above.", - studioPath.toAbsolutePath().toString())); - return; - } - - logger.info(String.format("Initialized OrientDB Studio at [%s].", studioPath.toAbsolutePath().toString())); - - Files.createDirectories(studioPath); - - String root = String.format("META-INF/resources/webjars/orientdb-studio/%s/", ORIENTDB_STUDIO_VERSION); - - try (JarFile jar = getStudioClasspathLocation()) - { - Enumeration enumEntries = jar.entries(); - while (enumEntries.hasMoreElements()) - { - JarEntry file = enumEntries.nextElement(); - if (!file.getName().startsWith(root)) - { - continue; - } - - Path filePath = studioPath.resolve(file.getName().replace(root, "")); - if (file.isDirectory()) - { - Files.createDirectories(filePath); - continue; - } - - try (InputStream is = jar.getInputStream(file)) - { - try (FileOutputStream fos = new java.io.FileOutputStream(filePath.toFile())) - { - while (is.available() > 0) - { - fos.write(is.read()); - } - } - } - } - } - } - - protected Path resolvePath(String path) - { - return Paths.get(path).toAbsolutePath().normalize(); - } - - private void init() - throws Exception - { - logger.info(String.format("Initialized Embedded OrientDB server.")); - - // Don't touch below line. Don't move it down the code. It needs to be - // called before OServerMain.create() - // the size is in Kb - OGlobalConfiguration.NETWORK_BINARY_MAX_CONTENT_LENGTH.setValue(65536); - - - server = OServerMain.create(); - serverConfiguration = new OServerConfiguration(); - - // OServerHookConfiguration hookConfiguration = new - // OServerHookConfiguration(); - // serverConfiguration.hooks = Arrays.asList(new - // OServerHookConfiguration[] { hookConfiguration }); - // hookConfiguration.clazz = GenericEntityHook.class.getName(); - - OServerNetworkListenerConfiguration binaryListener = new OServerNetworkListenerConfiguration(); - binaryListener.ipAddress = serverProperties.getHost(); - binaryListener.portRange = serverProperties.getPort(); - binaryListener.protocol = "binary"; - binaryListener.socket = "default"; - - OServerNetworkProtocolConfiguration binaryProtocol = new OServerNetworkProtocolConfiguration(); - binaryProtocol.name = "binary"; - binaryProtocol.implementation = "com.orientechnologies.orient.server.network.protocol.binary.ONetworkProtocolBinary"; - - // prepare network configuration - OServerNetworkConfiguration networkConfiguration = new OServerNetworkConfiguration(); - - networkConfiguration.protocols = new LinkedList<>(); - networkConfiguration.protocols.add(binaryProtocol); - - networkConfiguration.listeners = new LinkedList<>(); - networkConfiguration.listeners.add(binaryListener); - - // add users (incl system-level root user) - List users = new LinkedList<>(); - users.add(buildUser(serverProperties.getUsername(), serverProperties.getPassword(), "*")); - - System.setProperty("ORIENTDB_ROOT_PASSWORD", serverProperties.getPassword()); - - String serverDatabasePath = resolvePath(serverProperties.getPath()).toString(); - // add other properties - List properties = new LinkedList<>(); - properties.add(buildProperty("server.database.path", serverDatabasePath)); - properties.add(buildProperty("plugin.dynamic", "false")); - properties.add(buildProperty("log.console.level", "info")); - properties.add(buildProperty("orientdb.www.path", serverDatabasePath)); - - serverConfiguration.network = networkConfiguration; - serverConfiguration.users = users.toArray(new OServerUserConfiguration[users.size()]); - serverConfiguration.properties = properties.toArray(new OServerEntryConfiguration[properties.size()]); - } - - private void activate() - throws Exception - { - if (!server.isActive()) - { - server.startup(serverConfiguration); - server.activate(); - } - } - - private OServerUserConfiguration buildUser(String name, - String password, - String resources) - { - OServerUserConfiguration user = new OServerUserConfiguration(); - user.name = name; - user.password = password; - user.resources = resources; - - return user; - } - - private OServerEntryConfiguration buildProperty(String name, - String value) - { - OServerEntryConfiguration property = new OServerEntryConfiguration(); - property.name = name; - property.value = value; - - return property; - } - - @PreDestroy - @Override - public void stop() - { - server.shutdown(); - } - -} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/InMemoryJanusGraphServer.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/InMemoryJanusGraphServer.java new file mode 100644 index 0000000..3bc7f1d --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/InMemoryJanusGraphServer.java @@ -0,0 +1,17 @@ +package org.strongbox.db.server; + +import java.util.function.Supplier; + +/** + * @author sbespalov + */ +public class InMemoryJanusGraphServer extends JanusGraphServer +{ + + public InMemoryJanusGraphServer(JanusGraphConfiguration janusGraphProperties, + Supplier idBlockQueueSupplier) + { + super(janusGraphProperties, idBlockQueueSupplier); + } + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphConfiguration.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphConfiguration.java new file mode 100644 index 0000000..85bf03c --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphConfiguration.java @@ -0,0 +1,8 @@ +package org.strongbox.db.server; + +public interface JanusGraphConfiguration +{ + + String getConfigLocation(); + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphProperties.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphProperties.java new file mode 100644 index 0000000..4fed70f --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphProperties.java @@ -0,0 +1,19 @@ +package org.strongbox.db.server; + +public class JanusGraphProperties implements JanusGraphConfiguration +{ + + private final String configLocation; + + public JanusGraphProperties(String configLocation) + { + this.configLocation = configLocation; + } + + @Override + public String getConfigLocation() + { + return configLocation; + } + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphServer.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphServer.java new file mode 100644 index 0000000..c0d82b3 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphServer.java @@ -0,0 +1,139 @@ +package org.strongbox.db.server; + +import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.function.Supplier; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.janusgraph.core.JanusGraph; +import org.janusgraph.diskstorage.configuration.ReadConfiguration; +import org.janusgraph.diskstorage.configuration.backend.CommonsConfiguration; +import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; +import org.janusgraph.graphdb.configuration.builder.GraphDatabaseConfigurationBuilder; +import org.janusgraph.graphdb.database.StandardJanusGraph; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.strongbox.db.server.janusgraph.CustomGraphDatabaseConfiguration; + +/** + * @author sbespalov + * + */ +public abstract class JanusGraphServer implements EmbeddedDbServer +{ + private static final Logger logger = LoggerFactory.getLogger(JanusGraphServer.class); + + private final JanusGraphConfiguration janusGraphProperties; + private final Supplier idBlockQueueSupplier; + private volatile JanusGraph janusGraph; + + public JanusGraphServer(JanusGraphConfiguration janusGraphProperties, Supplier idBlockQueueSupplier) + { + this.janusGraphProperties = janusGraphProperties; + this.idBlockQueueSupplier = idBlockQueueSupplier; + } + + public JanusGraph getJanusGraph() + { + return janusGraph; + } + + @PostConstruct + public synchronized void start() + throws Exception + { + try + { + provideJanusGraphInstance(); + } + catch (Exception e) + { + stop(); + throw new RuntimeException("Unable to start the embedded DB server!", e); + } + } + + protected JanusGraph provideJanusGraphInstance() + throws Exception + { + if (janusGraph != null) + { + return janusGraph; + } + + JanusGraph janusGraphLocal = buildJanusGraph(janusGraphProperties); + + try + { + Method removeHookMethod = StandardJanusGraph.class.getDeclaredMethod("removeHook"); + removeHookMethod.setAccessible(true); + removeHookMethod.invoke(janusGraphLocal); + } + catch (Exception e) + { + janusGraphLocal.close(); + throw e; + } + + return this.janusGraph = janusGraphLocal; + } + + protected JanusGraph buildJanusGraph(JanusGraphConfiguration configuration) + { + String configLocation = configuration.getConfigLocation(); + try + { + URL configLocationUrl = new URL(configLocation); + ReadConfiguration janusGraphLocalConfig = new CommonsConfiguration(new PropertiesConfiguration(configLocationUrl)); + GraphDatabaseConfigurationBuilder configBuilder = new GraphDatabaseConfigurationBuilder(); + GraphDatabaseConfiguration janusGraphDbConfig = configBuilder.build(janusGraphLocalConfig); + + return new StandardJanusGraph(new CustomGraphDatabaseConfiguration(janusGraphDbConfig, idBlockQueueSupplier)); + } + catch (MalformedURLException|ConfigurationException e) + { + throw new RuntimeException(String.format("Invalid configuration [%s].", configLocation), e); + } + } + + @PreDestroy + @Override + public synchronized void stop() + throws Exception + { + try + { + closeJanusGraph(); + } + catch (Exception e) + { + logger.error("Failed to close JanusGraph", e); + } + + logger.info("JanusGraph instance stopped!"); + } + + private void closeJanusGraph() + { + if (janusGraph == null) + { + logger.info("Skip closing JanusGraph.."); + return; + } + logger.info("Shutting JanusGraph instance.."); + try + { + janusGraph.close(); + } + finally + { + janusGraph = null; + } + } + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphWithEmbeddedCassandra.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphWithEmbeddedCassandra.java new file mode 100644 index 0000000..19f4a3a --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphWithEmbeddedCassandra.java @@ -0,0 +1,129 @@ +package org.strongbox.db.server; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.function.Supplier; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.apache.cassandra.service.CassandraDaemon; +import org.apache.cassandra.service.StorageService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.strongbox.db.server.cassandra.CassandraEmbeddedPropertiesLoader; + +/** + * Embedded JanusGraph+Cassandra server + * + * @author sbespalov + */ +public class JanusGraphWithEmbeddedCassandra extends JanusGraphWithRemoteCassandra +{ + + private static final Logger logger = LoggerFactory.getLogger(JanusGraphWithEmbeddedCassandra.class); + + private final CassandraEmbeddedConfiguration cassandraEmbeddedProperties; + private volatile CassandraDaemon cassandraDaemon; + + public JanusGraphWithEmbeddedCassandra(CassandraEmbeddedConfiguration cassandraEmbeddedProperties, + JanusGraphConfiguration janusGraphProperties, + Supplier idBlockQueueSupplier) + { + super(janusGraphProperties, idBlockQueueSupplier); + this.cassandraEmbeddedProperties = cassandraEmbeddedProperties; + } + + public CassandraDaemon getCassandraDaemon() + { + return cassandraDaemon; + } + + @PostConstruct + public synchronized void start() + throws Exception + { + try + { + provideCassandraInstance(); + } + catch (Exception e) + { + stop(); + throw new RuntimeException("Unable to start the embedded DB server!", e); + } + + super.start(); + } + + private CassandraDaemon provideCassandraInstance() + { + if (cassandraDaemon != null) + { + return cassandraDaemon; + } + + Class configLoaderClass = CassandraEmbeddedPropertiesLoader.bootstrap(cassandraEmbeddedProperties); + System.setProperty("cassandra.config.loader", configLoaderClass.getName()); + + System.setProperty("cassandra-foreground", "true"); + System.setProperty("cassandra.native.epoll.enabled", "true"); + System.setProperty("cassandra.unsafesystem", "true"); + + CassandraDaemon cassandraDaemonLocal = new CassandraDaemon(true); + cassandraDaemonLocal.activate(); + + StorageService.instance.removeShutdownHook(); + + return this.cassandraDaemon = cassandraDaemonLocal; + } + + @PreDestroy + @Override + public synchronized void stop() + throws Exception + { + super.stop(); + + try + { + closeCassandra(); + } + catch (Exception e) + { + logger.error("Failed to close Cassandra", e); + } + + logger.info("Cassandra embedded service stopped!"); + } + + private void closeCassandra() + throws IOException, + InterruptedException, + ExecutionException + { + if (cassandraDaemon == null) + { + logger.info("Skip closing cassandra daemon.."); + return; + } + logger.info("Shutting down cassandra daemon.."); + try + { + StorageService.instance.drain(); + } + catch (Exception e) + { + logger.error("Failed to drain cassandra service.", e); + } + try + { + cassandraDaemon.deactivate(); + } + finally + { + cassandraDaemon = null; + } + } + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphWithRemoteCassandra.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphWithRemoteCassandra.java new file mode 100644 index 0000000..4ed344a --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/JanusGraphWithRemoteCassandra.java @@ -0,0 +1,44 @@ +package org.strongbox.db.server; + +import java.util.function.Supplier; + +import org.janusgraph.core.JanusGraph; +import org.janusgraph.core.JanusGraphFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author sbespalov + */ +public class JanusGraphWithRemoteCassandra extends JanusGraphServer +{ + + private static final Logger logger = LoggerFactory.getLogger(JanusGraphWithEmbeddedCassandra.class); + + public JanusGraphWithRemoteCassandra(JanusGraphConfiguration janusGraphProperties, + Supplier idBlockQueueSupplier) + { + super(janusGraphProperties, idBlockQueueSupplier); + } + + @Override + protected JanusGraph provideJanusGraphInstance() + throws Exception + { + for (int i = 0; i < 20; i++) + { + try + { + return super.provideJanusGraphInstance(); + } + catch (Exception e) + { + Thread.sleep(500); + logger.info(String.format("Retry JanusGraph instance initialization with [%s] attempt...", i)); + } + } + + return super.provideJanusGraphInstance(); + } + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbServer.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbServer.java deleted file mode 100644 index 276c9cf..0000000 --- a/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbServer.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.strongbox.db.server; - -public interface OrientDbServer -{ - - void start(); - - void stop(); - -} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbServerConfiguration.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbServerConfiguration.java deleted file mode 100644 index b750679..0000000 --- a/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbServerConfiguration.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.strongbox.db.server; - -public interface OrientDbServerConfiguration -{ - - String getUsername(); - - String getPassword(); - - String getProtocol(); - - String getHost(); - - String getPort(); - - String getDatabase(); - - String getPath(); - - String getUrl(); - -} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbServerProperties.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbServerProperties.java deleted file mode 100644 index a717d0c..0000000 --- a/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbServerProperties.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.strongbox.db.server; - -public class OrientDbServerProperties implements OrientDbServerConfiguration -{ - - private String username; - private String password; - private String protocol; - private String host; - private String port; - private String database; - private String path; - - @Override - public String getUsername() - { - return username; - } - - public void setUsername(String username) - { - this.username = username; - } - - @Override - public String getPassword() - { - return password; - } - - public void setPassword(String password) - { - this.password = password; - } - - @Override - public String getProtocol() - { - return protocol; - } - - public void setProtocol(String protocol) - { - this.protocol = protocol; - } - - @Override - public String getHost() - { - return host; - } - - public void setHost(String host) - { - this.host = host; - } - - @Override - public String getPort() - { - return port; - } - - public void setPort(String port) - { - this.port = port; - } - - @Override - public String getDatabase() - { - return database; - } - - public void setDatabase(String database) - { - this.database = database; - } - - @Override - public String getPath() - { - return path; - } - - public void setPath(String path) - { - this.path = path; - } - - @Override - public String getUrl() - { - if ("memory".equals(protocol)) - { - return String.format("%s:%s", protocol, database); - } - - StringBuilder sb = new StringBuilder(); - sb.append(protocol); - if (host != null) - { - sb.append(":").append(host); - } - if (port != null) - { - sb.append(":").append(port); - } - sb.append("/").append(database); - - return sb.toString(); - } - -} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbStudioConfiguration.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbStudioConfiguration.java deleted file mode 100644 index 7444ab8..0000000 --- a/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbStudioConfiguration.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.strongbox.db.server; - -public interface OrientDbStudioConfiguration -{ - - boolean isEnabled(); - - String getIpAddress(); - - int getPort(); - - String getPath(); - -} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbStudioProperties.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbStudioProperties.java deleted file mode 100644 index d285304..0000000 --- a/strongbox-db-server/src/main/java/org/strongbox/db/server/OrientDbStudioProperties.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.strongbox.db.server; - -public class OrientDbStudioProperties implements OrientDbStudioConfiguration -{ - - private boolean enabled; - private String ipAddress; - private int port; - private String path; - - /* (non-Javadoc) - * @see org.strongbox.db.server.OrientDbStudioConfiguration#isEnabled() - */ - @Override - public boolean isEnabled() - { - return enabled; - } - - public void setEnabled(boolean studioEnabled) - { - this.enabled = studioEnabled; - } - - /* (non-Javadoc) - * @see org.strongbox.db.server.OrientDbStudioConfiguration#getIpAddress() - */ - @Override - public String getIpAddress() - { - return ipAddress; - } - - public void setIpAddress(String studioIpAddress) - { - this.ipAddress = studioIpAddress; - } - - /* (non-Javadoc) - * @see org.strongbox.db.server.OrientDbStudioConfiguration#getPort() - */ - @Override - public int getPort() - { - return port; - } - - public void setPort(int studioPort) - { - this.port = studioPort; - } - - /* (non-Javadoc) - * @see org.strongbox.db.server.OrientDbStudioConfiguration#getPath() - */ - @Override - public String getPath() - { - return path; - } - - public void setPath(String studioPath) - { - this.path = studioPath; - } - -} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/cassandra/CassandraEmbeddedPropertiesLoader.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/cassandra/CassandraEmbeddedPropertiesLoader.java new file mode 100644 index 0000000..743c5ae --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/cassandra/CassandraEmbeddedPropertiesLoader.java @@ -0,0 +1,66 @@ +package org.strongbox.db.server.cassandra; + +import java.net.MalformedURLException; +import java.net.URL; + +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.YamlConfigurationLoader; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.strongbox.db.server.CassandraEmbeddedConfiguration; + +public class CassandraEmbeddedPropertiesLoader extends YamlConfigurationLoader +{ + + private static CassandraEmbeddedConfiguration configuration; + + public static synchronized Class bootstrap(CassandraEmbeddedConfiguration configuration) + { + if (CassandraEmbeddedPropertiesLoader.configuration != null) + { + throw new IllegalStateException(); + } + + CassandraEmbeddedPropertiesLoader.configuration = configuration; + + return CassandraEmbeddedPropertiesLoader.class; + } + + @Override + public Config loadConfig() + throws ConfigurationException + { + CassandraEmbeddedConfiguration configurationLocal; + if ((configurationLocal = CassandraEmbeddedPropertiesLoader.configuration) == null) + { + throw new IllegalStateException(); + } + + Config config; + try + { + config = super.loadConfig(new URL(configurationLocal.getConfigLocation())); + } + catch (MalformedURLException e) + { + throw new ConfigurationException(e.getMessage(), e); + } + + String storageFolder = configuration.getStorageRoot(); + // ------------------------ + // Storage locations + // ------------------------ + config.data_file_directories = new String[] { String.format("%s/cassandra/data", storageFolder) }; + config.saved_caches_directory = String.format("%s/cassandra/saved_caches", storageFolder); + config.commitlog_directory = String.format("%s/cassandra/commitlog", storageFolder); + // CommitLogSegments are moved to this directory on flush if + // cdc_enabled: true and the segment contains + // mutations for a CDC-enabled table. This should be placed on a + // separate spindle than the data directories. + // If not set, the default directory is $CASSANDRA_HOME/data/cdc_raw. + config.cdc_raw_directory = String.format("%s/cassandra/cdc", storageFolder); + config.hints_directory = String.format("%s/cassandra/hints", storageFolder); + + return config; + } + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/janusgraph/CustomGraphDatabaseConfiguration.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/janusgraph/CustomGraphDatabaseConfiguration.java new file mode 100644 index 0000000..49ecf1e --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/janusgraph/CustomGraphDatabaseConfiguration.java @@ -0,0 +1,38 @@ +package org.strongbox.db.server.janusgraph; + +import java.util.function.Supplier; + +import org.janusgraph.diskstorage.Backend; +import org.janusgraph.diskstorage.configuration.BasicConfiguration; +import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; +import org.janusgraph.diskstorage.configuration.backend.CommonsConfiguration; +import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; +import org.janusgraph.graphdb.database.idassigner.VertexIDAssigner; + +/** + * @author sbespalov + */ +public class CustomGraphDatabaseConfiguration extends GraphDatabaseConfiguration +{ + + private final Supplier idBlockQueueSupplier; + + public CustomGraphDatabaseConfiguration(GraphDatabaseConfiguration target, Supplier idBlockQueueSupplier) + { + super(new CommonsConfiguration(target.getConfigurationAtOpen()), + new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS, + new CommonsConfiguration(target.getLocalConfiguration()), + BasicConfiguration.Restriction.NONE), + target.getUniqueGraphId(), + target.getConfiguration()); + + this.idBlockQueueSupplier = idBlockQueueSupplier; + } + + @Override + public VertexIDAssigner getIDAssigner(Backend backend) + { + return new TransactionalVertexIDAssigner(getConfiguration(), backend.getIDAuthority(), backend.getStoreFeatures(), idBlockQueueSupplier); + } + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/db/server/janusgraph/TransactionalVertexIDAssigner.java b/strongbox-db-server/src/main/java/org/strongbox/db/server/janusgraph/TransactionalVertexIDAssigner.java new file mode 100644 index 0000000..cd2f369 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/db/server/janusgraph/TransactionalVertexIDAssigner.java @@ -0,0 +1,177 @@ +package org.strongbox.db.server.janusgraph; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.function.Supplier; + +import org.janusgraph.core.VertexLabel; +import org.janusgraph.diskstorage.BackendException; +import org.janusgraph.diskstorage.IDAuthority; +import org.janusgraph.diskstorage.IDBlock; +import org.janusgraph.diskstorage.configuration.Configuration; +import org.janusgraph.diskstorage.keycolumnvalue.KeyRange; +import org.janusgraph.diskstorage.keycolumnvalue.StoreFeatures; +import org.janusgraph.graphdb.database.idassigner.IDBlockSizer; +import org.janusgraph.graphdb.database.idassigner.IDPool; +import org.janusgraph.graphdb.database.idassigner.VertexIDAssigner; +import org.janusgraph.graphdb.idmanagement.IDManager; +import org.janusgraph.graphdb.internal.InternalRelation; +import org.janusgraph.graphdb.internal.InternalVertex; + +/** + * Provides separate {@link VertexIDAssigner} instances for `idBlockQueue`s + * which is usually supplied as part of a transaction. This allows for separate + * {@link IDPool} for different application threads to eliminate the influence + * of threads on each other. + * + * @author sbespalov + */ +public class TransactionalVertexIDAssigner extends VertexIDAssigner +{ + + public static final String QUEUE_DEFAULT = "default"; + public static final String TX_RESOURCE_QUEUE = TransactionalVertexIDAssigner.class.getName() + ".queue"; + + private Map queueMap = new ConcurrentHashMap<>(); + private final Function newInstanceFactory; + private final Supplier idBlockQueueSupplier; + + /** + * This will create 3 groups of VertexIDAssigner - default, cron-job and super (since we're invoking the super constructor) + */ + public TransactionalVertexIDAssigner(Configuration config, + IDAuthority idAuthority, + StoreFeatures idAuthFeatures, + Supplier idBlockQueueSupplier) + { + super(config, idAuthority, idAuthFeatures); + newInstanceFactory = (queueId) -> new VertexIDAssigner(config, new ReadOnlyIDAuthority(idAuthority), idAuthFeatures); + this.idBlockQueueSupplier = idBlockQueueSupplier; + } + + @Override + public IDManager getIDManager() + { + return super.getIDManager(); + } + + @Override + public synchronized void close() + { + queueMap.values().forEach(VertexIDAssigner::close); + // close any connections made by the VertexIDAssigner + super.close(); + } + + @Override + public void assignID(InternalRelation relation) + { + VertexIDAssigner currentIDAssigner = getCurrentIDAssigner(); + if (currentIDAssigner == this) + { + super.assignID(relation); + } + else + { + currentIDAssigner.assignID(relation); + } + } + + @Override + public void assignID(InternalVertex vertex, + VertexLabel label) + { + VertexIDAssigner currentIDAssigner = getCurrentIDAssigner(); + if (currentIDAssigner == this) + { + super.assignID(vertex, label); + } + else + { + currentIDAssigner.assignID(vertex, label); + } + } + + @Override + public void assignIDs(Iterable addedRelations) + { + VertexIDAssigner currentIDAssigner = getCurrentIDAssigner(); + if (currentIDAssigner == this) + { + super.assignIDs(addedRelations); + } + else + { + currentIDAssigner.assignIDs(addedRelations); + } + } + + protected VertexIDAssigner getCurrentIDAssigner() + { + + return getCurrentQueueId().map((id) -> queueMap.computeIfAbsent(id, newInstanceFactory)).orElse(this); + } + + private Optional getCurrentQueueId() + { + return Optional.ofNullable(idBlockQueueSupplier.get()); + } + + /** + * This wrapper class needed to avoid + * {@link IDAuthority#setIDBlockSizer(IDBlockSizer)} call within + * {@link VertexIDAssigner} constructor which fails for additional + * {@link VertexIDAssigner} instances attached to `idBlockQueue`s. + * + * @author sbespalov + */ + private class ReadOnlyIDAuthority implements IDAuthority + { + + final IDAuthority target; + + public ReadOnlyIDAuthority(IDAuthority target) + { + this.target = target; + } + + public IDBlock getIDBlock(int partition, + int idNamespace, + Duration timeout) + throws BackendException + { + return target.getIDBlock(partition, idNamespace, timeout); + } + + public List getLocalIDPartition() + throws BackendException + { + return target.getLocalIDPartition(); + } + + public void setIDBlockSizer(IDBlockSizer sizer) + { + + } + + public void close() + throws BackendException + { + } + + public String getUniqueID() + { + return target.getUniqueID(); + } + + public boolean supportsInterruption() + { + return target.supportsInterruption(); + } + + } +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/util/Commons.java b/strongbox-db-server/src/main/java/org/strongbox/util/Commons.java new file mode 100644 index 0000000..4bb8379 --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/util/Commons.java @@ -0,0 +1,49 @@ +package org.strongbox.util; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Date; + +public class Commons +{ + + public static LocalDateTime toLocalDateTime(Date date) + { + if (date == null) + { + return null; + } + return date.toInstant() + .atZone(ZoneId.systemDefault()) + .toLocalDateTime(); + } + + public static Date toDate(LocalDateTime date) + { + if (date == null) + { + return null; + } + return Date.from(date.atZone(ZoneId.systemDefault()).toInstant()); + } + + public static LocalDateTime toLocalDateTime(Long value) + { + if (value == null) + { + return null; + } + return Instant.ofEpochMilli(value).atZone(ZoneId.systemDefault()).toLocalDateTime(); + } + + public static Long toLong(LocalDateTime date) + { + if (date == null) + { + return null; + } + return date.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + } + +} diff --git a/strongbox-db-server/src/main/java/org/strongbox/util/ConfigurationUtils.java b/strongbox-db-server/src/main/java/org/strongbox/util/ConfigurationUtils.java new file mode 100644 index 0000000..ba99bfd --- /dev/null +++ b/strongbox-db-server/src/main/java/org/strongbox/util/ConfigurationUtils.java @@ -0,0 +1,33 @@ +package org.strongbox.util; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; + +import org.strongbox.db.server.JanusGraphServer; + +/** + * @author sbespalov + * + */ +public class ConfigurationUtils +{ + + public static void extractConfigurationFile(String rootFolder, + String configFileName) + throws IOException + { + try (InputStream in = JanusGraphServer.class.getResourceAsStream(String.format("/etc/conf/%s", configFileName))) + { + Path configRoot = Paths.get(rootFolder).resolve("etc").resolve("conf"); + Files.createDirectories(configRoot); + Files.copy(in, + configRoot.resolve(configFileName), + StandardCopyOption.REPLACE_EXISTING); + } + } + +} diff --git a/strongbox-db-server/src/main/resources/etc/conf/cassandra.yaml b/strongbox-db-server/src/main/resources/etc/conf/cassandra.yaml new file mode 100644 index 0000000..dcb4000 --- /dev/null +++ b/strongbox-db-server/src/main/resources/etc/conf/cassandra.yaml @@ -0,0 +1,188 @@ +# ------------------------ +# Generic configuration +# ------------------------ +cluster_name: Test Cluster +snapshot_before_compaction: false +auto_snapshot: false + +## ------------------------ +## Security +## ------------------------ +authenticator: PasswordAuthenticator +authorizer: AllowAllAuthorizer +permissions_validity_in_ms: 2000 + +## ------------------------ +## Network +## ------------------------ +## start_rpc - disable thrift - it's deprecated and should not be used! +listen_address: 127.0.0.1 +storage_port: 7010 +ssl_storage_port: 7011 +start_native_transport: true +native_transport_port: 49142 +native_transport_max_threads: 256 + +## Coordinator related +read_request_timeout_in_ms: 5000 +range_request_timeout_in_ms: 10000 +write_request_timeout_in_ms: 2000 +cas_contention_timeout_in_ms: 1000 +truncate_request_timeout_in_ms: 60000 +request_timeout_in_ms: 10000 +cross_node_timeout: false + +## Addresses of hosts that are deemed contact points. +## Cassandra nodes use this list of hosts to find each other and learn +## the topology of the ring. You must change this if you are running +## multiple nodes! +## - seeds is actually a comma-delimited list of addresses. +## Ex: "seeds: ,," +seed_provider: + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + - seeds: "127.0.0.1" + +## ------------------------ +## Disk +## ------------------------ +disk_optimization_strategy: ssd +disk_failure_policy: stop +## trickle_fsync is good on SSDs, probably bad on hdd +trickle_fsync: true +trickle_fsync_interval_in_kb: 10240 +concurrent_compactors: 2 +## compaction_throughput_mb_per_sec - 128+ for SSD, 16 for hdd +compaction_throughput_mb_per_sec: 128 +## - Usually (16 × number_of_drives) +concurrent_reads: 32 +## - Writes in Cassandra are rarely I/O bound, so the ideal number of +## concurrent writes depends on the number +## of CPU cores on the node. The recommended value is 8 × +## number_of_cpu_cores. +concurrent_writes: 16 +## - Counter writes read the current values before incrementing and +## writing them back. +## The recommended value is (16 × number_of_drives). +concurrent_counter_writes: 32 +## - Limit on the number of concurrent materialized view writes. Set +## this to the lesser of concurrent reads or +## concurrent writes, because there is a read involved in each +## materialized view write. (Default: 32) +concurrent_materialized_view_writes: 32 + +## - (Default 1024KB ) Total maximum throttle for replaying hints. +## Throttling is reduced proportionally to +## the number of nodes in the cluster. +## batchlog_replay_throttle_in_kb: 1024 + +## Cache and index settings +## - (Default: 64) Granularity of the index of rows within a partition. +## For huge rows, decrease this setting to +## improve seek time. If you use key cache, be careful not to make this +## setting too large because key cache +## will be overwhelmed. If you're unsure of the size of the rows, it's +## best to use the default setting. +column_index_size_in_kb: 32 ## default 64 +column_index_cache_size_in_kb: 1024 + +## Memtable (in-memory structures where Cassandra buffers writes) +## https://cassandra.apache.org/doc/latest/architecture/storage_engine.html#memtables +memtable_allocation_type: offheap_objects +## - Smaller of number of disks or number of cores with a minimum of 2 +## and a maximum of 8 +## If your data directories are backed by SSDs, increase this setting to +## the number of cores. +memtable_flush_writers: 2 +## - The compaction process opens SSTables before they are completely +## written and uses them in place of +## the prior SSTables for any range previously written. This setting +## helps to smoothly transfer reads +## between the SSTables by reducing page cache churn and keeps hot rows +## hot. +## default 50 +sstable_preemptive_open_interval_in_mb: 64 + +## Enable / disable CDC functionality on a per-node basis. This modifies +## the logic used for write path allocation rejection +## (standard: never reject. cdc: reject Mutation containing a +## CDC-enabled table if at space limit in cdc_raw_directory). +cdc_enabled: false + +## The default option is “periodic” where writes may be ack'ed +## immediately and the CommitLog is simply synced +## every commitlog_sync_period_in_ms milliseconds. What this means, is +## the commit is +commitlog_sync: periodic +## sync log every 3s. +commitlog_sync_period_in_ms: 3000 +commitlog_total_space_in_mb: 32 +commitlog_segment_size_in_mb: 32 +max_mutation_size_in_kb: 16384 +## ideal_consistency_level: LOCAL_QUORUM +## ## available in cassandra 4 + +## ------------------------ +## Replication +## ------------------------ + +## A partitioner determines how data is distributed across the nodes in +## the cluster (including replicas). +## Basically, a partitioner is a function for deriving a token +## representing a row from its partition key, +## typically by hashing. Each row of data is then distributed across the +## cluster by the value of the token. +## +## Default: Murmur3Partitioner using MurmurHash which uses 64-bit +## hashing function and allows for possible +## range of hash values is from -2^63 to +2^63-1. +## WARNING: You cannot change the partitioner in existing clusters that +## use a different partitioner. +## +partitioner: org.apache.cassandra.dht.Murmur3Partitioner + +## hinted_handoff is used to optimize cluster consistency process and +## anti-entropy (the synchronization of +## replica data on nodes to ensure that the data is fresh) when a +## replica-owning node is not available to accept +## the write operation (i.e. network issues/etc). This operation DOES +## NOT guarantee successful write operations, +## except when a client application uses consistency level of `ANY`. +## WARNING: This MUST be enabled for HA setups! It's currently disabled, +## because we're using single nodes setups. +hinted_handoff_enabled: false + +## Endpoint snitch +## This teaches Cassandra enough about your network's topology so it can +## route requests efficiently and spread +## replicas by grouping machines nto `data centers` and `racks`. +## WARNING: Switching this option CAN CAUSE DATA LOSS (read manual) +## Default: SimpleSnitch (leave it as is for now) +endpoint_snitch: SimpleSnitch +dynamic_snitch_update_interval_in_ms: 100 +dynamic_snitch_reset_interval_in_ms: 600000 +dynamic_snitch_badness_threshold: 0.1 + +## Back-pressure settings # If enabled, the coordinator will apply the +## back-pressure strategy specified below to +## each mutation sent to replicas, with the aim of reducing pressure on +## overloaded replicas. +## Should be configured usually for cluster setup. +back_pressure_enabled: false +## back_pressure_strategy: +## - class_name: org.apache.cassandra.net.RateBasedBackPressure +## parameters: +## - high_ratio: 0.90 +## factor: 5 +## flow: FAST + + +## internode_compression controls whether traffic between nodes is +## compressed +## all - all traffic is compressed +## dc - traffic between nodes is compressed +## none - no compression (suitable for single nodes) +internode_compression: none + +## Wanr for GC pauses longer than 500ms (usually means heap is near limit) +gc_warn_threshold_in_ms: 500 diff --git a/strongbox-db-server/src/main/resources/etc/conf/janusgraph-cassandra.properties b/strongbox-db-server/src/main/resources/etc/conf/janusgraph-cassandra.properties new file mode 100644 index 0000000..89f2e8b --- /dev/null +++ b/strongbox-db-server/src/main/resources/etc/conf/janusgraph-cassandra.properties @@ -0,0 +1,48 @@ +## Disabling database cache since it might lead to inconsistencies. +cache.db-cache = false +## This is a percentage of the JVM memory - 0.4 means 40% of -Xmx! +cache.db-cache-size = 0 +## can be 0 for local database +cache.db-cache-clean-wait = 0 +cache.db-cache-time = 60000 +cache.tx-cache-size = 50000 +cache.tx-dirty-size = 1000 +## default: 10000 +ids.block-size = 5000 +## default: 10 +ids.num-partitions = 3 +## default: 120000 ms +ids.renew-timeout = 300000 +## percentage; default: 0.3 +ids.renew-percentage = 0.2 +ids.authority.conflict-avoidance-mode = GLOBAL_AUTO +storage.backend = cql +storage.hostname = 127.0.0.1 +storage.port = 49142 +storage.username = cassandra +storage.password = cassandra +## Whether JanusGraph should attempt to parallelize storage operations +storage.parallel-backend-ops = true +storage.cql.keyspace = strongbox +storage.cql.only-use-local-consistency-for-system-operations = true +storage.cql.local-core-connections-per-host = 1 +storage.cql.local-max-connections-per-host = 10 +storage.cql.read-consistency-level = ONE +storage.cql.write-consistency-level = ONE +storage.cassandra.replication-factor = 1 +storage.cassandra.compaction-strategy-class = SizeTieredCompactionStrategy +storage.cassandra.compaction-strategy-options = +## thrift is deprecated, but can't be 0 for cassandra 3! +storage.cassandra.frame-size-mb = 0 +## disable sstable_compression prevent additional unnecessary IO (disk space is cheap) +storage.cassandra.compression-type = +## default 100 (probably ok for cluster, bad for single node) +storage.lock.wait-time = 5 +storage.lock.retries = 20 +tx.log-tx = true +## schema +schema.default = none +schema.constraints = true +## enable metrics +metrics.enabled = true +metrics.jmx.enabled = true \ No newline at end of file diff --git a/strongbox-db-server/src/main/resources/etc/conf/janusgraph-inmemory.properties b/strongbox-db-server/src/main/resources/etc/conf/janusgraph-inmemory.properties new file mode 100644 index 0000000..d857182 --- /dev/null +++ b/strongbox-db-server/src/main/resources/etc/conf/janusgraph-inmemory.properties @@ -0,0 +1,4 @@ +storage.backend = inmemory +## schema +schema.default = none +schema.constraints = true