From 977478e66a83fbbd5ac14b85dc00698051c8233a Mon Sep 17 00:00:00 2001 From: nhuray Date: Thu, 12 Apr 2012 15:16:25 +0200 Subject: [PATCH] Implements Template Support : - Upgrade elasticsearch version to 0.19-3-SNAPSHOT to use this feature https://github.com/elasticsearch/elasticsearch/issues/1860 - Rename forceReinit to forceMapping in ElasticsearchAbstractClientFactoryBean - Add Test ElasticsearchTemplateTest.java --- .gitignore | 2 + README.textile | 4 +- pom.xml | 2 +- ...lasticsearchAbstractClientFactoryBean.java | 169 ++++++++++++++++-- .../xml/ElasticsearchTemplateTest.java | 34 ++++ src/test/resources/es-context.xml | 2 +- .../es6/_template/twitter_template.json | 16 ++ .../xml/es-mapping-failed-test-context.xml | 2 +- .../xml/es-mapping-test-context.xml | 2 +- .../xml/es-settings-failed-test-context.xml | 2 +- .../xml/es-settings-test-context.xml | 2 +- .../xml/es-template-test-context.xml | 26 +++ 12 files changed, 243 insertions(+), 20 deletions(-) create mode 100644 src/test/java/fr/pilato/spring/elasticsearch/xml/ElasticsearchTemplateTest.java create mode 100644 src/test/resources/es6/_template/twitter_template.json create mode 100644 src/test/resources/fr/pilato/spring/elasticsearch/xml/es-template-test-context.xml diff --git a/.gitignore b/.gitignore index 9b08022f..4076a06d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ /.classpath /.settings /target +*.iml +*.idea diff --git a/README.textile b/README.textile index 24cddb12..c791c6fd 100644 --- a/README.textile +++ b/README.textile @@ -160,7 +160,7 @@ If merging fails, the factory will not start. h3. Force rebuild mappings (use with caution) For test purpose or for continuous integration, you could force the factory to clean the previous @type@ when starting the client. -It will *remove all your datas* for that @type@. Just set @forceReinit@ property to @true@. +It will *remove all your datas* for that @type@. Just set @forceMapping@ property to @true@. bc. @@ -169,5 +169,5 @@ bc. twitter/tweet - + diff --git a/pom.xml b/pom.xml index 5cd8a9f5..0b51d483 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ 3.1.1.RELEASE - 0.19.1 + 0.19.3-SNAPSHOT diff --git a/src/main/java/fr/pilato/spring/elasticsearch/ElasticsearchAbstractClientFactoryBean.java b/src/main/java/fr/pilato/spring/elasticsearch/ElasticsearchAbstractClientFactoryBean.java index 60ca6afb..208ca0a9 100644 --- a/src/main/java/fr/pilato/spring/elasticsearch/ElasticsearchAbstractClientFactoryBean.java +++ b/src/main/java/fr/pilato/spring/elasticsearch/ElasticsearchAbstractClientFactoryBean.java @@ -12,21 +12,23 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; -import org.elasticsearch.action.admin.indices.close.CloseIndexRequestBuilder; -import org.elasticsearch.action.admin.indices.close.CloseIndexResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequestBuilder; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.settings.UpdateSettingsRequestBuilder; +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.InitializingBean; +import org.springframework.util.Assert; /** * An abstract {@link FactoryBean} used to create an ElasticSearch @@ -79,8 +81,14 @@ * alltheworld:rss * * - * + * + * + * rss_template + * + * + * * + * * * * @@ -131,7 +139,9 @@ public abstract class ElasticsearchAbstractClientFactoryBean extends Elasticsear protected Client client; - protected boolean forceReinit; + protected boolean forceMapping; + + protected boolean forceTemplate; protected boolean mergeMapping; @@ -141,6 +151,8 @@ public abstract class ElasticsearchAbstractClientFactoryBean extends Elasticsear protected String[] aliases; + protected String[] templates; + protected String classpathRoot = "/es"; // TODO Let the user decide @@ -158,12 +170,21 @@ public abstract class ElasticsearchAbstractClientFactoryBean extends Elasticsear /** * Set to true if you want to force reinit indexes/mapping - * @param forceReinit + * @param forceMapping */ - public void setForceReinit(boolean forceReinit) { - this.forceReinit = forceReinit; + public void setForceMapping(boolean forceMapping) { + this.forceMapping = forceMapping; } + /** + * Set to true if you want to force recreate templates + * + * @param forceTemplate + */ + public void setForceTemplate(boolean forceTemplate) { + this.forceTemplate = forceTemplate; + } + /** * Set to true if you want to try to merge mappings * @param mergeMapping @@ -220,6 +241,28 @@ public void setMappings(String[] mappings) { public void setAliases(String[] aliases) { this.aliases = aliases; } + + /** + * Define templates you want to manage with this factory
+ *

+ * Example :
+ * + * 

+	 * {@code
+	 * 
+	 *  
+	 *   template_1
+	 *   template_2
+	 *  
+	 * 
+	 * }
+	 *
+ * + * @param templates list of template + */ + public void setTemplates(String[] templates) { + this.templates = templates; + } /** * Classpath root for index and mapping files (default : /es) @@ -244,6 +287,7 @@ public void afterPropertiesSet() throws Exception { logger.info("Starting ElasticSearch client"); client = buildClient(); + initTemplates(); initMappings(); initAliases(); } @@ -256,7 +300,7 @@ public void destroy() throws Exception { client.close(); } } catch (final Exception e) { - logger.error("Error closing Elasticsearch client: ", e); + logger.error("Error closing ElasticSearch client: ", e); } } @@ -275,9 +319,29 @@ public boolean isSingleton() { return true; } + /** + * Init templates if needed. + *

+ * Note that you can force to recreate template using + * {@link #setForceTemplate(boolean)} + * + * @throws Exception + */ + private void initTemplates() throws Exception { + if (templates != null && templates.length > 0) { + for (int i = 0; i < templates.length; i++) { + String template = templates[i]; + Assert.hasText(template, "Can not read template in [" + + templates[i] + + "]. Check that templates is not empty."); + createTemplate(template, forceTemplate); + } + } + } + /** * Init mapping if needed. - *

Note that you can force to reinit mapping using {@link #setForceReinit(boolean)} + *

Note that you can force to reinit mapping using {@link #setForceMapping(boolean)} * @throws Exception */ private void initMappings() throws Exception { @@ -317,7 +381,7 @@ private void initMappings() throws Exception { for (Iterator iterator = mappings.iterator(); iterator .hasNext();) { String type = iterator.next(); - pushMapping(index, type, forceReinit, mergeMapping); + pushMapping(index, type, forceMapping, mergeMapping); } } } @@ -369,6 +433,59 @@ private void createAlias(String alias, String index) throws Exception { if (!response.acknowledged()) throw new Exception("Could not define alias [" + alias + "] for index [" + index + "]."); if (logger.isTraceEnabled()) logger.trace("/createAlias("+alias+","+index+")"); } + + /** + * Create a template if needed + * + * @param template template name + * @param force force recreate template + * @throws Exception + */ + private void createTemplate(String template, boolean force) + throws Exception { + if (logger.isTraceEnabled()) + logger.trace("createTemplate(" + template + ")"); + checkClient(); + + // If template already exists and if we are in force mode, we delete the + // type and its mapping + if (force && isTemplateExist(template)) { + if (logger.isDebugEnabled()) + logger.debug("Force remove template [" + template + "]"); + // Remove template in ElasticSearch ! + final DeleteIndexTemplateResponse response = client.admin() + .indices().prepareDeleteTemplate(template).execute() + .actionGet(); + } + + // Read the template json file if exists and use it + String source = readTemplate(template); + if (source != null) { + if (logger.isTraceEnabled()) + logger.trace("Template [" + template + "]=" + source); + // Create template + final PutIndexTemplateResponse response = client.admin().indices() + .preparePutTemplate(template).setSource(source).execute() + .actionGet(); + if (!response.acknowledged()) { + throw new Exception("Could not define template [" + template + + "]."); + } else { + if (logger.isDebugEnabled()) { + logger.debug("Template [" + template + + "] successfully created."); + } + } + } else { + if (logger.isWarnEnabled()) { + logger.warn("No template definition for [" + template + + "]. Ignoring."); + } + } + + if (logger.isTraceEnabled()) + logger.trace("/createTemplate(" + template + ")"); + } /** * Check if an index already exists @@ -395,6 +512,23 @@ private boolean isMappingExist(String index, String type) { if (mdd != null) return true; return false; } + + /** + * Check if a template already exists + * + * @param template template name + * @return true if template exists + */ + private boolean isTemplateExist(String template) { + ClusterState cs = client.admin().cluster().prepareState() + .setFilterIndexTemplates(template).execute().actionGet() + .getState(); + final IndexTemplateMetaData mdd = cs.getMetaData().templates() + .get(template); + + if (mdd != null) return true; + return false; + } /** * Define a type for a given index and if exists with its mapping definition @@ -486,9 +620,8 @@ private void createIndex(String index) throws Exception { } /** - * Create a new index in Elasticsearch + * Create a new index in ElasticSearch * @param index Index name - * @param merge Try to merge settings ? * @throws Exception */ private void mergeIndexSettings(String index) throws Exception { @@ -532,6 +665,18 @@ private String readMapping(String index, String type) throws Exception { return readFileInClasspath(classpathRoot + "/" + index + "/" + type + jsonFileExtension); } + /** + * Read the template.
+ * Shortcut to readFileInClasspath(classpathRoot + "/_template/" + template + jsonFileExtension); + * + * @param template Template name + * @return Template if exists. Null otherwise. + * @throws Exception + */ + private String readTemplate(String template) throws Exception { + return readFileInClasspath(classpathRoot + "/_template/" + template + jsonFileExtension); + } + /** * Read settings for an index.
* Shortcut to readFileInClasspath(classpathRoot + "/" + index + "/" + indexSettingsFileName); diff --git a/src/test/java/fr/pilato/spring/elasticsearch/xml/ElasticsearchTemplateTest.java b/src/test/java/fr/pilato/spring/elasticsearch/xml/ElasticsearchTemplateTest.java new file mode 100644 index 00000000..890c9085 --- /dev/null +++ b/src/test/java/fr/pilato/spring/elasticsearch/xml/ElasticsearchTemplateTest.java @@ -0,0 +1,34 @@ +package fr.pilato.spring.elasticsearch.xml; + +import org.elasticsearch.client.Client; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +public class ElasticsearchTemplateTest { + + static protected ConfigurableApplicationContext ctx; + + @BeforeClass + static public void setup() { + ctx = new ClassPathXmlApplicationContext( + "fr/pilato/spring/elasticsearch/xml/es-template-test-context.xml"); + } + + @AfterClass + static public void tearDown() { + if (ctx != null) { + ctx.close(); + } + } + + @Test + public void test_transport_client() { + Client client = ctx.getBean("esClient", Client.class); + Assert.assertNotNull("Client must not be null...", client); + } + +} diff --git a/src/test/resources/es-context.xml b/src/test/resources/es-context.xml index da6eaeaf..dc14927f 100644 --- a/src/test/resources/es-context.xml +++ b/src/test/resources/es-context.xml @@ -26,7 +26,7 @@ http://www.springframework.org/schema/context http://www.springframework.org/sch alltheworld:rss - + \ No newline at end of file diff --git a/src/test/resources/es6/_template/twitter_template.json b/src/test/resources/es6/_template/twitter_template.json new file mode 100644 index 00000000..01552210 --- /dev/null +++ b/src/test/resources/es6/_template/twitter_template.json @@ -0,0 +1,16 @@ +{ + "template" : "twee*", + "settings" : { + "number_of_shards" : 1 + }, + "mappings" : { + "tweet" : { + "properties" : { + "message" : { + "type" : "string", + "store" : "yes" + } + } + } + } +} \ No newline at end of file diff --git a/src/test/resources/fr/pilato/spring/elasticsearch/xml/es-mapping-failed-test-context.xml b/src/test/resources/fr/pilato/spring/elasticsearch/xml/es-mapping-failed-test-context.xml index 147cf625..2e071a3f 100644 --- a/src/test/resources/fr/pilato/spring/elasticsearch/xml/es-mapping-failed-test-context.xml +++ b/src/test/resources/fr/pilato/spring/elasticsearch/xml/es-mapping-failed-test-context.xml @@ -20,7 +20,7 @@ twitter/tweet - + twitter/tweet - + twitter/tweet - + twitter/tweet - + + + + + + + + + + + twitter_template + + + + + + \ No newline at end of file