Elasticsearch实践（二）在Springboot微服务中集成搜索服务

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
@NoRepositoryBean
public interface CrudRepository<T, ID extends Serializable> extends Repository<T, ID> {

    /**
     * Saves a given entity. Use the returned instance for further operations as the save operation might have changed the
     * entity instance completely.
     * 
     * @param entity
     * @return the saved entity
     */
    <S extends T> S save(S entity);

    /**
     * Saves all given entities.
     * 
     * @param entities
     * @return the saved entities
     * @throws IllegalArgumentException in case the given entity is {@literal null}.
     */
    <S extends T> Iterable<S> save(Iterable<S> entities);

    /**
     * Retrieves an entity by its id.
     * 
     * @param id must not be {@literal null}.
     * @return the entity with the given id or {@literal null} if none found
     * @throws IllegalArgumentException if {@code id} is {@literal null}
     */
    T findOne(ID id);

    /**
     * Returns whether an entity with the given id exists.
     * 
     * @param id must not be {@literal null}.
     * @return true if an entity with the given id exists, {@literal false} otherwise
     * @throws IllegalArgumentException if {@code id} is {@literal null}
     */
    boolean exists(ID id);

    /**
     * Returns all instances of the type.
     * 
     * @return all entities
     */
    Iterable<T> findAll();

    /**
     * Returns all instances of the type with the given IDs.
     * 
     * @param ids
     * @return
     */
    Iterable<T> findAll(Iterable<ID> ids);

    /**
     * Returns the number of entities available.
     * 
     * @return the number of entities
     */
    long count();

    /**
     * Deletes the entity with the given id.
     * 
     * @param id must not be {@literal null}.
     * @throws IllegalArgumentException in case the given {@code id} is {@literal null}
     */
    void delete(ID id);

    /**
     * Deletes a given entity.
     * 
     * @param entity
     * @throws IllegalArgumentException in case the given entity is {@literal null}.
     */
    void delete(T entity);

    /**
     * Deletes the given entities.
     * 
     * @param entities
     * @throws IllegalArgumentException in case the given {@link Iterable} is {@literal null}.
     */
    void delete(Iterable<? extends T> entities);

    /**
     * Deletes all entities managed by the repository.
     */
    void deleteAll();
    ...
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;

import java.net.InetAddress;

/**
 * 使用的是es 2.4.4 版本，因为springboot 1.5.x,以及目前版本最多支持到es 2.x。
 * <p>
 * Created by lijingyao on 2017/5/17 16:32.
 */
@Configuration
@EnableElasticsearchRepositories(basePackages = "com.puregold.ms")
public class SearchConfig {

    // 假设使用三个node,(一主两备)的配置。在实际的生产环境，需在properties文件中替换成实际ip(内网或者外网ip)
    @Value("${elasticsearch.host1}")
    private String esHost;// master node

    @Value("${elasticsearch.host2:}") 
    private String esHost2;//replica node

    @Value("${elasticsearch.host3:}")
    private String esHost3;//replica node

    @Value("${elasticsearch.port}")
    private int esPort;

    @Value("${elasticsearch.clustername}")
    private String esClusterName;

    @Bean
    public TransportClient transportClient() throws Exception {

        Settings settings = Settings.settingsBuilder()
                .put("cluster.name", esClusterName)
                .build();

        TransportClient transportClient = TransportClient.builder()
                .settings(settings)
                .build()
                .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost), esPort));
        if (StringUtils.isNotEmpty(esHost2)) {
            transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost2), esPort));
        }
        if (StringUtils.isNotEmpty(esHost3)) {
            transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esHost3), esPort));
        }
        return transportClient;
    }

    @Bean
    public ElasticsearchTemplate elasticsearchTemplate() throws Exception {
        return new ElasticsearchTemplate(transportClient());
    }

}     

1
nested exception is java.lang.IllegalArgumentException: Parent ID property should be String  

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
@Document(indexName = OrderDocument.INDEX, type = OrderDocument.ORDER_TYPE, refreshInterval = "-1")
public class OrderDocument {


    public static final String INDEX = "orders-test";
    public static final String ORDER_TYPE = "order-document";
    public static final String DETAIL_TYPE = "order-detail-document";


    @Id
    private String id;

    // 订单备注，不需要分词，可以搜索
    @Field(type = FieldType.String, index = FieldIndex.not_analyzed)
    private String note;

    // 订单名称，可以通过ik 分词器进行分词
    @Field(type = FieldType.String, searchAnalyzer = "ik", analyzer = "ik")
    private String name;


    // 订单价格
    @Field(type = FieldType.Long)
    private Long price;

    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getNote() {
        return note;
    }

    public void setNote(String note) {
        this.note = note;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public Long getPrice() {
        return price;
    }

    public void setPrice(Long price) {
        this.price = price;
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
@Document(indexName = OrderDocument.INDEX, type = OrderDocument.DETAIL_TYPE, shards = 10, replicas = 2, refreshInterval = "-1")
public class DetailOrderDocument {


    @Id
    private String id;

    // 指定主订单关联的父子关系
    @Field(type = FieldType.String, store = true)
    @Parent(type = OrderDocument.ORDER_TYPE)
    private String parentId;


    // 子订单价格
    @Field(type = FieldType.Long)
    private Long price;

    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getParentId() {
        return parentId;
    }

    public void setParentId(String parentId) {
        this.parentId = parentId;
    }

    public Long getPrice() {
        return price;
    }

    public void setPrice(Long price) {
        this.price = price;
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
{
  "orders-test" : {
    "aliases" : { },
    "mappings" : {
      "order-detail-document" : {
        "_parent" : {
          "type" : "order-document" },
        "_routing" : {
          "required" : true },
        "properties" : {
          "parentId" : { "type" : "string", "store" : true },
          "price" : { "type" : "long" } }
      },
      "order-document" : {
        "properties" : {
          "name" : { "type" : "string", "analyzer" : "ik" },
          "note" : { "type" : "string", "index" : "not_analyzed" },
          "price" : { "type" : "long" } }
      }
    },
    "settings" : {
      "index" : {
        "refresh_interval" : "-1",
        "number_of_shards" : "10",
        "creation_date" : "1511403448676",
        "store" : {
          "type" : "fs" },
        "number_of_replicas" : "2",
        "uuid" : "sHA5s7kEQA2AWCAA8-aBlQ",
        "version" : {
          "created" : "2040499" }
      }
    },
    "warmers" : { }
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
{
  "error" : {
    "root_cause" : [ {
      "type" : "routing_missing_exception",
      "reason" : "routing is required for [XX]/[YY]/[yourid]",
      "index" : "forests"
    } ],
    "type" : "routing_missing_exception",
    "reason" : "routing is required for [XX]/[YY]/[yourid]",
    "index" : "forests"
  },
  "status" : 400
}

1
2
3
4
5
6
7
8
9
public interface DetailOrderDocumentRepository extends ElasticsearchRepository<DetailOrderDocument, String> {


    List<DetailOrderDocument> findByParentId(String parentId, Sort sort);

    DetailOrderDocument findById(String id);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
@Component
public class OrderManager {

    @Autowired
    private ElasticsearchTemplate elasticsearchTemplate;


    public Page<OrderDocument> queryPagedOrders(Integer pageNo, Integer pageSize, String name, Long minPrice, Long maxPrice) {
        // 默认，价格升序（为了支持丰富的排序场景，建议将所有可能的排序规则放到统一的enum中
        Pageable pageable = new PageRequest(pageNo, pageSize, new Sort(new Sort.Order(Sort.Direction.ASC, "price")));

        NativeSearchQueryBuilder nbq = new NativeSearchQueryBuilder().withIndices(OrderDocument.INDEX).withTypes(OrderDocument
                .ORDER_TYPE).withSearchType(SearchType.DEFAULT).withPageable(pageable);


        BoolQueryBuilder bqb = boolQuery();
        // 匹配订单name
        if (StringUtils.isNotEmpty(name)) {
            bqb.must(termQuery("name", name));
        }
        // 查询价格区间 minPrice<=price<=maxPrice
        if (minPrice != null && minPrice >= 0) {
            bqb.filter(rangeQuery("price").gte(minPrice));
        }
        if (maxPrice != null && maxPrice >= 0) {
            bqb.filter(rangeQuery("price").lte(maxPrice));
        }

        Page<OrderDocument> page = elasticsearchTemplate.queryForPage(nbq.withQuery(bqb).build(), OrderDocument.class);

        return page;
    }


}

1
curl -XDELETE 'http://yourip:9200/orders-test/?pretty'

1
2
    @Field(type = FieldType.String, searchAnalyzer = "ik", analyzer = "ik")
    private String name;

1
2
3
Failed to execute phase [query], all shards failed; shardFailures {[X-XXXXX][YYYY][0]: RemoteTransportException[[your-node][yourip:9300][indices:data/read/search[phase/query]]]; nested:
 QueryPhaseExecutionException[Result window is too large, from + size must be less than or equal to: [10000] but was [99020].
  See the scroll api for a more efficient way to request large data sets. This limit can be set by changing the [index.max_result_window] index level parameter.]; }{[X-XXXXX][YYYY][1]: RemoteTransportException[[

1
curl -XPUT "http://your_cluster:9200/index_name/_settings" -d '{ "index" : { "max_result_window" : 500000 } }'  

1
String scrollId = elasticsearchTemplate.scan(nbq.withQuery(bqb).build(), 1000, false);   

1
Page<OrderDocument> page = elasticsearchTemplate.scroll(scrollId, 2000L, OrderDocument.class);

1
2
3
4
5
6
7
8
9
10
11
12
13
curl -XDELETE 'http://your_cluster:9200/orders-test/order-detail-document/_query?routing=parent_order_id&pretty' -H 'Content-Type: application/json' -d'
{
   "query": {
      "bool": {
         "must": [
               { "term" : 
               { "id" : "detail_order_id" } 
               }
         ]
      }
   }
}
'

1
http://your_cluster:9200/orders-test/order-detail-document/_search?&_routing= parent_order_id&q=id:detail_order_id&pretty