- Which version?
0.26 - What is exactly the problem?
GET _cat/indices/operate-list-view-0.26.0_alias
- Please describe your expected behavior?
The new index in date format should be generated automatically but not, operate-list-view-0.26.0_2021-03-15 was created by me manually today. - What is exactly the actual behavior?
No new index in date format generates everyday, operate-list-view-0.26.0_ index is bigger and bigger. - What have you tried so far to solve this issue?
I have tried to configure rollover strategy but the index operate-list-view-0.26.0_ not matches the rollover index format which should be end with number. And the data retention in operation should be the root cause. - How does your configuration look like?
index settings is as below:
{
“settings”: {
“index”: {
“number_of_shards”: “1”,
“provided_name”: “operate-list-view-0.26.0_”,
“creation_date”: “1613650427670”,
“analysis”: {
“normalizer”: {
“case_insensitive”: {
“filter”: “lowercase”
}
}
},
“number_of_replicas”: “2”,
“uuid”: “63T-qeyMT0SOHi-iJCf1SQ”,
“version”: {
“created”: “6080199”
}
}
},
“defaults”: {
“index”: {
“max_inner_result_window”: “100”,
“unassigned”: {
“node_left”: {
“delayed_timeout”: “1m”
}
},
“max_terms_count”: “65536”,
“lifecycle”: {
“name”: “”,
“rollover_alias”: “”,
“indexing_complete”: “false”
},
“routing_partition_size”: “1”,
“max_docvalue_fields_search”: “100”,
“merge”: {
“scheduler”: {
“max_thread_count”: “4”,
“auto_throttle”: “true”,
“max_merge_count”: “9”
},
“policy”: {
“reclaim_deletes_weight”: “2.0”,
“floor_segment”: “2mb”,
“max_merge_at_once_explicit”: “30”,
“max_merge_at_once”: “10”,
“max_merged_segment”: “5gb”,
“expunge_deletes_allowed”: “10.0”,
“segments_per_tier”: “10.0”,
“deletes_pct_allowed”: “33.0”
}
},
“max_refresh_listeners”: “1000”,
“max_regex_length”: “1000”,
“load_fixed_bitset_filters_eagerly”: “true”,
“number_of_routing_shards”: “5”,
“write”: {
“wait_for_active_shards”: “1”
},
“mapping”: {
“coerce”: “false”,
“nested_fields”: {
“limit”: “50”
},
“depth”: {
“limit”: “20”
},
“ignore_malformed”: “false”,
“total_fields”: {
“limit”: “1000”
}
},
“source_only”: “false”,
“soft_deletes”: {
“enabled”: “false”,
“retention”: {
“operations”: “0”
},
“retention_lease”: {
“period”: “12h”
}
},
“max_script_fields”: “32”,
“query”: {
“default_field”: [
“"
],
“parse”: {
“allow_unmapped_fields”: “true”
}
},
“format”: “0”,
“frozen”: “false”,
“sort”: {
“missing”: [],
“mode”: [],
“field”: [],
“order”: []
},
“priority”: “1”,
“codec”: “default”,
“max_rescore_window”: “10000”,
“max_adjacency_matrix_filters”: “100”,
“gc_deletes”: “60s”,
“optimize_auto_generated_id”: “true”,
“max_ngram_diff”: “1”,
“translog”: {
“generation_threshold_size”: “64mb”,
“flush_threshold_size”: “512mb”,
“sync_interval”: “5s”,
“retention”: {
“size”: “512mb”,
“age”: “12h”
},
“durability”: “REQUEST”
},
“auto_expand_replicas”: “false”,
“mapper”: {
“dynamic”: “true”
},
“requests”: {
“cache”: {
“enable”: “true”
}
},
“data_path”: “”,
“highlight”: {
“max_analyzed_offset”: “-1”
},
“routing”: {
“rebalance”: {
“enable”: “all”
},
“allocation”: {
“enable”: “all”,
“total_shards_per_node”: “-1”
}
},
“search”: {
“slowlog”: {
“level”: “TRACE”,
“threshold”: {
“fetch”: {
“warn”: “-1”,
“trace”: “-1”,
“debug”: “-1”,
“info”: “-1”
},
“query”: {
“warn”: “-1”,
“trace”: “-1”,
“debug”: “-1”,
“info”: “-1”
}
}
},
“throttled”: “false”
},
“fielddata”: {
“cache”: “node”
},
“default_pipeline”: “_none”,
“max_slices_per_scroll”: “1024”,
“shard”: {
“check_on_startup”: “false”
},
“xpack”: {
“watcher”: {
“template”: {
“version”: “”
}
},
“version”: “”,
“ccr”: {
“following_index”: “false”
}
},
“percolator”: {
“map_unmapped_fields_as_text”: “false”,
“map_unmapped_fields_as_string”: “false”
},
“allocation”: {
“max_retries”: “5”
},
“refresh_interval”: “1s”,
“indexing”: {
“slowlog”: {
“reformat”: “true”,
“threshold”: {
“index”: {
“warn”: “-1”,
“trace”: “-1”,
“debug”: “-1”,
“info”: “-1”
}
},
“source”: “1000”,
“level”: “TRACE”
}
},
“compound_format”: “0.1”,
“blocks”: {
“metadata”: “false”,
“read”: “false”,
“read_only_allow_delete”: “false”,
“read_only”: “false”,
“write”: “false”
},
“max_result_window”: “10000”,
“store”: {
“stats_refresh_interval”: “10s”,
“type”: “”,
“fs”: {
“fs_lock”: “native”
},
“preload”: []
},
“queries”: {
“cache”: {
“enabled”: “true”
}
},
“ttl”: {
“disable_purge”: “false”
},
“warmer”: {
“enabled”: “true”
},
“max_shingle_diff”: “3”,
“query_string”: {
“lenient”: “false”
}
}
}
}
Mapping is as below:
{
“mapping”: {
“_doc”: {
“dynamic”: “strict”,
“dynamic_templates”: [
{
“string_template”: {
“path_match”: "”,
“match_mapping_type”: “string”,
“mapping”: {
“index_options”: “docs”,
“norms”: {
“enabled”: false
},
“type”: “string”
}
}
}
],
“properties”: {
“activityId”: {
“type”: “keyword”,
“eager_global_ordinals”: true
},
“activityState”: {
“type”: “keyword”
},
“activityType”: {
“type”: “keyword”
},
“batchOperationIds”: {
“type”: “keyword”
},
“bpmnProcessId”: {
“type”: “keyword”
},
“endDate”: {
“type”: “date”,
“format”: “date_time”
},
“errorMessage”: {
“type”: “text”
},
“id”: {
“type”: “keyword”
},
“incidentJobKey”: {
“type”: “keyword”
},
“incidentKey”: {
“type”: “long”
},
“joinRelation”: {
“type”: “join”,
“eager_global_ordinals”: true,
“relations”: {
“workflowInstance”: [
“activity”,
“variable”
]
}
},
“key”: {
“type”: “long”
},
“partitionId”: {
“type”: “integer”
},
“scopeKey”: {
“type”: “long”
},
“startDate”: {
“type”: “date”,
“format”: “date_time”
},
“state”: {
“type”: “keyword”
},
“varName”: {
“type”: “keyword”
},
“varValue”: {
“type”: “keyword”,
“ignore_above”: 8191
},
“workflowInstanceKey”: {
“type”: “long”
},
“workflowKey”: {
“type”: “long”
},
“workflowName”: {
“type”: “keyword”,
“normalizer”: “case_insensitive”
},
“workflowVersion”: {
“type”: “long”
}
}
}
}
}
Stats is as blow:
{
“_shards”: {
“total”: 3,
“successful”: 3,
“failed”: 0
},
“stats”: {
“uuid”: “63T-qeyMT0SOHi-iJCf1SQ”,
“primaries”: {
“docs”: {
“count”: 588656580,
“deleted”: 33996389
},
“store”: {
“size_in_bytes”: 695594367710
},
“indexing”: {
“index_total”: 97252336,
“index_time_in_millis”: 17094132,
“index_current”: 0,
“index_failed”: 28,
“delete_total”: 9860936,
“delete_time_in_millis”: 1206047,
“delete_current”: 0,
“noop_update_total”: 0,
“is_throttled”: false,
“throttle_time_in_millis”: 0
},
“get”: {
“total”: 96771833,
“time_in_millis”: 16794661,
“exists_total”: 16825228,
“exists_time_in_millis”: 1710470,
“missing_total”: 79946605,
“missing_time_in_millis”: 15084191,
“current”: 4
},
“search”: {
“open_contexts”: 1,
“query_total”: 299873,
“query_time_in_millis”: 87759555,
“query_current”: 1,
“fetch_total”: 299853,
“fetch_time_in_millis”: 174578,
“fetch_current”: 0,
“scroll_total”: 8846,
“scroll_time_in_millis”: 2407970,
“scroll_current”: 0,
“suggest_total”: 0,
“suggest_time_in_millis”: 0,
“suggest_current”: 0
},
“merges”: {
“current”: 1,
“current_docs”: 4001493,
“current_size_in_bytes”: 5452965014,
“total”: 15060,
“total_time_in_millis”: 112964442,
“total_docs”: 362811590,
“total_size_in_bytes”: 472625139514,
“total_stopped_time_in_millis”: 0,
“total_throttled_time_in_millis”: 105059014,
“total_auto_throttle_in_bytes”: 5242880
},
“refresh”: {
“total”: 89982,
“total_time_in_millis”: 2303886,
“listeners”: 0
},
“flush”: {
“total”: 257,
“periodic”: 256,
“total_time_in_millis”: 1394188
},
“warmer”: {
“current”: 1,
“total”: 89700,
“total_time_in_millis”: 284880431
},
“query_cache”: {
“memory_size_in_bytes”: 0,
“total_count”: 0,
“hit_count”: 0,
“miss_count”: 0,
“cache_size”: 0,
“cache_count”: 0,
“evictions”: 0
},
“fielddata”: {
“memory_size_in_bytes”: 166116928,
“evictions”: 0
},
“completion”: {
“size_in_bytes”: 0
},
“segments”: {
“count”: 177,
“memory_in_bytes”: 497583956,
“terms_memory_in_bytes”: 340360864,
“stored_fields_memory_in_bytes”: 142745040,
“term_vectors_memory_in_bytes”: 0,
“norms_memory_in_bytes”: 10944,
“points_memory_in_bytes”: 14341872,
“doc_values_memory_in_bytes”: 125236,
“index_writer_memory_in_bytes”: 9983165,
“version_map_memory_in_bytes”: 1022060,
“fixed_bit_set_memory_in_bytes”: 0,
“max_unsafe_auto_id_timestamp”: -1,
“file_sizes”: {}
},
“translog”: {
“operations”: 388321,
“size_in_bytes”: 555250982,
“uncommitted_operations”: 108290,
“uncommitted_size_in_bytes”: 150718454,
“earliest_last_modified_age”: 0
},
“request_cache”: {
“memory_size_in_bytes”: 0,
“evictions”: 0,
“hit_count”: 808,
“miss_count”: 3386
},
“recovery”: {
“current_as_source”: 0,
“current_as_target”: 0,
“throttle_time_in_millis”: 45168044
}
},
“total”: {
“docs”: {
“count”: 1765968343,
“deleted”: 101622695
},
“store”: {
“size_in_bytes”: 2082722678351
},
“indexing”: {
“index_total”: 273090548,
“index_time_in_millis”: 47945306,
“index_current”: 0,
“index_failed”: 28,
“delete_total”: 27193880,
“delete_time_in_millis”: 3187114,
“delete_current”: 0,
“noop_update_total”: 0,
“is_throttled”: false,
“throttle_time_in_millis”: 0
},
“get”: {
“total”: 96771833,
“time_in_millis”: 16794661,
“exists_total”: 16825228,
“exists_time_in_millis”: 1710470,
“missing_total”: 79946605,
“missing_time_in_millis”: 15084191,
“current”: 4
},
“search”: {
“open_contexts”: 1,
“query_total”: 392598,
“query_time_in_millis”: 196803610,
“query_current”: 1,
“fetch_total”: 392564,
“fetch_time_in_millis”: 396280,
“fetch_current”: 0,
“scroll_total”: 20328,
“scroll_time_in_millis”: 5810304,
“scroll_current”: 0,
“suggest_total”: 0,
“suggest_time_in_millis”: 0,
“suggest_current”: 0
},
“merges”: {
“current”: 5,
“current_docs”: 7381615,
“current_size_in_bytes”: 10080605825,
“total”: 38618,
“total_time_in_millis”: 317904759,
“total_docs”: 982429893,
“total_size_in_bytes”: 1287505273370,
“total_stopped_time_in_millis”: 0,
“total_throttled_time_in_millis”: 295480289,
“total_auto_throttle_in_bytes”: 15728640
},
“refresh”: {
“total”: 252810,
“total_time_in_millis”: 6331464,
“listeners”: 0
},
“flush”: {
“total”: 747,
“periodic”: 744,
“total_time_in_millis”: 3879852
},
“warmer”: {
“current”: 2,
“total”: 252006,
“total_time_in_millis”: 790443988
},
“query_cache”: {
“memory_size_in_bytes”: 0,
“total_count”: 0,
“hit_count”: 0,
“miss_count”: 0,
“cache_size”: 0,
“cache_count”: 0,
“evictions”: 0
},
“fielddata”: {
“memory_size_in_bytes”: 335738904,
“evictions”: 0
},
“completion”: {
“size_in_bytes”: 0
},
“segments”: {
“count”: 561,
“memory_in_bytes”: 1493009966,
“terms_memory_in_bytes”: 1021193841,
“stored_fields_memory_in_bytes”: 428346352,
“term_vectors_memory_in_bytes”: 0,
“norms_memory_in_bytes”: 33728,
“points_memory_in_bytes”: 43028569,
“doc_values_memory_in_bytes”: 407476,
“index_writer_memory_in_bytes”: 31378592,
“version_map_memory_in_bytes”: 2843243,
“fixed_bit_set_memory_in_bytes”: 0,
“max_unsafe_auto_id_timestamp”: -1,
“file_sizes”: {}
},
“translog”: {
“operations”: 1172525,
“size_in_bytes”: 1680882976,
“uncommitted_operations”: 565623,
“uncommitted_size_in_bytes”: 801480188,
“earliest_last_modified_age”: 0
},
“request_cache”: {
“memory_size_in_bytes”: 0,
“evictions”: 0,
“hit_count”: 1590,
“miss_count”: 7661
},
“recovery”: {
“current_as_source”: 0,
“current_as_target”: 0,
“throttle_time_in_millis”: 57770685
}
}
}
}
Hello @NewAlice,
Thanks for the question. Could you please provide the Operate configuration?
This is the application.yml file, but maybe you provided the configuration via environment variables. Also useful would be logging messages, especially with archive in the message text.
Kind regards,
Ralf