Reduce/Increase throughtput based on monitoring segments

[juarezr]

Reducing server indexing overhead

Indexing could start failing when solrcopy would be used to aggressively.
For improving this one could look a data in the segments handler in admin API.
The strategy can be monitoring the size and age of merge candidate segments.

Example

The address http://localhost:8983/solr/corename/admin/segments?wt=json would return a json response like:

{
  "responseHeader":{
    "status":0,
    "QTime":3
  },
  "segments":{
    "_dnc":{
      "name":"_dnc",
      "delCount":0,
      "sizeInBytes":3720192411,
      "size":1352743,
      "age":"2020-05-15T04:08:23.499Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_d4q":{
      "name":"_d4q",
      "delCount":538,
      "sizeInBytes":694162377,
      "size":214347,
      "age":"2020-05-15T03:51:56.820Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_eju":{
      "name":"_eju",
      "delCount":6070,
      "sizeInBytes":855393299,
      "size":265206,
      "age":"2020-05-15T04:57:48.340Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_dsx":{
      "name":"_dsx",
      "delCount":0,
      "sizeInBytes":503792200,
      "size":149989,
      "age":"2020-05-15T04:13:45.125Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_e8s":{
      "name":"_e8s",
      "delCount":47776,
      "sizeInBytes":648225090,
      "size":196282,
      "age":"2020-05-15T04:27:55.925Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_e43":{
      "name":"_e43",
      "delCount":0,
      "sizeInBytes":520414059,
      "size":155389,
      "age":"2020-05-15T04:23:38.009Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_fef":{
      "name":"_fef",
      "delCount":0,
      "sizeInBytes":725333779,
      "size":222873,
      "age":"2020-05-15T05:24:41.320Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_es7":{
      "name":"_es7",
      "delCount":0,
      "sizeInBytes":513762350,
      "size":159011,
      "age":"2020-05-15T05:05:08.497Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_f0u":{
      "name":"_f0u",
      "delCount":0,
      "sizeInBytes":573389084,
      "size":172592,
      "age":"2020-05-15T05:12:38.196Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_g0p":{
      "name":"_g0p",
      "delCount":45940,
      "sizeInBytes":1024868184,
      "size":322497,
      "age":"2020-05-15T05:44:38.097Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_f8w":{
      "name":"_f8w",
      "delCount":0,
      "sizeInBytes":497816681,
      "size":147568,
      "age":"2020-05-15T05:19:41.182Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_gaz":{
      "name":"_gaz",
      "delCount":0,
      "sizeInBytes":593980446,
      "size":180392,
      "age":"2020-05-15T05:52:56.207Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_fkn":{
      "name":"_fkn",
      "delCount":3275,
      "sizeInBytes":12405668,
      "size":3451,
      "age":"2020-05-15T05:29:46.512Z",
      "source":"merge",
      "version":"6.6.6",
      "mergeCandidate":true
    },
    "_g42":{
      "name":"_g42",
      "delCount":123,
      "sizeInBytes":133100032,
      "size":37831,
      "age":"2020-05-15T05:47:16.496Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_gf8":{
      "name":"_gf8",
      "delCount":0,
      "sizeInBytes":276107099,
      "size":79162,
      "age":"2020-05-15T05:56:58.199Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_gp6":{
      "name":"_gp6",
      "delCount":0,
      "sizeInBytes":101241802,
      "size":29038,
      "age":"2020-05-15T06:06:45.908Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_gkg":{
      "name":"_gkg",
      "delCount":0,
      "sizeInBytes":593948603,
      "size":179423,
      "age":"2020-05-15T06:02:00.224Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_gw6":{
      "name":"_gw6",
      "delCount":0,
      "sizeInBytes":227529468,
      "size":65338,
      "age":"2020-05-15T06:13:29.840Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_grn":{
      "name":"_grn",
      "delCount":0,
      "sizeInBytes":549726114,
      "size":167534,
      "age":"2020-05-15T06:09:19.418Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_h1e":{
      "name":"_h1e",
      "delCount":0,
      "sizeInBytes":521633536,
      "size":156034,
      "age":"2020-05-15T06:19:10.628Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_gv2":{
      "name":"_gv2",
      "delCount":0,
      "sizeInBytes":78213126,
      "size":22363,
      "age":"2020-05-15T06:12:37.908Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_gxh":{
      "name":"_gxh",
      "delCount":0,
      "sizeInBytes":96406063,
      "size":26971,
      "age":"2020-05-15T06:15:24.332Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_h6p":{
      "name":"_h6p",
      "delCount":0,
      "sizeInBytes":171039601,
      "size":48299,
      "age":"2020-05-15T06:23:30.900Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_h20":{
      "name":"_h20",
      "delCount":0,
      "sizeInBytes":65716159,
      "size":19190,
      "age":"2020-05-15T06:19:40.164Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_h3n":{
      "name":"_h3n",
      "delCount":0,
      "sizeInBytes":84534197,
      "size":24092,
      "age":"2020-05-15T06:21:01.420Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_h34":{
      "name":"_h34",
      "delCount":0,
      "sizeInBytes":43293118,
      "size":12481,
      "age":"2020-05-15T06:20:35.132Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_h52":{
      "name":"_h52",
      "delCount":0,
      "sizeInBytes":59819241,
      "size":17372,
      "age":"2020-05-15T06:22:06.211Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_h5v":{
      "name":"_h5v",
      "delCount":0,
      "sizeInBytes":72945656,
      "size":21005,
      "age":"2020-05-15T06:22:53.226Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_h7j":{
      "name":"_h7j",
      "delCount":0,
      "sizeInBytes":71065724,
      "size":20723,
      "age":"2020-05-15T06:24:14.704Z",
      "source":"merge",
      "version":"6.6.6"
    },
    "_h6q":{
      "name":"_h6q",
      "delCount":0,
      "sizeInBytes":2253642,
      "size":654,
      "age":"2020-05-15T06:23:39.117Z",
      "source":"flush",
      "version":"6.6.6",
      "mergeCandidate":true
    },
    "_h6s":{
      "name":"_h6s",
      "delCount":0,
      "sizeInBytes":4712345,
      "size":1316,
      "age":"2020-05-15T06:23:48.008Z",
      "source":"flush",
      "version":"6.6.6",
      "mergeCandidate":true
    },
    "_h78":{
      "name":"_h78",
      "delCount":0,
      "sizeInBytes":3324666,
      "size":910,
      "age":"2020-05-15T06:24:07.306Z",
      "source":"flush",
      "version":"6.6.6",
      "mergeCandidate":true
    },
    "_h7s":{
      "name":"_h7s",
      "delCount":0,
      "sizeInBytes":4712917,
      "size":1312,
      "age":"2020-05-15T06:24:33.932Z",
      "source":"merge",
      "version":"6.6.6",
      "mergeCandidate":true
    },
    "_h7i":{
      "name":"_h7i",
      "delCount":0,
      "sizeInBytes":4906686,
      "size":1389,
      "age":"2020-05-15T06:24:22.213Z",
      "source":"flush",
      "version":"6.6.6",
      "mergeCandidate":true
    },
    "_h7k":{
      "name":"_h7k",
      "delCount":0,
      "sizeInBytes":3548433,
      "size":1027,
      "age":"2020-05-15T06:24:21.420Z",
      "source":"flush",
      "version":"6.6.6",
      "mergeCandidate":true
    },
    "_h7h":{
      "name":"_h7h",
      "delCount":0,
      "sizeInBytes":4420269,
      "size":1299,
      "age":"2020-05-15T06:24:22.029Z",
      "source":"flush",
      "version":"6.6.6",
      "mergeCandidate":true
    },
    "_h7q":{
      "name":"_h7q",
      "delCount":0,
      "sizeInBytes":8349820,
      "size":2400,
      "age":"2020-05-15T06:24:33.616Z",
      "source":"flush",
      "version":"6.6.6"
    },
    "_h7p":{
      "name":"_h7p",
      "delCount":0,
      "sizeInBytes":8634659,
      "size":2468,
      "age":"2020-05-15T06:24:33.823Z",
      "source":"flush",
      "version":"6.6.6"
    },
    "_h7r":{
      "name":"_h7r",
      "delCount":0,
      "sizeInBytes":390310,
      "size":109,
      "age":"2020-05-15T06:24:34.421Z",
      "source":"flush",
      "version":"6.6.6",
      "mergeCandidate":true
    },
    "_h7t":{
      "name":"_h7t",
      "delCount":0,
      "sizeInBytes":4729144,
      "size":1395,
      "age":"2020-05-15T06:24:37.886Z",
      "source":"flush",
      "version":"6.6.6",
      "mergeCandidate":true
    }
  }
}