Compare commits

Home / Compare 9b37ea77 .. 5c990152

grey: no significant change wrt stddev
red: change exceeding max stddev
purple: some change, not seemingly significant

Aggregation: measureAggregations

Factorallocated MiBpeak allocated MiBtime_s
cache=n
dps=1
iters=10
policies=100
mean54.055-54.057=-0.002
σ0.369-0.368=0.000
min53.912-53.914=-0.001
max55.104-55.105=-0.001
mean415.000-418.000=-3.000
σ0.000
min415.000-418.000=-3.000
max415.000-418.000=-3.000
mean0.140-0.141=-0.001
σ0.003-0.003=-0.000
min0.137-0.138=-0.001
max0.145-0.146=-0.002
cache=y
dps=1
iters=10
policies=100
mean43.394-43.395=-0.001
σ0.007-0.008=-0.001
min43.384-43.385=-0.001
max43.406-43.409=-0.002
mean415.000-418.000=-3.000
σ0.000
min415.000-418.000=-3.000
max415.000-418.000=-3.000
mean0.120-0.123=-0.003
σ0.003-0.004=-0.000
min0.115-0.120=-0.005
max0.127-0.128=-0.001

CSV - column_set() on indexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=column_set() on indexed
rows=200000
mean36.614
σ0.000
min36.614
max36.615
mean8887.000
σ0.000
min8887.000
max8887.000
mean13.041
σ0.414
min12.484
max13.646
iterations=10
operation_display=column_set() on indexed, repeat
rows=200000
mean36.614
σ0.000
min36.614
max36.614
mean8887.000
σ0.000
min8887.000
max8887.000
mean12.630
σ0.558
min11.817
max13.250

CSV - column_set() on unindexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=column_set() on unindexed, dense, repeat
rows=200000
mean62.074-62.074=0.000
σ0.000-0.000=-0.000
min62.074-62.074=-0.000
max62.074-62.075=-0.001
mean5090.500-5111.700=-21.200
σ2537.543-2610.220=-72.677
min1350.000-1356.000=-6.000
max8720.000-8533.000=187.000
mean106.462-105.202=1.260
σ10.347-12.332=-1.985
min86.812-82.420=4.392
max118.592-119.873=-1.282
iterations=10
operation_display=column_set() on unindexed, sparse, repeat
rows=200000
mean70.664
σ0.002
min70.662
max70.668
mean8887.000
σ0.000
min8887.000
max8887.000
mean201.607
σ202.266
min135.531
max777.237

CSV - column_set(), warmup

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=column_set() on unindexed, dense
rows=200000
mean15109.632-15109.632=-0.000
σ3.862-3.862=0.000
min15108.410-15108.410=-0.000
max15120.624
mean5090.500-5111.700=-21.200
σ2537.543-2610.220=-72.677
min1350.000-1356.000=-6.000
max8720.000-8533.000=187.000
mean10337.545-9947.788=389.757
σ1791.830-2278.230=-486.400
min6665.147-6660.158=4.990
max12334.214-12548.469=-214.254
iterations=10
operation_display=column_set() on unindexed, sparse
rows=200000
mean14060.732
σ0.002
min14060.731
max14060.736
mean8887.000
σ0.000
min8887.000
max8887.000
mean8432.210
σ2113.917
min5430.379
max11452.268

CSV - loading a lazy table

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=load_product() with indexes
rows=200000
mean8744.535
σ0.000
min8744.534
max8744.535
mean8887.000
σ0.000
min8887.000
max8887.000
mean4508.807
σ1149.775
min3501.342
max6004.661
iterations=10
operation_display=load_product(), no indexes
rows=200000
mean2524.581
σ0.002
min2524.579
max2524.583
mean8887.000
σ0.000
min8887.000
max8887.000
mean1455.235
σ982.668
min1024.309
max4019.295
iterations=10
operation_display=load_product(), with indexes
rows=200000
mean8744.546-8744.546=0.000
σ0.035-0.035=0.000
min8744.534-8744.534=0.000
max8744.646-8744.645=0.001
mean4786.100-4712.700=73.400
σ2639.804-2572.096=67.708
min1071.000-1077.000=-6.000
max8720.000-8533.000=187.000
mean4689.171-4775.949=-86.777
σ998.950-1217.652=-218.702
min3535.299-3503.051=32.248
max6054.907-6902.864=-847.957
iterations=10
operation_display=load_product()with indexes
rows=200000
mean8744.532-8744.532=0.000
σ0.002-0.002=-0.000
min8744.531-8744.530=0.001
max8744.535-8744.535=0.000
mean8720.000-8568.400=151.600
σ0.000-111.945=-111.945
min8720.000-8533.000=187.000
max8720.000-8887.000=-167.000
mean4511.299-4766.504=-255.205
σ1240.546-1450.613=-210.067
min3531.234-3503.778=27.456
max6819.521-7240.200=-420.680

CSV - table_records()

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=table_records(), no indexes
rows=200000
mean41.902
σ0.002
min41.900
max41.904
mean8887.000
σ0.000
min8887.000
max8887.000
mean14.663
σ1.234
min11.274
max15.476
iterations=10
operation_display=table_records(), no indexes, repeat
rows=200000
mean41.900
σ0.002
min41.900
max41.903
mean8887.000
σ0.000
min8887.000
max8887.000
mean13.678
σ1.386
min10.765
max15.490
iterations=10
operation_display=table_records(), repeat
rows=200000
mean41.901-41.901=0.000
σ0.002-0.002=-0.000
min41.899-41.899=0.000
max41.904-41.904=0.000
mean8720.000-8568.400=151.600
σ0.000-111.945=-111.945
min8720.000-8533.000=187.000
max8720.000-8887.000=-167.000
mean13.510-14.316=-0.805
σ1.948-1.559=0.389
min11.423-11.153=0.270
max15.899-15.727=0.173

CSV - table_records(), warmup

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=table_records()
rows=200000
mean15088.238-15088.238=-0.000
σ0.002-0.002=-0.000
min15088.236
max15088.241-15088.242=-0.001
mean8720.000-8568.400=151.600
σ0.000-111.945=-111.945
min8720.000-8533.000=187.000
max8720.000-8887.000=-167.000
mean9271.608-9874.848=-603.239
σ2417.933-2196.310=221.623
min6584.099-6499.327=84.772
max12218.058-12671.283=-453.226

CSV - vertical_lookup_list() on indexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_list() on indexed, after column_set(), repeat
rows=200000
mean26.398-26.398=0.000
σ0.001-0.001=-0.000
min26.397-26.397=0.000
max26.401
mean5090.500-5111.700=-21.200
σ2537.543-2610.220=-72.677
min1350.000-1356.000=-6.000
max8720.000-8533.000=187.000
mean7.292-7.018=0.274
σ0.577-0.587=-0.010
min6.009-6.257=-0.248
max8.024-7.744=0.280
iterations=10
operation_display=vertical_lookup_list() on indexed, after table_records(), repeat
rows=200000
mean26.398-26.398=0.000
σ0.002-0.002=-0.000
min26.397-26.397=-0.000
max26.401
mean8720.000-8568.400=151.600
σ0.000-111.945=-111.945
min8720.000-8533.000=187.000
max8720.000-8887.000=-167.000
mean6.939-6.700=0.239
σ0.329-0.453=-0.124
min6.233-5.803=0.430
max7.324-7.372=-0.048
iterations=10
operation_display=vertical_lookup_list() on indexed, repeat
rows=200000
mean26.398
σ0.000
min26.398
max26.398
mean8887.000
σ0.000
min8887.000
max8887.000
mean8.214
σ0.387
min7.620
max8.732

CSV - vertical_lookup_list() on unindexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_list() on unindexed missing Int
rows=200000
mean40.944
σ0.003
min40.942
max40.949
mean8887.000
σ0.000
min8887.000
max8887.000
mean97.451
σ11.624
min75.908
max107.512
iterations=10
operation_display=vertical_lookup_list() on unindexed missing Int, repeat
rows=200000
mean40.943
σ0.002
min40.942
max40.946
mean8887.000
σ0.000
min8887.000
max8887.000
mean96.683
σ12.760
min70.713
max107.623
iterations=10
operation_display=vertical_lookup_list() on unindexed unique Int near end
rows=200000
mean40.949
σ0.002
min40.947
max40.952
mean8887.000
σ0.000
min8887.000
max8887.000
mean99.850
σ8.029
min80.545
max107.427
iterations=10
operation_display=vertical_lookup_list() on unindexed unique Int near end, repeat
rows=200000
mean40.948
σ0.001
min40.947
max40.951
mean8887.000
σ0.000
min8887.000
max8887.000
mean128.542
σ91.142
min76.398
max386.685

CSV - vertical_lookup_list(), mini warmup

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_list() on indexed, after column_set()
rows=200000
mean27.602
σ0.001
min27.602
max27.603
mean5111.700
σ2610.220
min1356.000
max8533.000
mean9.870
σ0.691
min8.827
max10.616
iterations=10
operation_display=vertical_lookup_list() on indexed, after table_records()
rows=200000
mean27.602
σ0.002
min27.602
max27.605
mean8568.400
σ111.945
min8533.000
max8887.000
mean9.641
σ0.611
min8.272
max10.239

CSV - vertical_lookup_list(), warmup

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_list() on indexed
rows=200000
mean15073.940
σ0.000
min15073.939
max15073.940
mean8887.000
σ0.000
min8887.000
max8887.000
mean9658.605
σ2392.196
min6483.387
max12229.084

CSV - vertical_lookup_row_list() on indexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_row_list() on 2 indexed
rows=200000
mean25.634-25.633=0.000
σ0.003-0.002=0.001
min25.631-25.631=0.000
max25.639-25.637=0.003
mean8720.000-8568.400=151.600
σ0.000-111.945=-111.945
min8720.000-8533.000=187.000
max8720.000-8887.000=-167.000
mean5.282-5.149=0.133
σ0.212-0.189=0.023
min4.967-4.945=0.022
max5.577-5.540=0.037
iterations=10
operation_display=vertical_lookup_row_list() on 2 indexed, repeat
rows=200000
mean25.628-25.628=0.000
σ0.002-0.002=-0.000
min25.626-25.626=0.000
max25.630-25.630=-0.000
mean8720.000-8568.400=151.600
σ0.000-111.945=-111.945
min8720.000-8533.000=187.000
max8720.000-8887.000=-167.000
mean5.168-5.039=0.129
σ0.180-0.185=-0.004
min4.921-4.874=0.047
max5.415-5.368=0.047
iterations=10
operation_display=vertical_lookup_row_list() on 3 (1 unindexed)
rows=200000
mean25.932-25.932=-0.000
σ0.002-0.002=-0.000
min25.930-25.930=0.000
max25.935-25.935=-0.000
mean8720.000-8568.400=151.600
σ0.000-111.945=-111.945
min8720.000-8533.000=187.000
max8720.000-8887.000=-167.000
mean5.215-5.158=0.057
σ0.205-0.217=-0.012
min4.912-4.923=-0.011
max5.448-5.504=-0.056
iterations=10
operation_display=vertical_lookup_row_list() on 3 (1 unindexed), repeat
rows=200000
mean25.931-25.931=-0.000
σ0.002-0.002=-0.000
min25.930-25.930=-0.000
max25.934
mean8720.000-8568.400=151.600
σ0.000-111.945=-111.945
min8720.000-8533.000=187.000
max8720.000-8887.000=-167.000
mean5.209-5.096=0.112
σ0.193-0.166=0.027
min4.899-4.840=0.059
max5.429-5.350=0.079

CSV - vertical_lookup_row_list() on unindexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_row_list() on unindexed small unique Int
rows=200000
mean66.671
σ0.001
min66.670
max66.673
mean8887.000
σ0.000
min8887.000
max8887.000
mean150.728
σ17.399
min113.634
max165.000
iterations=10
operation_display=vertical_lookup_row_list() on unindexed small unique Int, repeat
rows=200000
mean66.670
σ0.002
min66.670
max66.673
mean8887.000
σ0.000
min8887.000
max8887.000
mean150.718
σ17.331
min114.385
max165.539

CSV internal benchmark: pointedIndexFromKvs

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=pointedIndexFromKvs
rows=200000
mean172.661-172.663=-0.002
σ0.012-0.007=0.004
min172.643-172.650=-0.007
max172.677-172.674=0.003
mean485.900-491.000=-5.100
σ1.449-0.000=1.449
min485.000-491.000=-6.000
max488.000-491.000=-3.000
mean388.337-391.865=-3.527
σ35.851-26.246=9.605
min320.105-344.574=-24.468
max420.314-420.228=0.086
iterations=10
operation_display=pointedIndexFromKvs
rows=400000
mean357.411-357.404=0.007
σ0.143-0.135=0.009
min357.321
max357.627-357.643=-0.016
mean562.000-569.000=-7.000
σ0.000
min562.000-569.000=-7.000
max562.000-569.000=-7.000
mean765.476-777.539=-12.063
σ55.854-58.339=-2.486
min628.821-652.487=-23.666
max804.955-867.770=-62.815

Marine Hull: transactPolicy

Factorallocated MiBpeak allocated MiBtime
datapoints=1
iterations=20
mean1006.875-1006.876=-0.001
σ0.751-0.751=0.001
min1006.406-1006.403=0.003
max1009.979-1009.977=0.002
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean1.578-1.569=0.009
σ0.160-0.140=0.020
min1.302-1.463=-0.162
max2.037-1.995=0.042

Parser: Comma Sep

Factorallocated MiBpeak allocated MiBtime (ms)
name=ambiguous_item.art
mean0.883
σ0.003
min0.882
max0.887
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean0.310-0.308=0.001
σ0.107-0.124=-0.017
min0.256-0.246=0.010
max0.501-0.529=-0.028
name=error_case.art
mean2.339
σ0.003
min2.337
max2.342
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean0.635-0.831=-0.196
σ0.079-0.130=-0.050
min0.591-0.642=-0.051
max0.775-0.993=-0.218
name=heavy_item.art
mean1.524
σ0.002
min1.522
max1.526
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean0.527-0.536=-0.009
σ0.051-0.045=0.006
min0.491-0.497=-0.006
max0.615-0.607=0.008
name=nested_use.art
mean2.361-2.361=0.000
σ0.002-0.002=-0.000
min2.360
max2.364
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean0.834-0.847=-0.013
σ0.043-0.042=0.001
min0.803-0.813=-0.010
max0.907-0.917=-0.010

Parser: Real World

Factorallocated MiBpeak allocated MiBtime (sec)
name=aon_gpf25_7k_loc.art
mean964.837-964.829=0.008
σ0.033-0.027=0.006
min964.810-964.802=0.008
max964.892-964.864=0.028
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean0.788-0.828=-0.040
σ0.035-0.023=0.012
min0.733-0.788=-0.055
max0.828-0.841=-0.013
name=cdr_enhanced_4k_loc.art
mean885.513-885.511=0.002
σ0.008-0.006=0.002
min885.504
max885.523-885.519=0.004
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean0.732-0.720=0.012
σ0.005-0.020=-0.015
min0.725-0.685=0.040
max0.737-0.734=0.004

Parser: Record Parsing

Factorallocated MiBpeak allocated MiBtime (ms)
name=record_render_large_success.art
mean7.716-7.716=-0.000
σ0.002-0.002=-0.000
min7.715
max7.721
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean4.214-4.211=0.003
σ0.078-0.084=-0.006
min4.076-4.042=0.034
max4.468-4.543=-0.075
name=record_render_large_syntax_err.art
mean7.665-7.665=0.000
σ0.002-0.002=0.000
min7.664
max7.669-7.668=0.001
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean4.222-4.203=0.018
σ0.080-0.084=-0.005
min4.060-4.060=-0.001
max4.557-4.513=0.044
name=record_render_success.art
mean2.291
σ0.002
min2.290
max2.294
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean0.739-0.729=0.010
σ0.020-0.021=-0.001
min0.710-0.675=0.035
max0.880-0.842=0.038
name=record_render_syntax_err.art
mean2.166
σ0.002
min2.165
max2.169
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean0.701-0.703=-0.002
σ0.019-0.014=0.004
min0.645-0.675=-0.030
max0.810-0.785=0.025

SoV import

Factorallocated MiBpeak allocated MiBtime_s
iters=10
rows=1000
mean295.376-295.519=-0.142
σ0.191-0.232=-0.041
min295.242-295.276=-0.035
max295.900-296.094=-0.194
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean0.595-0.566=0.029
σ0.050-0.066=-0.016
min0.512-0.498=0.014
max0.688-0.712=-0.023

Workflow: transactPolicy

Factorallocated MiBpeak allocated MiBtime
iters=5
new-facts=1
prior-facts=1
mean285.383-285.368=0.015
σ0.028-0.025=0.003
min285.347-285.335=0.012
max285.416-285.402=0.014
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean0.394-0.394=-0.000
σ0.008-0.007=0.000
min0.388-0.390=-0.002
max0.408-0.407=0.000
iters=5
new-facts=1
prior-facts=6402
mean1295.771-1295.797=-0.026
σ0.059-0.076=-0.018
min1295.681-1295.693=-0.011
max1295.816-1295.908=-0.092
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean1.932-2.090=-0.158
σ0.083-0.180=-0.097
min1.817-1.821=-0.004
max1.999-2.266=-0.268
iters=5
new-facts=6402
prior-facts=1
mean1539.979-1539.981=-0.003
σ6.904-6.890=0.014
min1536.718-1536.729=-0.011
max1552.326-1552.304=0.021
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean2.340-2.318=0.022
σ0.213-0.176=0.037
min2.102-2.117=-0.014
max2.645-2.578=0.067
iters=5
new-facts=6402
prior-facts=6402
mean1857.694-1857.648=0.046
σ0.041-0.070=-0.030
min1857.651-1857.558=0.093
max1857.746-1857.723=0.022
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean3.097-3.024=0.073
σ0.254-0.110=0.144
min2.850-2.854=-0.003
max3.525-3.143=0.382

newProductH/cdr_auton/broker

Factortime_s
iterations=5
mean1.231-1.614=-0.384
σ0.294-0.235=0.059
min0.933-1.234=-0.301
max1.642-1.841=-0.198

policy addition benchmark

Factorallocated MiBdb queriespeak allocated MiBtime
policies=2
mean445.741-445.738=0.003
σ580.932-580.932=-0.000
min155.273-155.269=0.003
max1317.139-1317.136=0.002
mean96.250
σ8.500
min92.000
max109.000
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean0.542-0.562=-0.020
σ0.554-0.583=-0.029
min0.263-0.266=-0.003
max1.373-1.436=-0.063
policies=20
mean1552.960-1552.986=-0.025
σ0.268-0.335=-0.067
min1552.763-1552.759=0.003
max1553.355-1553.482=-0.128
mean920.000
σ0.000
min920.000
max920.000
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean2.609-2.611=-0.001
σ0.016-0.028=-0.012
min2.593-2.592=0.000
max2.632-2.653=-0.021
policies=200
mean15538.035-15538.012=0.023
σ0.133-0.167=-0.033
min15537.869-15537.847=0.023
max15538.187-15538.243=-0.057
mean9200.000
σ0.000
min9200.000
max9200.000
mean8720.000-8887.000=-167.000
σ0.000
min8720.000-8887.000=-167.000
max8720.000-8887.000=-167.000
mean26.326-26.320=0.006
σ0.091-0.158=-0.066
min26.261-26.123=0.138
max26.460-26.451=0.009