Compare commits

Home / Compare dff7c7cf .. 9b37ea77

grey: no significant change wrt stddev
red: change exceeding max stddev
purple: some change, not seemingly significant

Aggregation: measureAggregations

Factorallocated MiBpeak allocated MiBtime_s
cache=n
dps=1
iters=10
policies=100
mean54.057-54.056=0.001
σ0.368-0.370=-0.002
min53.914-53.914=-0.000
max55.105-55.109=-0.004
mean418.000-421.000=-3.000
σ0.000
min418.000-421.000=-3.000
max418.000-421.000=-3.000
mean0.141-0.140=0.000
σ0.003-0.003=-0.000
min0.138-0.137=0.001
max0.146-0.147=-0.001
cache=y
dps=1
iters=10
policies=100
mean43.395-43.393=0.002
σ0.008-0.006=0.002
min43.385-43.382=0.003
max43.409-43.403=0.006
mean418.000-421.000=-3.000
σ0.000
min418.000-421.000=-3.000
max418.000-421.000=-3.000
mean0.123-0.121=0.002
σ0.004-0.002=0.001
min0.120-0.119=0.001
max0.128-0.128=0.001

CSV - column_set() on indexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=column_set() on indexed
rows=200000
mean36.614
σ0.000
min36.614
max36.615
mean8887.000
σ0.000
min8887.000
max8887.000
mean13.041
σ0.414
min12.484
max13.646
iterations=10
operation_display=column_set() on indexed, repeat
rows=200000
mean36.614
σ0.000
min36.614
max36.614
mean8887.000
σ0.000
min8887.000
max8887.000
mean12.630
σ0.558
min11.817
max13.250

CSV - column_set() on unindexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=column_set() on unindexed, dense, repeat
rows=200000
mean62.074-62.078=-0.004
σ0.000-0.000=0.000
min62.074-62.077=-0.004
max62.075-62.078=-0.003
mean5111.700-5061.300=50.400
σ2610.220-2522.200=88.020
min1356.000-1351.000=5.000
max8533.000-8558.000=-25.000
mean105.202-102.779=2.423
σ12.332-11.525=0.807
min82.420-84.424=-2.004
max119.873-115.975=3.898
iterations=10
operation_display=column_set() on unindexed, sparse, repeat
rows=200000
mean70.664
σ0.002
min70.662
max70.668
mean8887.000
σ0.000
min8887.000
max8887.000
mean201.607
σ202.266
min135.531
max777.237

CSV - column_set(), warmup

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=column_set() on unindexed, dense
rows=200000
mean15109.632-15109.632=0.000
σ3.862-3.863=-0.000
min15108.410-15108.410=0.000
max15120.624-15120.625=-0.000
mean5111.700-5061.300=50.400
σ2610.220-2522.200=88.020
min1356.000-1351.000=5.000
max8533.000-8558.000=-25.000
mean9947.788-10457.631=-509.843
σ2278.230-1869.933=408.297
min6660.158-6692.107=-31.949
max12548.469-12544.884=3.585
iterations=10
operation_display=column_set() on unindexed, sparse
rows=200000
mean14060.732
σ0.002
min14060.731
max14060.736
mean8887.000
σ0.000
min8887.000
max8887.000
mean8432.210
σ2113.917
min5430.379
max11452.268

CSV - loading a lazy table

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=load_product() with indexes
rows=200000
mean8744.535
σ0.000
min8744.534
max8744.535
mean8887.000
σ0.000
min8887.000
max8887.000
mean4508.807
σ1149.775
min3501.342
max6004.661
iterations=10
operation_display=load_product(), no indexes
rows=200000
mean2524.581
σ0.002
min2524.579
max2524.583
mean8887.000
σ0.000
min8887.000
max8887.000
mean1455.235
σ982.668
min1024.309
max4019.295
iterations=10
operation_display=load_product(), with indexes
rows=200000
mean8744.546-8744.542=0.004
σ0.035-0.036=-0.001
min8744.534-8744.530=0.004
max8744.645-8744.644=0.001
mean4712.700-4760.600=-47.900
σ2572.096-2538.158=33.939
min1077.000-1075.000=2.000
max8533.000-8558.000=-25.000
mean4775.949-5124.386=-348.438
σ1217.652-1448.553=-230.901
min3503.051-3531.828=-28.778
max6902.864-7660.100=-757.236
iterations=10
operation_display=load_product()with indexes
rows=200000
mean8744.532-8744.532=-0.000
σ0.002-0.002=0.000
min8744.530-8744.530=-0.000
max8744.535
mean8568.400-8558.000=10.400
σ111.945-0.000=111.945
min8533.000-8558.000=-25.000
max8887.000-8558.000=329.000
mean4766.504-4199.033=567.471
σ1450.613-1071.097=379.516
min3503.778-3506.357=-2.579
max7240.200-6461.268=778.932

CSV - table_records()

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=table_records(), no indexes
rows=200000
mean41.902
σ0.002
min41.900
max41.904
mean8887.000
σ0.000
min8887.000
max8887.000
mean14.663
σ1.234
min11.274
max15.476
iterations=10
operation_display=table_records(), no indexes, repeat
rows=200000
mean41.900
σ0.002
min41.900
max41.903
mean8887.000
σ0.000
min8887.000
max8887.000
mean13.678
σ1.386
min10.765
max15.490
iterations=10
operation_display=table_records(), repeat
rows=200000
mean41.901-41.901=-0.000
σ0.002-0.002=-0.000
min41.899
max41.904-41.904=-0.000
mean8568.400-8558.000=10.400
σ111.945-0.000=111.945
min8533.000-8558.000=-25.000
max8887.000-8558.000=329.000
mean14.316-12.410=1.905
σ1.559-1.841=-0.282
min11.153-10.529=0.624
max15.727-14.739=0.988

CSV - table_records(), warmup

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=table_records()
rows=200000
mean15088.238-15088.238=0.000
σ0.002-0.002=0.000
min15088.236-15088.236=-0.000
max15088.242-15088.241=0.001
mean8568.400-8558.000=10.400
σ111.945-0.000=111.945
min8533.000-8558.000=-25.000
max8887.000-8558.000=329.000
mean9874.848-9369.417=505.431
σ2196.310-2024.262=172.048
min6499.327-6564.246=-64.920
max12671.283-11685.514=985.769

CSV - vertical_lookup_list() on indexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_list() on indexed, after column_set(), repeat
rows=200000
mean26.398-26.398=0.000
σ0.001-0.000=0.001
min26.397-26.398=-0.000
max26.401-26.398=0.003
mean5111.700-5061.300=50.400
σ2610.220-2522.200=88.020
min1356.000-1351.000=5.000
max8533.000-8558.000=-25.000
mean7.018-6.649=0.369
σ0.587-0.499=0.088
min6.257-5.504=0.753
max7.744-7.354=0.390
iterations=10
operation_display=vertical_lookup_list() on indexed, after table_records(), repeat
rows=200000
mean26.398-26.399=-0.000
σ0.002-0.002=-0.000
min26.397-26.397=0.000
max26.401-26.402=-0.000
mean8568.400-8558.000=10.400
σ111.945-0.000=111.945
min8533.000-8558.000=-25.000
max8887.000-8558.000=329.000
mean6.700-6.038=0.661
σ0.453-0.344=0.108
min5.803-5.507=0.296
max7.372-6.556=0.817
iterations=10
operation_display=vertical_lookup_list() on indexed, repeat
rows=200000
mean26.398
σ0.000
min26.398
max26.398
mean8887.000
σ0.000
min8887.000
max8887.000
mean8.214
σ0.387
min7.620
max8.732

CSV - vertical_lookup_list() on unindexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_list() on unindexed missing Int
rows=200000
mean40.944
σ0.003
min40.942
max40.949
mean8887.000
σ0.000
min8887.000
max8887.000
mean97.451
σ11.624
min75.908
max107.512
iterations=10
operation_display=vertical_lookup_list() on unindexed missing Int, repeat
rows=200000
mean40.943
σ0.002
min40.942
max40.946
mean8887.000
σ0.000
min8887.000
max8887.000
mean96.683
σ12.760
min70.713
max107.623
iterations=10
operation_display=vertical_lookup_list() on unindexed unique Int near end
rows=200000
mean40.949
σ0.002
min40.947
max40.952
mean8887.000
σ0.000
min8887.000
max8887.000
mean99.850
σ8.029
min80.545
max107.427
iterations=10
operation_display=vertical_lookup_list() on unindexed unique Int near end, repeat
rows=200000
mean40.948
σ0.001
min40.947
max40.951
mean8887.000
σ0.000
min8887.000
max8887.000
mean128.542
σ91.142
min76.398
max386.685

CSV - vertical_lookup_list(), mini warmup

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_list() on indexed, after column_set()
rows=200000
mean27.602
σ0.001
min27.602
max27.603
mean5111.700
σ2610.220
min1356.000
max8533.000
mean9.870
σ0.691
min8.827
max10.616
iterations=10
operation_display=vertical_lookup_list() on indexed, after table_records()
rows=200000
mean27.602
σ0.002
min27.602
max27.605
mean8568.400
σ111.945
min8533.000
max8887.000
mean9.641
σ0.611
min8.272
max10.239

CSV - vertical_lookup_list(), warmup

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_list() on indexed
rows=200000
mean15073.940
σ0.000
min15073.939
max15073.940
mean8887.000
σ0.000
min8887.000
max8887.000
mean9658.605
σ2392.196
min6483.387
max12229.084

CSV - vertical_lookup_row_list() on indexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_row_list() on 2 indexed
rows=200000
mean25.633-25.633=-0.000
σ0.002-0.002=0.000
min25.631
max25.637-25.635=0.001
mean8568.400-8558.000=10.400
σ111.945-0.000=111.945
min8533.000-8558.000=-25.000
max8887.000-8558.000=329.000
mean5.149-4.730=0.420
σ0.189-0.151=0.038
min4.945-4.515=0.429
max5.540-4.974=0.566
iterations=10
operation_display=vertical_lookup_row_list() on 2 indexed, repeat
rows=200000
mean25.628-25.628=0.000
σ0.002-0.002=0.000
min25.626-25.626=-0.000
max25.630-25.630=0.000
mean8568.400-8558.000=10.400
σ111.945-0.000=111.945
min8533.000-8558.000=-25.000
max8887.000-8558.000=329.000
mean5.039-4.668=0.372
σ0.185-0.182=0.002
min4.874-4.473=0.401
max5.368-4.929=0.439
iterations=10
operation_display=vertical_lookup_row_list() on 3 (1 unindexed)
rows=200000
mean25.932-25.932=0.000
σ0.002-0.002=0.000
min25.930-25.930=0.000
max25.935-25.935=-0.000
mean8568.400-8558.000=10.400
σ111.945-0.000=111.945
min8533.000-8558.000=-25.000
max8887.000-8558.000=329.000
mean5.158-4.608=0.550
σ0.217-0.117=0.100
min4.923-4.504=0.418
max5.504-4.867=0.637
iterations=10
operation_display=vertical_lookup_row_list() on 3 (1 unindexed), repeat
rows=200000
mean25.931-25.931=0.000
σ0.002-0.002=0.000
min25.930-25.930=-0.000
max25.934
mean8568.400-8558.000=10.400
σ111.945-0.000=111.945
min8533.000-8558.000=-25.000
max8887.000-8558.000=329.000
mean5.096-4.645=0.452
σ0.166-0.188=-0.021
min4.840-4.470=0.370
max5.350-5.098=0.252

CSV - vertical_lookup_row_list() on unindexed

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=vertical_lookup_row_list() on unindexed small unique Int
rows=200000
mean66.671
σ0.001
min66.670
max66.673
mean8887.000
σ0.000
min8887.000
max8887.000
mean150.728
σ17.399
min113.634
max165.000
iterations=10
operation_display=vertical_lookup_row_list() on unindexed small unique Int, repeat
rows=200000
mean66.670
σ0.002
min66.670
max66.673
mean8887.000
σ0.000
min8887.000
max8887.000
mean150.718
σ17.331
min114.385
max165.539

CSV internal benchmark: pointedIndexFromKvs

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=pointedIndexFromKvs
rows=200000
mean172.663-172.664=-0.001
σ0.007-0.010=-0.002
min172.650-172.643=0.006
max172.674-172.677=-0.003
mean491.000
σ0.000
min491.000
max491.000
mean391.865-396.817=-4.952
σ26.246-21.156=5.089
min344.574-354.761=-10.188
max420.228-418.579=1.649
iterations=10
operation_display=pointedIndexFromKvs
rows=400000
mean357.404-357.406=-0.001
σ0.135-0.121=0.013
min357.321
max357.643-357.618=0.025
mean569.000-567.000=2.000
σ0.000
min569.000-567.000=2.000
max569.000-567.000=2.000
mean777.539-727.259=50.280
σ58.339-98.191=-39.852
min652.487-584.080=68.407
max867.770-810.932=56.838

Marine Hull: transactPolicy

Factorallocated MiBpeak allocated MiBtime
datapoints=1
iterations=20
mean1006.876-1006.875=0.001
σ0.751-0.748=0.002
min1006.403-1006.405=-0.002
max1009.977-1009.967=0.010
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean1.569-1.599=-0.031
σ0.140-0.126=0.014
min1.463-1.476=-0.013
max1.995-1.924=0.071

Parser: Comma Sep

Factorallocated MiBpeak allocated MiBtime (ms)
name=ambiguous_item.art
mean0.883
σ0.003
min0.882
max0.887
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean0.308-0.313=-0.005
σ0.124-0.129=-0.005
min0.246-0.246=-0.000
max0.529-0.543=-0.014
name=error_case.art
mean2.339
σ0.003
min2.337
max2.342
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean0.831-0.637=0.195
σ0.130-0.086=0.044
min0.642-0.585=0.057
max0.993-0.788=0.205
name=heavy_item.art
mean1.524
σ0.002
min1.522
max1.526
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean0.536-0.538=-0.002
σ0.045-0.056=-0.012
min0.497-0.497=-0.001
max0.607-0.636=-0.029
name=nested_use.art
mean2.361
σ0.002
min2.360
max2.364
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean0.847-0.849=-0.001
σ0.042-0.039=0.003
min0.813-0.825=-0.012
max0.917-0.917=-0.000

Parser: Real World

Factorallocated MiBpeak allocated MiBtime (sec)
name=aon_gpf25_7k_loc.art
mean964.829-964.831=-0.002
σ0.027-0.022=0.005
min964.802-964.807=-0.005
max964.864-964.859=0.005
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean0.828-0.819=0.009
σ0.023-0.025=-0.002
min0.788-0.785=0.003
max0.841-0.841=0.000
name=cdr_enhanced_4k_loc.art
mean885.511-885.511=0.000
σ0.006-0.006=0.000
min885.504
max885.519-885.519=0.000
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean0.720-0.714=0.006
σ0.020-0.023=-0.003
min0.685-0.675=0.010
max0.734-0.732=0.001

Parser: Record Parsing

Factorallocated MiBpeak allocated MiBtime (ms)
name=record_render_large_success.art
mean7.716-7.716=0.000
σ0.002-0.002=0.000
min7.715
max7.721
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean4.211-4.226=-0.016
σ0.084-0.098=-0.013
min4.042-4.071=-0.029
max4.543-4.777=-0.234
name=record_render_large_syntax_err.art
mean7.665
σ0.002
min7.664
max7.668
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean4.203-4.273=-0.070
σ0.084-0.081=0.003
min4.060-4.125=-0.065
max4.513-4.548=-0.035
name=record_render_success.art
mean2.291
σ0.002
min2.290
max2.294
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean0.729-0.740=-0.010
σ0.021-0.025=-0.004
min0.675-0.544=0.130
max0.842-0.828=0.013
name=record_render_syntax_err.art
mean2.166
σ0.002
min2.165
max2.169
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean0.703-0.705=-0.002
σ0.014-0.017=-0.003
min0.675-0.671=0.003
max0.785-0.796=-0.011

SoV import

Factorallocated MiBpeak allocated MiBtime_s
iters=10
rows=1000
mean295.519-295.606=-0.087
σ0.232-0.207=0.025
min295.276-295.466=-0.190
max296.094-296.179=-0.085
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean0.566-0.575=-0.009
σ0.066-0.074=-0.008
min0.498-0.505=-0.007
max0.712-0.730=-0.019

Workflow: transactPolicy

Factorallocated MiBpeak allocated MiBtime
iters=5
new-facts=1
prior-facts=1
mean285.368-285.371=-0.003
σ0.025-0.030=-0.005
min285.335-285.343=-0.008
max285.402-285.419=-0.017
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean0.394-0.400=-0.006
σ0.007-0.014=-0.006
min0.390-0.390=0.000
max0.407-0.416=-0.009
iters=5
new-facts=1
prior-facts=6402
mean1295.797-1295.883=-0.085
σ0.076-0.076=0.000
min1295.693-1295.781=-0.088
max1295.908-1295.968=-0.060
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean2.090-2.252=-0.162
σ0.180-0.194=-0.013
min1.821-1.956=-0.135
max2.266-2.496=-0.230
iters=5
new-facts=6402
prior-facts=1
mean1539.981-1540.011=-0.030
σ6.890-6.885=0.006
min1536.729-1536.798=-0.068
max1552.304-1552.325=-0.020
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean2.318-2.455=-0.136
σ0.176-0.234=-0.058
min2.117-2.115=0.002
max2.578-2.688=-0.110
iters=5
new-facts=6402
prior-facts=6402
mean1857.648-1857.707=-0.059
σ0.070-0.031=0.040
min1857.558-1857.654=-0.096
max1857.723-1857.729=-0.006
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean3.024-3.083=-0.059
σ0.110-0.206=-0.097
min2.854-2.869=-0.015
max3.143-3.407=-0.264

newProductH/cdr_auton/broker

Factortime_s
iterations=5
mean1.614-1.565=0.049
σ0.235-0.251=-0.016
min1.234-1.308=-0.073
max1.841-1.885=-0.044

policy addition benchmark

Factorallocated MiBdb queriespeak allocated MiBtime
policies=2
mean445.738-445.740=-0.003
σ580.932-580.934=-0.002
min155.269-155.270=-0.000
max1317.136-1317.141=-0.005
mean96.250
σ8.500
min92.000
max109.000
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean0.562-0.566=-0.004
σ0.583-0.582=0.001
min0.266-0.271=-0.005
max1.436-1.438=-0.002
policies=20
mean1552.986-1552.982=0.003
σ0.335-0.341=-0.006
min1552.759-1552.747=0.012
max1553.482-1553.488=-0.005
mean920.000
σ0.000
min920.000
max920.000
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean2.611-2.616=-0.006
σ0.028-0.027=0.001
min2.592-2.581=0.012
max2.653-2.638=0.014
policies=200
mean15538.012-15537.984=0.027
σ0.167-0.083=0.084
min15537.847-15537.865=-0.018
max15538.243-15538.051=0.192
mean9200.000
σ0.000
min9200.000
max9200.000
mean8887.000-8558.000=329.000
σ0.000
min8887.000-8558.000=329.000
max8887.000-8558.000=329.000
mean26.320-26.376=-0.056
σ0.158-0.064=0.094
min26.123-26.295=-0.171
max26.451-26.439=0.012