Compare commits

Home / Compare e73cd0b9 .. 6e5da052

grey: no significant change wrt stddev
red: change exceeding max stddev
purple: some change, not seemingly significant

Aggregation: measureAggregations

Factorallocated MiBpeak allocated MiBtime_s
cache=n
dps=1
iters=10
policies=100
mean54.055-54.059=-0.004
σ0.370-0.370=0.000
min53.911-53.919=-0.009
max55.107-55.111=-0.004
mean412.700-414.000=-1.300
σ0.483-0.000=0.483
min412.000-414.000=-2.000
max413.000-414.000=-1.000
mean0.134-0.149=-0.015
σ0.005-0.008=-0.004
min0.130-0.137=-0.007
max0.141-0.160=-0.019
cache=y
dps=1
iters=10
policies=100
mean43.394-43.397=-0.003
σ0.005-0.007=-0.002
min43.386-43.386=0.000
max43.404-43.408=-0.003
mean413.000-414.000=-1.000
σ0.000
min413.000-414.000=-1.000
max413.000-414.000=-1.000
mean0.121-0.128=-0.007
σ0.004-0.010=-0.005
min0.113-0.116=-0.003
max0.128-0.142=-0.014

CSV - indexed operations

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=column_set() on indexed
rows=200000
mean70.650
σ0.002
min70.647
max70.652
mean8706.000
σ0.000
min8706.000
max8706.000
mean150.130
σ9.439
min126.369
max158.266
iterations=10
operation_display=column_set() on indexed, repeat
rows=200000
mean70.649
σ0.002
min70.647
max70.651
mean8706.000
σ0.000
min8706.000
max8706.000
mean148.823
σ10.128
min122.855
max156.796
iterations=10
operation_display=table_records(), no indexes
rows=200000
mean41.929
σ0.000
min41.928
max41.929
mean8706.000
σ0.000
min8706.000
max8706.000
mean14.005
σ1.984
min11.093
max15.606
iterations=10
operation_display=table_records(), no indexes, repeat
rows=200000
mean41.931
σ0.003
min41.928
max41.936
mean8706.000
σ0.000
min8706.000
max8706.000
mean12.743
σ1.585
min10.480
max14.760
iterations=10
operation_display=table_records(), repeat
rows=200000
mean41.929
σ0.002
min41.928
max41.932
mean5028.000
σ2354.218
min1339.000
max8706.000
mean14.261
σ1.681
min11.200
max15.822
iterations=10
operation_display=vertical_lookup_list() on indexed, after table_records()
rows=200000
mean27.606
σ0.002
min27.604
max27.608
mean5028.000
σ2354.218
min1339.000
max8706.000
mean9.543
σ0.582
min8.303
max10.355
iterations=10
operation_display=vertical_lookup_list() on indexed, after table_records(), repeat
rows=200000
mean26.401
σ0.002
min26.400
max26.404
mean5028.000
σ2354.218
min1339.000
max8706.000
mean6.997
σ0.424
min6.348
max7.483
iterations=10
operation_display=vertical_lookup_list() on indexed, repeat
rows=200000
mean41.729
σ0.002
min41.727
max41.731
mean8706.000
σ0.000
min8706.000
max8706.000
mean65.519
σ2.267
min60.694
max68.798
iterations=10
operation_display=vertical_lookup_row_list() on 2 indexed
rows=200000
mean25.665
σ0.002
min25.664
max25.669
mean5028.000
σ2354.218
min1339.000
max8706.000
mean6.144
σ0.068
min6.005
max6.223
iterations=10
operation_display=vertical_lookup_row_list() on 2 indexed, repeat
rows=200000
mean25.660
σ0.002
min25.659
max25.663
mean5028.000
σ2354.218
min1339.000
max8706.000
mean5.521
σ0.299
min4.985
max5.864
iterations=10
operation_display=vertical_lookup_row_list() on 3 (1 unindexed)
rows=200000
mean25.968
σ0.003
min25.966
max25.974
mean5028.000
σ2354.218
min1339.000
max8706.000
mean5.388
σ0.254
min5.076
max5.706
iterations=10
operation_display=vertical_lookup_row_list() on 3 (1 unindexed), repeat
rows=200000
mean25.967
σ0.002
min25.966
max25.970
mean5028.000
σ2354.218
min1339.000
max8706.000
mean5.373
σ0.226
min5.035
max5.645
iterations=8
operation_display=column_set() on indexed
rows=200000
mean36.618
σ0.002
min36.616
max36.620
mean4391.500
σ2048.755
min1350.000
max6795.000
mean12.893
σ0.260
min12.516
max13.294
iterations=8
operation_display=column_set() on indexed, repeat
rows=200000
mean36.618
σ0.002
min36.616
max36.620
mean4391.500
σ2048.755
min1350.000
max6795.000
mean12.575
σ0.619
min11.992
max13.667
iterations=8
operation_display=table_records(), no indexes
rows=200000
mean41.930
σ0.002
min41.928
max41.932
mean6795.000
σ0.000
min6795.000
max6795.000
mean14.297
σ1.748
min11.589
max15.960
iterations=8
operation_display=table_records(), no indexes, repeat
rows=200000
mean41.930
σ0.002
min41.928
max41.933
mean6795.000
σ0.000
min6795.000
max6795.000
mean13.386
σ2.055
min10.317
max15.814
iterations=8
operation_display=table_records(), repeat
rows=200000
mean41.929
σ0.002
min41.928
max41.932
mean4391.500
σ2048.755
min1350.000
max6795.000
mean13.105
σ2.172
min11.086
max16.062
iterations=8
operation_display=vertical_lookup_list() on indexed, after table_records()
rows=200000
mean27.605
σ0.002
min27.604
max27.608
mean4391.500
σ2048.755
min1350.000
max6795.000
mean9.700
σ1.460
min8.655
max13.026
iterations=8
operation_display=vertical_lookup_list() on indexed, after table_records(), repeat
rows=200000
mean26.400
σ0.001
min26.400
max26.404
mean4391.500
σ2048.755
min1350.000
max6795.000
mean7.016
σ1.259
min6.238
max10.037
iterations=8
operation_display=vertical_lookup_row_list() on 2 indexed
rows=200000
mean25.666
σ0.002
min25.664
max25.669
mean4391.500
σ2048.755
min1350.000
max6795.000
mean5.978
σ0.158
min5.777
max6.272
iterations=8
operation_display=vertical_lookup_row_list() on 2 indexed, repeat
rows=200000
mean25.660
σ0.002
min25.659
max25.663
mean4391.500
σ2048.755
min1350.000
max6795.000
mean5.470
σ0.223
min5.033
max5.679
iterations=8
operation_display=vertical_lookup_row_list() on 3 (1 unindexed)
rows=200000
mean25.968
σ0.002
min25.966
max25.970
mean4391.500
σ2048.755
min1350.000
max6795.000
mean5.501
σ0.240
min5.119
max5.740
iterations=8
operation_display=vertical_lookup_row_list() on 3 (1 unindexed), repeat
rows=200000
mean25.967
σ0.001
min25.966
max25.970
mean4391.500
σ2048.755
min1350.000
max6795.000
mean5.496
σ0.293
min4.965
max5.794

CSV - loading a lazy table

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=load_product(), no indexes
rows=200000
mean2524.581
σ0.002
min2524.579
max2524.583
mean8706.000
σ0.000
min8706.000
max8706.000
mean1557.086
σ828.001
min1029.348
max3261.872
iterations=10
operation_display=load_product(), with indexes
rows=200000
mean8744.543
σ0.036
min8744.530
max8744.644
mean4531.300
σ2389.416
min1069.000
max8706.000
mean4558.634
σ1300.006
min3507.488
max7336.001
iterations=8
operation_display=load_product(), no indexes
rows=200000
mean2524.582
σ0.003
min2524.579
max2524.587
mean6795.000
σ0.000
min6795.000
max6795.000
mean1634.754
σ775.549
min1040.497
max3056.552
iterations=8
operation_display=load_product(), with indexes
rows=200000
mean8744.546
σ0.042
min8744.530
max8744.649
mean3851.125
σ2009.976
min1072.000
max6795.000
mean4697.701
σ1458.914
min3537.445
max7053.579

CSV - unindexed operations

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=column_set() on unindexed, dense
rows=200000
mean62.078
σ0.002
min62.076
max62.081
mean5028.000
σ2354.218
min1339.000
max8706.000
mean105.694
σ9.882
min86.199
max114.310
iterations=10
operation_display=column_set() on unindexed, dense, repeat
rows=200000
mean62.078
σ0.002
min62.076
max62.080
mean5028.000
σ2354.218
min1339.000
max8706.000
mean106.347
σ10.106
min87.061
max115.292
iterations=10
operation_display=vertical_lookup_list() on unindexed missing Int
rows=200000
mean40.945
σ0.001
min40.944
max40.946
mean8706.000
σ0.000
min8706.000
max8706.000
mean98.741
σ15.373
min69.723
max109.278
iterations=10
operation_display=vertical_lookup_list() on unindexed missing Int, repeat
rows=200000
mean40.945
σ0.000
min40.944
max40.946
mean8706.000
σ0.000
min8706.000
max8706.000
mean144.601
σ149.221
min69.827
max567.166
iterations=10
operation_display=vertical_lookup_list() on unindexed unique Int near end
rows=200000
mean40.952
σ0.002
min40.949
max40.955
mean8706.000
σ0.000
min8706.000
max8706.000
mean126.632
σ80.380
min69.474
max352.613
iterations=10
operation_display=vertical_lookup_list() on unindexed unique Int near end, repeat
rows=200000
mean40.952
σ0.002
min40.950
max40.953
mean8706.000
σ0.000
min8706.000
max8706.000
mean98.897
σ15.306
min69.368
max109.610
iterations=10
operation_display=vertical_lookup_row_list() on unindexed small unique Int
rows=200000
mean66.688
σ0.002
min66.686
max66.690
mean8706.000
σ0.000
min8706.000
max8706.000
mean146.041
σ25.860
min104.058
max167.578
iterations=10
operation_display=vertical_lookup_row_list() on unindexed small unique Int, repeat
rows=200000
mean66.688
σ0.002
min66.686
max66.690
mean8706.000
σ0.000
min8706.000
max8706.000
mean145.911
σ26.361
min102.700
max168.075
iterations=8
operation_display=column_set() on unindexed, dense
rows=200000
mean62.078
σ0.002
min62.076
max62.081
mean4391.500
σ2048.755
min1350.000
max6795.000
mean100.723
σ13.773
min87.134
max124.363
iterations=8
operation_display=column_set() on unindexed, dense, repeat
rows=200000
mean62.078
σ0.002
min62.076
max62.080
mean4391.500
σ2048.755
min1350.000
max6795.000
mean97.103
σ9.056
min88.666
max113.215
iterations=8
operation_display=vertical_lookup_list() on unindexed missing Int
rows=200000
mean40.946
σ0.002
min40.944
max40.948
mean6795.000
σ0.000
min6795.000
max6795.000
mean93.614
σ17.214
min67.687
max107.761
iterations=8
operation_display=vertical_lookup_list() on unindexed missing Int, repeat
rows=200000
mean40.946
σ0.002
min40.945
max40.948
mean6795.000
σ0.000
min6795.000
max6795.000
mean93.647
σ17.177
min67.286
max107.639
iterations=8
operation_display=vertical_lookup_list() on unindexed unique Int near end
rows=200000
mean40.951
σ0.002
min40.949
max40.953
mean6795.000
σ0.000
min6795.000
max6795.000
mean98.325
σ15.774
min73.927
max111.657
iterations=8
operation_display=vertical_lookup_list() on unindexed unique Int near end, repeat
rows=200000
mean40.952
σ0.002
min40.949
max40.955
mean6795.000
σ0.000
min6795.000
max6795.000
mean93.908
σ17.367
min67.453
max108.285
iterations=8
operation_display=vertical_lookup_row_list() on unindexed small unique Int
rows=200000
mean66.686
σ0.001
min66.686
max66.690
mean6795.000
σ0.000
min6795.000
max6795.000
mean146.464
σ24.804
min109.439
max167.209
iterations=8
operation_display=vertical_lookup_row_list() on unindexed small unique Int, repeat
rows=200000
mean66.687
σ0.002
min66.686
max66.689
mean6795.000
σ0.000
min6795.000
max6795.000
mean146.548
σ24.714
min108.735
max167.121

CSV - warmup

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=table_records()
rows=200000
mean15089.487
σ3.862
min15088.265
max15100.479
mean5028.000
σ2354.218
min1339.000
max8706.000
mean9979.491
σ1673.270
min6539.077
max12012.031
iterations=10
operation_display=vertical_lookup_list() on indexed
rows=200000
mean14031.796
σ0.000
min14031.796
max14031.797
mean8706.000
σ0.000
min8706.000
max8706.000
mean7569.530
σ1993.927
min5377.453
max10185.543
iterations=8
operation_display=table_records()
rows=200000
mean15089.794
σ4.318
min15088.265
max15100.480
mean4391.500
σ2048.755
min1350.000
max6795.000
mean9559.495
σ2336.670
min6620.976
max11912.341
iterations=8
operation_display=vertical_lookup_list() on indexed
rows=200000
mean14031.797
σ0.001
min14031.795
max14031.799
mean6795.000
σ0.000
min6795.000
max6795.000
mean8436.716
σ1343.884
min6050.054
max10316.984

CSV internal benchmark: pointedIndexFromKvs

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=10
operation_display=pointedIndexFromKvs
rows=200000
mean172.665
σ0.011
min172.650
max172.679
mean485.000
σ0.000
min485.000
max485.000
mean380.174
σ45.873
min269.425
max418.173
iterations=10
operation_display=pointedIndexFromKvs
rows=400000
mean357.401
σ0.128
min357.321
max357.611
mean562.000
σ0.000
min562.000
max562.000
mean785.393
σ46.350
min717.387
max862.992
iterations=8
operation_display=pointedIndexFromKvs
rows=200000
mean172.662
σ0.015
min172.639
max172.680
mean485.125
σ0.354
min485.000
max486.000
mean378.753
σ40.246
min302.113
max419.826
iterations=8
operation_display=pointedIndexFromKvs
rows=400000
mean357.464
σ0.154
min357.321
max357.662
mean571.000
σ0.000
min571.000
max571.000
mean741.316
σ81.075
min614.155
max859.086

Marine Hull: transactPolicy

Factorallocated MiBpeak allocated MiBtime
datapoints=1
iterations=20
mean1006.876-1006.876=0.000
σ0.751-0.750=0.000
min1006.404-1006.405=-0.001
max1009.977-1009.978=-0.002
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean1.500-1.610=-0.110
σ0.084-0.146=-0.062
min1.415-1.443=-0.028
max1.828-2.008=-0.181

Parser: Comma Sep

Factorallocated MiBpeak allocated MiBtime (ms)
name=ambiguous_item.art
mean0.883-0.883=-0.001
σ0.002-0.003=-0.000
min0.882
max0.887
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean0.383-0.179=0.204
σ0.110-0.095=0.015
min0.264-0.127=0.138
max0.496-0.348=0.148
name=error_case.art
mean2.339-2.339=0.001
σ0.003-0.003=-0.000
min2.337
max2.342
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean0.894-0.360=0.534
σ0.111-0.070=0.041
min0.825-0.316=0.509
max1.084-0.484=0.600
name=heavy_item.art
mean1.523-1.524=-0.001
σ0.001-0.002=-0.001
min1.522
max1.525-1.526=-0.001
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean0.579-0.270=0.310
σ0.062-0.127=-0.065
min0.509-0.196=0.313
max0.653-0.493=0.160
name=nested_use.art
mean2.362-2.361=0.001
σ0.002-0.002=0.000
min2.360
max2.364
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean0.841-0.829=0.012
σ0.038-0.039=-0.001
min0.807-0.799=0.008
max0.904-0.896=0.008

Parser: Real World

Factorallocated MiBpeak allocated MiBtime (sec)
name=aon_gpf25_7k_loc.art
mean964.830-964.829=0.000
σ0.023-0.037=-0.014
min964.802
max964.855-964.893=-0.038
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean0.784-0.810=-0.026
σ0.064-0.051=0.012
min0.689-0.726=-0.038
max0.853-0.852=0.001
name=cdr_enhanced_4k_loc.art
mean885.512-885.511=0.001
σ0.008-0.007=0.001
min885.504
max885.524-885.519=0.005
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean0.720-0.618=0.101
σ0.024-0.053=-0.029
min0.678-0.534=0.144
max0.737-0.679=0.058

Parser: Record Parsing

Factorallocated MiBpeak allocated MiBtime (ms)
name=record_render_large_success.art
mean7.716-7.716=0.000
σ0.002-0.002=0.000
min7.715
max7.719
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean4.215-3.483=0.732
σ0.092-0.106=-0.014
min4.063-3.289=0.774
max4.525-3.799=0.726
name=record_render_large_syntax_err.art
mean7.665-7.665=-0.000
σ0.002-0.002=-0.000
min7.664
max7.668-7.669=-0.002
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean4.201-3.469=0.732
σ0.077-0.087=-0.011
min4.052-3.252=0.800
max4.436-3.672=0.764
name=record_render_success.art
mean2.291
σ0.002
min2.290
max2.294
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean0.740-0.521=0.219
σ0.016-0.015=0.002
min0.701-0.492=0.209
max0.840-0.621=0.219
name=record_render_syntax_err.art
mean2.166
σ0.002
min2.165
max2.169
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean0.704-0.478=0.226
σ0.016-0.061=-0.045
min0.666-0.271=0.395
max0.782-0.568=0.214

SoV import

Factorallocated MiBpeak allocated MiBtime_s
iters=10
rows=1000
mean295.705-295.548=0.157
σ0.229-0.202=0.028
min295.534-295.407=0.127
max296.331-296.083=0.248
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean0.575-0.580=-0.006
σ0.048-0.028=0.020
min0.507-0.525=-0.018
max0.668-0.617=0.051

Workflow: transactPolicy

Factorallocated MiBpeak allocated MiBtime
iters=5
new-facts=1
prior-facts=1
mean285.381-285.371=0.010
σ0.037-0.019=0.019
min285.323-285.346=-0.023
max285.419-285.397=0.022
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean0.390-0.386=0.004
σ0.008-0.008=-0.001
min0.387-0.380=0.007
max0.404-0.400=0.003
iters=5
new-facts=1
prior-facts=6402
mean1295.779-1295.799=-0.020
σ0.066-0.081=-0.015
min1295.682-1295.692=-0.010
max1295.829-1295.907=-0.078
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean2.097-2.056=0.041
σ0.119-0.163=-0.044
min1.969-1.830=0.138
max2.227-2.266=-0.039
iters=5
new-facts=6402
prior-facts=1
mean1539.963-1539.989=-0.026
σ6.869-6.855=0.014
min1536.736-1536.753=-0.017
max1552.247-1552.247=0.000
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean2.267-2.290=-0.023
σ0.111-0.198=-0.087
min2.119-2.102=0.018
max2.393-2.543=-0.150
iters=5
new-facts=6402
prior-facts=6402
mean1857.678-1857.656=0.022
σ0.054-0.060=-0.005
min1857.603-1857.586=0.017
max1857.731-1857.733=-0.002
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean3.097-3.037=0.060
σ0.223-0.208=0.016
min2.839-2.844=-0.005
max3.457-3.378=0.078

newProductH/cdr_auton/broker

Factortime_s
iterations=5
mean1.602-1.618=-0.017
σ0.213-0.283=-0.070
min1.322-1.288=0.034
max1.835-1.900=-0.064

policy addition benchmark

Factorallocated MiBdb queriespeak allocated MiBtime
policies=2
mean445.741-445.741=0.000
σ580.932-580.935=-0.003
min155.272-155.271=0.001
max1317.139-1317.143=-0.004
mean96.250
σ8.500
min92.000
max109.000
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean0.545-0.562=-0.017
σ0.551-0.585=-0.034
min0.267-0.268=-0.002
max1.373-1.440=-0.068
policies=20
mean1552.978-1553.002=-0.023
σ0.327-0.333=-0.006
min1552.770-1552.776=-0.007
max1553.465-1553.495=-0.030
mean920.000
σ0.000
min920.000
max920.000
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean2.607-2.613=-0.006
σ0.034-0.031=0.004
min2.572-2.588=-0.017
max2.650-2.654=-0.004
policies=200
mean15537.947-15538.046=-0.099
σ0.176-0.105=0.072
min15537.797-15537.982=-0.185
max15538.183-15538.201=-0.018
mean9200.000
σ0.000
min9200.000
max9200.000
mean6795.000-8706.000=-1911.000
σ0.000
min6795.000-8706.000=-1911.000
max6795.000-8706.000=-1911.000
mean26.186-26.268=-0.082
σ0.059-0.206=-0.147
min26.100-26.110=-0.010
max26.227-26.569=-0.341