Compare commits

Home / Compare cec473e8 .. a2d29984

grey: no significant change wrt stddev
red: change exceeding max stddev
purple: some change, not seemingly significant

Aggregation: measureAggregations

Factorallocated MiBpeak allocated MiBtime_s
cache=n
dps=1
iters=10
policies=100
mean54.576-54.617=-0.041
σ0.349-0.351=-0.002
min54.443-54.485=-0.042
max55.569-55.617=-0.048
mean419.000-417.600=1.400
σ0.000-0.843=-0.843
min419.000-416.000=3.000
max419.000-418.000=1.000
mean0.140-0.141=-0.001
σ0.005-0.004=0.001
min0.137-0.136=0.001
max0.149-0.147=0.002
cache=y
dps=1
iters=10
policies=100
mean43.706-43.784=-0.078
σ0.006-0.006=-0.000
min43.700-43.776=-0.075
max43.719-43.793=-0.074
mean419.000-418.000=1.000
σ0.000
min419.000-418.000=1.000
max419.000-418.000=1.000
mean0.121-0.123=-0.002
σ0.004-0.004=0.000
min0.117-0.115=0.002
max0.130-0.131=-0.001

CSV - indexed operations

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=8
operation_display=column_set() on indexed
rows=200000
mean36.617
σ0.002
min36.615
max36.620
mean4171.875
σ1899.226
min1552.000
max7205.000
mean12.318
σ0.418
min11.863
max13.010
iterations=8
operation_display=column_set() on indexed, repeat
rows=200000
mean36.617
σ0.002
min36.616
max36.620
mean4291.875
σ1978.942
min1552.000
max7205.000
mean103.269
σ255.890
min12.575
max736.563
iterations=8
operation_display=table_records(), no indexes
rows=200000
mean41.930
σ0.002
min41.928
max41.932
mean7205.000
σ0.000
min7205.000
max7205.000
mean14.246
σ1.215
min11.716
max15.070
iterations=8
operation_display=table_records(), no indexes, repeat
rows=200000
mean41.930
σ0.002
min41.928
max41.932
mean7205.000
σ0.000
min7205.000
max7205.000
mean12.890
σ1.110
min10.806
max14.162
iterations=8
operation_display=table_records(), repeat
rows=200000
mean41.929
σ0.002
min41.927
max41.932
mean4171.875
σ1899.226
min1552.000
max7205.000
mean12.721
σ1.458
min10.784
max14.688
iterations=8
operation_display=vertical_lookup_list() on indexed, after table_records()
rows=200000
mean27.605
σ0.002
min27.603
max27.607
mean4171.875
σ1899.226
min1552.000
max7205.000
mean8.542
σ0.254
min8.131
max8.847
iterations=8
operation_display=vertical_lookup_list() on indexed, after table_records(), repeat
rows=200000
mean26.401
σ0.002
min26.399
max26.403
mean4171.875
σ1899.226
min1552.000
max7205.000
mean6.603
σ0.299
min6.097
max6.924
iterations=8
operation_display=vertical_lookup_row_list() on 2 indexed
rows=200000
mean25.666
σ0.002
min25.664
max25.670
mean4291.875
σ1978.942
min1552.000
max7205.000
mean5.721
σ0.593
min5.386
max7.154
iterations=8
operation_display=vertical_lookup_row_list() on 2 indexed, repeat
rows=200000
mean25.661
σ0.002
min25.659
max25.663
mean4291.875
σ1978.942
min1552.000
max7205.000
mean5.415
σ0.541
min5.071
max6.731
iterations=8
operation_display=vertical_lookup_row_list() on 3 (1 unindexed)
rows=200000
mean25.969
σ0.002
min25.967
max25.972
mean4291.875
σ1978.942
min1552.000
max7205.000
mean5.373
σ0.403
min5.135
max6.357
iterations=8
operation_display=vertical_lookup_row_list() on 3 (1 unindexed), repeat
rows=200000
mean25.968
σ0.002
min25.967
max25.971
mean4291.875
σ1978.942
min1552.000
max7205.000
mean5.260
σ0.392
min5.053
max6.208
op=column_set() on indexed
mean36.618
σ0.002
min36.615
max36.620
mean4154.500
σ1902.195
min1557.000
max7314.000
mean12.744
σ0.243
min12.401
max13.147
op=column_set() on indexed, repeat
mean36.617
σ0.002
min36.616
max36.620
mean4154.500
σ1902.195
min1557.000
max7314.000
mean13.145
σ0.403
min12.258
max13.685
op=table_records(), no indexes
mean41.932
σ0.001
min41.929
max41.933
mean7314.000
σ0.000
min7314.000
max7314.000
mean15.091
σ0.959
min12.763
max15.671
op=table_records(), repeat
mean41.928
σ0.002
min41.927
max41.932
mean4154.500
σ1902.195
min1557.000
max7314.000
mean13.098
σ1.765
min10.799
max15.619
op=vertical_lookup_list() on indexed, after table_records()
mean27.653
σ0.136
min27.603
max27.990
mean4154.500
σ1902.195
min1557.000
max7314.000
mean9.184
σ0.560
min8.564
max10.282
op=vertical_lookup_list() on indexed, after table_records(), repeat
mean26.400
σ0.002
min26.399
max26.403
mean4154.500
σ1902.195
min1557.000
max7314.000
mean6.841
σ0.370
min6.199
max7.202
op=vertical_lookup_row_list() on 2 indexed
mean25.666
σ0.002
min25.665
max25.668
mean4154.500
σ1902.195
min1557.000
max7314.000
mean6.140
σ0.172
min5.827
max6.405
op=vertical_lookup_row_list() on 3 (1 unindexed)
mean25.968
σ0.002
min25.967
max25.972
mean4154.500
σ1902.195
min1557.000
max7314.000
mean5.893
σ0.102
min5.739
max6.057

CSV - loading a lazy table

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=8
operation_display=load_product(), no indexes
rows=200000
mean2524.595
σ0.002
min2524.594
max2524.598
mean7205.000
σ0.000
min7205.000
max7205.000
mean1194.621
σ415.565
min1026.290
max2222.801
iterations=8
operation_display=load_product(), with indexes
rows=200000
mean8744.561
σ0.040
min8744.545
max8744.661
mean3598.500
σ1819.402
min1076.000
max5839.000
mean4953.536
σ1596.041
min3502.194
max7014.892
op=load_product(), no indexes
mean2524.606
σ0.000
min2524.606
max2524.606
mean7314.000
σ0.000
min7314.000
max7314.000
mean1039.728
σ6.251
min1030.739
max1048.902
op=load_product(), with indexes
mean8744.722
σ0.469
min8744.554
max8745.884
mean3575.625
σ1799.306
min1044.000
max5805.000
mean5052.547
σ1714.060
min3517.703
max7341.368

CSV - unindexed operations

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=8
operation_display=column_set() on unindexed, dense
rows=200000
mean62.077
σ0.002
min62.076
max62.080
mean4171.875
σ1899.226
min1552.000
max7205.000
mean82.281
σ10.566
min71.253
max100.320
iterations=8
operation_display=column_set() on unindexed, dense, repeat
rows=200000
mean62.077
σ0.002
min62.076
max62.080
mean4171.875
σ1899.226
min1552.000
max7205.000
mean83.051
σ10.630
min71.199
max100.913
iterations=8
operation_display=vertical_lookup_list() on unindexed missing Int
rows=200000
mean40.945
σ0.002
min40.944
max40.947
mean7205.000
σ0.000
min7205.000
max7205.000
mean102.087
σ9.650
min83.615
max107.880
iterations=8
operation_display=vertical_lookup_list() on unindexed missing Int, repeat
rows=200000
mean40.945
σ0.002
min40.943
max40.948
mean7205.000
σ0.000
min7205.000
max7205.000
mean102.137
σ9.924
min83.384
max108.399
iterations=8
operation_display=vertical_lookup_list() on unindexed unique Int near end
rows=200000
mean40.951
σ0.002
min40.949
max40.953
mean7205.000
σ0.000
min7205.000
max7205.000
mean104.397
σ9.173
min89.736
max112.698
iterations=8
operation_display=vertical_lookup_list() on unindexed unique Int near end, repeat
rows=200000
mean40.950
σ0.002
min40.948
max40.954
mean7205.000
σ0.000
min7205.000
max7205.000
mean102.220
σ9.571
min84.061
max108.520
iterations=8
operation_display=vertical_lookup_row_list() on unindexed small unique Int
rows=200000
mean66.687
σ0.002
min66.685
max66.690
mean7205.000
σ0.000
min7205.000
max7205.000
mean226.229
σ194.656
min131.978
max706.800
iterations=8
operation_display=vertical_lookup_row_list() on unindexed small unique Int, repeat
rows=200000
mean66.687
σ0.002
min66.685
max66.689
mean7205.000
σ0.000
min7205.000
max7205.000
mean153.685
σ18.279
min121.863
max168.643
op=column_set() on unindexed, dense
mean62.078
σ0.002
min62.076
max62.080
mean4154.500
σ1902.195
min1557.000
max7314.000
mean86.030
σ11.458
min70.835
max99.395
op=vertical_lookup_list() on unindexed missing Int
mean40.944
σ0.001
min40.944
max40.945
mean7314.000
σ0.000
min7314.000
max7314.000
mean106.306
σ8.829
min85.413
max112.486
op=vertical_lookup_row_list() on unindexed small unique Int
mean66.686
σ0.000
min66.685
max66.687
mean7314.000
σ0.000
min7314.000
max7314.000
mean162.020
σ13.091
min130.471
max169.779

CSV - warmup

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=8
operation_display=table_records()
rows=200000
mean11656.833
σ4.318
min11655.305
max11667.519
mean4171.875
σ1899.226
min1552.000
max7205.000
mean7004.260
σ1173.716
min4918.654
max8314.334
iterations=8
operation_display=vertical_lookup_list() on indexed
rows=200000
mean11560.065
σ0.001
min11560.064
max11560.068
mean7205.000
σ0.000
min7205.000
max7205.000
mean5671.153
σ1525.791
min4357.434
max8525.665
op=table_records()
mean11656.836
σ4.324
min11655.305
max11667.538
mean4154.500
σ1902.195
min1557.000
max7314.000
mean6582.482
σ1335.135
min4895.283
max8182.013
op=vertical_lookup_list() on indexed
mean11560.065
σ0.001
min11560.064
max11560.066
mean7314.000
σ0.000
min7314.000
max7314.000
mean5142.594
σ1277.726
min4289.536
max7892.758

CSV internal benchmark: pointedIndexFromKvs

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=8
operation_display=pointedIndexFromKvs
rows=200000
mean172.668
σ0.011
min172.642
max172.681
mean488.125
σ0.354
min488.000
max489.000
mean400.904
σ20.720
min371.589
max424.511
iterations=8
operation_display=pointedIndexFromKvs
rows=400000
mean357.458
σ0.153
min357.321
max357.633
mean565.000
σ0.000
min565.000
max565.000
mean801.328
σ50.571
min707.854
max890.570
op=pointedIndexFromKvs
rows=200000
mean172.559
σ0.001
min172.558
max172.562
mean478.000
σ0.000
min478.000
max478.000
mean382.407
σ16.626
min358.065
max399.625
op=pointedIndexFromKvs
rows=400000
mean357.322
σ0.001
min357.321
max357.325
mean560.000
σ0.000
min560.000
max560.000
mean741.261
σ67.584
min606.913
max804.348

Marine Hull: transactPolicy

Factorallocated MiBpeak allocated MiBtime
datapoints=1
iterations=20
mean1008.501-1008.084=0.417
σ0.760-0.761=-0.001
min1008.025-1007.610=0.415
max1011.641-1011.231=0.411
mean1058.650-7205.000=-6146.350
σ314.440-0.000=314.440
min563.000-7205.000=-6642.000
max1521.000-7205.000=-5684.000
mean1.522-1.477=0.046
σ0.061-0.095=-0.034
min1.470-1.380=0.089
max1.725-1.712=0.013

Parser: Comma Sep

Factorallocated MiBpeak allocated MiBtime (ms)
name=ambiguous_item.art
mean0.913-0.883=0.030
σ0.020-0.002=0.018
min0.903-0.881=0.021
max0.949-0.886=0.063
mean402.000-7205.000=-6803.000
σ0.000
min402.000-7205.000=-6803.000
max402.000-7205.000=-6803.000
mean0.295-0.311=-0.016
σ0.291-0.127=0.164
min0.135-0.236=-0.101
max0.811-0.535=0.276
name=error_case.art
mean2.339-2.333=0.006
σ0.012-0.003=0.010
min2.333-2.332=0.001
max2.361-2.337=0.024
mean402.000-7205.000=-6803.000
σ0.000
min402.000-7205.000=-6803.000
max402.000-7205.000=-6803.000
mean0.479-0.787=-0.307
σ0.316-0.171=0.145
min0.331-0.617=-0.286
max1.044-1.009=0.035
name=heavy_item.art
mean1.522-1.521=0.001
σ0.002-0.002=0.000
min1.521-1.520=0.001
max1.525-1.524=0.001
mean402.000-7205.000=-6803.000
σ0.000
min402.000-7205.000=-6803.000
max402.000-7205.000=-6803.000
mean0.220-0.528=-0.308
σ0.016-0.048=-0.032
min0.208-0.487=-0.279
max0.248-0.601=-0.353
name=nested_use.art
mean2.357-2.355=0.001
σ0.002
min2.356-2.355=0.001
max2.360-2.358=0.001
mean402.000-7205.000=-6803.000
σ0.000
min402.000-7205.000=-6803.000
max402.000-7205.000=-6803.000
mean0.333-0.836=-0.503
σ0.011-0.045=-0.034
min0.324-0.801=-0.477
max0.352-0.912=-0.560

Parser: Real World

Factorallocated MiBpeak allocated MiBtime (sec)
name=aon_gpf25_7k_loc.art
mean964.741-963.541=1.200
σ0.011-0.020=-0.009
min964.736-963.523=1.213
max964.760-963.569=1.191
mean436.200-7205.000=-6768.800
σ5.541-0.000=5.541
min431.000-7205.000=-6774.000
max442.000-7205.000=-6763.000
mean0.549-0.833=-0.284
σ0.031-0.025=0.006
min0.523-0.788=-0.265
max0.602-0.851=-0.248
name=cdr_enhanced_4k_loc.art
mean885.814-884.853=0.961
σ0.002-0.006=-0.004
min885.812-884.847=0.965
max885.816-884.860=0.956
mean442.000-7205.000=-6763.000
σ0.000
min442.000-7205.000=-6763.000
max442.000-7205.000=-6763.000
mean0.474-0.733=-0.259
σ0.030-0.004=0.026
min0.436-0.728=-0.292
max0.511-0.738=-0.226

Parser: Record Parsing

Factorallocated MiBpeak allocated MiBtime (ms)
name=record_render_large_success.art
mean7.708-7.706=0.001
σ0.002-0.002=-0.000
min7.707-7.705=0.001
max7.711-7.711=-0.001
mean442.000-7205.000=-6763.000
σ0.000
min442.000-7205.000=-6763.000
max442.000-7205.000=-6763.000
mean2.552-4.289=-1.736
σ0.074-0.112=-0.039
min2.444-4.123=-1.679
max2.892-4.726=-1.834
name=record_render_large_syntax_err.art
mean7.656-7.655=0.001
σ0.002
min7.655-7.654=0.001
max7.659-7.658=0.001
mean442.000-7205.000=-6763.000
σ0.000
min442.000-7205.000=-6763.000
max442.000-7205.000=-6763.000
mean2.608-4.260=-1.652
σ0.090-0.093=-0.002
min2.421-4.124=-1.703
max2.926-4.554=-1.629
name=record_render_success.art
mean2.289-2.288=0.001
σ0.002
min2.288-2.287=0.001
max2.292-2.291=0.001
mean442.000-7205.000=-6763.000
σ0.000
min442.000-7205.000=-6763.000
max442.000-7205.000=-6763.000
mean0.306-0.755=-0.449
σ0.009-0.021=-0.012
min0.289-0.716=-0.428
max0.363-0.870=-0.506
name=record_render_syntax_err.art
mean2.165-2.163=0.001
σ0.002
min2.164-2.163=0.001
max2.168-2.166=0.001
mean442.000-7205.000=-6763.000
σ0.000
min442.000-7205.000=-6763.000
max442.000-7205.000=-6763.000
mean0.294-0.716=-0.422
σ0.019-0.022=-0.003
min0.278-0.603=-0.325
max0.463-0.837=-0.373

SoV import

Factorallocated MiBpeak allocated MiBtime_s
iters=10
rows=1000
mean298.310-297.509=0.802
σ0.945-0.195=0.750
min297.794-297.384=0.410
max300.979-298.034=2.944
mean441.100-7205.000=-6763.900
σ13.337-0.000=13.337
min420.000-7205.000=-6785.000
max460.000-7205.000=-6745.000
mean0.620-0.576=0.045
σ0.008-0.038=-0.030
min0.610-0.518=0.092
max0.637-0.616=0.021

Timed compile per phase

Factorallocated MiBpeak allocated MiBtime (ms)
phase=buildDeclTree
mean222.008
σ0.000
min222.008
max222.008
mean1748.000
σ0.000
min1748.000
max1748.000
mean203.335
σ1.688
min200.719
max206.077
phase=macro expansion
mean6.130
σ0.000
min6.130
max6.130
mean1748.000
σ0.000
min1748.000
max1748.000
mean11.400
σ1.197
min10.125
max12.574
phase=parser
mean1042.098
σ0.000
min1042.098
max1042.099
mean1748.000
σ0.000
min1748.000
max1748.000
mean727.355
σ87.029
min564.822
max844.427
phase=processProdDef
mean6169.660
σ0.001
min6169.659
max6169.661
mean1749.000
σ2.828
min1748.000
max1756.000
mean4968.745
σ1092.179
min3484.979
max6998.011
phase=renaming
mean21.671
σ0.000
min21.671
max21.671
mean1748.000
σ0.000
min1748.000
max1748.000
mean40.141
σ7.638
min31.497
max47.425
phase=type checker
mean958.457
σ0.066
min958.360
max958.547
mean1748.000
σ0.000
min1748.000
max1748.000
mean813.244
σ60.004
min718.981
max872.961

Workflow: transactPolicy

Factorallocated MiBpeak allocated MiBtime
iters=5
new-facts=1
prior-facts=1
mean293.379-289.722=3.656
σ0.035-0.028=0.007
min293.331-289.680=3.651
max293.416-289.748=3.667
mean445.000-7205.000=-6760.000
σ0.000
min445.000-7205.000=-6760.000
max445.000-7205.000=-6760.000
mean0.406-0.396=0.010
σ0.006-0.006=0.000
min0.398-0.388=0.009
max0.411-0.403=0.007
iters=5
new-facts=1
prior-facts=6402
mean1303.493-1299.654=3.839
σ0.078-0.068=0.010
min1303.363-1299.581=3.781
max1303.562-1299.766=3.796
mean1086.000-7205.000=-6119.000
σ181.119-0.000=181.119
min840.000-7205.000=-6365.000
max1305.000-7205.000=-5900.000
mean2.073-2.018=0.055
σ0.157-0.156=0.000
min1.829-1.824=0.005
max2.242-2.251=-0.009
iters=5
new-facts=6402
prior-facts=1
mean1546.511-1542.626=3.885
σ4.028-4.067=-0.039
min1544.532-1540.576=3.955
max1553.710-1549.893=3.817
mean614.400-7205.000=-6590.600
σ76.474-0.000=76.474
min497.000-7205.000=-6708.000
max701.000-7205.000=-6504.000
mean2.268-2.327=-0.059
σ0.185-0.234=-0.049
min2.156-2.080=0.075
max2.594-2.658=-0.064
iters=5
new-facts=6402
prior-facts=6402
mean1865.441-1861.608=3.832
σ0.037-0.059=-0.022
min1865.393-1861.526=3.867
max1865.479-1861.654=3.824
mean1739.400-7205.000=-5465.600
σ222.542-0.000=222.542
min1442.000-7205.000=-5763.000
max1950.000-7205.000=-5255.000
mean2.953-2.938=0.015
σ0.128-0.073=0.055
min2.851-2.880=-0.029
max3.161-3.044=0.117

newProductH/cdr_auton/broker

Factortime_s
iterations=5
mean1.825-1.810=0.015
σ0.117-0.109=0.008
min1.628-1.662=-0.033
max1.914-1.924=-0.011

policy addition benchmark

Factorallocated MiBdb queriespeak allocated MiBtime
policies=2
mean447.980-446.145=1.834
σ579.240-577.063=2.177
min158.357-157.612=0.745
max1316.839-1311.739=5.100
mean96.250
σ8.500
min92.000
max109.000
mean443.000-7205.000=-6762.000
σ0.000
min443.000-7205.000=-6762.000
max443.000-7205.000=-6762.000
mean0.538-0.540=-0.002
σ0.552-0.545=0.007
min0.260-0.266=-0.006
max1.366-1.357=0.008
policies=20
mean1583.922-1576.411=7.510
σ0.363-0.353=0.010
min1583.670-1576.175=7.495
max1584.458-1576.935=7.523
mean920.000
σ0.000
min920.000
max920.000
mean446.000-7205.000=-6759.000
σ2.000-0.000=2.000
min443.000-7205.000=-6762.000
max447.000-7205.000=-6758.000
mean2.648-2.628=0.020
σ0.020-0.016=0.004
min2.627-2.605=0.022
max2.675-2.640=0.035
policies=200
mean15847.097-15772.281=74.815
σ0.096-0.111=-0.014
min15846.964-15772.156=74.808
max15847.181-15772.408=74.773
mean9200.000
σ0.000
min9200.000
max9200.000
mean498.500-7205.000=-6706.500
σ4.509-0.000=4.509
min493.000-7205.000=-6712.000
max504.000-7205.000=-6701.000
mean26.549-26.564=-0.016
σ0.167-0.250=-0.082
min26.358-26.285=0.073
max26.761-26.795=-0.033