Compare commits

Home / Compare 26f3d994 .. 66570b53

grey: no significant change wrt stddev
red: change exceeding max stddev
purple: some change, not seemingly significant

Aggregation: measureAggregations

Factorallocated MiBpeak allocated MiBtime_s
cache=n
dps=1
iters=10
policies=100
mean54.583-53.703=0.880
σ0.344-0.369=-0.025
min54.447-53.559=0.887
max55.562-54.753=0.809
mean414.000
σ0.000
min414.000
max414.000
mean0.141-0.139=0.002
σ0.004-0.004=0.000
min0.138-0.135=0.003
max0.148-0.144=0.004
cache=y
dps=1
iters=10
policies=100
mean43.705-43.058=0.648
σ0.005-0.005=-0.000
min43.696-43.051=0.645
max43.711-43.064=0.647
mean414.000
σ0.000
min414.000
max414.000
mean0.122-0.119=0.004
σ0.005-0.005=0.000
min0.115-0.113=0.002
max0.130-0.126=0.004

CSV - indexed operations

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=8
operation_display=column_set() on indexed
rows=200000
mean36.617
σ0.002
min36.615
max36.619
mean4284.750
σ1976.153
min1548.000
max7210.000
mean104.094
σ257.601
min12.714
max741.623
iterations=8
operation_display=column_set() on indexed, repeat
rows=200000
mean36.618
σ0.002
min36.616
max36.620
mean4284.750
σ1976.153
min1548.000
max7210.000
mean14.368
σ2.744
min13.081
max21.144
iterations=8
operation_display=table_records(), no indexes
rows=200000
mean41.930
σ0.002
min41.928
max41.932
mean7210.000
σ0.000
min7210.000
max7210.000
mean14.929
σ0.909
min13.321
max15.761
iterations=8
operation_display=table_records(), no indexes, repeat
rows=200000
mean41.929
σ0.002
min41.928
max41.932
mean7210.000
σ0.000
min7210.000
max7210.000
mean13.724
σ1.096
min12.215
max14.890
iterations=8
operation_display=table_records(), repeat
rows=200000
mean41.929
σ0.002
min41.927
max41.932
mean4167.500
σ1898.698
min1548.000
max7210.000
mean13.887
σ1.521
min11.368
max15.699
iterations=8
operation_display=vertical_lookup_list() on indexed, after table_records()
rows=200000
mean27.605
σ0.002
min27.603
max27.607
mean4167.500
σ1898.698
min1548.000
max7210.000
mean9.208
σ0.425
min8.550
max9.796
iterations=8
operation_display=vertical_lookup_list() on indexed, after table_records(), repeat
rows=200000
mean26.400
σ0.001
min26.399
max26.403
mean4167.500
σ1898.698
min1548.000
max7210.000
mean6.556
σ0.183
min6.325
max6.815
iterations=8
operation_display=vertical_lookup_row_list() on 2 indexed
rows=200000
mean25.666
σ0.002
min25.664
max25.668
mean4284.750
σ1976.153
min1548.000
max7210.000
mean6.271
σ0.504
min5.910
max7.494
iterations=8
operation_display=vertical_lookup_row_list() on 2 indexed, repeat
rows=200000
mean25.661
σ0.002
min25.659
max25.663
mean4284.750
σ1976.153
min1548.000
max7210.000
mean6.007
σ0.593
min5.664
max7.447
iterations=8
operation_display=vertical_lookup_row_list() on 3 (1 unindexed)
rows=200000
mean25.969
σ0.002
min25.967
max25.971
mean4284.750
σ1976.153
min1548.000
max7210.000
mean6.057
σ0.577
min5.693
max7.473
iterations=8
operation_display=vertical_lookup_row_list() on 3 (1 unindexed), repeat
rows=200000
mean25.968
σ0.002
min25.967
max25.971
mean4284.750
σ1976.153
min1548.000
max7210.000
mean5.921
σ0.635
min5.599
max7.481
op=column_set() on indexed
mean36.618
σ0.002
min36.615
max36.620
mean4156.875
σ1907.091
min1556.000
max7327.000
mean13.049
σ0.248
min12.710
max13.312
op=column_set() on indexed, repeat
mean36.617
σ0.002
min36.615
max36.620
mean4156.875
σ1907.091
min1556.000
max7327.000
mean13.176
σ0.390
min12.269
max13.495
op=table_records(), no indexes
mean41.931
σ0.001
min41.929
max41.932
mean7327.000
σ0.000
min7327.000
max7327.000
mean14.299
σ0.797
min12.377
max14.767
op=table_records(), repeat
mean41.928
σ0.002
min41.927
max41.931
mean4156.875
σ1907.091
min1556.000
max7327.000
mean12.502
σ1.674
min10.924
max14.542
op=vertical_lookup_list() on indexed, after table_records()
mean27.653
σ0.136
min27.603
max27.990
mean4156.875
σ1907.091
min1556.000
max7327.000
mean9.064
σ0.358
min8.600
max9.558
op=vertical_lookup_list() on indexed, after table_records(), repeat
mean26.400
σ0.002
min26.399
max26.403
mean4156.875
σ1907.091
min1556.000
max7327.000
mean7.090
σ0.094
min6.928
max7.262
op=vertical_lookup_row_list() on 2 indexed
mean25.666
σ0.002
min25.665
max25.669
mean4156.875
σ1907.091
min1556.000
max7327.000
mean6.119
σ0.142
min5.955
max6.390
op=vertical_lookup_row_list() on 3 (1 unindexed)
mean25.968
σ0.002
min25.967
max25.972
mean4156.875
σ1907.091
min1556.000
max7327.000
mean5.787
σ0.066
min5.675
max5.901

CSV - loading a lazy table

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=8
operation_display=load_product(), no indexes
rows=200000
mean2524.596
σ0.002
min2524.594
max2524.598
mean7210.000
σ0.000
min7210.000
max7210.000
mean1420.424
σ628.629
min1041.276
max2780.107
iterations=8
operation_display=load_product(), with indexes
rows=200000
mean8744.560
σ0.041
min8744.545
max8744.661
mean3590.500
σ1812.038
min1072.000
max5815.000
mean4911.557
σ1619.976
min3535.764
max7380.228
op=load_product(), no indexes
mean2524.606
σ0.000
min2524.606
max2524.607
mean7327.000
σ0.000
min7327.000
max7327.000
mean1051.076
σ8.368
min1033.361
max1059.857
op=load_product(), with indexes
mean8744.721
σ0.469
min8744.553
max8745.883
mean3578.125
σ1803.844
min1044.000
max5818.000
mean4927.222
σ1661.246
min3545.894
max7425.330

CSV - unindexed operations

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=8
operation_display=column_set() on unindexed, dense
rows=200000
mean62.077
σ0.001
min62.076
max62.080
mean4167.500
σ1898.698
min1548.000
max7210.000
mean80.302
σ9.015
min72.245
max96.712
iterations=8
operation_display=column_set() on unindexed, dense, repeat
rows=200000
mean62.077
σ0.002
min62.076
max62.080
mean4167.500
σ1898.698
min1548.000
max7210.000
mean81.227
σ9.174
min72.636
max98.263
iterations=8
operation_display=vertical_lookup_list() on unindexed missing Int
rows=200000
mean40.945
σ0.002
min40.944
max40.949
mean7210.000
σ0.000
min7210.000
max7210.000
mean104.456
σ8.671
min86.251
max109.896
iterations=8
operation_display=vertical_lookup_list() on unindexed missing Int, repeat
rows=200000
mean40.945
σ0.002
min40.943
max40.947
mean7210.000
σ0.000
min7210.000
max7210.000
mean104.361
σ8.514
min86.493
max109.953
iterations=8
operation_display=vertical_lookup_list() on unindexed unique Int near end
rows=200000
mean40.950
σ0.002
min40.949
max40.954
mean7210.000
σ0.000
min7210.000
max7210.000
mean107.302
σ7.515
min92.529
max114.508
iterations=8
operation_display=vertical_lookup_list() on unindexed unique Int near end, repeat
rows=200000
mean40.950
σ0.002
min40.948
max40.953
mean7210.000
σ0.000
min7210.000
max7210.000
mean104.348
σ8.553
min86.201
max109.952
iterations=8
operation_display=vertical_lookup_row_list() on unindexed small unique Int
rows=200000
mean66.687
σ0.003
min66.686
max66.693
mean7210.000
σ0.000
min7210.000
max7210.000
mean215.674
σ158.430
min137.571
max606.814
iterations=8
operation_display=vertical_lookup_row_list() on unindexed small unique Int, repeat
rows=200000
mean66.686
σ0.001
min66.685
max66.689
mean7210.000
σ0.000
min7210.000
max7210.000
mean154.211
σ18.537
min117.959
max167.857
op=column_set() on unindexed, dense
mean62.078
σ0.002
min62.076
max62.081
mean4156.875
σ1907.091
min1556.000
max7327.000
mean81.987
σ11.064
min71.717
max104.343
op=vertical_lookup_list() on unindexed missing Int
mean40.944
σ0.001
min40.944
max40.945
mean7327.000
σ0.000
min7327.000
max7327.000
mean104.157
σ8.751
min84.555
max111.782
op=vertical_lookup_row_list() on unindexed small unique Int
mean66.686
σ0.000
min66.685
max66.687
mean7327.000
σ0.000
min7327.000
max7327.000
mean161.194
σ12.559
min131.019
max168.117

CSV - warmup

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=8
operation_display=table_records()
rows=200000
mean11656.833
σ4.319
min11655.305
max11667.523
mean4167.500
σ1898.698
min1548.000
max7210.000
mean6881.016
σ1371.091
min4905.550
max8435.507
iterations=8
operation_display=vertical_lookup_list() on indexed
rows=200000
mean11560.066
σ0.002
min11560.064
max11560.069
mean7210.000
σ0.000
min7210.000
max7210.000
mean5960.613
σ1440.546
min4389.494
max7774.767
op=table_records()
mean11656.836
σ4.325
min11655.305
max11667.538
mean4156.875
σ1907.091
min1556.000
max7327.000
mean6456.411
σ1204.164
min5004.905
max8175.039
op=vertical_lookup_list() on indexed
mean11560.065
σ0.001
min11560.064
max11560.066
mean7327.000
σ0.000
min7327.000
max7327.000
mean5845.889
σ1449.121
min4451.405
max7692.845

CSV internal benchmark: pointedIndexFromKvs

Factorallocated MiBpeak allocated MiBtime (ms)
iterations=8
operation_display=pointedIndexFromKvs
rows=200000
mean172.668
σ0.009
min172.654
max172.680
mean485.000
σ0.000
min485.000
max485.000
mean402.631
σ18.662
min364.364
max421.959
iterations=8
operation_display=pointedIndexFromKvs
rows=400000
mean357.466
σ0.157
min357.321
max357.646
mean561.000
σ0.000
min561.000
max561.000
mean790.811
σ58.801
min647.279
max832.521
op=pointedIndexFromKvs
rows=200000
mean172.559
σ0.001
min172.558
max172.562
mean477.000
σ0.000
min477.000
max477.000
mean366.453
σ16.533
min344.278
max381.278
op=pointedIndexFromKvs
rows=400000
mean357.322
σ0.001
min357.321
max357.325
mean560.000
σ0.000
min560.000
max560.000
mean723.319
σ52.158
min629.155
max779.147

Marine Hull: transactPolicy

Factorallocated MiBpeak allocated MiBtime
datapoints=1
iterations=20
mean1008.499-1005.001=3.499
σ0.760-0.751=0.009
min1008.025-1004.528=3.497
max1011.641-1008.104=3.537
mean1057.350-7210.000=-6152.650
σ309.652-0.000=309.652
min585.000-7210.000=-6625.000
max1511.000-7210.000=-5699.000
mean1.541-1.594=-0.053
σ0.070-0.115=-0.045
min1.481-1.475=0.005
max1.751-1.882=-0.132

Parser: Comma Sep

Factorallocated MiBpeak allocated MiBtime (ms)
name=ambiguous_item.art
mean0.913-0.883=0.030
σ0.020-0.002=0.018
min0.903-0.881=0.021
max0.949-0.886=0.063
mean401.000-7210.000=-6809.000
σ0.000
min401.000-7210.000=-6809.000
max401.000-7210.000=-6809.000
mean0.345-0.423=-0.078
σ0.392-0.143=0.249
min0.141-0.343=-0.202
max1.044-0.677=0.366
name=error_case.art
mean2.339-2.333=0.006
σ0.012-0.003=0.010
min2.333-2.332=0.001
max2.361-2.337=0.024
mean401.000-7210.000=-6809.000
σ0.000
min401.000-7210.000=-6809.000
max401.000-7210.000=-6809.000
mean0.424-0.889=-0.464
σ0.197-0.105=0.093
min0.319-0.802=-0.483
max0.776-1.069=-0.293
name=heavy_item.art
mean1.522-1.521=0.001
σ0.002-0.002=0.000
min1.521-1.520=0.001
max1.525-1.524=0.001
mean401.000-7210.000=-6809.000
σ0.000
min401.000-7210.000=-6809.000
max401.000-7210.000=-6809.000
mean0.220-0.539=-0.319
σ0.023-0.042=-0.019
min0.204-0.504=-0.300
max0.259-0.604=-0.345
name=nested_use.art
mean2.357-2.355=0.001
σ0.002
min2.356-2.355=0.001
max2.360-2.358=0.001
mean401.000-7210.000=-6809.000
σ0.000
min401.000-7210.000=-6809.000
max401.000-7210.000=-6809.000
mean0.332-0.840=-0.507
σ0.018-0.048=-0.031
min0.319-0.795=-0.477
max0.363-0.915=-0.552

Parser: Real World

Factorallocated MiBpeak allocated MiBtime (sec)
name=aon_gpf25_7k_loc.art
mean964.741-963.548=1.193
σ0.011-0.024=-0.014
min964.736-963.523=1.213
max964.760-963.580=1.180
mean435.800-7210.000=-6774.200
σ6.017-0.000=6.017
min430.000-7210.000=-6780.000
max442.000-7210.000=-6768.000
mean0.511-0.817=-0.305
σ0.056-0.043=0.013
min0.450-0.765=-0.315
max0.592-0.852=-0.260
name=cdr_enhanced_4k_loc.art
mean885.814-884.854=0.960
σ0.002-0.006=-0.004
min885.812-884.847=0.965
max885.816-884.861=0.955
mean442.000-7210.000=-6768.000
σ0.000
min442.000-7210.000=-6768.000
max442.000-7210.000=-6768.000
mean0.450-0.730=-0.280
σ0.066-0.024=0.042
min0.386-0.687=-0.301
max0.533-0.744=-0.211

Parser: Record Parsing

Factorallocated MiBpeak allocated MiBtime (ms)
name=record_render_large_success.art
mean7.708-7.706=0.001
σ0.002-0.002=-0.000
min7.707-7.705=0.001
max7.711-7.709=0.001
mean442.000-7210.000=-6768.000
σ0.000
min442.000-7210.000=-6768.000
max442.000-7210.000=-6768.000
mean2.596-4.296=-1.700
σ0.086-0.109=-0.024
min2.447-3.679=-1.233
max2.882-4.630=-1.748
name=record_render_large_syntax_err.art
mean7.656-7.655=0.001
σ0.002
min7.655-7.654=0.001
max7.659-7.658=0.001
mean442.000-7210.000=-6768.000
σ0.000
min442.000-7210.000=-6768.000
max442.000-7210.000=-6768.000
mean2.561-4.265=-1.704
σ0.071-0.078=-0.006
min2.450-4.145=-1.695
max2.810-4.568=-1.758
name=record_render_success.art
mean2.289-2.288=0.001
σ0.002
min2.288-2.287=0.001
max2.292-2.291=0.001
mean442.000-7210.000=-6768.000
σ0.000
min442.000-7210.000=-6768.000
max442.000-7210.000=-6768.000
mean0.305-0.747=-0.442
σ0.007-0.018=-0.011
min0.291-0.713=-0.422
max0.347-0.861=-0.514
name=record_render_syntax_err.art
mean2.165-2.163=0.001
σ0.002
min2.164-2.163=0.001
max2.168-2.166=0.001
mean442.000-7210.000=-6768.000
σ0.000
min442.000-7210.000=-6768.000
max442.000-7210.000=-6768.000
mean0.290-0.716=-0.426
σ0.012-0.021=-0.009
min0.277-0.624=-0.346
max0.362-0.812=-0.450

SoV import

Factorallocated MiBpeak allocated MiBtime_s
iters=10
rows=1000
mean298.112-294.251=3.861
σ1.056-0.200=0.856
min297.684-294.070=3.614
max301.112-294.764=6.348
mean445.100-7210.000=-6764.900
σ14.693-0.000=14.693
min422.000-7210.000=-6788.000
max466.000-7210.000=-6744.000
mean0.615-0.580=0.036
σ0.011-0.030=-0.019
min0.601-0.511=0.090
max0.630-0.604=0.026

Timed compile per phase

Factorallocated MiBpeak allocated MiBtime (ms)
phase=buildDeclTree
mean222.008
σ0.000
min222.008
max222.008
mean1741.000
σ0.000
min1741.000
max1741.000
mean204.539
σ3.368
min200.385
max212.177
phase=macro expansion
mean6.130
σ0.000
min6.130
max6.130
mean1741.000
σ0.000
min1741.000
max1741.000
mean11.541
σ2.047
min7.305
max12.756
phase=parser
mean1042.099
σ0.000
min1042.099
max1042.099
mean1741.000
σ0.000
min1741.000
max1741.000
mean677.100
σ89.088
min485.083
max728.195
phase=processProdDef
mean6169.659
σ0.001
min6169.659
max6169.660
mean1741.000
σ0.000
min1741.000
max1741.000
mean5494.515
σ1639.562
min3374.577
max7361.589
phase=renaming
mean21.671
σ0.000
min21.671
max21.671
mean1741.000
σ0.000
min1741.000
max1741.000
mean45.617
σ3.861
min38.823
max49.152
phase=type checker
mean958.457
σ0.066
min958.360
max958.547
mean1741.000
σ0.000
min1741.000
max1741.000
mean819.376
σ74.722
min690.330
max879.160

Workflow: transactPolicy

Factorallocated MiBpeak allocated MiBtime
iters=5
new-facts=1
prior-facts=1
mean293.687-281.455=12.232
σ0.044-0.034=0.010
min293.631-281.401=12.229
max293.737-281.495=12.242
mean444.000-7210.000=-6766.000
σ0.000
min444.000-7210.000=-6766.000
max444.000-7210.000=-6766.000
mean0.404-0.393=0.011
σ0.007-0.006=0.001
min0.399-0.386=0.013
max0.417-0.400=0.017
iters=5
new-facts=1
prior-facts=6402
mean1303.780-1291.479=12.301
σ0.063-0.087=-0.025
min1303.714-1291.366=12.348
max1303.868-1291.583=12.285
mean1099.400-7210.000=-6110.600
σ191.003-0.000=191.003
min837.000-7210.000=-6373.000
max1329.000-7210.000=-5881.000
mean2.030-1.940=0.089
σ0.199-0.067=0.132
min1.822-1.868=-0.046
max2.267-2.020=0.247
iters=5
new-facts=6402
prior-facts=1
mean1546.825-1534.343=12.482
σ4.031-4.063=-0.032
min1544.812-1532.301=12.511
max1554.029-1541.602=12.427
mean618.000-7210.000=-6592.000
σ80.567-0.000=80.567
min497.000-7210.000=-6713.000
max713.000-7210.000=-6497.000
mean2.275-2.267=0.008
σ0.184-0.238=-0.054
min2.142-2.087=0.055
max2.598-2.659=-0.061
iters=5
new-facts=6402
prior-facts=6402
mean1865.810-1853.338=12.472
σ0.050-0.078=-0.028
min1865.753-1853.229=12.523
max1865.887-1853.427=12.460
mean1730.600-7210.000=-5479.400
σ177.046-0.000=177.046
min1480.000-7210.000=-5730.000
max1893.000-7210.000=-5317.000
mean3.063-3.173=-0.110
σ0.270-0.237=0.033
min2.881-2.871=0.011
max3.542-3.386=0.156

newProductH/cdr_auton/broker

Factortime_s
iterations=5
mean1.775-1.659=0.116
σ0.197-0.180=0.017
min1.521-1.369=0.152
max1.952-1.799=0.153

policy addition benchmark

Factorallocated MiBdb queriespeak allocated MiBtime
policies=2
mean450.088-442.428=7.661
σ579.264-577.063=2.201
min160.454-153.895=6.559
max1318.985-1308.023=10.962
mean102.250-96.250=6.000
σ8.500
min98.000-92.000=6.000
max115.000-109.000=6.000
mean444.000-7210.000=-6766.000
σ0.000
min444.000-7210.000=-6766.000
max444.000-7210.000=-6766.000
mean0.535-0.558=-0.023
σ0.533-0.586=-0.053
min0.266-0.264=0.002
max1.335-1.436=-0.101
policies=20
mean1604.881-1539.203=65.677
σ0.403-0.316=0.087
min1604.599-1538.983=65.616
max1605.476-1539.671=65.805
mean980.000-920.000=60.000
σ0.000
min980.000-920.000=60.000
max980.000-920.000=60.000
mean446.250-7210.000=-6763.750
σ1.500-0.000=1.500
min444.000-7210.000=-6766.000
max447.000-7210.000=-6763.000
mean2.667-2.587=0.080
σ0.007-0.029=-0.023
min2.657-2.571=0.086
max2.672-2.631=0.040
policies=200
mean16057.069-15400.260=656.808
σ0.119-0.089=0.030
min16056.916-15400.155=656.761
max16057.206-15400.372=656.834
mean9800.000-9200.000=600.000
σ0.000
min9800.000-9200.000=600.000
max9800.000-9200.000=600.000
mean497.000-7210.000=-6713.000
σ5.598-0.000=5.598
min490.000-7210.000=-6720.000
max502.000-7210.000=-6708.000
mean27.117-26.123=0.994
σ0.120-0.118=0.002
min26.986-26.022=0.964
max27.268-26.295=0.974