上一篇，我们借助Sqlite分析了2000w条数据，分析结果不错，但有一个问题：造数据很耗时（花费了2个小时）、sql查询同样很耗时（分组汇总查询一次大概需要30秒）。
那么，我们可以尝试下直接在内存中分析，那么会不会更快一些呢？
上代码：
package com.lnwazg.a;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.RandomUtils;
import com.lnwazg.kit.describe.DescribeUtils;
public class Test
{
public static void main(String[] args)
{
long begin = System.currentTimeMillis();
System.out.println("begin...");
//构造2000w条数据
int[] data = new int[20000000];
for (int i = 0; i < data.length; i++)
{
data[i] = RandomUtils.nextInt(0, 1000);
}
System.out.println("OK");
//统计每个随机数的出现次数
Map<Integer, Integer> numMap = new HashMap<>();
for (int i = 0; i < data.length; i++)
{
if (!numMap.containsKey(data[i]))
{
numMap.put(data[i], 0);
}
numMap.put(data[i], numMap.get(data[i]) + 1);
}
long end = System.currentTimeMillis();
System.out.println("Cost " + (end - begin) + " millseconds");
//打印结果
DescribeUtils.describeMap(numMap);
}
}
结果是：
begin...
OK
Cost 819 millseconds
key:0 value:20039
key:1 value:20375
key:2 value:20048
key:3 value:19975
key:4 value:20132
key:5 value:19932
key:6 value:19964
key:7 value:19919
key:8 value:19974
key:9 value:20067
key:10 value:19889
key:11 value:19912
key:12 value:20047
key:13 value:19891
key:14 value:20000
key:15 value:19896
key:16 value:20011
key:17 value:19838
key:18 value:19971
key:19 value:19802
key:20 value:19934
key:21 value:20113
key:22 value:20015
key:23 value:20069
key:24 value:19960
key:25 value:20066
key:26 value:19835
key:27 value:20011
key:28 value:20022
key:29 value:20176
key:30 value:20083
key:31 value:19962
key:32 value:19981
key:33 value:19954
key:34 value:19918
key:35 value:20013
key:36 value:19889
key:37 value:19942
key:38 value:20043
key:39 value:20258
key:40 value:19986
key:41 value:19999
key:42 value:19896
key:43 value:20058
key:44 value:19892
key:45 value:20049
key:46 value:20021
key:47 value:20241
key:48 value:20099
key:49 value:19935
key:50 value:19857
key:51 value:19952
key:52 value:20126
key:53 value:19913
key:54 value:20002
key:55 value:20163
key:56 value:20041
key:57 value:20170
key:58 value:20142
key:59 value:20001
key:60 value:20004
key:61 value:20002
key:62 value:20015
key:63 value:20070
key:64 value:20213
key:65 value:20057
key:66 value:19993
key:67 value:19978
key:68 value:19901
key:69 value:19916
key:70 value:20367
key:71 value:20225
key:72 value:19949
key:73 value:20058
key:74 value:20151
key:75 value:19630
key:76 value:19916
key:77 value:19941
key:78 value:20147
key:79 value:20172
key:80 value:20063
key:81 value:20033
key:82 value:20199
key:83 value:19686
key:84 value:19985
key:85 value:20014
key:86 value:20119
key:87 value:20121
key:88 value:19944
key:89 value:19918
key:90 value:20061
key:91 value:20048
key:92 value:20079
key:93 value:20125
key:94 value:20254
key:95 value:19979
key:96 value:20003
key:97 value:20152
key:98 value:20083
key:99 value:20092
key:100 value:20039
key:101 value:19810
key:102 value:20057
key:103 value:20064
key:104 value:20263
key:105 value:20081
key:106 value:19828
key:107 value:20030
key:108 value:20056
key:109 value:19998
key:110 value:20008
key:111 value:19757
key:112 value:20008
key:113 value:20133
key:114 value:19837
key:115 value:20037
key:116 value:19661
key:117 value:19966
key:118 value:20134
key:119 value:19829
key:120 value:19769
key:121 value:20154
key:122 value:19907
key:123 value:20219
key:124 value:20047
key:125 value:20026
key:126 value:19904
key:127 value:19975
key:128 value:19944
key:129 value:19873
key:130 value:20039
key:131 value:19789
key:132 value:19979
key:133 value:19985
key:134 value:19861
key:135 value:20347
key:136 value:19913
key:137 value:19988
key:138 value:20179
key:139 value:20044
key:140 value:19736
key:141 value:19828
key:142 value:19788
key:143 value:19780
key:144 value:20057
key:145 value:19950
key:146 value:19997
key:147 value:19910
key:148 value:20318
key:149 value:20119
key:150 value:20018
key:151 value:20305
key:152 value:19871
key:153 value:20019
key:154 value:19879
key:155 value:20011
key:156 value:19952
key:157 value:19956
key:158 value:20007
key:159 value:19847
key:160 value:20047
key:161 value:20135
key:162 value:19892
key:163 value:20119
key:164 value:20042
key:165 value:20045
key:166 value:19996
key:167 value:19903
key:168 value:20086
key:169 value:19884
key:170 value:19943
key:171 value:20175
key:172 value:20165
key:173 value:19930
key:174 value:19957
key:175 value:19927
key:176 value:19965
key:177 value:19590
key:178 value:20159
key:179 value:19980
key:180 value:20148
key:181 value:19798
key:182 value:20147
key:183 value:19992
key:184 value:19957
key:185 value:20078
key:186 value:20008
key:187 value:20098
key:188 value:20339
key:189 value:19856
key:190 value:19823
key:191 value:20294
key:192 value:20189
key:193 value:20171
key:194 value:20181
key:195 value:20051
key:196 value:19985
key:197 value:20157
key:198 value:19888
key:199 value:19969
key:200 value:19874
key:201 value:19974
key:202 value:20053
key:203 value:20303
key:204 value:20088
key:205 value:20147
key:206 value:19860
key:207 value:20099
key:208 value:19991
key:209 value:19885
key:210 value:19899
key:211 value:20347
key:212 value:19888
key:213 value:20203
key:214 value:20166
key:215 value:19937
key:216 value:19921
key:217 value:20035
key:218 value:20021
key:219 value:20131
key:220 value:20225
key:221 value:20075
key:222 value:19983
key:223 value:19947
key:224 value:19802
key:225 value:20070
key:226 value:19985
key:227 value:19952
key:228 value:19931
key:229 value:20017
key:230 value:20167
key:231 value:20053
key:232 value:20000
key:233 value:20075
key:234 value:19937
key:235 value:20142
key:236 value:19737
key:237 value:19900
key:238 value:20175
key:239 value:19916
key:240 value:20159
key:241 value:20303
key:242 value:19641
key:243 value:20033
key:244 value:19835
key:245 value:19902
key:246 value:20028
key:247 value:19871
key:248 value:20031
key:249 value:19839
key:250 value:19810
key:251 value:20141
key:252 value:19922
key:253 value:19903
key:254 value:20277
key:255 value:20167
key:256 value:20099
key:257 value:20167
key:258 value:19902
key:259 value:20269
key:260 value:20056
key:261 value:20396
key:262 value:19786
key:263 value:20015
key:264 value:19914
key:265 value:20038
key:266 value:19709
key:267 value:20070
key:268 value:20063
key:269 value:19979
key:270 value:19467
key:271 value:19841
key:272 value:20010
key:273 value:19741
key:274 value:20113
key:275 value:19928
key:276 value:19889
key:277 value:19986
key:278 value:20011
key:279 value:20022
key:280 value:20143
key:281 value:20154
key:282 value:19998
key:283 value:20090
key:284 value:19935
key:285 value:20267
key:286 value:19585
key:287 value:20143
key:288 value:19790
key:289 value:19818
key:290 value:19982
key:291 value:20014
key:292 value:20191
key:293 value:19800
key:294 value:19839
key:295 value:20077
key:296 value:19961
key:297 value:19763
key:298 value:19862
key:299 value:19643
key:300 value:19899
key:301 value:20108
key:302 value:20114
key:303 value:20119
key:304 value:19890
key:305 value:20007
key:306 value:19986
key:307 value:19861
key:308 value:20244
key:309 value:19835
key:310 value:19900
key:311 value:19847
key:312 value:20085
key:313 value:19872
key:314 value:19840
key:315 value:20044
key:316 value:19848
key:317 value:19909
key:318 value:19688
key:319 value:19824
key:320 value:19903
key:321 value:20026
key:322 value:20192
key:323 value:20023
key:324 value:20049
key:325 value:19958
key:326 value:20061
key:327 value:20009
key:328 value:20051
key:329 value:20063
key:330 value:20122
key:331 value:19901
key:332 value:19844
key:333 value:20074
key:334 value:19987
key:335 value:20178
key:336 value:20036
key:337 value:20088
key:338 value:20137
key:339 value:20014
key:340 value:19900
key:341 value:20147
key:342 value:19950
key:343 value:19873
key:344 value:20064
key:345 value:20136
key:346 value:20171
key:347 value:19935
key:348 value:19955
key:349 value:20078
key:350 value:19862
key:351 value:20072
key:352 value:19946
key:353 value:20013
key:354 value:19912
key:355 value:20051
key:356 value:19734
key:357 value:20121
key:358 value:20062
key:359 value:19994
key:360 value:19934
key:361 value:19824
key:362 value:20164
key:363 value:19855
key:364 value:20113
key:365 value:20049
key:366 value:19971
key:367 value:19938
key:368 value:19909
key:369 value:19784
key:370 value:20167
key:371 value:19724
key:372 value:19945
key:373 value:20003
key:374 value:19789
key:375 value:19832
key:376 value:19796
key:377 value:20277
key:378 value:19945
key:379 value:20202
key:380 value:20050
key:381 value:20066
key:382 value:19979
key:383 value:19937
key:384 value:19936
key:385 value:20171
key:386 value:19969
key:387 value:19806
key:388 value:19964
key:389 value:20143
key:390 value:19947
key:391 value:20204
key:392 value:20131
key:393 value:20164
key:394 value:20009
key:395 value:20142
key:396 value:20257
key:397 value:19899
key:398 value:19996
key:399 value:20025
key:400 value:19861
key:401 value:19937
key:402 value:19932
key:403 value:19873
key:404 value:20147
key:405 value:20190
key:406 value:20264
key:407 value:20155
key:408 value:20043
key:409 value:19866
key:410 value:20036
key:411 value:19799
key:412 value:20112
key:413 value:20179
key:414 value:20015
key:415 value:20215
key:416 value:20274
key:417 value:19910
key:418 value:20063
key:419 value:19936
key:420 value:19889
key:421 value:19974
key:422 value:19995
key:423 value:19925
key:424 value:20058
key:425 value:20108
key:426 value:20052
key:427 value:19836
key:428 value:20009
key:429 value:20009
key:430 value:19910
key:431 value:20114
key:432 value:19978
key:433 value:19869
key:434 value:19914
key:435 value:19948
key:436 value:19847
key:437 value:20058
key:438 value:19958
key:439 value:20297
key:440 value:20017
key:441 value:20040
key:442 value:20013
key:443 value:19969
key:444 value:19973
key:445 value:19964
key:446 value:20051
key:447 value:19933
key:448 value:19922
key:449 value:20112
key:450 value:19860
key:451 value:20235
key:452 value:20320
key:453 value:19833
key:454 value:19850
key:455 value:20148
key:456 value:19982
key:457 value:19905
key:458 value:20163
key:459 value:19788
key:460 value:20120
key:461 value:20045
key:462 value:20051
key:463 value:20078
key:464 value:20074
key:465 value:20179
key:466 value:19887
key:467 value:19988
key:468 value:19957
key:469 value:19999
key:470 value:20011
key:471 value:20043
key:472 value:20275
key:473 value:19944
key:474 value:20074
key:475 value:19758
key:476 value:20185
key:477 value:20081
key:478 value:20008
key:479 value:19647
key:480 value:20315
key:481 value:20029
key:482 value:20070
key:483 value:19990
key:484 value:19918
key:485 value:20011
key:486 value:20085
key:487 value:19759
key:488 value:20025
key:489 value:20204
key:490 value:19899
key:491 value:20050
key:492 value:19746
key:493 value:19614
key:494 value:19935
key:495 value:20002
key:496 value:19931
key:497 value:20269
key:498 value:19990
key:499 value:20057
key:500 value:19991
key:501 value:20201
key:502 value:19948
key:503 value:19997
key:504 value:20015
key:505 value:19990
key:506 value:19869
key:507 value:19897
key:508 value:20324
key:509 value:20162
key:510 value:19808
key:511 value:20019
key:512 value:19809
key:513 value:20004
key:514 value:20138
key:515 value:19717
key:516 value:20166
key:517 value:20055
key:518 value:20013
key:519 value:20021
key:520 value:19793
key:521 value:19800
key:522 value:19973
key:523 value:19954
key:524 value:19919
key:525 value:19990
key:526 value:19921
key:527 value:20011
key:528 value:19962
key:529 value:19967
key:530 value:19938
key:531 value:20077
key:532 value:19700
key:533 value:20299
key:534 value:20054
key:535 value:20243
key:536 value:20139
key:537 value:19870
key:538 value:19989
key:539 value:19975
key:540 value:19813
key:541 value:19887
key:542 value:19911
key:543 value:20047
key:544 value:20096
key:545 value:19851
key:546 value:19975
key:547 value:20092
key:548 value:19793
key:549 value:20062
key:550 value:20068
key:551 value:19607
key:552 value:19896
key:553 value:20164
key:554 value:20164
key:555 value:20019
key:556 value:20060
key:557 value:19944
key:558 value:19813
key:559 value:20185
key:560 value:19844
key:561 value:20248
key:562 value:19755
key:563 value:20011
key:564 value:20057
key:565 value:19882
key:566 value:20219
key:567 value:19844
key:568 value:20015
key:569 value:19924
key:570 value:20044
key:571 value:19829
key:572 value:19886
key:573 value:19922
key:574 value:19855
key:575 value:19833
key:576 value:19991
key:577 value:20187
key:578 value:20027
key:579 value:20056
key:580 value:19990
key:581 value:20332
key:582 value:20000
key:583 value:19696
key:584 value:19881
key:585 value:19925
key:586 value:19948
key:587 value:20095
key:588 value:20096
key:589 value:19780
key:590 value:19957
key:591 value:19964
key:592 value:20141
key:593 value:19876
key:594 value:20024
key:595 value:20003
key:596 value:20192
key:597 value:20138
key:598 value:19967
key:599 value:20112
key:600 value:19889
key:601 value:19968
key:602 value:20083
key:603 value:19979
key:604 value:19850
key:605 value:20251
key:606 value:19928
key:607 value:19907
key:608 value:19751
key:609 value:19690
key:610 value:20004
key:611 value:20184
key:612 value:20214
key:613 value:19817
key:614 value:20176
key:615 value:20090
key:616 value:20199
key:617 value:20134
key:618 value:20001
key:619 value:19884
key:620 value:19975
key:621 value:19924
key:622 value:19960
key:623 value:19848
key:624 value:20080
key:625 value:20056
key:626 value:20202
key:627 value:19961
key:628 value:19910
key:629 value:19834
key:630 value:20132
key:631 value:19884
key:632 value:19916
key:633 value:20206
key:634 value:20116
key:635 value:20017
key:636 value:20025
key:637 value:20055
key:638 value:19854
key:639 value:20067
key:640 value:19803
key:641 value:20014
key:642 value:20040
key:643 value:20102
key:644 value:19788
key:645 value:19758
key:646 value:20023
key:647 value:19953
key:648 value:20171
key:649 value:20059
key:650 value:20084
key:651 value:20049
key:652 value:19984
key:653 value:19832
key:654 value:19954
key:655 value:20140
key:656 value:19942
key:657 value:20049
key:658 value:19890
key:659 value:20031
key:660 value:20030
key:661 value:20236
key:662 value:20110
key:663 value:20112
key:664 value:20039
key:665 value:19791
key:666 value:19945
key:667 value:20085
key:668 value:19971
key:669 value:19856
key:670 value:20060
key:671 value:20304
key:672 value:19969
key:673 value:19680
key:674 value:20185
key:675 value:20030
key:676 value:20184
key:677 value:19935
key:678 value:19879
key:679 value:20057
key:680 value:20155
key:681 value:20107
key:682 value:19833
key:683 value:20139
key:684 value:20254
key:685 value:20122
key:686 value:20142
key:687 value:20231
key:688 value:20110
key:689 value:20094
key:690 value:19778
key:691 value:20143
key:692 value:19854
key:693 value:19907
key:694 value:19898
key:695 value:19893
key:696 value:19993
key:697 value:20080
key:698 value:20114
key:699 value:20063
key:700 value:20045
key:701 value:19874
key:702 value:19957
key:703 value:19834
key:704 value:20035
key:705 value:19904
key:706 value:20217
key:707 value:19913
key:708 value:19744
key:709 value:20038
key:710 value:20022
key:711 value:20141
key:712 value:19845
key:713 value:19994
key:714 value:19900
key:715 value:20244
key:716 value:19785
key:717 value:20241
key:718 value:19921
key:719 value:19949
key:720 value:19714
key:721 value:19773
key:722 value:19994
key:723 value:20020
key:724 value:19913
key:725 value:20036
key:726 value:19977
key:727 value:20006
key:728 value:20190
key:729 value:19828
key:730 value:20107
key:731 value:19977
key:732 value:19801
key:733 value:19937
key:734 value:20029
key:735 value:19942
key:736 value:19893
key:737 value:19989
key:738 value:19903
key:739 value:20062
key:740 value:19886
key:741 value:20189
key:742 value:20006
key:743 value:20090
key:744 value:20014
key:745 value:19983
key:746 value:19906
key:747 value:19838
key:748 value:20029
key:749 value:19923
key:750 value:20028
key:751 value:19838
key:752 value:19870
key:753 value:19907
key:754 value:19972
key:755 value:19989
key:756 value:20068
key:757 value:20033
key:758 value:20110
key:759 value:19737
key:760 value:20056
key:761 value:19843
key:762 value:19949
key:763 value:19917
key:764 value:19808
key:765 value:20162
key:766 value:19942
key:767 value:19953
key:768 value:19893
key:769 value:19850
key:770 value:20188
key:771 value:19999
key:772 value:20288
key:773 value:20017
key:774 value:19948
key:775 value:19806
key:776 value:19960
key:777 value:19893
key:778 value:19789
key:779 value:19921
key:780 value:20013
key:781 value:20064
key:782 value:20074
key:783 value:19779
key:784 value:19931
key:785 value:20013
key:786 value:19978
key:787 value:19893
key:788 value:20000
key:789 value:19944
key:790 value:20004
key:791 value:20110
key:792 value:19747
key:793 value:19914
key:794 value:19937
key:795 value:20011
key:796 value:20083
key:797 value:19963
key:798 value:19988
key:799 value:19930
key:800 value:20362
key:801 value:20198
key:802 value:20013
key:803 value:20156
key:804 value:20070
key:805 value:19838
key:806 value:20178
key:807 value:19750
key:808 value:19934
key:809 value:19785
key:810 value:20082
key:811 value:20074
key:812 value:20150
key:813 value:19795
key:814 value:20186
key:815 value:20006
key:816 value:20228
key:817 value:19790
key:818 value:20246
key:819 value:20075
key:820 value:20032
key:821 value:20139
key:822 value:20025
key:823 value:20008
key:824 value:19974
key:825 value:19950
key:826 value:20138
key:827 value:19892
key:828 value:20114
key:829 value:19897
key:830 value:20002
key:831 value:20033
key:832 value:20051
key:833 value:19960
key:834 value:19866
key:835 value:20028
key:836 value:20177
key:837 value:20021
key:838 value:20235
key:839 value:19794
key:840 value:20133
key:841 value:20204
key:842 value:20006
key:843 value:19929
key:844 value:20011
key:845 value:20091
key:846 value:20115
key:847 value:20067
key:848 value:20326
key:849 value:20015
key:850 value:20072
key:851 value:20148
key:852 value:20142
key:853 value:19987
key:854 value:19843
key:855 value:19837
key:856 value:20061
key:857 value:20042
key:858 value:20045
key:859 value:19707
key:860 value:19915
key:861 value:19935
key:862 value:20042
key:863 value:19942
key:864 value:20027
key:865 value:19930
key:866 value:20070
key:867 value:19901
key:868 value:20098
key:869 value:19824
key:870 value:20140
key:871 value:19838
key:872 value:20194
key:873 value:19846
key:874 value:19936
key:875 value:19967
key:876 value:20035
key:877 value:20030
key:878 value:19927
key:879 value:20227
key:880 value:19983
key:881 value:20039
key:882 value:20091
key:883 value:19757
key:884 value:19663
key:885 value:19951
key:886 value:20206
key:887 value:19926
key:888 value:20083
key:889 value:19899
key:890 value:19946
key:891 value:19923
key:892 value:20049
key:893 value:20014
key:894 value:19966
key:895 value:19890
key:896 value:19828
key:897 value:20062
key:898 value:20041
key:899 value:20212
key:900 value:20074
key:901 value:20142
key:902 value:20028
key:903 value:19791
key:904 value:20065
key:905 value:19943
key:906 value:19712
key:907 value:19791
key:908 value:19861
key:909 value:19690
key:910 value:20118
key:911 value:20067
key:912 value:20019
key:913 value:19929
key:914 value:19810
key:915 value:19885
key:916 value:19766
key:917 value:19618
key:918 value:20049
key:919 value:20094
key:920 value:19847
key:921 value:19916
key:922 value:20250
key:923 value:20223
key:924 value:19925
key:925 value:20098
key:926 value:19898
key:927 value:20063
key:928 value:19659
key:929 value:20216
key:930 value:20144
key:931 value:19938
key:932 value:20206
key:933 value:19987
key:934 value:19894
key:935 value:20065
key:936 value:20174
key:937 value:20035
key:938 value:20156
key:939 value:20000
key:940 value:19785
key:941 value:19981
key:942 value:19895
key:943 value:20011
key:944 value:19796
key:945 value:20191
key:946 value:20161
key:947 value:20085
key:948 value:20017
key:949 value:19858
key:950 value:20043
key:951 value:20053
key:952 value:19950
key:953 value:20125
key:954 value:19868
key:955 value:19971
key:956 value:20085
key:957 value:20011
key:958 value:20072
key:959 value:19960
key:960 value:19881
key:961 value:19934
key:962 value:19945
key:963 value:20149
key:964 value:19925
key:965 value:20161
key:966 value:19968
key:967 value:20232
key:968 value:20164
key:969 value:19920
key:970 value:20017
key:971 value:20159
key:972 value:19958
key:973 value:19973
key:974 value:20071
key:975 value:20051
key:976 value:19991
key:977 value:19967
key:978 value:20161
key:979 value:20158
key:980 value:20204
key:981 value:19818
key:982 value:20072
key:983 value:19884
key:984 value:19920
key:985 value:19995
key:986 value:19985
key:987 value:20176
key:988 value:20052
key:989 value:20290
key:990 value:20000
key:991 value:20066
key:992 value:20180
key:993 value:20110
key:994 value:19852
key:995 value:20102
key:996 value:20107
key:997 value:19789
key:998 value:19972
key:999 value:19807
结论：
内存计算因为少了网络传输、数据库操作、sql查询等时间的消耗，纯内存计算的效率惊人的高。
从构造数据，到分类汇总，仅消耗819毫秒时间！相比前篇的做法接近2小时的时间，快了将近10000倍有余！
内存计算胜在其算法可以高度定制优化，缺点则是通用性比较差。
传统的关系数据库的方式虽然查询速度较慢，但是数据可以持久化保存，并且除了统计次数之外还可以做很多其他的事情！
但是总体来说，内存计算的效率之高，着实给我留下了深刻的印象。由此，我对Spark在大数据分析领域的表现充满了期待！
