計(jì)算json文件的電影總分

package com.doit.sumrate;public class UserRateBean {private String movie;private Integer rate;private String timeStamp;private String uid;public String getMovie() {return movie;}public void setMovie(String movie) {this.movie = movie;}public Integer getRate() {return rate;}public void setRate(Integer rate) {this.rate = rate;}public String getTimeStamp() {return timeStamp;}public void setTimeStamp(String timeStamp) {this.timeStamp = timeStamp;}public String getUid() {return uid;}public void setUid(String uid) {this.uid = uid;}@Overridepublic String toString() {return "UserRateBean{" +"movie='" + movie + '\'' +", rate=" + rate +", timeStamp='" + timeStamp + '\'' +", uid='" + uid + '\'' +'}';} } package com.doit.sumrate; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.codehaus.jackson.map.ObjectMapper; import java.io.IOException; public class UserRateDriver {public static class UserRateMap extends Mapper<LongWritable,Text, Text, IntWritable>{ObjectMapper objectMapper = new ObjectMapper();protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {String line = value.toString();//{"movie":"1193","rate":"5","timeStamp":"978300760","uid":"1"}UserRateBean userRateBean = objectMapper.readValue(line, UserRateBean.class);String movie = userRateBean.getMovie();Integer rate = userRateBean.getRate();context.write(new Text(movie),new IntWritable(rate));}}public static class UserRateReducer extends Reducer<Text, IntWritable,Text, IntWritable> {protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {int sum = 0;for (IntWritable v : values) {sum = sum + v.get();}context.write(key, new IntWritable(sum));}}public static void main(String[] args) throws InterruptedException, IOException, ClassNotFoundException {Configuration conf = new Configuration();conf.set("yarn.resorcemanager.hostname", "node01");conf.set("fs.deafutFS", "hdfs://node01:9000/");Job job = Job.getInstance(conf);job.setJarByClass(UserRateDriver.class);job.setMapperClass(UserRateMap.class);job.setReducerClass(UserRateReducer.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(IntWritable.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(IntWritable.class);job.setInputFormatClass(TextInputFormat.class);job.setOutputFormatClass(TextOutputFormat.class);FileInputFormat.setInputPaths(job,new Path(args[0]));FileOutputFormat.setOutputPath(job,new Path(args[1]));job.submit();boolean b = job.waitForCompletion(true);System.exit(b?0:1);}} [root@node01 home]# hadoop fs -mkdir -p /lll/yyy/input/ [root@node01 home]# hadoop fs -put rating.json /lll/yyy/input/ [root@node01 home]# hadoop fs -cat /lll/yyy/input/rating.json [root@node01 home]# hadoop jar movitrate.jar /lll/yyy/input/rating.json /lll/yyy/output/ [root@node01 home]# hadoop fs -ls /lll/yyy/input/rating.json/ [root@node01 home]# hadoop fs -cat /lll/yyy/output/part-r-00000/ 469 204 47 4669 470 66 471 2175 472 119 473 336 474 3718 475 1217 476 94 477 676 478 59 479 372 48 1137 480 10057 481 994 482 621 483 277 484 77 485 1254 486 180 487 162 488 132 489 240 49 101 490 615 491 703 492 1246 493 1075 494 1392 495 142 496 119 497 2668 498 129 499 135 5 890 50 8054 500 2866 501 339 502 300 503 48 504 526 505 172 506 398 507 1085 508 2185 509 2234 510 101 511 472 512 514 513 209 514 1222 515 1763 516 712 517 1213 518 366 519 392 52 1569 520 1096 521 394 522 296 523 428 524 1721 525 67 526 20 527 10392 528 171 529 2864 53 38 530 7 531 1274 532 1186 533 555 534 1097 535 1149 536 174 537 637 538 1204 539 3708 54 105 540 375 541 7692 542 551 543 2308 544 341 545 2 546 656 547 357 548 349 549 600 55 138 550 537 551 3703 552 1714 553 2142 554 68 555 2585 556 558 557 9 558 365 559 15 56 18 560 29 561 47 562 1816 563 55 564 102 565 191 566 38 567 50 568 135 569 293 57 337 570 74 571 122 572 6 573 74 574 603 575 302 576 3 577 163 578 9 579 2 58 2051 580 201 581 571 582 6 583 103 584 4 585 1215 586 2116 587 3930 588 5118 589 10751 59 22 590 5681 591 4 592 5153 593 11219 594 2934 595 4119 596 1868 597 3809 598 34 599 1079 6 3646 60 1147 600 90 601 3 602 169 603 174 605 749 606 207 607 4 608 10692 609 186 61 201 610 2192 611 534 612 190 613 298 614 24 615 271 616 932 617 62 618 19 619 83 62 2000 621 156 623 9 624 4 626 43 627 436 628 1890 63 317 630 20 631 156 632 52 633 72 634 5 635 279 637 356 638 61 639 204 64 179 640 190 641 1 642 2 643 6 644 2 645 72 647 1567 648 5242 649 67 65 295 650 309 651 2 652 26 653 1974 655 1 656 100 657 6 658 3 659 93 66 162 660 3 661 1819 662 382 663 902 664 24 665 128 666 10 667 96 668 207 669 96 67 13 670 247 671 1678 672 4 673 1475 674 1164 678 1299 679 7 68 207 680 215 681 59 682 14 684 1 685 99 687 9 688 485 69 1166 690 4 691 375 692 224 694 373 695 429 696 13 697 217 698 11 7 1562 70 2885 700 482 701 4 702 99 703 16 704 97 705 138 706 5 707 849 708 2332 709 438 71 202 710 119 711 155 712 7 714 595 715 188 716 75 717 4 718 115 719 684 72 338 720 1939 722 103 724 1284 725 254 726 27 728 817 729 11 73 855 730 1 731 134 732 17 733 4989 734 9 735 339 736 3523 737 609 74 357 741 1284 742 440 743 275 744 6 745 2970 746 124 747 110 748 1314 749 11 75 23 750 6083 751 53 753 101 754 38 755 34 756 12 757 15 758 4 759 242 76 508 760 116 761 343 762 651 763 3 764 242 765 450 766 357 767 77 769 39 77 128 771 68 774 4 775 29 776 12 778 2974 779 140 78 163 780 6073 781 348 782 370 783 1257 784 1051 785 2661 786 1437 787 15 788 2840 789 6 79 297 790 3 791 7 792 4 793 26 796 10 798 552 799 1132 8 205 80 199 800 2572 801 375 802 1493 803 210 804 694 805 1235 806 190 807 42 808 146 809 270 81 525 810 176 811 35 813 118 814 3 815 12 818 484 82 345 820 13 821 4 823 13 824 110 826 1 827 6 828 157 829 300 83 122 830 871 831 77 832 1962 833 170 834 8 835 322 836 629 837 758 838 1944 839 299 84 65 840 128 841 85 842 391 843 1 844 25 846 142 847 13 848 536 849 1283 85 660 850 110 851 444 852 1981 853 12 854 29 858 10059 859 2 86 801 860 36 861 959 862 113 863 35 864 43 865 2 866 2187 867 53 868 3 869 130 87 168 870 107 872 3 874 25 875 149 876 60 877 79 878 4 879 670 88 565 880 713 881 202 882 160 884 6 885 111 886 306 887 5 888 106 889 5 89 704 891 432 892 587 893 255 895 2 896 175 897 368 898 2503 899 3217 9 271 90 22 900 1321 901 634 902 2642 903 3867 904 4700 905 1601 906 766 907 412 908 5765 909 1721 910 3578 911 1280 912 7365 913 4585 914 2642 915 1447 916 1792 917 283 918 717 919 7298 92 206 920 4621 921 1176 922 2111 923 4898 924 6982 925 39 926 1715 927 334 928 1623 929 352 93 199 930 1911 931 806 932 747 933 1817 934 1112 935 258 936 538 937 165 938 677 939 35 94 678 940 1502 941 389 942 1227 943 681 944 368 945 1041 946 620 947 772 948 488 949 855 95 1835 950 1238 951 1687 952 969 953 3134 954 1624 955 1722 956 142 957 120 958 44 959 217 96 51 960 69 961 80 962 39 963 193 964 173 965 1031 966 24 967 75 968 2625 969 4494 97 165 970 126 971 1097 972 19 973 321 974 60 975 3 976 200 977 4 978 394 98 12 980 14 981 11 982 298 984 88 985 15 986 836 987 116 988 233 989 5 99 173 990 139 991 599 992 52 993 28 994 1843 996 744 997 94 998 280 999 939 ........ ........ ........

總結(jié)

以上是生活随笔為你收集整理的Hadoop Day06~MapReduce计算json文件的电影总分的全部?jī)?nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。