安科网

MapReduce设计模式之概要设计模式

gougouzhang

gougouzhang

2019-11-05

关注 关注

什么是概要设计模式

概要分析师将相似数据分组到一起并执行类似统计计算、索引生成或简单计数等后续的分析操作。

概要设计模式有哪些分类？

(1)数值概要 (2)倒排索引概要 (3)计数器计数等等。

数值概要

包括最大值、最小值、平均数、方差和中位数等等。

注意：MapReduce的内容结果输出，如果父文件夹已经存在，会报文件已存在错误，每次重新输出文件，如果都手动删除，会比较麻烦，可以自己写一个删除文件的工具类。或者hadoop中有一个FileUtil.fullyDelete()方法可以删除文件;以下是自己写的删除文件夹的代码：

import java.io.File;

/**
 * @Author bluesnail95
 * @Date 2019/7/14 23:31
 * @Description
 */
public class FileUtil {

    /**
     * 删除文件
     * @param fileName 文件名称
     */
    public static void deleteFile(String fileName) {
        File file = new File(fileName);
        if(!file.exists()) {
            return;
        }
        if(file.isFile()) {
            file.delete();
        }else if(file.isDirectory()) {
            File[] fileList = file.listFiles();
            for (int i = 0; i < fileList.length; i++) {
                fileList[i].delete();
            }
            file.delete();
        }
    }
}

最大值/最小值/计数

import org.apache.hadoop.io.Writable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;

/**
 * @Author bluesnail95
 * @Date 2019/7/14 9:57
 * @Description
 */
public class MinMaxCountData implements Writable {

    //日期
    private Date createDate;
    //用户标识
    private String userId;

    private final static SimpleDateFormat frmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");

    public MinMaxCountData() {
    }

    public MinMaxCountData(Date createDate, String userId) {
        this.createDate = createDate;
        this.userId = userId;
    }

    public Date getCreateDate() {
        return createDate;
    }

    public void setCreateDate(Date createDate) {
        this.createDate = createDate;
    }

    public String getUserId() {
        return userId;
    }

    public void setUserId(String userId) {
        this.userId = userId;
    }

    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeLong(createDate.getTime());
        dataOutput.writeBytes(userId);
    }

    public void readFields(DataInput dataInput) throws IOException {
        createDate = new Date(dataInput.readLong());
        userId = dataInput.readLine();
    }

    @Override
    public String toString() {
        return "MinMaxCountData{" +
                "createDate=" + createDate +
                ", userId='" + userId + '\'' +
                '}';
    }
}
import org.apache.hadoop.io.Writable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;

/**
 * @Author bluesnail95
 * @Date 2019/7/14 9:36
 * @Description
 */
public class MinMaxCountTuple implements Writable {
    //最小日期
    private Date min = null;
    //最大日期
    private Date max = null;
    //计数
    private long count = 0;

    private final static SimpleDateFormat frmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");

    public Date getMin() {
        return min;
    }

    public void setMin(Date min) {
        this.min = min;
    }

    public Date getMax() {
        return max;
    }

    public void setMax(Date max) {
        this.max = max;
    }

    public long getCount() {
        return count;
    }

    public void setCount(long count) {
        this.count = count;
    }

    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeLong(min.getTime());
        dataOutput.writeLong(max.getTime());
        dataOutput.writeLong(count);
    }

    public void readFields(DataInput dataInput) throws IOException {
        min = new Date(dataInput.readLong());
        max = new Date(dataInput.readLong());
        count  = dataInput.readLong();
    }

    public String toString() {
        return frmt.format(min) + "\t" + frmt.format(max) + "\t" + count;
    }
}
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.codehaus.jackson.map.ObjectMapper;

import java.io.File;
import java.text.SimpleDateFormat;

/**
 * @Author bluesnail95
 * @Date 2019/7/14 10:02
 * @Description
 */
public class MinMaxCountMain {

    public static class MinMaxCountMapper extends Mapper<Object,Text,Text,MinMaxCountTuple> {

        private Text userId = new Text();
        private MinMaxCountTuple minMaxCountTuple = new MinMaxCountTuple();
        private final static SimpleDateFormat frmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");

        public void map(Object key,Text value,Context context){
            try {
                ObjectMapper objectMapper = new ObjectMapper();
                objectMapper.setDateFormat(frmt);
                MinMaxCountData minMaxCountData = objectMapper.readValue(value.toString(), MinMaxCountData.class);
                minMaxCountTuple.setCount(1);
                minMaxCountTuple.setMin(minMaxCountData.getCreateDate());
                minMaxCountTuple.setMax(minMaxCountData.getCreateDate());
                userId.set(minMaxCountData.getUserId());
                context.write(userId, minMaxCountTuple);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    public static class MinMaxCountReducer extends Reducer<Text, MinMaxCountTuple, Text, MinMaxCountTuple> {
        private MinMaxCountTuple minMaxCountTuple = new MinMaxCountTuple();

        public void reduce(Text key,Iterable<MinMaxCountTuple> values,Context context) {
            try {
                long sum = 0;
                for (MinMaxCountTuple value : values) {
                    if(minMaxCountTuple.getMin() == null || value.getMin().compareTo(minMaxCountTuple.getMin()) < 0 ) {
                        minMaxCountTuple.setMin(value.getMin());
                    }
                    if(minMaxCountTuple.getMax() == null || value.getMax().compareTo(minMaxCountTuple.getMax()) > 0 ) {
                        minMaxCountTuple.setMax(value.getMax());
                    }
                    sum += value.getCount();
                }
                minMaxCountTuple.setCount(sum);
                context.write(key, minMaxCountTuple);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    public static void main(String[] args) {
        Configuration conf = new Configuration();
        try {
            Job job = Job.getInstance(conf, "NumericalSummarization：MinMaxCount");
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(MinMaxCountTuple.class);
            job.setJarByClass(MinMaxCountMain.class);
            job.setMapperClass(MinMaxCountMapper.class);
            job.setCombinerClass(MinMaxCountReducer.class);
            job.setReducerClass(MinMaxCountReducer.class);
            FileInputFormat.addInputPath(job, new Path(args[0]));
            File outputFile = new File(args[1]);
            if(outputFile.exists()){
                outputFile.delete();
            }
            FileOutputFormat.setOutputPath(job, new Path(args[1]));
            System.exit(job.waitForCompletion(true) ? 0 : 1);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

MapReduce设计模式之概要设计模式

MapReduce设计模式之概要设计模式

MapReduce设计模式之概要设计模式

平均值

import org.apache.hadoop.io.Writable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;

/**
 * @Author bluesnail95
 * @Date 2019/7/14 21:51
 * @Description
 */
public class CountAverageData implements Writable {

    //日期
    private Date creationDate;

    //文本
    private String text;

    private final static SimpleDateFormat frmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");

    public CountAverageData() {
    }

    public CountAverageData(Date creationDate, String text) {
        this.creationDate = creationDate;
        this.text = text;
    }

    public Date getCreationDate() {
        return creationDate;
    }

    public void setCreationDate(Date creationDate) {
        this.creationDate = creationDate;
    }

    public String getText() {
        return text;
    }

    public void setText(String text) {
        this.text = text;
    }

    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeBytes(frmt.format(creationDate));
        dataOutput.writeBytes(text);
    }

    public void readFields(DataInput dataInput) throws IOException {
        try {
            System.out.println(dataInput);
            creationDate = frmt.parse(dataInput.toString());
            text = dataInput.readLine();
        } catch (ParseException e) {
            e.printStackTrace();
        }
    }

    @Override
    public String toString() {
        return "{" +
                "creationDate=" + creationDate +
                ", text='" + text + '\'' +
                '}';
    }
}
import org.apache.hadoop.io.Writable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

/**
 * @Author bluesnail95
 * @Date 2019/7/14 21:36
 * @Description
 */
public class CountAverageTuple implements Writable {

    //计数
    private long count;

    //平均值
    private float average;

    public long getCount() {
        return count;
    }

    public void setCount(long count) {
        this.count = count;
    }

    public float getAverage() {
        return average;
    }

    public void setAverage(float average) {
        this.average = average;
    }

    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeLong(count);
        dataOutput.writeFloat(average);
    }

    public void readFields(DataInput dataInput) throws IOException {
        count = dataInput.readLong();
        average = dataInput.readFloat();
    }

    @Override
    public String toString() {
        return "{" +
                "count=" + count +
                ", average=" + average +
                '}';
    }
}
import file.FileUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.codehaus.jackson.map.ObjectMapper;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;

/**
 * @Author bluesnail95
 * @Date 2019/7/14 21:40
 * @Description
 */
public class CountAverageMain {

    public static class CountAverageMapper extends Mapper<Object, Text, IntWritable, CountAverageTuple> {

        private IntWritable outHour = new IntWritable();
        private CountAverageTuple countAverageTuple = new CountAverageTuple();
        private final static SimpleDateFormat frmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");

        public void map(Object key,Text value,Context context) {
            ObjectMapper objectMapper = new ObjectMapper();
            objectMapper.setDateFormat(frmt);
            try {
                CountAverageData countAverageData = objectMapper.readValue(value.toString(), CountAverageData.class);
                Calendar calendar = Calendar.getInstance();
                Date creationDate = countAverageData.getCreationDate();
                calendar.setTime(creationDate);
                int hour = calendar.get(Calendar.HOUR_OF_DAY);
                outHour.set(hour);
                countAverageTuple.setAverage(countAverageData.getText().length());
                countAverageTuple.setCount(1);
                context.write(outHour, countAverageTuple);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    public static class CountAverageReducer extends Reducer<IntWritable, CountAverageTuple,IntWritable, CountAverageTuple> {

        private CountAverageTuple result = new CountAverageTuple();

        public void reduce(IntWritable key, Iterable<CountAverageTuple> values,Context context) {
            float sum = 0;
            long count = 0;
            for(CountAverageTuple countAverageTuple : values) {
                count += countAverageTuple.getCount();
                sum += countAverageTuple.getCount() * countAverageTuple.getAverage();
            }
            result.setAverage(sum / count);
            result.setCount(count);
            try {
                context.write(key, result);
            } catch (IOException e) {
                e.printStackTrace();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }


    public static void main(String[] args) {

        Configuration configuration = new Configuration();
        try {
            Job job = Job.getInstance(configuration, "CountAverage");
            job.setJarByClass(CountAverageMain.class);
            job.setMapperClass(CountAverageMapper.class);
            job.setCombinerClass(CountAverageReducer.class);
            job.setReducerClass(CountAverageReducer.class);
            job.setOutputKeyClass(IntWritable.class);
            job.setOutputValueClass(CountAverageTuple.class);
            FileInputFormat.addInputPath(job, new Path(args[0]));
            FileUtil.deleteFile(args[1]);
            FileOutputFormat.setOutputPath(job, new Path(args[1]));
            System.exit(job.waitForCompletion(true )? 0 : 1);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

MapReduce设计模式之概要设计模式

MapReduce设计模式之概要设计模式

MapReduce设计模式之概要设计模式

中位数和方差

import org.apache.hadoop.io.Writable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

/**
 * @Author bluesnail95
 * @Date 2019/7/16 6:33
 * @Description
 */
public class MedianStdDevTuple implements Writable {

    private float median;

    private float stdDev;

    public float getMedian() {
        return median;
    }

    public void setMedian(float median) {
        this.median = median;
    }

    public float getStdDev() {
        return stdDev;
    }

    public void setStdDev(float stdDev) {
        this.stdDev = stdDev;
    }

    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeFloat(median);
        dataOutput.writeFloat(stdDev);
    }

    public void readFields(DataInput dataInput) throws IOException {
        median = dataInput.readFloat();
        stdDev = dataInput.readFloat();
    }

    @Override
    public String toString() {
        return "{" +
                "median=" + median +
                ", stdDev=" + stdDev +
                '}';
    }
}
import file.FileUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.codehaus.jackson.map.ObjectMapper;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;

/**
 * @Author bluesnail95
 * @Date 2019/7/16 6:18
 * @Description
 */
public class MedianStdDevMain {

    private final static SimpleDateFormat frmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");

    public static class MedianStdDevMapper extends Mapper<Object, Text,IntWritable, IntWritable> {

        private IntWritable outhour = new IntWritable();
        private IntWritable outlength = new IntWritable();

        public void map(Object key,Text value,Context context) {
            ObjectMapper objectMapper = new ObjectMapper();
            objectMapper.setDateFormat(frmt);
            try {
                CountAverageData countAverageData = objectMapper.readValue(value.toString(), CountAverageData.class);
                Date creationDate = countAverageData.getCreationDate();
                Calendar calendar = Calendar.getInstance();
                calendar.setTime(creationDate);
                int hour = calendar.get(Calendar.HOUR_OF_DAY);
                int length = countAverageData.getText().length();
                outhour.set(hour);
                outlength.set(length);
                context.write(outhour, outlength);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    public static class MadianStdDevReducer extends Reducer<IntWritable, IntWritable, IntWritable, MedianStdDevTuple> {

        private ArrayList<Float> lengths = new ArrayList<Float>();
        private MedianStdDevTuple medianStdDevTuple = new MedianStdDevTuple();

        public void reduce(IntWritable key, Iterable<IntWritable> values, Context context) {
            int sum = 0;
            int count = 0;
            try {
                for (IntWritable value : values) {
                    sum += value.get();
                    count++;
                    lengths.add((float) value.get());
                }

                //进行排序
                Collections.sort(lengths);
                //求中位数
                if(count == 1 || count % 2 == 0) {
                    medianStdDevTuple.setMedian(lengths.get(count/2));
                }else {
                    medianStdDevTuple.setMedian((lengths.get(count / 2 - 1) + lengths.get(count / 2)) / 2.0f);
                }
                //求平均值
                float mean = sum / count;
                float sumOfSquare = 0.0f;
                //求标准差
                for(Float value: lengths) {
                    sumOfSquare += (value - mean) * (value - mean);
                }
                if(count == 1) {
                    medianStdDevTuple.setStdDev(0);
                }else{
                    medianStdDevTuple.setStdDev((float)Math.sqrt(sumOfSquare / (count - 1)));
                }

                context.write(key, medianStdDevTuple);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    public static void main(String[] args) {
        Configuration configuration = new Configuration();
        try {
            Job job = Job.getInstance(configuration, "CountAverage");
            job.setJarByClass(MedianStdDevMain.class);
            job.setMapperClass(MedianStdDevMapper.class);
            job.setReducerClass(MadianStdDevReducer.class);
            job.setOutputKeyClass(IntWritable.class);
            job.setOutputValueClass(IntWritable.class);
            FileInputFormat.addInputPath(job, new Path(args[0]));
            FileUtil.deleteFile(args[1]);
            FileOutputFormat.setOutputPath(job, new Path(args[1]));
            System.exit(job.waitForCompletion(true )? 0 : 1);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

中位数方差升级版

import file.FileUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.codehaus.jackson.map.ObjectMapper;


import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * @Author bluesnail95
 * @Date 2019/7/16 21:28
 * @Description
 */
public class MedianStdDevUpgradeMain {

    private final static SimpleDateFormat frmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");

    public static class MedianStdDevUpgradeMapper extends Mapper<Object, Text, IntWritable, SortedMapWritable> {
        private IntWritable outHour = new IntWritable();
        private LongWritable one = new LongWritable(1);
        private IntWritable lengths = new IntWritable();

        public void map(Object key,Text value,Context context) {
            ObjectMapper objectMapper = new ObjectMapper();
            objectMapper.setDateFormat(frmt);
            try {
                CountAverageData countAverageData = objectMapper.readValue(value.toString(), CountAverageData.class);
                Date creationDate = countAverageData.getCreationDate();
                Calendar calendar = Calendar.getInstance();
                calendar.setTime(creationDate);
                outHour.set(calendar.get(Calendar.HOUR_OF_DAY));
                lengths.set(countAverageData.getText().length());
                SortedMapWritable sortedMapWritable = new SortedMapWritable();
                sortedMapWritable.put(lengths,one);
                context.write(outHour, sortedMapWritable);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    public static class MedianStdDevUpgradeCombiner extends Reducer<IntWritable,SortedMapWritable,IntWritable,SortedMapWritable> {

        protected void reduce(IntWritable key,Iterable<SortedMapWritable> values,Context context) {
            SortedMapWritable outValue = new SortedMapWritable();

            try {
                for (SortedMapWritable sortedMapWritable : values) {
                    Set<Map.Entry<WritableComparable,Writable>> set = sortedMapWritable.entrySet();
                    Iterator<Map.Entry<WritableComparable,Writable>> iterator = set.iterator();
                    while(iterator.hasNext()) {
                        Map.Entry<WritableComparable,Writable> entry = iterator.next();

                       LongWritable count = (LongWritable) outValue.get(entry.getKey());
                       if(count != null) {
                            count.set(count.get() + ((LongWritable)entry.getValue()).get());
                            outValue.put(entry.getKey(), count);
                       }else{
                           outValue.put(entry.getKey(),new LongWritable(((LongWritable)entry.getValue()).get()));
                       }

                    }
                }
                context.write(key, outValue);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    public static class MedianStdDevUpgradeReducer extends Reducer<IntWritable,SortedMapWritable,IntWritable,MedianStdDevTuple> {
        private MedianStdDevTuple medianStdDevTuple = new MedianStdDevTuple();
        private TreeMap<Integer, Long> lengthCounts = new TreeMap<Integer, Long>();

        public void reduce(IntWritable key,Iterable<SortedMapWritable> values,Context context) {
            float sum = 0;
            long total = 0;
           lengthCounts.clear();
            medianStdDevTuple.setStdDev(0);
            medianStdDevTuple.setMedian(0);

            for(SortedMapWritable sortedMapWritable : values) {
                Set<Map.Entry<WritableComparable,Writable>> set = sortedMapWritable.entrySet();
                Iterator<Map.Entry<WritableComparable,Writable>> iterator =  set.iterator();
               while (iterator.hasNext()) {
                   Map.Entry<WritableComparable,Writable> writableEntry = iterator.next();
                   int length = ((IntWritable)writableEntry.getKey()).get();
                   long count = ((LongWritable)writableEntry.getValue()).get();

                   total += count;
                   sum += count * length;
                   Long sortedCount = lengthCounts.get(length);
                   if(sortedCount == null) {
                       lengthCounts.put(length, count);
                   }else{
                       lengthCounts.put(length, count + sortedCount);
                   }
               }
            }

            long medianIndex = total / 2;
            long previousCount = 0;
            long count = 0;
            long prevKey = 0;
            for(Map.Entry<Integer, Long> entry:lengthCounts.entrySet()) {
                count = previousCount + entry.getValue();
                if(previousCount <= medianIndex && medianIndex < count) {
                    if(total % 2 == 0 && previousCount == medianIndex) {
                        medianStdDevTuple.setMedian((entry.getKey() + prevKey) / 2.0f);
                    }else{
                        medianStdDevTuple.setMedian(entry.getKey());
                    }
                    break;
                }
                previousCount = count;
                prevKey = entry.getKey();
            }

            float mean = sum / total;
            float sumOfSquares = 0.0f;
            for(Map.Entry<Integer, Long> entry:lengthCounts.entrySet()) {
                sumOfSquares += (entry.getKey() - mean) * (entry.getKey() - mean) * entry.getValue();
            }
            if(total == 1) {
                medianStdDevTuple.setStdDev(0);
            }else{
                medianStdDevTuple.setStdDev((float)Math.sqrt((sumOfSquares / (total - 1))));
            }
            try {
                context.write(key, medianStdDevTuple);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    public static void main(String[] args) {
        Configuration configuration = new Configuration();
        try {
            Job job = Job.getInstance(configuration, "MedianStdDevUpgrade");
            job.setJarByClass(MedianStdDevUpgradeMain.class);
            job.setMapperClass(MedianStdDevUpgradeMapper.class);
            job.setCombinerClass(MedianStdDevUpgradeCombiner.class);
            job.setReducerClass(MedianStdDevUpgradeReducer.class);
            job.setOutputKeyClass(IntWritable.class);
            job.setOutputValueClass(SortedMapWritable.class);
            FileInputFormat.addInputPath(job, new Path(args[0]));
            FileUtil.deleteFile(args[1]);
            FileOutputFormat.setOutputPath(job, new Path(args[1]));
            System.exit(job.waitForCompletion(true )? 0 : 1);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

MapReduce设计模式之概要设计模式

MapReduce设计模式之概要设计模式

MapReduce设计模式之概要设计模式

倒排索引概要

import java.io.Serializable;

/**
 * @Author bluesnail95
 * @Date 2019/7/18 23:22
 * @Description
 */
public class ExtractorData implements Serializable {

    private String link;

    private String id;

    public String getLink() {
        return link;
    }

    public void setLink(String link) {
        this.link = link;
    }

    public String getId() {
        return id;
    }

    public void setId(String id) {
        this.id = id;
    }
}
import file.FileUtil;
import numericalSummarization.MedianStdDevMain;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.codehaus.jackson.map.ObjectMapper;

import java.io.IOException;
import java.util.Iterator;

/**
 * @Author bluesnail95
 * @Date 2019/7/18 23:18
 * @Description
 */
public class ExtractorMain  {

    public static class ExtractorMapper extends Mapper<Object, Text,Text,Text> {

        private Text link = new Text();
        private Text id = new Text();

        public void map(Object key,Text value,Context context) {
            ObjectMapper objectMapper = new ObjectMapper();
            try {
                ExtractorData extractorData = objectMapper.readValue(value.toString(), ExtractorData.class);
                link.set(extractorData.getLink());
                id.set(extractorData.getId());
                context.write(link,id);
            } catch (Exception e) {
                e.printStackTrace();
            }

        }
    }

    public static class ExtractorReducer extends Reducer<Text,Text,Text,Text> {
        private Text link = new Text();
        private Text ids = new Text();

        public void reduce(Text key, Iterable<Text> values,Context context) {
            StringBuilder buffer = new StringBuilder("");
            for(Text value:values) {
                buffer.append(value.toString());
                buffer.append(",");
            }
            ids.set(buffer.toString().substring(0, buffer.length() - 1));
            link.set(key.toString());
            try {
                context.write(link,ids);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    public static void main(String[] args) {
        Configuration configuration = new Configuration();
        try {
            Job job = Job.getInstance(configuration, "ExtractorMain");
            job.setJarByClass(ExtractorMain.class);
            job.setMapperClass(ExtractorMapper.class);
            job.setCombinerClass(ExtractorReducer.class);
            job.setReducerClass(ExtractorReducer.class);
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Text.class);
            FileInputFormat.addInputPath(job, new Path(args[0]));
            FileUtil.deleteFile(args[1]);
            FileOutputFormat.setOutputPath(job, new Path(args[1]));
            System.exit(job.waitForCompletion(true )? 0 : 1);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

MapReduce设计模式之概要设计模式

MapReduce设计模式之概要设计模式

计数器技术

import file.FileUtil;
import invertedIndex.ExtractorMain;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.util.Arrays;
import java.util.HashSet;

/**
 * @Author bluesnail95
 * @Date 2019/7/19 6:31
 * @Description
 */
public class CountNumUsersByStateMain {

    public static class CountNumUsersByStateMapper extends Mapper<Object, Text, NullWritable,NullWritable> {
        public static final String STATE_COUNTER_GROUP = "State";
        public static final String UNKNOWN_COUNTER = "Unknown";
        public static final String NULL_OR_EMPTY_COUNTER = "Null or Empty";

        private String stateArray[] = {"BeiJing","ShangHai","ShenZhen","GuangZhou"};

        private HashSet<String> stateSet = new HashSet<String>(Arrays.asList(stateArray));

        public void map(Object key,Text value,Context context) {
            String state = value.toString();
            if(state != null && StringUtils.isNoneBlank(state)) {
                if(stateSet.contains(state)) {
                    context.getCounter(STATE_COUNTER_GROUP,state).increment(1);
                }else{
                    context.getCounter(STATE_COUNTER_GROUP,UNKNOWN_COUNTER).increment(1);
                }
            }else {
                context.getCounter(STATE_COUNTER_GROUP,NULL_OR_EMPTY_COUNTER).increment(1);
            }
        }
    }

    public static void main(String[] args) {
        Configuration configuration = new Configuration();
        try {
            Job job = Job.getInstance(configuration, "CountNumUsersByState");
            job.setJarByClass(CountNumUsersByStateMain.class);
            job.setMapperClass(CountNumUsersByStateMapper.class);
            job.setOutputKeyClass(NullWritable.class);
            job.setOutputValueClass(NullWritable.class);
            FileInputFormat.addInputPath(job, new Path(args[0]));
            FileUtil.deleteFile(args[1]);
            FileOutputFormat.setOutputPath(job, new Path(args[1]));
            int code = job.waitForCompletion(true )? 0 : 1;
            if(code == 0) {
                for(Counter counter:job.getCounters().getGroup(CountNumUsersByStateMapper.STATE_COUNTER_GROUP)) {
                    System.out.println(counter.getDisplayName() + "\t" + counter.getValue());
                }
            }
            FileSystem.get(configuration).delete(new Path(args[1]),true);
            System.exit(code);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

MapReduce设计模式之概要设计模式

参考资料

《MapReduce设计模式》

概要设计 mapreduce

gougouzhang

gougouzhang

0 关注 0 粉丝 0 动态

关注 关注

相关推荐

工程文档编写

现在很多企业业务开展都离不开项目管理，项目文档管理，是指在一个项目运行过程中将提交的各类文档进行收集管理控制的过程。工程项目保存的文档要涵盖项目可研、总体设计、基础设计、详细设计等整个项目周期，其中包括项目系统管理、文档版本控制、文档质量管理等管理内容。项

myblogdhy 2020-07-08

数据库应用系统功能设计与实施

可采用层次结构图表示软件总体结构，体现自顶向下、逐步求精的设计思想。将每个子系统进一步划分为功能模块，定义各功能模块的数据结构、相互间交互关系。

jameszgw 2020-06-14

『互联网架构』软件架构-软件系统设计（一）

按照正常的互联网玩法，产品经理原型画好进行需求评审，评审完后，需要把需求丢给技术经理，或者技术负责人，进行一整套的概要设计，然后针对概要设计评审，概要评审后进行开发。这次咱们一起说说概要设计的体系结构。软件系统设计在很多人眼里就是写文档，写文档是一种负担，

THEEYE 2020-06-06

理顺软件开发各个环节-13（开发管理-概要设计和详细设计）

　　概要设计，用于子系统或模块设计，也可用新增业务需求的跨子系统设计。　　由于软件总体设计从宏观上架构软件，距离开发实现，还有许多需要细化之处。　　另外，概要设计与代码实现的联系更紧密一些，如代码分层、核心的对象类及关系等。软件需求规格书；　　详细设计，能

chvnetcom 2020-05-29

第五弹——概要设计和详细设计说明书的诞生

在之前的需求规格说明书完美告一段落之后，我们又将面临着完成概要设计和详细设计说明书的任务。我们通过小组内分工的方式有条有理的在五一假期前各自完成了自己的部分并由小组中的负责人负责将所有内容整合到一起。就这样，既有分工又有讨论和监督，我们狮子山街一号的概要设

jameszgw 2020-05-07

第四章、数据库应用系统功能设计与实施

DBAS功能设计包括应用软件设中数据库事务设计和应用程序设计。规划分析阶段提出系统功能、性能及时向的约束对软件做了评估和选择建议，需求对系统功能性能提出了具体，为保证DBAS功能性能顺利实现，总体设计阶段需要对软硬件设备做出合理选择，并举行初步配置设计。

chvnetcom 2020-02-18

软件生存周期模型之V模型

RAD模型是软件开发过程中的一个重要模型，通过开发和测试同时进行的方式来缩短开发周期，提高开发效率。其形状像一个字母V，故称为V模型。又称软件测试的V模型。V模型是一个著名的、以测试为驱动的开发模型，该模型强调开发过程中测试贯穿始终，是瀑布模型的一个变体。

Henryztong 2020-02-02

软件工程---常用总结

软件工程的框架可概括为：目标、过程和原则。瀑布模型产生的历史背景是20世界70年代出现的软件危机，该模型将软件开发分为若干阶段，由于其类似于瀑布从上到下的过程，故称其为“瀑布模型”。开始需要把需求做到最全，惧怕用户测试中的反馈,惧怕需求变更。V模型是在瀑布

VanTYS 2020-01-12

如何评测软件工程知识技能水平？

编程语言是软件工程师的必备节能，不同方向的开发需要使用到不同的编程语言，因此软件工程师应该对各种编程语言都有一定的了解，同时要有至少两门能够熟练掌握的语言，并且 能深入理解各种编程思想，从而为以后的软件开发奠定坚实基础。软件开发往往是一个团队协作。耦合性，

嵌入式移动开发 2020-01-06

如何评测软件工程知识技能水平？

您认为一个软件工程师的软件工程知识技能水平高低主要体现在哪些方面，请您简要总结一下。软件生存周期模型确立了软件开发和演绎中各阶段的次序限制以及各阶段或机动的准则，确立开发过程所遵守的规定和限制，便于各种活动的协调，便于各种人员的有效通信，有利于活动重用，有

嵌入式移动开发 2020-01-05

软件设计---软件开发新手应该需要了解的内容

要养成职业化能力的意识。软件开发阶段一般是包括需求分析、总体分析、概要和详细设计、编码、测试、安装及维护。每一个人对每一个阶段的理解不同以及做法不同。一定要理解三阶段、五视图，尽管在实际开发中并不会全部使用到，也要有对应的概念。技术观点上来看软件设计应该包

随心而作 2019-12-29

架构设计文档模板--概要设计

分解软件模块，定义模块描述，模块交互关系（尽量使用数据耦合，少使用控制耦合，限制使用公共环境耦合，完全不用内容耦合。）--调用深度，宽度---控制好

水龙吟的备忘录 2019-11-14

软件开发流程（转载）

产品需求定义的目标是：“清楚地描述要做的产品是什么样的？不涉及具体实现方法。”，其定义过程如下图所示。按照需求，将软件逐级细化，分解为不必再分解的的模块，每个模块完成一定的功能，为一个或多个父模块服务，也接受一个或多个子模块的服务。模块的概念，和编程语言中

uileader 2017-08-08

网上商城-概要设计

根据需求文档，界面原型整理出主要功能点。主要是画图，包括功能架构，系统架构，包图，类图，流程图，E-R图，时序图。常用工具包括visio，powerdesigner，rose。

zjxbjh 2015-09-06

北京985院校，毕业3年，2.5年银行支付项目经理，求升级！

现为公司支付产品线项目经理、支付产品线骨干技术成员，现场、远程负责参与的银行项目7个。由于自我追求等原因，现恳请高人推荐，求升级，非常感谢！

zjxbjh 2013-03-21

概要设计与详细设计的区别

概要设计、详细设计(二)设计的内容

李晨璐十五期 2015-04-21

在大学中的学习和思考

在学校的学习中，学习软件的开发流程，并且知道软件开发过程中的产生文档，对软件开发队伍的管理等。对自己提出问题，仔细思考这些问题，完善自己的软件开发链，建立起完善的软件开发概念模型。如果以后忘了，还能及时的进行复习。产生的文档有系统用例图、系统序列流程图、用

laj0 2014-03-04

软件设计说明书

不涉及具体实现方法。用户能看得明白，开发人员也可据此进行下面的工作。并且需要指出关键技术难点等。

laj0 2012-10-19

详解软件项目管理流程的每一步

了解项目干系人及其利害关系。所有项目组成员是否到位，如到位则拿到项目开发人员的简历，详细了解每个开发人员的情况。根据项目需求规格列出项目功能列表，并根据开发人员技术等情况创建WBS。根据项目时间、资源等情况规划项目初步开发计划。确定各种软硬件需求，如：版本

yanglin 2012-05-16

概要设计说明

做软件到一定层次了，就要考虑到设计了，设计了很久，就是不系统，系统的设计需要一个记录，记录就用文档，那么对项目所有包括技术上的设计都记录下来，我们就可以理解为软件的概要设计了。这是项目小组今后共同作战的基础，有了开发规范和程序模块之间和项目成员彼此之间的接

agjllxchjy 2011-10-28
gougouzhang

gougouzhang

W3CSchool教程
HTML 教程
CSS 教程
Bootstrap 教程
Javascript 教程
jQuery 教程
后端教程
C 教程
Java 教程
PHP 教程
Python 教程
Go 教程
移动开发
Android 教程
Swift 教程
Kotlin 教程
jQuery Mobile 教程
ionic 教程
关于我们
新闻动态
联系方式
招聘英才
安科实验室
帮助与反馈

安科网(Ancii)，中国第一极客网

Copyright © 2013 - 2019 Ancii.com

京ICP备18063983号 京公网安备11010802014868号