hadoop mapreduce partitioner not invoked -



need mapreduce job, custom partitioner never invoked. checked million times, no result. used work while ago, have no idea why isn't. appreicated.
adding code (it doesn't work either custom keys input either easy cases).
mapper outputs right values 100%, , partitioner skipped.

//import of libs import org.apache.hadoop.conf.configured; import org.apache.hadoop.conf.configuration; import org.apache.hadoop.mapreduce.job; import org.apache.hadoop.mapreduce.reducer; import org.apache.hadoop.mapreduce.partitioner; import org.apache.hadoop.hbase.mapreduce.tablemapper; import org.apache.hadoop.hbase.mapreduce.tablemapreduceutil; ...  public class hbasecounttest extends configured implements tool { ....  static class mymapper extends tablemapper<text,text> {     @override     public void map(immutablebyteswritable rowkey,result result, context context) throws ioexception {         ...... //dropping calculations         context.write(new text(gender), new text(capsstr)); // right here, checked.     }  }      public static class mypartitioner extends partitioner<text, text> {     @override     public int getpartition(text key, text value, int numreducetasks) { //getpartitioner never invoked         system.out.println("partitioner started");          string heur = value.tostring().split(":")[0];          int h = integer.parseint(heur);         if (h<10) {             ... return... //dropping calculations         } else if (h>9 && h<19) {             ...         } else              {             ...             }     }  }  @override public int run(string[] arg0) throws exception {     job job = job.getinstance(getconf(), "jobname1");     job.setnumreducetasks(3);     job.setjarbyclass(getclass());     configuration conf = job.getconfiguration();     hbaseconfiguration.merge(conf, hbaseconfiguration.create(conf));     conf.addresource("/home/hadoop/training/cdh4/hadoop-2.0.0-cdh4.0.0/conf/hadoop-local.xml");     conf.addresource("/home/hadoop/training/cdh4/hadoop-2.0.0-cdh4.0.0/conf/mapred-site.xml");     filesystem fs = filesystem.get(getconf());     if (fs.exists(new path(arg0[0]))) {         fs.delete(new path(arg0[0]));     }     scan scan = new scan();     scan.addcolumn(tobytes(famname), tobytes(colnamepage));     scan.addcolumn(tobytes(famname), tobytes(colnametime));     scan.addcolumn(tobytes(famname1), tobytes(colnameregion));     scan.addcolumn(tobytes(famname1), tobytes(colnamegender));     tablemapreduceutil.inittablemapperjob(tablename, scan, mymapper.class, text.class, text.class, job);      job.setpartitionerclass(mypartitioner.class);     job.setreducerclass(myreducer.class);     job.setoutputformatclass(textoutputformat.class);     textoutputformat.setoutputpath(job, new path(arg0[0]));     job.setoutputkeyclass(textoutputformat.class);     job.setoutputvalueclass(textoutputformat.class);             job.setnumreducetasks(3);     return job.waitforcompletion(true)?0:1; }  } 

thanks lot in advance,
alex

try set number of reducers number greater number of unique keys.


Comments

Popular posts from this blog

c# - How to get the current UAC mode -

postgresql - Lazarus + Postgres: incomplete startup packet -

javascript - Ajax jqXHR.status==0 fix error -