hadoop - How are hive udf, udaf, udtfs written in java debugged in an ide like eclipse? -


for e.g debugging pig udfs works : http://ben-tech.blogspot.ie/2011/08/how-to-debug-pig-udfs-in-eclipse.html
have hive script in use udaf failing step through udf code.

junit can debugged eclipse ide., since java class.

consider udf.

example 1

class simplehelloworldudfexample extends udf {         public text evaluate(text input) {     if(input == null) return null;     return new text("hello " + input.tostring());   } } 

junit test method this...

@test public void testudfnullcheck() {   simplehelloworldudfexample example = new simplehelloworldudfexample();   assert.assertnull(example.evaluate(null)); } 

example 2

package com.hive.udftest  import java.util.list;  import org.apache.hadoop.hive.ql.exec.udfargumentexception; import org.apache.hadoop.hive.ql.exec.udfargumentlengthexception; import org.apache.hadoop.hive.ql.metadata.hiveexception; import org.apache.hadoop.hive.ql.udf.generic.genericudf; import org.apache.hadoop.hive.serde2.objectinspector.listobjectinspector; import org.apache.hadoop.hive.serde2.objectinspector.objectinspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.primitiveobjectinspectorfactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.stringobjectinspector;  class hiveudftest extends genericudf {    listobjectinspector listoi;   stringobjectinspector elementoi;    @override   public string getdisplaystring(string[] arg0) {     return "arraycontainsexample()"; // should better   }    @override   public objectinspector initialize(objectinspector[] arguments) throws udfargumentexception {     if (arguments.length != 2) {       throw new udfargumentlengthexception("arraycontainsexample takes 2 arguments: list<t>, t");     }     // 1. check received right object types.     objectinspector = arguments[0];     objectinspector b = arguments[1];     if (!(a instanceof listobjectinspector) || !(b instanceof stringobjectinspector)) {       throw new udfargumentexception("first argument must list / array, second argument must string");     }     this.listoi = (listobjectinspector) a;     this.elementoi = (stringobjectinspector) b;      // 2. check list contains strings     if(!(listoi.getlistelementobjectinspector() instanceof stringobjectinspector)) {       throw new udfargumentexception("first argument must list of strings");     }      // return type of our function boolean, provide correct object inspector     return primitiveobjectinspectorfactory.javabooleanobjectinspector;   }    @override   public object evaluate(deferredobject[] arguments) throws hiveexception {      // list , string deferred objects using object inspectors     list<string> list = (list<string>) this.listoi.getlist(arguments[0].get());     string arg = elementoi.getprimitivejavaobject(arguments[1].get());      // check nulls     if (list == null || arg == null) {       return null;     }      // see if our list contains value need     for(string s: list) {       if (arg.equals(s)) return new boolean(true);     }     return new boolean(false);   }  } 

junit test case be

package com.hive.udftest  import java.util.arraylist; import java.util.list;  import junit.framework.assert;  import org.apache.hadoop.hive.ql.metadata.hiveexception; import org.apache.hadoop.hive.ql.udf.generic.genericudf.deferredjavaobject; import org.apache.hadoop.hive.ql.udf.generic.genericudf.deferredobject; import org.apache.hadoop.hive.serde2.objectinspector.objectinspector; import org.apache.hadoop.hive.serde2.objectinspector.objectinspectorfactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.javabooleanobjectinspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.primitiveobjectinspectorfactory; import org.junit.test;  public class hiveudftesttest {     @test   public void testcomplexudfreturnscorrectvalues() throws hiveexception {      // set models need     hiveudftest example = new hiveudftest();     objectinspector stringoi = primitiveobjectinspectorfactory.javastringobjectinspector;     objectinspector listoi = objectinspectorfactory.getstandardlistobjectinspector(stringoi);     javabooleanobjectinspector resultinspector = (javabooleanobjectinspector) example.initialize(new objectinspector[]{listoi, stringoi});      // create actual udf arguments     list<string> list = new arraylist<string>();     list.add("a");     list.add("b");     list.add("c");      // test our results      // value exists     object result = example.evaluate(new deferredobject[]{new deferredjavaobject(list), new deferredjavaobject("a")});     assert.assertequals(true, resultinspector.get(result));      // value doesn't exist     object result2 = example.evaluate(new deferredobject[]{new deferredjavaobject(list), new deferredjavaobject("d")});     assert.assertequals(false, resultinspector.get(result2));      // arguments null     object result3 = example.evaluate(new deferredobject[]{new deferredjavaobject(null), new deferredjavaobject(null)});     assert.assertnull(result3);   } } 

similar way udaf,udtf well...


Comments

Popular posts from this blog

javascript - Laravel datatable invalid JSON response -

java - Exception in thread "main" org.springframework.context.ApplicationContextException: Unable to start embedded container; -

sql server 2008 - My Sql Code Get An Error Of Msg 245, Level 16, State 1, Line 1 Conversion failed when converting the varchar value '8:45 AM' to data type int -