hadoop - How are hive udf, udaf, udtfs written in java debugged in an ide like eclipse? -
for e.g debugging pig udfs works : http://ben-tech.blogspot.ie/2011/08/how-to-debug-pig-udfs-in-eclipse.html
have hive script in use udaf failing step through udf code.
junit can debugged eclipse ide., since java class.
consider udf.
example 1
class simplehelloworldudfexample extends udf { public text evaluate(text input) { if(input == null) return null; return new text("hello " + input.tostring()); } }
junit test method this...
@test public void testudfnullcheck() { simplehelloworldudfexample example = new simplehelloworldudfexample(); assert.assertnull(example.evaluate(null)); }
example 2
package com.hive.udftest import java.util.list; import org.apache.hadoop.hive.ql.exec.udfargumentexception; import org.apache.hadoop.hive.ql.exec.udfargumentlengthexception; import org.apache.hadoop.hive.ql.metadata.hiveexception; import org.apache.hadoop.hive.ql.udf.generic.genericudf; import org.apache.hadoop.hive.serde2.objectinspector.listobjectinspector; import org.apache.hadoop.hive.serde2.objectinspector.objectinspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.primitiveobjectinspectorfactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.stringobjectinspector; class hiveudftest extends genericudf { listobjectinspector listoi; stringobjectinspector elementoi; @override public string getdisplaystring(string[] arg0) { return "arraycontainsexample()"; // should better } @override public objectinspector initialize(objectinspector[] arguments) throws udfargumentexception { if (arguments.length != 2) { throw new udfargumentlengthexception("arraycontainsexample takes 2 arguments: list<t>, t"); } // 1. check received right object types. objectinspector = arguments[0]; objectinspector b = arguments[1]; if (!(a instanceof listobjectinspector) || !(b instanceof stringobjectinspector)) { throw new udfargumentexception("first argument must list / array, second argument must string"); } this.listoi = (listobjectinspector) a; this.elementoi = (stringobjectinspector) b; // 2. check list contains strings if(!(listoi.getlistelementobjectinspector() instanceof stringobjectinspector)) { throw new udfargumentexception("first argument must list of strings"); } // return type of our function boolean, provide correct object inspector return primitiveobjectinspectorfactory.javabooleanobjectinspector; } @override public object evaluate(deferredobject[] arguments) throws hiveexception { // list , string deferred objects using object inspectors list<string> list = (list<string>) this.listoi.getlist(arguments[0].get()); string arg = elementoi.getprimitivejavaobject(arguments[1].get()); // check nulls if (list == null || arg == null) { return null; } // see if our list contains value need for(string s: list) { if (arg.equals(s)) return new boolean(true); } return new boolean(false); } }
junit test case be
package com.hive.udftest import java.util.arraylist; import java.util.list; import junit.framework.assert; import org.apache.hadoop.hive.ql.metadata.hiveexception; import org.apache.hadoop.hive.ql.udf.generic.genericudf.deferredjavaobject; import org.apache.hadoop.hive.ql.udf.generic.genericudf.deferredobject; import org.apache.hadoop.hive.serde2.objectinspector.objectinspector; import org.apache.hadoop.hive.serde2.objectinspector.objectinspectorfactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.javabooleanobjectinspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.primitiveobjectinspectorfactory; import org.junit.test; public class hiveudftesttest { @test public void testcomplexudfreturnscorrectvalues() throws hiveexception { // set models need hiveudftest example = new hiveudftest(); objectinspector stringoi = primitiveobjectinspectorfactory.javastringobjectinspector; objectinspector listoi = objectinspectorfactory.getstandardlistobjectinspector(stringoi); javabooleanobjectinspector resultinspector = (javabooleanobjectinspector) example.initialize(new objectinspector[]{listoi, stringoi}); // create actual udf arguments list<string> list = new arraylist<string>(); list.add("a"); list.add("b"); list.add("c"); // test our results // value exists object result = example.evaluate(new deferredobject[]{new deferredjavaobject(list), new deferredjavaobject("a")}); assert.assertequals(true, resultinspector.get(result)); // value doesn't exist object result2 = example.evaluate(new deferredobject[]{new deferredjavaobject(list), new deferredjavaobject("d")}); assert.assertequals(false, resultinspector.get(result2)); // arguments null object result3 = example.evaluate(new deferredobject[]{new deferredjavaobject(null), new deferredjavaobject(null)}); assert.assertnull(result3); } }
similar way udaf,udtf well...
Comments
Post a Comment