Apache Pig的UDF返回值问题
今天写了关于Pig的EvalFunc UDF函数，结果一执行，发现返回值，总是bag类型，我就纳闷了，我明明指定了返回是String类型，怎么会变成Bag类型呢？经查找，发现拷贝的问题，由于先前写的UDF函数，返回值是多个，而现在的这个是一个，所以导致，我在pig脚本里面，进行强转string类型出错，发现问题后，设置返回类型为DataType.CHARARRAY问题得以解决。案例（一），输入值为多个参数，返回也为多个参数
package com.easy.pig;
import com.easy.similar.model.ResultModel;
import com.easy.similar.tools.TextBuildID;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.*;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import java.io.IOException;
/**
* Created by qindongliang on 2015/9/29.
* 根据一篇内容返回md5和关键词words
*/
public class FingerUDFextends EvalFunc<DataBag>{
/**tuple实例**/
TupleFactory mTupleFactory = TupleFactory.getInstance();
/**Bag实例*/
BagFactory mBagFactory = BagFactory.getInstance();
/**md5构建**/
TextBuildID textBuildID = new TextBuildID();
@Override
public DataBag exec(Tuple tuple) throws IOException {
try {
DataBag output = mBagFactory.newDefaultBag();
Object o = tuple.get(0);
//返回多少句子
int num_sentence = Integer.parseInt(tuple.get(1).toString());
//返回几个关键词
int num_words = Integer.parseInt(tuple.get(2).toString());
//使用指纹算法 1还是分词算法 2
int type=Integer.parseInt(tuple.get(3).toString());
//参与扩展计算的指纹,null的直接返回空
String ext=tuple.get(4)==null?"":tuple.get(4).toString();
//设置句子数量
textBuildID.top_N_Sentence=num_sentence;
//设置返回指纹数量
textBuildID.top_N_Words=num_words;
//设置去重算法类型
textBuildID.type=type;
ResultModel ro=textBuildID.buildID(o.toString(),ext);
output.add(mTupleFactory.newTuple(ro.getMd5()));//获取md5值
output.add(mTupleFactory.newTuple(ro.getWords()));//获取指纹关键词
return output;
} catch (Exception ee) {
// error handling goes here
ee.printStackTrace();
}
return null;
}
/**描述scheaml形式*/
public Schema outputSchema(Schema input) {
try{
Schema bagSchema = new Schema();
bagSchema.add(new Schema.FieldSchema("token", DataType.CHARARRAY));
return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
bagSchema, DataType.BAG));
}catch (Exception e){
return null;
}
}
}
案例（二），输入值为String，返回也为String
package com.easy.pig;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import java.io.IOException;
/**
*处理url
*/
public class UrlCvUDF extends EvalFunc<String>{
@Override
public String exec(Tuple tuple) throws IOException {
try {
Object o = tuple.get(0);
if(o!=null){
//判断是否为指定url开头的来源
if((o+"").startsWith("http://www.court.gov.cn")){
return "1";
}else{
return "2";
}
}
} catch (Exception ee) {
ee.printStackTrace();
}
//url 为null 则返回0
return "0";
}
//
/**描述scheaml形式*/
public Schema outputSchema(Schema input) {
try{
Schema bagSchema = new Schema();
bagSchema.add(new Schema.FieldSchema("token", DataType.CHARARRAY));
//注意此处返回值要与泛型里面的对应
return new Schema(new Schema.FieldSchema(getSchemaName(this.getClass().getName().toLowerCase(), input),
bagSchema, DataType.CHARARRAY));
}catch (Exception e){
e.printStackTrace();
return null;
}
}
}
案例一的pig脚本：
--SET debug 'on'
--REGISTER ./aa.jar
REGISTER ./udf-pig-similarty-hbase-1.0-SNAPSHOT-jar-with-dependencies.jar
REGISTER ./pig-udf-extend-1.0.1-SNAPSHOT-jar-with-dependencies.jar
--mkdir /user/webmaster/crawldb/finger/
--rmf /user/webmaster/crawldb/finger/
mkdir /user/webmaster/search/monitor/finger-data;
rmf/user/webmaster/search/monitor/finger-data;
set job.name 'pig-hbase-build-index'
a = load 'hbase://ETLDB' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('content:conn content:casenum,meta:isdelete','-loadKey true');
--a = foreach a generate FLATTEN(com.easy.pig.FingerUDF((chararray)$1));
-- 4个参数0：文章内容 1：前n最长的句子数，2：前n关键词，3：判重算法， 1=>指纹 2=>分词
a = foreach a generate $0 as rowkey:chararray , BagToString(com.easy.pig.FingerUDF((chararray)$1,5,8,1,$2),'@') as info:chararray,$2 as casenum:chararray , $3 as isdel:chararray ;
--a = foreach a generate $2 as num:chararray;
--a = limit a 50;
--dump a;
--describe a;
a = foreach a generate $0 as rowkey:chararray , STRSPLIT(info,'@',2).$0 as finger_md5:chararray ,STRSPLIT(info,'@',2).$1 as finger_content:chararray ,casenum,isdel ;
--describe a;
store a into'/user/webmaster/search/monitor/finger-data'using com.pig.support.lucene.LuceneStore('row:true:false,finger_md5:true:false,finger_content:true:false,casenum:true:false,isdel:true:false','default');
案例二的pig脚本：
--SET debug 'on'
--REGISTER ./aa.jar
REGISTER ./udf-pig-similarty-hbase-1.0-SNAPSHOT-jar-with-dependencies.jar
REGISTER ./pig-udf-extend-1.0.1-SNAPSHOT-jar-with-dependencies.jar
--mkdir /user/webmaster/crawldb/finger/
--rmf /user/webmaster/crawldb/finger/
mkdir /user/webmaster/search/monitor/finger-data;
rmf/user/webmaster/search/monitor/finger-data;
set job.name 'pig-hbase-build-index'
a = load 'hbase://ETLDB' USING org.apache.pig.backend.hadoop.hbase.HBaseStorage('content:conn content:casenum meta:isdelete content:url','-loadKey true');
--a = foreach a generate FLATTEN(com.easy.pig.FingerUDF((chararray)$1));
-- 4个参数0：文章内容 1：前n最长的句子数，2：前n关键词，3：判重算法， 1=>指纹 2=>分词
a = foreach a generate $0 as rowkey:chararray , BagToString(com.easy.pig.FingerUDF((chararray)$1,5,8,1,''),'@') as info:chararray,$2 as casenum:chararray , $3 as isdel:chararray, com.easy.pig.UrlCvUDF((chararray)$4)as source:chararray ;
--a = foreach a generate $2 as num:chararray;
a = limit a 11;
dump a;
describe a;
--describe a;
--a = foreach a generate $0 as rowkey:chararray , STRSPLIT(info,'@',2).$0 as finger_md5:chararray ,STRSPLIT(info,'@',2).$1 as finger_content:chararray ,casenum,isdel ;
--describe a;
--store a into'/user/webmaster/search/monitor/finger-data'using com.pig.support.lucene.LuceneStore('row:true:false,finger_md5:true:false,finger_content:true:false,casenum:true:false,isdel:true:false','default');
页:
[1]