MLSQL NLP Example

-- download from file server.
-- run command as DownloadExt.`` where 
-- from="public/SogouCS.reduced.tar" and
-- to="/tmp/nlp/sogo";
-- or you can use command line:
--  !saveUploadFileToHome public/SogouCS.reduced.tar /tmp/nlp/sogo;

-- load data with xml format
load xml.`/tmp/nlp/sogo/news_sohusite_xml.dat` where rowTag="doc" and charset="GBK" as xmlData; 


--extract `sports` from url[http://sports.sohu.com/20070422/n249599819.shtml]
select temp.* from (select split(split(url,"/")[2],"\\.")[0] as labelStr,content from xmlData) as temp 
where temp.labelStr is not null 
as rawData;
-- try to use the follow sql to explore how many label we have and how they looks like.
--
-- select distinct(split(split(url,"/")[2],"\\.")[0]) as labelStr from rawData as output;
-- select split(split(url,"/")[2],"\\.")[0] as labelStr,url from rawData as output;

-- train a model which can map label to number and vice versa
train rawData as StringIndex.`/tmp/nlp/label_mapping` where inputCol="labelStr"and
outputCol="label" ;

-- convert label to number 
predict rawData as StringIndex.`/tmp/nlp/label_mapping` as rawDataWithLabel;

-- you can use register to convert a model to a functioin
register StringIndex.`/tmp/nlp/label_mapping` as convert_label; 

-- we can reduce the dataset. Because if there are too much data but just get  limited resource 
-- it may take too long. you can use command line 
-- or you can use raw ET:
--
-- run xmlData as RateSampler.`` 
-- where labelCol="url" and sampleRate="0.9,0.1" 
-- as xmlDataArray;
!split rawDataWithLabel by label with "0.9,0.1" named xmlDataArray;
-- then we fetch the xmlDataArray with position one to get the 10% data.
select * from xmlDataArray where __split__=1 as miniXmlData;

-- we can save the result data, because it really take much time.
save overwrite miniXmlData as parquet.`/tmp/nlp/miniXmlData`;

load parquet.`/tmp/nlp/miniXmlData` as miniXmlData;
-- select * from miniXmlData limit 10 as output;

--convert the content to tfidf format
train miniXmlData as TfIdfInPlace.`/tmp/nlp/tfidf` where inputCol="content" as trainData;

save overwrite trainData as parquet.`/tmp/nlp/trainData`;
load parquet.`/tmp/nlp/trainData` as trainData;
-- again register  a model as a functioin
register TfIdfInPlace.`/tmp/nlp/tfidf` as tfidf_predict;

-- use algorithm RandomForest to train 
train trainData as RandomForest.`/tmp/nlp/rf` where 
keepVersion="true" 
and fitParam.0.featuresCol="content" 
and fitParam.0.inputLabel="labelCol"
and fitParam.0.maxDepth="4"
and fitParam.0.checkpointInterval="100"
;

-- register  RF model as a functioin
register RandomForest.`/tmp/nlp/rf` as rf_predict;

-- end to end predict; you can also deploy this as a API service
select rf_predict(tfidf_predict("新闻不错")) as predicted as output;