简单的http请求数据保存到hdfs(代码片段)

package com.etl;

import java.io.IOException;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import okhttp3.Call;
import okhttp3.Callback;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;

public class LinkHttp 
    
    private  static Configuration conf = null;
    private static String fsName = "fs.defaultFS";
    private static String fsURI = null;

    public static void main(String[] args) throws Exception 
        
        String name = args[0];
        String uri =  args[1];
        String url = args[2];
        String targetFile = args[3];  //文件全路径
        
        //初始化
        init(name, uri);
        OkHttpClient client = new OkHttpClient();
        final Request request = new Request.Builder().url(url).get().build();
        Call call = client.newCall(request);
        call.enqueue(new Callback() 

            @Override
            public void onFailure(Call call, IOException e) 
                System.out.println("Fail");
            

            @Override
            public void onResponse(Call call, Response response) throws IOException 
                FileSystem fs = null;
                try 
                        Path dstPath = new Path(targetFile);
                        fs = FileSystem.get(conf);
                        FSDataOutputStream outputStream = fs.create(dstPath);    
                        if(response.isSuccessful()) 
            //                    System.out.println(response.body().string());
                                outputStream.write(response.body().bytes());        
                                outputStream.close();        
                                System.out.println("create file " + targetFile + " success!");        
                                //fs.close();
                        
                catch (Exception e)
                    e.printStackTrace();
                finally 
                    fs.close();
                
                System.out.println("run writeHdfs end");

                //关闭
                if(response.body()!=null) 
                    response.body().close();
                
            
        );
        
    

    private static void init(String name, String uri) 
            if(StringUtils.isNotBlank(fsName))
                fsName = name;
            
             fsURI  = uri;
             conf = new Configuration();
             conf.set(fsName, fsURI);
    

#!/bin/sh
name=fs.defaultFS        #固定不变
uri=dwpro-name1:8020     #hdfs文件系统地址
url=http://www.cnblogs.com/30go/   #待保存的http地址
targetPath=/tmp/test/king.txt   # 目标的文件名
java -Djava.ext.dirs=lib com.etl.LinkHttp $name $uri $url $targetPath  >> test.log 2>&1 &