使用okhttp工具集來開發(fā)：（如果文件已經(jīng)存在會報錯）

package com.etl;import java.io.IOException;import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path;import okhttp3.Call; import okhttp3.Callback; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response;public class LinkHttp {private static Configuration conf = null;private static String fsName = "fs.defaultFS";private static String fsURI = null;public static void main(String[] args) throws Exception {String name = args[0];String uri = args[1];String url = args[2];String targetFile = args[3]; //文件全路徑//初始化 init(name, uri);OkHttpClient client = new OkHttpClient();final Request request = new Request.Builder().url(url).get().build();Call call = client.newCall(request);call.enqueue(new Callback() {@Overridepublic void onFailure(Call call, IOException e) {System.out.println("Fail");}@Overridepublic void onResponse(Call call, Response response) throws IOException {FileSystem fs = null;try {Path dstPath = new Path(targetFile);fs = FileSystem.get(conf);FSDataOutputStream outputStream = fs.create(dstPath); if(response.isSuccessful()) {// System.out.println(response.body().string()); outputStream.write(response.body().bytes()); outputStream.close(); System.out.println("create file " + targetFile + " success!"); //fs.close(); }}catch (Exception e){e.printStackTrace();}finally {fs.close();}System.out.println("run writeHdfs end");//關閉if(response.body()!=null) {response.body().close();}}});}private static void init(String name, String uri) {if(StringUtils.isNotBlank(fsName)){fsName = name;}fsURI = uri;conf = new Configuration();conf.set(fsName, fsURI);}}

配置啟動腳本如下：

#!/bin/sh name=fs.defaultFS #固定不變 uri=dwpro-name1:8020 #hdfs文件系統(tǒng)地址 url=http://www.cnblogs.com/30go/ #待保存的http地址 targetPath=/tmp/test/king.txt # 目標的文件名 java -Djava.ext.dirs=lib com.etl.LinkHttp \ ${name} ${uri} ${url} ${targetPath} >> test.log 2>&1 &

?

轉(zhuǎn)載于:https://www.cnblogs.com/30go/p/9762527.html

超強干貨來襲 云風專訪：近40年碼齡，通宵達旦的技術人生

總結

以上是生活随笔為你收集整理的简单的Http请求数据保存到Hdfs的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。