louzin
1 year ago
9 changed files with 245 additions and 5 deletions
@ -0,0 +1,60 @@
|
||||
package com.louzin.hdfsapi.util; |
||||
|
||||
import org.apache.hadoop.conf.Configuration; |
||||
import org.apache.hadoop.fs.*; |
||||
import org.junit.After; |
||||
import org.junit.Before; |
||||
import org.junit.Test; |
||||
|
||||
import java.io.IOException; |
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
public class hdfsapi { |
||||
static FileSystem hdfs=null; |
||||
static{ |
||||
// 构造一个配置参数对象,设置一个参数:要访问的HDFS的URI
|
||||
Configuration conf = new Configuration(); |
||||
// 指定使用HDFS访问
|
||||
conf.set("fs.defaultFS", "hdfs://local1:8020"); |
||||
// 进行客户端身份的设置(root为虚拟机的用户名,hadoop集群节点的其中一个都可以)
|
||||
System.setProperty("HADOOP_USER_NAME", "root"); |
||||
// 通过FileSystem的静态get()方法获取HDFS文件系统客户端对象
|
||||
try { |
||||
hdfs = FileSystem.get(conf); |
||||
} catch (IOException e) { |
||||
System.out.println("实例化失败!"); |
||||
} |
||||
} |
||||
|
||||
@Before |
||||
public void init() throws IOException { |
||||
// 构造一个配置参数对象,设置一个参数:要访问的HDFS的URI
|
||||
Configuration conf = new Configuration(); |
||||
// 指定使用HDFS访问
|
||||
conf.set("fs.defaultFS", "hdfs://local1:8020"); |
||||
// 进行客户端身份的设置(root为虚拟机的用户名,hadoop集群节点的其中一个都可以)
|
||||
System.setProperty("HADOOP_USER_NAME", "root"); |
||||
// 通过FileSystem的静态get()方法获取HDFS文件系统客户端对象
|
||||
hdfs = FileSystem.get(conf); |
||||
} |
||||
//扫描采集日志的路径
|
||||
@Test |
||||
public List<String> scan() throws IOException { |
||||
List<String> lists=new ArrayList(); |
||||
Path path=new Path("/flume/events/"); |
||||
FileStatus[] fileStatuses=hdfs.listStatus(path);//events下的每日文件夹
|
||||
for (FileStatus fs:fileStatuses) { |
||||
String[] str=String.valueOf(fs.getPath()).split("8020"); |
||||
lists.add(str[1]); |
||||
} |
||||
close(); |
||||
return lists; |
||||
} |
||||
@After |
||||
public void close() throws IOException { |
||||
hdfs.close(); |
||||
} |
||||
} |
||||
|
||||
|
@ -0,0 +1,18 @@
|
||||
package com.louzin.hiveapi; |
||||
|
||||
import java.sql.ResultSet; |
||||
import java.sql.SQLException; |
||||
|
||||
public class test { |
||||
private static HiveConnecter hiveConnecter; |
||||
public static void main(String[] args) throws SQLException, ClassNotFoundException { |
||||
hiveConnecter=HiveConnecter.getInstance(); |
||||
hiveConnecter.getConn(); |
||||
String sql="show tables"; |
||||
ResultSet rs = hiveConnecter.executeQueryBYStatement(sql); |
||||
while (rs.next()){ |
||||
System.out.println(rs.getString(1)); |
||||
} |
||||
hiveConnecter.closeAll(); |
||||
} |
||||
} |
@ -1,3 +1,27 @@
|
||||
package com.louzin.webserver.controller; |
||||
|
||||
import org.springframework.stereotype.Controller; |
||||
import org.springframework.web.bind.annotation.*; |
||||
import com.louzin.hiveapi.util.jdbcHiveConnect; |
||||
import com.louzin.hdfsapi.util.hdfsapi; |
||||
import java.sql.SQLException; |
||||
import java.util.List; |
||||
|
||||
@Controller |
||||
@ResponseBody |
||||
@RequestMapping("/dev_api") |
||||
@CrossOrigin |
||||
public class mycontroller { |
||||
@GetMapping("/test") |
||||
public String testMethod(){ |
||||
return "local:/dev_api/test"; |
||||
} |
||||
@PostMapping("/getAllTable") |
||||
public List<String> saupl() throws SQLException, ClassNotFoundException { |
||||
return new jdbcHiveConnect().showAllTable(); |
||||
} |
||||
@PostMapping("/scan") |
||||
public List<String> scanFolder() throws Exception{ |
||||
return new hdfsapi().scan(); |
||||
} |
||||
} |
||||
|
@ -0,0 +1,9 @@
|
||||
package com.louzin.webserver.dao; |
||||
|
||||
import java.sql.SQLException; |
||||
|
||||
public class Daotest { |
||||
public static void main(String[] args) throws SQLException, ClassNotFoundException { |
||||
System.out.println(new hiveDao().getAllTable()); |
||||
} |
||||
} |
@ -0,0 +1,31 @@
|
||||
package com.louzin.webserver.dao; |
||||
import com.louzin.hiveapi.HiveConnecter; |
||||
|
||||
import java.sql.ResultSet; |
||||
import java.sql.SQLException; |
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
public class hiveDao { |
||||
private HiveConnecter hiveConnecter; |
||||
public hiveDao() throws SQLException, ClassNotFoundException { |
||||
hiveConnecter=HiveConnecter.getInstance(); |
||||
} |
||||
public List<String> getAllTable(){ |
||||
String sql="show tables"; |
||||
List<String> tblist=new ArrayList(); |
||||
try { |
||||
hiveConnecter.getConn(); |
||||
ResultSet rs = hiveConnecter.executeQueryBYStatement(sql); |
||||
while (rs.next()){ |
||||
tblist.add(rs.getString(1)); |
||||
} |
||||
}catch (Exception e){ |
||||
System.out.println(e); |
||||
} |
||||
finally { |
||||
hiveConnecter.closeAll(); |
||||
} |
||||
return tblist; |
||||
} |
||||
} |
Loading…
Reference in new issue