Hbase_各种过滤器使用

1.常规比较器

• 行键过滤器
• 列簇过滤器
• 列过滤器
• 列值过滤器

package com.shujia;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;import java.io.IOException;
import java.util.List;/*** @author WangTao* @date 2022/6/11 21:39*/
public class HbaseFilterComparatorDemo {private HConnection conn  = null;private HBaseAdmin hAdmin = null;@Beforepublic void connect(){try {//1.获取Hadoop的相关配置环境Configuration conf = new Configuration();//获取zookeeper的配置conf.set("hbase.zookeeper.quorum", "master:2181,node1:2181,node2:2181");//获取与Hbase 的连接，这个连接是将来可以用户获取hbase表的conn = HConnectionManager.createConnection(conf);//将来我们要对表做DDL相关的操作，而对表的操作在hbase架构中是有HMaster的hAdmin = new HBaseAdmin(conf);System.out.println("建立连接成功："+conn+",HMaster建立连接成功："+hAdmin);} catch (IOException e) {e.printStackTrace();}}/*** 行键过滤器* 通过RowFilter与BinaryComparator过滤比rowKey 1500100010小的所有值出来*/@Testpublic void RowFilter(){try {//获取表的实例HTableInterface students = conn.getTable("students");BinaryComparator binaryComparator = new BinaryComparator("1500100010".getBytes());//创建一个行键过滤器的对象RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.LESS, binaryComparator);Scan scan = new Scan();scan.setFilter(rowFilter);ResultScanner scanner = students.getScanner(scan);print2(scanner);} catch (IOException e) {e.printStackTrace();}}/*** 列簇过滤器案例1：通过FamilyFilter与SubstringComparator查寻列簇名包含in的所有列簇下面的数据*/@Testpublic void FamilyFilter1(){try {//获取表的实例HTableInterface students = conn.getTable("students");//创建一个比较器对象//只要列簇名中包含了 in ，就把该列簇下的所有的列查询出来SubstringComparator substringComparator = new SubstringComparator("nf");//创建一个列簇过滤器FamilyFilter familyFilter = new FamilyFilter(CompareFilter.CompareOp.EQUAL, substringComparator);Scan scan = new Scan();scan.setFilter(familyFilter);//获取数据ResultScanner scanner = students.getScanner(scan);print2(scanner);} catch (IOException e) {e.printStackTrace();}}/*** 列簇过滤器案例2：通过FamilyFilter与BinaryPrefixComparator 过滤以 i 开头的列簇下的所有数据*/@Testpublic void FamilyFilter2(){try {//获取表的实例HTableInterface students = conn.getTable("students");//创建前缀比较器BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator("i".getBytes());//创建列簇比较器FamilyFilter familyFilter = new FamilyFilter(CompareFilter.CompareOp.EQUAL, binaryPrefixComparator);Scan scan = new Scan();scan.setFilter(familyFilter);ResultScanner scanner = students.getScanner(scan);print2(scanner);} catch (IOException e) {e.printStackTrace();}}/*** 列过滤器案例1：通过QualifierFilter与SubstringComparator查询列名包含 ge 的列的值*/@Testpublic void QualifierFilter1(){try {//获取表的实例HTableInterface students = conn.getTable("students");//创建包含比较器//age//genderSubstringComparator substringComparator = new SubstringComparator("ge");//创建一个过滤器QualifierFilter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, substringComparator);Scan scan = new Scan();scan.setFilter(qualifierFilter);//调用方法读取数据ResultScanner scanner = students.getScanner(scan);print2(scanner);} catch (IOException e) {e.printStackTrace();}}/*** 列过滤器案例2：通过QualifierFilter与SubstringComparator查询列名包含 am 的列的值*/@Testpublic void QualifierFilter2(){try {//获取表的实例HTableInterface students = conn.getTable("students");SubstringComparator substringComparator = new SubstringComparator("am");//创建列过滤器QualifierFilter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, substringComparator);Scan scan = new Scan();scan.setFilter(qualifierFilter);ResultScanner scanner = students.getScanner(scan);print2(scanner);} catch (IOException e) {e.printStackTrace();}}/***  列值过滤器案例1：通过ValueFilter与BinaryPrefixComparator过滤出所有的cell中以"张"开头的同学**/@Testpublic void ValueFilter1(){try {//获取表的实例HTableInterface students = conn.getTable("students");//创建前缀比较器BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator("张".getBytes());//创建列值过滤器对象ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, binaryPrefixComparator);Scan scan = new Scan();scan.setFilter(valueFilter);ResultScanner scanner = students.getScanner(scan);//print2(scanner);//因为 ResultScanner 类继承了迭代器//使用增强for循环遍历for (Result rs : scanner) {String id = Bytes.toString(rs.getRow());System.out.println("当前行的rowkey为："+id);//继续使用增强for循环到每一行中的每一个单元格(列)//获取每一行的单元格for (Cell cell : rs.listCells()) {//获取该单元格属于的列簇String family = Bytes.toString(CellUtil.cloneFamily(cell));//获取该单元格的列名String colName = Bytes.toString(CellUtil.cloneQualifier(cell));//获取单元格的列值String value = Bytes.toString(CellUtil.cloneValue(cell));System.out.println(family+":"+colName+"的值为："+value);}}} catch (IOException e) {e.printStackTrace();}}/*** 列值过滤器案例2：> 过滤出文科的学生，只会返回以文科开头的数据列，其他列的数据不符合条件，不会返回*/@Testpublic void valueFilter2(){try {//获取表的实例HTableInterface students = conn.getTable("students");//创建正则比较器RegexStringComparator regexStringComparator = new RegexStringComparator("^文科.*");//创建列值过滤器ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, regexStringComparator);Scan scan = new Scan();scan.setFilter(valueFilter);ResultScanner scanner = students.getScanner(scan);print(scanner);} catch (IOException e) {e.printStackTrace();}}/*** 专门用来打印数据的方法*/public void print(ResultScanner scanner) throws IOException {Result rs = null;while((rs = scanner.next()) != null){String id = Bytes.toString(rs.getRow());List<Cell> cells = rs.listCells();System.out.println("id"+id);System.out.println("\t");for (Cell cell : cells) {//    String s = Bytes.toString(cell.getValue());String col = Bytes.toString(CellUtil.cloneFamily(cell));String s = Bytes.toString(CellUtil.cloneValue(cell));System.out.println(col+":"+s);System.out.println("\t");}}}/*** 专门用来打印数据的方法2*/public void print2(ResultScanner scanner) throws IOException {Result rs = null;while((rs = scanner.next())!= null){String id = Bytes.toString(rs.getRow());String name = Bytes.toString(rs.getValue("info".getBytes(), "name".getBytes()));String age = Bytes.toString(rs.getValue("info".getBytes(), "age".getBytes()));String gender = Bytes.toString(rs.getValue("info".getBytes(), "gender".getBytes()));String clazz = Bytes.toString(rs.getValue("info".getBytes(), "clazz".getBytes()));System.out.println("学号：" + id + ",姓名：" + name + ",年龄：" + age + ",性别：" + gender + ",班级：" + clazz);}}@Afterpublic void close(){if(conn!=null){try {conn.close();} catch (IOException e) {e.printStackTrace();}System.out.println("conn已经关闭...");}if(hAdmin!=null){try {hAdmin.close();} catch (IOException e) {e.printStackTrace();}System.out.println("hMaster已经关闭...");}}
}

2.专用过滤器和分页包装过滤器

• 单列值过滤器
• 单列值排除过滤器
• rowkey前缀过滤器
• 分页过滤器

package com.shujia;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;import java.io.IOException;
import java.util.List;
/*** @author WangTao* @date 2022/6/12 9:36*/
public class HbaseFilterSpecialDemo {private HConnection conn;private HBaseAdmin hAdmin;@Beforepublic void connect() {try {//1、获取Hadoop的相关配置环境Configuration conf = new Configuration();//2、获取zookeeper的配置conf.set("hbase.zookeeper.quorum", "master:2181,node1:2181,node2:2181");//获取与Hbase的连接，这个连接是将来可以用户获取hbase表的conn = HConnectionManager.createConnection(conf);//将来我们要对表做DDL相关操作，而对表的操作在hbase架构中是有HMasterhAdmin = new HBaseAdmin(conf);System.out.println("建立连接成功:" + conn + ", HMaster获取成功：" + hAdmin);} catch (IOException e) {e.printStackTrace();}}/*** 单列值过滤器* SingleColumnValueFilter会返回满足条件的cell所在行的所有cell的值(即会返回一行数据)* <p>* 通过SingleColumnValueFilter与查询文科班所有学生信息*/@Testpublic void SingleColumnValueFilter(){try {//获取表的实例HTableInterface students = conn.getTable("students");//创建一个正则比较器RegexStringComparator regexStringComparator = new RegexStringComparator("^文科.*");//创建一个单列值过滤器对象SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter("info".getBytes(),"clazz".getBytes(),CompareFilter.CompareOp.EQUAL,regexStringComparator);Scan scan = new Scan();scan.setFilter(singleColumnValueFilter);ResultScanner scanner = students.getScanner(scan);print2(scanner);} catch (IOException e) {e.printStackTrace();}}/*** 列值排除过滤器* 与SingleColumnValueFilter相反，会排除掉指定的列，其他的列全部返回* <p>* 通过SingleColumnValueExcludeFilter与BinaryComparator查询文科一班所有学生信息，最终不返回clazz列*/@Testpublic void SingleColumnValueExcludeFilter(){try {//获取表的实例HTableInterface students = conn.getTable("students");//创建一个二进制比较器对象BinaryComparator binaryComparator = new BinaryComparator("文科一班".getBytes());//创建一个列值排除过滤器SingleColumnValueExcludeFilter singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter("info".getBytes(),"clazz".getBytes(),CompareFilter.CompareOp.EQUAL,binaryComparator);Scan scan = new Scan();scan.setFilter(singleColumnValueExcludeFilter);ResultScanner scanner = students.getScanner(scan);print2(scanner);} catch (IOException e) {e.printStackTrace();}}/*** rowkey前缀过滤器* <p>* 通过PrefixFilter查询以150010008开头的所有前缀的rowkey*/@Testpublic void PrefixFilter() {try {//获取表的实例HTableInterface students = conn.getTable("students");//创建rowkey前缀过滤器PrefixFilter prefixFilter = new PrefixFilter("150010008".getBytes());Scan scan = new Scan();scan.setFilter(prefixFilter);ResultScanner scanner = students.getScanner(scan);print2(scanner);} catch (IOException e) {e.printStackTrace();}}/*** 分页过滤器* 通过PageFilter查询三页的数据，每页十条*/@Testpublic void pageFilter(){try {//获取表的实例HTableInterface students = conn.getTable("students");//定义要查询的页数int pageNum=4;//定义每页的条数int pageSize = 10;Scan scan = new Scan();//定义一开始的行String current_page_start_row = "";for (int i = 1; i < pageNum; i++) {System.out.println("=======================当前是第"+i+"页=============================");//创建一个分页过滤器PageFilter pageFilter = new PageFilter(pageSize);scan.setFilter(pageFilter);ResultScanner scanner = students.getScanner(scan);for (Result result : scanner) {current_page_start_row = Bytes.toString(result.getRow());//告诉扫描器是从那一行开始获取数据scan.withStartRow((current_page_start_row+0).getBytes());PageFilter pageFilter1 = new PageFilter(pageSize);scan.setFilter(pageFilter1);//获取idString id = Bytes.toString(result.getRow());String name = Bytes.toString(result.getValue("info".getBytes(), "name".getBytes()));String age = Bytes.toString(result.getValue("info".getBytes(), "age".getBytes()));String gender = Bytes.toString(result.getValue("info".getBytes(), "gender".getBytes()));String clazz = Bytes.toString(result.getValue("info".getBytes(), "clazz".getBytes()));System.out.println("学号：" + id + ",姓名：" + name + ",年龄：" + age + ",性别：" + gender + ",班级：" + clazz);}}} catch (IOException e) {e.printStackTrace();}}@Testpublic void baoZhuang1() throws IOException {HTableInterface students = conn.getTable("students");Filter filter1 = new RowFilter(CompareFilter.CompareOp.NOT_EQUAL,new BinaryComparator("1500100009".getBytes()));//不做包装Scan scan = new Scan();scan.setFilter(filter1);ResultScanner scanner1 = students.getScanner(scan);Result rs = scanner1.next();while (rs != null) {String id = Bytes.toString(rs.getRow());String name = Bytes.toString(rs.getValue("info".getBytes(), "name".getBytes()));String age = Bytes.toString(rs.getValue("info".getBytes(), "age".getBytes()));String gender = Bytes.toString(rs.getValue("info".getBytes(), "gender".getBytes()));String clazz = Bytes.toString(rs.getValue("info".getBytes(), "clazz".getBytes()));System.out.println(id + "\t" + name + "\t" + age + "\t" + gender + "\t" + clazz + "\t");rs = scanner1.next();}System.out.println("-----------------------------------------------------------------------------");// 使用 WhileMatchFilter 进行包装Filter filter2 = new WhileMatchFilter(filter1);scan.setFilter(filter2);ResultScanner scanner = students.getScanner(scan);Result rs2 = scanner.next();while (rs2 != null) {String id = Bytes.toString(rs2.getRow());String name = Bytes.toString(rs2.getValue("info".getBytes(), "name".getBytes()));String age = Bytes.toString(rs2.getValue("info".getBytes(), "age".getBytes()));String gender = Bytes.toString(rs2.getValue("info".getBytes(), "gender".getBytes()));String clazz = Bytes.toString(rs2.getValue("info".getBytes(), "clazz".getBytes()));System.out.println(id + "\t" + name + "\t" + age + "\t" + gender + "\t" + clazz + "\t");rs2 = scanner.next();}}/*** 专门用来打印数据的方法*/public void print(ResultScanner scanner) throws IOException {Result rs = null;while((rs = scanner.next()) != null){String id = Bytes.toString(rs.getRow());List<Cell> cells = rs.listCells();System.out.println("id"+id);System.out.println("\t");for (Cell cell : cells) {//    String s = Bytes.toString(cell.getValue());String col = Bytes.toString(CellUtil.cloneFamily(cell));String s = Bytes.toString(CellUtil.cloneValue(cell));System.out.println(col+":"+s);System.out.println("\t");}}}/*** 专门用来打印数据的方法2*/public void print2(ResultScanner scanner) throws IOException {Result rs = null;while((rs = scanner.next())!= null){String id = Bytes.toString(rs.getRow());String name = Bytes.toString(rs.getValue("info".getBytes(), "name".getBytes()));String age = Bytes.toString(rs.getValue("info".getBytes(), "age".getBytes()));String gender = Bytes.toString(rs.getValue("info".getBytes(), "gender".getBytes()));String clazz = Bytes.toString(rs.getValue("info".getBytes(), "clazz".getBytes()));System.out.println("学号：" + id + ",姓名：" + name + ",年龄：" + age + ",性别：" + gender + ",班级：" + clazz);}}@Afterpublic void close() {if (conn != null) {try {conn.close();} catch (IOException e) {e.printStackTrace();}System.out.println("conn连接已经关闭.....");}if (hAdmin != null) {try {hAdmin.close();} catch (IOException e) {e.printStackTrace();}System.out.println("HMaster已经关闭......");}}}

3.组合比较器

package com.shujia;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.*;
import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;import java.io.IOException;
import java.util.List;
/*** @author WangTao* @date 2022/6/12 10:49*//***  组合过滤器的使用***  1 通过运用4种比较器过滤出姓于，年纪大于23岁，性别为女，且是理科的学生。****  2 过滤出学号是以15001001开头的文科学生*/
public class HbaseFileCombinationDemo {private HConnection conn;private HBaseAdmin hAdmin;@Beforepublic void connect() {try {//1、获取Hadoop的相关配置环境Configuration conf = new Configuration();//2、获取zookeeper的配置conf.set("hbase.zookeeper.quorum", "master:2181,node1:2181,node2:2181");//获取与Hbase的连接，这个连接是将来可以用户获取hbase表的conn = HConnectionManager.createConnection(conf);//将来我们要对表做DDL相关操作，而对表的操作在hbase架构中是有HMasterhAdmin = new HBaseAdmin(conf);System.out.println("建立连接成功:" + conn + ", HMaster获取成功：" + hAdmin);} catch (IOException e) {e.printStackTrace();}}/*** 需求：1 通过运用4种比较器过滤出姓于，年纪大于23岁，性别为女，且是理科的学生。** 正则比较器   RegexStringComparator* 包含比较器   SubstringComparator* 二进制前缀比较器   BinaryPrefixComparator* 二进制比较器      BinaryComparator**/@Testpublic void FilterDate1(){try {//获取表的实例HTableInterface students = conn.getTable("students");/*第一个过滤器，过滤出理科开头的班级*/RegexStringComparator regexStringComparator = new RegexStringComparator("^理科.*");//单列值过滤器SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter("info".getBytes(), "clazz".getBytes(),CompareFilter.CompareOp.EQUAL, regexStringComparator);/*第二个过滤器，过滤出性别是女生的*/SubstringComparator substringComparator = new SubstringComparator("女");SingleColumnValueFilter singleColumnValueFilter1 = new SingleColumnValueFilter("info".getBytes(), "gender".getBytes(),CompareFilter.CompareOp.EQUAL, substringComparator);/*第二个过滤器，过滤出年龄大于20岁的*/BinaryComparator binaryComparator = new BinaryComparator("20".getBytes());SingleColumnValueFilter singleColumnValueFilter2 = new SingleColumnValueFilter("info".getBytes(), "age".getBytes(),CompareFilter.CompareOp.GREATER, binaryComparator);/*第四个过滤器，过滤出姓于的学生*/BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator("于".getBytes());SingleColumnValueFilter singleColumnValueFilter3 = new SingleColumnValueFilter("info".getBytes(), "name".getBytes(),CompareFilter.CompareOp.EQUAL, binaryPrefixComparator);Scan scan = new Scan();//要想实现过个需求同时过滤，就需要创建多个过滤器，添加到一个过滤列表中//然后将过滤列表传给扫描器scanFilterList filterList = new FilterList();filterList.addFilter(singleColumnValueFilter);filterList.addFilter(singleColumnValueFilter1);filterList.addFilter(singleColumnValueFilter2);filterList.addFilter(singleColumnValueFilter3);scan.setFilter(filterList);ResultScanner scanner = students.getScanner(scan);print2(scanner);} catch (IOException e) {e.printStackTrace();}}/***  过滤出学号是以15001001开头的文科学生*/@Testpublic void filterData2(){try {//获取表的实例HTableInterface students = conn.getTable("students");/*创建一个过滤器，过滤是以15001001开头的rowkey*/BinaryPrefixComparator binaryPrefixComparator = new BinaryPrefixComparator("15001001".getBytes());RowFilter rowFilter = new RowFilter(CompareFilter.CompareOp.EQUAL, binaryPrefixComparator);/*创建第二个过滤器，过滤出文科的学生*/RegexStringComparator regexStringComparator = new RegexStringComparator("^文科。*");SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter("info".getBytes(),"clazz".getBytes(),CompareFilter.CompareOp.EQUAL,regexStringComparator);FilterList filterList = new FilterList();filterList.addFilter(rowFilter);filterList.addFilter(singleColumnValueFilter);Scan scan = new Scan();scan.setFilter(filterList);ResultScanner scanner = students.getScanner(scan);print2(scanner);} catch (IOException e) {e.printStackTrace();}}/*** 专门用来打印数据的方法*/public void print(ResultScanner scanner) throws IOException {Result rs = null;while ((rs = scanner.next()) != null) {String id = Bytes.toString(rs.getRow());List<Cell> cells = rs.listCells();System.out.print("id:" + id);System.out.print("\t");for (Cell cell : cells) {
//                String s = Bytes.toString(cell.getValue());String col = Bytes.toString(CellUtil.cloneQualifier(cell));String s = Bytes.toString(CellUtil.cloneValue(cell));System.out.print(col + ":" + s);System.out.print("\t");}System.out.println();//            String name = Bytes.toString(rs.getValue("info".getBytes(), "name".getBytes()));
//            String age = Bytes.toString(rs.getValue("info".getBytes(), "age".getBytes()));
//            String gender = Bytes.toString(rs.getValue("info".getBytes(), "gender".getBytes()));
//            String clazz = Bytes.toString(rs.getValue("info".getBytes(), "clazz".getBytes()));
//            System.out.println("学号：" + id + ",姓名：" + name + ",年龄：" + age + ",性别：" + gender + ",班级：" + clazz);}}/*** 专门用来打印数据的方法*/public void print2(ResultScanner scanner) throws IOException {Result rs = null;while ((rs = scanner.next()) != null) {String id = Bytes.toString(rs.getRow());String name = Bytes.toString(rs.getValue("info".getBytes(), "name".getBytes()));String age = Bytes.toString(rs.getValue("info".getBytes(), "age".getBytes()));String gender = Bytes.toString(rs.getValue("info".getBytes(), "gender".getBytes()));String clazz = Bytes.toString(rs.getValue("info".getBytes(), "clazz".getBytes()));System.out.println("学号：" + id + ",姓名：" + name + ",年龄：" + age + ",性别：" + gender + ",班级：" + clazz);}}@Afterpublic void close() {if (conn != null) {try {conn.close();} catch (IOException e) {e.printStackTrace();}System.out.println("conn连接已经关闭.....");}if (hAdmin != null) {try {hAdmin.close();} catch (IOException e) {e.printStackTrace();}System.out.println("HMaster已经关闭......");}}
}