tonglin0325的个人主页

将parquet schema转换成avro schema

tonglin0325

1.引入依赖

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
<!--parquet-->
<dependency>
    <groupId>org.apache.parquet</groupId>
    <artifactId>parquet-avro</artifactId>
    <version>1.10.0</version>
</dependency>
<dependency>
    <groupId>org.apache.parquet</groupId>
    <artifactId>parquet-hadoop</artifactId>
    <version>1.10.0</version>
</dependency>
<!--hadoop-->
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>2.7.3</version>
</dependency>

2.从parquet文件的footer读取parquet schema

1
2
3
4
5
6
7
8
9
10
11
12
import org.apache.hadoop.conf.Configuration;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.util.HadoopInputFile;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.schema.MessageType;

Configuration config = new Configuration();
        Path parquetPath = new Path("file:///Users/lintong/Downloads/xxxx.snappy.parquet");
        ParquetFileReader reader = ParquetFileReader.open(HadoopInputFile.fromPath(parquetPath, config));
        MessageType parquetSchema = reader.getFooter().getFileMetaData().getSchema();
        System.out.println(parquetSchema);

输出

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
message TestSerializer {
  optional binary string1 (UTF8);
  optional int32 int1;
  optional int32 tinyint1;
  optional int32 smallint1;
  optional int64 bigint1;
  optional boolean boolean1;
  optional double float1;
  optional double double1;
  optional group list1 (LIST) {
    repeated binary array (UTF8);
  }
  optional group map1 (LIST) {
    repeated group array {
      optional binary key (UTF8);
      optional int32 value;
    }
  }
  optional group struct1 {
    optional int32 sInt;
    optional boolean sBoolean;
    optional binary sString (UTF8);
  }
  optional binary enum1 (UTF8);
  optional int32 nullableint;
}

3.将parquet schema转换成avro schema

1
2
3
4
5
6
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.avro.Schema;

Schema avroSchema = new AvroSchemaConverter(config).convert(parquetSchema);
System.out.println(avroSchema);

输出

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
{
   "type":"record",
   "name":"TestSerializer",
   "fields":[
      {
         "name":"string1",
         "type":[
            "null",
            "string"
         ],
         "default":null
      },
      {
         "name":"int1",
         "type":[
            "null",
            "int"
         ],
         "default":null
      },
      {
         "name":"tinyint1",
         "type":[
            "null",
            "int"
         ],
         "default":null
      },
      {
         "name":"smallint1",
         "type":[
            "null",
            "int"
         ],
         "default":null
      },
      {
         "name":"bigint1",
         "type":[
            "null",
            "long"
         ],
         "default":null
      },
      {
         "name":"boolean1",
         "type":[
            "null",
            "boolean"
         ],
         "default":null
      },
      {
         "name":"float1",
         "type":[
            "null",
            "double"
         ],
         "default":null
      },
      {
         "name":"double1",
         "type":[
            "null",
            "double"
         ],
         "default":null
      },
      {
         "name":"list1",
         "type":[
            "null",
            {
               "type":"array",
               "items":"string"
            }
         ],
         "default":null
      },
      {
         "name":"map1",
         "type":[
            "null",
            {
               "type":"array",
               "items":{
                  "type":"record",
                  "name":"array",
                  "fields":[
                     {
                        "name":"key",
                        "type":[
                           "null",
                           "string"
                        ],
                        "default":null
                     },
                     {
                        "name":"value",
                        "type":[
                           "null",
                           "int"
                        ],
                        "default":null
                     }
                  ]
               }
            }
         ],
         "default":null
      },
      {
         "name":"struct1",
         "type":[
            "null",
            {
               "type":"record",
               "name":"struct1",
               "fields":[
                  {
                     "name":"sInt",
                     "type":[
                        "null",
                        "int"
                     ],
                     "default":null
                  },
                  {
                     "name":"sBoolean",
                     "type":[
                        "null",
                        "boolean"
                     ],
                     "default":null
                  },
                  {
                     "name":"sString",
                     "type":[
                        "null",
                        "string"
                     ],
                     "default":null
                  }
               ]
            }
         ],
         "default":null
      },
      {
         "name":"enum1",
         "type":[
            "null",
            "string"
         ],
         "default":null
      },
      {
         "name":"nullableint",
         "type":[
            "null",
            "int"
         ],
         "default":null
      }
   ]
}

参考：https://stackoverflow.com/questions/54159454/how-to-convert-parquet-schema-to-avro-in-java-scala

tonglin0325

标签

标签云

AWS Airbnb Android CDH Doris ELK Flink Git Grafana HAProxy HBase Hadoop Hive Hudi InfluxDB JDBC JMeter JVM Java JavaWeb LaTex Linkedin Linux ML ML Infra MPP Maven MySQL Nexus OpenTSDB Paper Play Prometheus Python React Redis RocksDB Scala Solr Spark SpringBoot SpringMVC Thrift YARN antlr arthas avro cassandra clickhouse confluent docker filebeat flume golang google gradle hexo impala jenkins k8s kafka kerberos kudu ldap mac mongo mybatis nginx nlp open-falcon openwrt parquet presto ranger scyllaDB twitter zookeeper 刷题 前端 图存储及计算 多线程 广告系统 开发工具 数据结构 杂谈 正则表达式 算法 系统设计 计算机网络 设计模式

文章归档

最近文章