tonglin0325的个人主页

SpringBoot学习笔记——动态代理

tonglin0325

代理模式是一种设计模式，提供了对目标对象额外的访问方式，即通过代理对象访问目标对象，这样可以在不修改原目标对象的前提下，提供额外的功能操作，扩展目标对象的功能。

ubuntu16.04安装haproxy

tonglin0325

清华镜像站

1 2	https://mirrors.tuna.tsinghua.edu.cn/ubuntu/pool/main/h/haproxy/

下载haproxy

1 2	https://mirrors.tuna.tsinghua.edu.cn/ubuntu/pool/main/h/haproxy/haproxy_1.6.3-1ubuntu0.3_amd64.deb

安装haproxy

1 2	sudo dpkg -i ./haproxy_1.6.3-1ubuntu0.3_amd64.deb

编辑配置文件

1 2	vim /etc/haproxy/haproxy.cfg

其默认配置如下

global
        log /dev/log    local0
        log /dev/log    local1 notice
        chroot /var/lib/haproxy
        stats socket /run/haproxy/admin.sock mode 660 level admin
        stats timeout 30s
        user haproxy
        group haproxy
        daemon

        # Default SSL material locations
        ca-base /etc/ssl/certs
        crt-base /etc/ssl/private

        # Default ciphers to use on SSL-enabled listening sockets.
        # For more information, see ciphers(1SSL). This list is from:
        #  https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
        ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
        ssl-default-bind-options no-sslv3

defaults
        log     global
        mode    http
        option  httplog
        option  dontlognull
        timeout connect 5000
        timeout client  50000
        timeout server  50000
        errorfile 400 /etc/haproxy/errors/400.http
        errorfile 403 /etc/haproxy/errors/403.http
        errorfile 408 /etc/haproxy/errors/408.http
        errorfile 500 /etc/haproxy/errors/500.http
        errorfile 502 /etc/haproxy/errors/502.http
        errorfile 503 /etc/haproxy/errors/503.http
        errorfile 504 /etc/haproxy/errors/504.http

yarn学习笔记——yarn api

tonglin0325

参考：Yarn 监控 - 监控任务运行状态（包括Spark，MR 所有在Yarn中运行的任务）

    //获取任务的applicationId
    public static String getAppId(String jobName) throws IOException {

        Configuration conf = new Configuration();
        System.setProperty("java.security.krb5.conf", "/etc/krb5.conf");
        conf.set("hadoop.security.authentication", "Kerberos");
        UserGroupInformation.setConfiguration(conf);
        UserGroupInformation.loginUserFromKeytab("hdfs@XXXX", "/home/xxxx/hdfs.keytab");

        YarnClient client = YarnClient.createYarnClient();
        client.init(conf);
        client.start();
        EnumSet<YarnApplicationState> appStates = EnumSet.noneOf(YarnApplicationState.class);

        if (appStates.isEmpty()) {
            appStates.add(YarnApplicationState.RUNNING);
            appStates.add(YarnApplicationState.ACCEPTED);
            appStates.add(YarnApplicationState.SUBMITTED);
        }

        List<ApplicationReport> appsReport = null;
        try {
            // 返回EnumSet<YarnApplicationState>中个人任务状态的所有任务
            appsReport = client.getApplications(appStates);
        } catch (YarnException | IOException e) {
            e.printStackTrace();
        }

        assert appsReport != null;

        for (ApplicationReport appReport : appsReport) {
            System.out.println(appReport);
            // 获取任务名
            String jn = appReport.getName();
            String applicationType = appReport.getApplicationType();
            if (jn.equals(jobName)) { // &amp;&amp; "Apache Flink".equals(applicationType)) {
                try {
                    client.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
                return appReport.getApplicationId().toString();
            }
        }
        try {
            client.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }

    // 根据任务的applicationId去获取任务的状态
    public static YarnApplicationState getState(String appId) throws IOException {

        Configuration conf = new Configuration();
        System.setProperty("java.security.krb5.conf", "/etc/krb5.conf");
        conf.set("hadoop.security.authentication", "Kerberos");
        UserGroupInformation.setConfiguration(conf);
        UserGroupInformation.loginUserFromKeytab("hdfs@XXXXX", "/home/xxxx/hdfs.keytab");

        YarnClient client = YarnClient.createYarnClient();
        client.init(conf);
        client.start();
        ApplicationId applicationId = ConverterUtils.toApplicationId(appId);
        YarnApplicationState yarnApplicationState = null;
        try {
            ApplicationReport applicationReport = client.getApplicationReport(applicationId);
            yarnApplicationState = applicationReport.getYarnApplicationState();
        } catch (YarnException | IOException e) {
            e.printStackTrace();
        }
        try {
            client.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return yarnApplicationState;
    }

    public static void main(String[] args) throws IOException, InterruptedException {
        String state = getAppId("job_xxxxx");
        System.out.println(state);
//        System.out.println(state == YarnApplicationState.RUNNING);

    }

输出

applicationId { id: 279 cluster_timestamp: 1620359479641 } 
user: "dl" 
queue: "root.xxxx" 
name: "xxx-flink" 
host: "xxxxx" 
rpc_port: 18000 
client_to_am_token { identifier: "xxxx@XXXX" password: ";xxxxx" kind: "YARN_CLIENT_TOKEN" service: "" } 
yarn_application_state: RUNNING 
trackingUrl: "http://xxxxx:8088/proxy/application_xxxxx/" 
diagnostics: "" 
startTime: 1620391776339 
finishTime: 0 
final_application_status: APP_UNDEFINED 
app_resource_Usage { num_used_containers: 4 num_reserved_containers: 0 used_resources { memory: 8192 virtual_cores: 7 } reserved_resources { memory: 0 virtual_cores: 0 } needed_resources { memory: 8192 virtual_cores: 7 } memory_seconds: 12703546778 vcore_seconds: 10855065 } 
originalTrackingUrl: "http://xxxx:18000" currentApplicationAttemptId { application_id { id: 279 cluster_timestamp: 1620359479641 } attemptId: 1 } progress: 1.0 
applicationType: "XXXX Flink" 
log_aggregation_status: LOG_NOT_START

　　

Flink学习笔记——反压

tonglin0325

参考：

1 2	http://wuchong.me/blog/2016/04/26/flink-internals-how-to-handle-backpressure/

Mybatis学习笔记——通用mapper

tonglin0325

在使用mybatis-generator自动生成mapper代码的时候，对于基本的增删改查方法可以通过继承通用mapper的方式进行简化，参考：MyBatis 通用 Mapper

SpringBoot学习笔记——spring security

tonglin0325

Spring Security是提供了认证，鉴权以及其他的安全特性的java框架，下面是Spring Security的使用教程

1.引入依赖

<dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-security</artifactId>
</dependency>

引入依赖用会发现请求所有的接口都会跳转到 /login，要求你进行账号密码的认证

avro序列化框架的schema evolution

tonglin0325

docker学习笔记——网络模式

tonglin0325

查看容器的网络模式

docker ps -a
CONTAINER ID        IMAGE                                    COMMAND                  CREATED             STATUS              PORTS               NAMES
3f6822d8f262        confluentinc/cp-schema-registry:latest   "/etc/confluent/dock&hellip;"   13 minutes ago      Up 13 minutes                           schema-registry

docker inspect 3f6822d8f262 | grep -i "network"
            "NetworkMode": "host",
        "NetworkSettings": {
            "Networks": {
                    "NetworkID": "5d40a7d178679339f87cc31965ba9a1c662c74ccea853945967d4303e4f9acc0",

docker总共有4种网络模式，从上到下隔离度下降：

1.Close容器，即none模式，运行在Close容器中的进程只能访问本地回环接口，隔离度最高

2.Bridge容器，即bridge模式，当容器中的进程需要访问外部网络的时候应该使用，且bridge是docker的default网络模式。

上传snapshots jar包到nexus仓库

tonglin0325

在nexus界面上可以手动上传release和hosted的包，但是无法手动上传snapshots的包

MySQL学习笔记——binlog

tonglin0325

1.docker部署MySQL

amd64的机器可以使用centos的MySQL5.7的镜像：https://hub.docker.com/r/centos/mysql-57-centos7/

arm64和amd64的机器也可以使用MySQL8.0的镜像：https://hub.docker.com/layers/library/mysql/8.0.29/images/sha256-44f98f4dd825a945d2a6a4b7b2f14127b5d07c5aaa07d9d232c2b58936fb76dc

启动MySQL5.7的容器

1 2	docker run --name mysqltest -p 3306:3306 -e MYSQL_ROOT_PASSWORD=123456 -d mysql:5.7.44

启动MySQL8.0的容器

1 2	docker run --name mysqltest -p 3306:3306 -e MYSQL_ROOT_PASSWORD=123456 -d mysql:8.0.29

如果想指定mysql配置和data挂载路径，可以先进入容器中将mysql的配置先拷贝出来

进入容器查看MySQL的配置路径

sh-4.4# mysql --help | grep my.cnf
                      order of preference, my.cnf, $MYSQL_TCP_PORT,
/etc/my.cnf /etc/mysql/my.cnf /usr/etc/my.cnf ~/.my.cnf

参考：Docker安装MySQL 并挂载数据及配置文件,设置远程访问权限

将配置/etc/my.cnf拷贝到宿主机

1 2	docker cp mysqltest:/etc/my.cnf /Users/lintong/Downloads/mysql8.0/config/

指定mysql配置和data挂载路径启动docker mysql

mysql8.0

docker run --name mysqltest \
-v /Users/lintong/Downloads/mysql8.0/config/my.cnf:/etc/my.cnf \
-v /Users/lintong/Downloads/mysql8.0/data:/var/lib/mysql \
-p 3306:3306 -e MYSQL_ROOT_PASSWORD=123456 -d mysql:8.0.29

mysql5.7

docker run --name mysqltest \
-v /Users/lintong/Downloads/mysql5.7/config/my.cnf:/etc/my.cnf \
-v /Users/lintong/Downloads/mysql5.7/data:/var/lib/mysql \
-p 3306:3306 -e MYSQL_ROOT_PASSWORD=123456 -d mysql:5.7.44

2.开启binlog

查看binlog是否开启，MySQL8.0默认是开启的

mysql>  show variables like '%log_bin%';
+---------------------------------+--------------------------------+
| Variable_name                   | Value                          |
+---------------------------------+--------------------------------+
| log_bin                         | ON                             |
| log_bin_basename                | /var/lib/mysql/mysql-bin       |
| log_bin_index                   | /var/lib/mysql/mysql-bin.index |
| log_bin_trust_function_creators | OFF                            |
| log_bin_use_v1_row_events       | OFF                            |
| sql_log_bin                     | ON                             |
+---------------------------------+--------------------------------+

低版本默认是关闭的

如果是mysql5.7的话，需要在my.cnf配置中添加如下配置，参考：MySQL-开启binlog

1 2	[mysqld]<br />log-bin=mysql-bin<br />server-id=1

其他配置

#设置日志格式
binlog_format = mixed
#设置binlog清理时间
expire_logs_days = 5
#binlog每个日志文件大小
max_binlog_size = 50m
#binlog缓存大小
binlog_cache_size = 4m
#最大binlog缓存大小
max_binlog_cache_size = 512m

参考：Docker内部MySQL开启binlog日志

可以在data目录下看到生成的binlog文件

使用命令查看binlog列表

mysql> show binary logs;
+------------------+-----------+
| Log_name         | File_size |
+------------------+-----------+
| mysql-bin.000001 |       177 |
| mysql-bin.000002 |   2947794 |
| mysql-bin.000003 |       154 |
+------------------+-----------+
3 rows in set (0.01 sec)

查看当前记录的binlog文件的文件名和偏移

mysql> show master status;
+------------------+----------+--------------+------------------+-------------------+
| File             | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set |
+------------------+----------+--------------+------------------+-------------------+
| mysql-bin.000004 |      154 |              |                  |                   |
+------------------+----------+--------------+------------------+-------------------+
1 row in set (0.00 sec)

可以使用mysqlbinlog命令来查看binlog，如果想镜像中自带mysqlbinlog命令，可以使用debian的镜像，比如

docker run --name mysqltest \
-v /Users/lintong/Downloads/mysql5.7/config/my.cnf:/etc/my.cnf \
-v /Users/lintong/Downloads/mysql5.7/data:/var/lib/mysql \
-p 3306:3306 -e MYSQL_ROOT_PASSWORD=123456 -d mysql:5.7-debian

查看指定binlog文件的内容，可以看到这里先创建了一个test database，然后create了一张名为user的表

mysql> show binlog events in 'mysql-bin.000005';
+------------------+-----+----------------+-----------+-------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Log_name         | Pos | Event_type     | Server_id | End_log_pos | Info                                                                                                                                                                                                                                        |
+------------------+-----+----------------+-----------+-------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| mysql-bin.000005 |   4 | Format_desc    |         1 |         123 | Server ver: 5.7.42-log, Binlog ver: 4                                                                                                                                                                                                       |
| mysql-bin.000005 | 123 | Previous_gtids |         1 |         154 |                                                                                                                                                                                                                                             |
| mysql-bin.000005 | 154 | Anonymous_Gtid |         1 |         219 | SET @@SESSION.GTID_NEXT= 'ANONYMOUS'                                                                                                                                                                                                        |
| mysql-bin.000005 | 219 | Query          |         1 |         313 | create database test                                                                                                                                                                                                                        |
| mysql-bin.000005 | 313 | Anonymous_Gtid |         1 |         378 | SET @@SESSION.GTID_NEXT= 'ANONYMOUS'                                                                                                                                                                                                        |
| mysql-bin.000005 | 378 | Query          |         1 |         675 | use `test`; create table user
(
    id       bigint unsigned auto_increment comment ''
        primary key,
    username varchar(128) not null comment '',
    email    varchar(128) not null comment ''
)
    comment '' charset = utf8mb4 |
+------------------+-----+----------------+-----------+-------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
6 rows in set (0.00 sec)

查看binlog的格式，binlog的格式有3种，分别为STATEMENT，ROW和MIXED

mysql> SHOW VARIABLES LIKE '%binlog_format%';
+---------------+-------+
| Variable_name | Value |
+---------------+-------+
| binlog_format | ROW   |
+---------------+-------+
1 row in set (0.01 sec)

可以使用mysqlbinlog命令将binlog导成sql文件

1 2	mysqlbinlog --no-defaults --base64-output=decode-rows -v /var/lib/mysql/mysql-bin.000005 > /tmp/binlog005.sql

也可以指定开始和结束时间来导出binlog，或者指定position

1 2	mysqlbinlog --no-defaults --base64-output=decode-rows -v --start-datetime='2024-08-16 00:00:00' --stop-datetime='2024-08-16 23:00:00' /var/lib/mysql/mysql-bin.000005 > /tmp/binlog005.sql

查看mysql的时区，可以看出使用的是UTC时间

mysql> show variables like '%time_zone%';
+------------------+--------+
| Variable_name    | Value  |
+------------------+--------+
| system_time_zone | UTC    |
| time_zone        | SYSTEM |
+------------------+--------+
2 rows in set (0.02 sec)

mysql> select now();
+---------------------+
| now()               |
+---------------------+
| 2024-08-17 15:29:48 |
+---------------------+
1 row in set (0.00 sec)