tonglin0325的个人主页

ubuntu16.04安装haproxy

tonglin0325

清华镜像站

1
2
https://mirrors.tuna.tsinghua.edu.cn/ubuntu/pool/main/h/haproxy/

下载haproxy

1
2
https://mirrors.tuna.tsinghua.edu.cn/ubuntu/pool/main/h/haproxy/haproxy_1.6.3-1ubuntu0.3_amd64.deb

安装haproxy

1
2
sudo dpkg -i ./haproxy_1.6.3-1ubuntu0.3_amd64.deb

编辑配置文件

1
2
vim /etc/haproxy/haproxy.cfg

其默认配置如下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
global
        log /dev/log    local0
        log /dev/log    local1 notice
        chroot /var/lib/haproxy
        stats socket /run/haproxy/admin.sock mode 660 level admin
        stats timeout 30s
        user haproxy
        group haproxy
        daemon

        # Default SSL material locations
        ca-base /etc/ssl/certs
        crt-base /etc/ssl/private

        # Default ciphers to use on SSL-enabled listening sockets.
        # For more information, see ciphers(1SSL). This list is from:
        #  https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
        ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
        ssl-default-bind-options no-sslv3

defaults
        log     global
        mode    http
        option  httplog
        option  dontlognull
        timeout connect 5000
        timeout client  50000
        timeout server  50000
        errorfile 400 /etc/haproxy/errors/400.http
        errorfile 403 /etc/haproxy/errors/403.http
        errorfile 408 /etc/haproxy/errors/408.http
        errorfile 500 /etc/haproxy/errors/500.http
        errorfile 502 /etc/haproxy/errors/502.http
        errorfile 503 /etc/haproxy/errors/503.http
        errorfile 504 /etc/haproxy/errors/504.http

全文 >>

yarn学习笔记——yarn api

tonglin0325

参考：Yarn 监控 - 监控任务运行状态 （包括Spark，MR 所有在Yarn中运行的任务）

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
    //获取任务的applicationId
    public static String getAppId(String jobName) throws IOException {

        Configuration conf = new Configuration();
        System.setProperty("java.security.krb5.conf", "/etc/krb5.conf");
        conf.set("hadoop.security.authentication", "Kerberos");
        UserGroupInformation.setConfiguration(conf);
        UserGroupInformation.loginUserFromKeytab("hdfs@XXXX", "/home/xxxx/hdfs.keytab");

        YarnClient client = YarnClient.createYarnClient();
        client.init(conf);
        client.start();
        EnumSet<YarnApplicationState> appStates = EnumSet.noneOf(YarnApplicationState.class);

        if (appStates.isEmpty()) {
            appStates.add(YarnApplicationState.RUNNING);
            appStates.add(YarnApplicationState.ACCEPTED);
            appStates.add(YarnApplicationState.SUBMITTED);
        }

        List<ApplicationReport> appsReport = null;
        try {
            // 返回EnumSet<YarnApplicationState>中个人任务状态的所有任务
            appsReport = client.getApplications(appStates);
        } catch (YarnException | IOException e) {
            e.printStackTrace();
        }

        assert appsReport != null;

        for (ApplicationReport appReport : appsReport) {
            System.out.println(appReport);
            // 获取任务名
            String jn = appReport.getName();
            String applicationType = appReport.getApplicationType();
            if (jn.equals(jobName)) { // &amp;&amp; "Apache Flink".equals(applicationType)) {
                try {
                    client.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
                return appReport.getApplicationId().toString();
            }
        }
        try {
            client.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }

    // 根据任务的applicationId去获取任务的状态
    public static YarnApplicationState getState(String appId) throws IOException {

        Configuration conf = new Configuration();
        System.setProperty("java.security.krb5.conf", "/etc/krb5.conf");
        conf.set("hadoop.security.authentication", "Kerberos");
        UserGroupInformation.setConfiguration(conf);
        UserGroupInformation.loginUserFromKeytab("hdfs@XXXXX", "/home/xxxx/hdfs.keytab");

        YarnClient client = YarnClient.createYarnClient();
        client.init(conf);
        client.start();
        ApplicationId applicationId = ConverterUtils.toApplicationId(appId);
        YarnApplicationState yarnApplicationState = null;
        try {
            ApplicationReport applicationReport = client.getApplicationReport(applicationId);
            yarnApplicationState = applicationReport.getYarnApplicationState();
        } catch (YarnException | IOException e) {
            e.printStackTrace();
        }
        try {
            client.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return yarnApplicationState;
    }

    public static void main(String[] args) throws IOException, InterruptedException {
        String state = getAppId("job_xxxxx");
        System.out.println(state);
//        System.out.println(state == YarnApplicationState.RUNNING);

    }

输出

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
applicationId { id: 279 cluster_timestamp: 1620359479641 } 
user: "dl" 
queue: "root.xxxx" 
name: "xxx-flink" 
host: "xxxxx" 
rpc_port: 18000 
client_to_am_token { identifier: "xxxx@XXXX" password: ";xxxxx" kind: "YARN_CLIENT_TOKEN" service: "" } 
yarn_application_state: RUNNING 
trackingUrl: "http://xxxxx:8088/proxy/application_xxxxx/" 
diagnostics: "" 
startTime: 1620391776339 
finishTime: 0 
final_application_status: APP_UNDEFINED 
app_resource_Usage { num_used_containers: 4 num_reserved_containers: 0 used_resources { memory: 8192 virtual_cores: 7 } reserved_resources { memory: 0 virtual_cores: 0 } needed_resources { memory: 8192 virtual_cores: 7 } memory_seconds: 12703546778 vcore_seconds: 10855065 } 
originalTrackingUrl: "http://xxxx:18000" currentApplicationAttemptId { application_id { id: 279 cluster_timestamp: 1620359479641 } attemptId: 1 } progress: 1.0 
applicationType: "XXXX Flink" 
log_aggregation_status: LOG_NOT_START

　　

全文 >>

SpringBoot学习笔记——spring security

tonglin0325

Spring Security是提供了认证，鉴权以及其他的安全特性的java框架，下面是Spring Security的使用教程

1.引入依赖

1
2
3
4
5
<dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-security</artifactId>
</dependency>

引入依赖用会发现请求所有的接口都会跳转到 /login，要求你进行账号密码的认证

全文 >>

docker学习笔记——网络模式

tonglin0325

查看容器的网络模式

1
2
3
4
5
6
7
8
9
10
docker ps -a
CONTAINER ID        IMAGE                                    COMMAND                  CREATED             STATUS              PORTS               NAMES
3f6822d8f262        confluentinc/cp-schema-registry:latest   "/etc/confluent/dock&hellip;"   13 minutes ago      Up 13 minutes                           schema-registry

docker inspect 3f6822d8f262 | grep -i "network"
            "NetworkMode": "host",
        "NetworkSettings": {
            "Networks": {
                    "NetworkID": "5d40a7d178679339f87cc31965ba9a1c662c74ccea853945967d4303e4f9acc0",

docker总共有4种网络模式，从上到下隔离度下降：

1.Close容器，即none模式，运行在Close容器中的进程只能访问本地回环接口，隔离度最高

2.Bridge容器，即bridge模式，当容器中的进程需要访问外部网络的时候应该使用，且bridge是docker的default网络模式。

全文 >>

tonglin0325

标签

标签云

AWS Airbnb Android CDH Doris ELK Flink Git Grafana HAProxy HBase Hadoop Hive Hudi InfluxDB JDBC JMeter JVM Java JavaWeb LaTex Linkedin Linux ML ML Infra MPP Maven MySQL Nexus OpenTSDB Paper Play Prometheus Python React Redis RocksDB Scala Solr Spark SpringBoot SpringMVC Thrift YARN antlr arthas avro cassandra clickhouse confluent docker filebeat flume golang google gradle hexo impala jenkins k8s kafka kerberos kudu ldap mac mongo mybatis nginx nlp open-falcon openwrt parquet presto ranger scyllaDB twitter zookeeper 刷题 前端 图存储及计算 多线程 广告系统 开发工具 数据结构 杂谈 正则表达式 算法 系统设计 计算机网络 设计模式

文章归档

最近文章