Flink通过flink-yarn远程提交任务到yarn集群-CSDN博客

本文链接：https://blog.csdn.net/clearlxj/article/details/120560130

该博客详细介绍了如何在YARN上部署Flink应用程序。首先，配置了Flink的本地目录，并解决了可能出现的用户函数加载异常问题。接着，指定了Flink集群和用户JAR的位置，然后初始化并启动YarnClient。在配置了Flink的检查点、JAR路径、日志设置等参数后，通过YarnClusterDescriptor部署了应用集群。最后，获取了应用ID和Web界面URL，并提供了退出操作的示例。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

 public void crateStreamTaskByFlinkClient() {
    //flink的本地配置目录，为了得到flink的配置
    // 如果出现org.apache.flink.streaming.runtime.tasks.StreamTaskException: Cannot instantiate user function.错误
    // 则在flink-config.yaml加入
    // classloader.resolve-order: parent-first
    String configurationDirectory = "/opt/module/flink-1.11.4/conf";
    // String configurationDirectory = "/home/lxj/workspace/Olt-Test/bigdata/bigdataserver/src/main/resources/flink/conf";

    //存放flink集群相关的jar包目录
    String flinkLibs = "hdfs://hadoop113:8020/jar/flink11/libs";
    //用户jar
    String userJarPath = "hdfs://hadoop113:8020/jar/userTask/Flink1.11-1.0-SNAPSHOT-jar-with-dependencies.jar";
    String flinkDistJar = "hdfs://hadoop113:8020/jar/flink11/libs/flink-dist_2.12-1.11.4.jar";

    YarnClient yarnClient = YarnClient.createYarnClient();
    YarnConfiguration yarnConfiguration = new YarnConfiguration();
    yarnClient.init(yarnConfiguration);
    yarnClient.start();

    // 设置日志的，没有的话看不到日志
    YarnClusterInformationRetriever clusterInformationRetriever = YarnClientYarnClusterInformationRetriever
        .create(yarnClient);

    //获取flink的配置
    Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration(
        configurationDirectory);

    flinkConfiguration.set(CheckpointingOptions.INCREMENTAL_CHECKPOINTS, true);

    flinkConfiguration.set(
        PipelineOptions.JARS,
        Collections.singletonList(userJarPath));

    Path remoteLib = new Path(flinkLibs);
    flinkConfiguration.set(
        YarnConfigOptions.PROVIDED_LIB_DIRS,
        Collections.singletonList(remoteLib.toString()));

    flinkConfiguration.set(
        YarnConfigOptions.FLINK_DIST_JAR,
        flinkDistJar);

    // 设置为application模式
    flinkConfiguration.set(
        DeploymentOptions.TARGET,
        YarnDeploymentTarget.APPLICATION.getName());

    // yarn application name
    flinkConfiguration.set(YarnConfigOptions.APPLICATION_NAME, "flink-application");

    YarnLogConfigUtil.setLogConfigFileInConfig(flinkConfiguration, configurationDirectory);

    ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
        .createClusterSpecification();

    // 设置用户jar的参数和主类
    ApplicationConfiguration appConfig = new ApplicationConfiguration(new String[] {"test"}, "com.starnet.server.bigdata.flink.WordCount");

    YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
        flinkConfiguration,
        yarnConfiguration,
        yarnClient,
        clusterInformationRetriever,
        true);

    try {
        ClusterClientProvider<ApplicationId> clusterClientProvider = yarnClusterDescriptor.deployApplicationCluster(
            clusterSpecification,
            appConfig);

        ClusterClient<ApplicationId> clusterClient = clusterClientProvider.getClusterClient();

        ApplicationId applicationId = clusterClient.getClusterId();
        String webInterfaceURL = clusterClient.getWebInterfaceURL();
        log.error("applicationId is {}", applicationId);
        log.error("webInterfaceURL is {}", webInterfaceURL);
        
        // 退出
        // yarnClusterDescriptor.killCluster(applicationId);
    } catch (Exception e){
        log.error(e.getMessage(), e);
    }
}