本文介绍如何通过EMR OpenAPI创建数据开发项目并对项目进行管理。
前提条件
- 已创建集群,详情请参见创建集群。
- 已获取集群ID,详情请参见查看集群列表与详情。
- 已创建AccessKey,详情请参见获取AccessKey。
- 已获取相应的SDK,Java SDK请参见SDK下载,Python SDK请参见安装SDK。
使用场景
您需要对E-MapReduce中创建的集群进行作业开发和编辑。此时可以在数据开发中创建项目,并在项目中进行作业的编辑和工作流的调度。新建项目之后,可以对项目进行管理,为项目关联集群资源以及添加项目成员。
示例
- Java
- 创建数据开发项目。在华东1(杭州)创建名为emr_openapi_demo_project的项目。
import com.aliyuncs.DefaultAcsClient; import com.aliyuncs.IAcsClient; import com.aliyuncs.exceptions.ClientException; import com.aliyuncs.exceptions.ServerException; import com.aliyuncs.profile.DefaultProfile; import com.google.gson.Gson; import java.util.*; import com.aliyuncs.emr.model.v20160408.*; public class CreateFlowProject { public static void main(String[] args) { DefaultProfile profile = DefaultProfile.getProfile("cn-hangzhou", "<accessKeyId>", "<accessSecret>"); IAcsClient client = new DefaultAcsClient(profile); CreateFlowProjectRequest request = new CreateFlowProjectRequest(); request.setRegionId("cn-hangzhou"); request.setName("emr_openapi_demo_project"); request.setDescription("Create flow project via CreateFlowProject"); try { CreateFlowProjectResponse response = client.getAcsResponse(request); System.out.println(new Gson().toJson(response)); } catch (ServerException e) { e.printStackTrace(); } catch (ClientException e) { System.out.println("ErrCode:" + e.getErrCode()); System.out.println("ErrMsg:" + e.getErrMsg()); System.out.println("RequestId:" + e.getRequestId()); } } }
在返回结果中找到项目ID。{ "RequestId": "BF47231B-29AE-4A4F-A1A3-10205ECD9634", "Id": "FP-D18E9976D5A5****" }
- 为了关联项目集群资源,可以先查询当前Region下数据开发可用的集群列表。
import com.aliyuncs.DefaultAcsClient; import com.aliyuncs.IAcsClient; import com.aliyuncs.exceptions.ClientException; import com.aliyuncs.exceptions.ServerException; import com.aliyuncs.profile.DefaultProfile; import com.google.gson.Gson; import java.util.*; import com.aliyuncs.emr.model.v20160408.*; public class ListFlowClusterAll { public static void main(String[] args) { DefaultProfile profile = DefaultProfile.getProfile("cn-hangzhou", "<accessKeyId>", "<accessSecret>"); IAcsClient client = new DefaultAcsClient(profile); ListFlowClusterAllRequest request = new ListFlowClusterAllRequest(); request.setRegionId("cn-hangzhou"); try { ListFlowClusterAllResponse response = client.getAcsResponse(request); System.out.println(new Gson().toJson(response)); } catch (ServerException e) { e.printStackTrace(); } catch (ClientException e) { System.out.println("ErrCode:" + e.getErrCode()); System.out.println("ErrMsg:" + e.getErrMsg()); System.out.println("RequestId:" + e.getRequestId()); } } }
在返回结果中找到目标集群的集群ID。返回结果以JSON为例,集群ID的查找路径如下。Clusters -> ClusterInfo -> Id -> 集群ID
- 为项目emr_openapi_demo_project添加集群设置,使项目中的工作流运行在添加的集群环境中。也可以设置提交队列、提交用户和提交机器(当前只支持Gateway和Master机器)。
import com.aliyuncs.DefaultAcsClient; import com.aliyuncs.IAcsClient; import com.aliyuncs.exceptions.ClientException; import com.aliyuncs.exceptions.ServerException; import com.aliyuncs.profile.DefaultProfile; import com.google.gson.Gson; import java.util.*; import com.aliyuncs.emr.model.v20160408.*; public class CreateFlowProjectClusterSetting { public static void main(String[] args) { DefaultProfile profile = DefaultProfile.getProfile("cn-hangzhou", "<accessKeyId>", "<accessSecret>"); IAcsClient client = new DefaultAcsClient(profile); CreateFlowProjectClusterSettingRequest request = new CreateFlowProjectClusterSettingRequest(); request.setRegionId("cn-hangzhou"); request.setProjectId("FP-D18E9976D5A5****"); request.setClusterId("C-B503DDB15B34****"); try { CreateFlowProjectClusterSettingResponse response = client.getAcsResponse(request); System.out.println(new Gson().toJson(response)); } catch (ServerException e) { e.printStackTrace(); } catch (ClientException e) { System.out.println("ErrCode:" + e.getErrCode()); System.out.println("ErrMsg:" + e.getErrMsg()); System.out.println("RequestId:" + e.getRequestId()); } } }
- 为项目添加RAM用户。下面示例将RAM用户test添加到项目emr_openapi_demo_project中,使其有开发项目的权限。
import com.aliyuncs.DefaultAcsClient; import com.aliyuncs.IAcsClient; import com.aliyuncs.exceptions.ClientException; import com.aliyuncs.exceptions.ServerException; import com.aliyuncs.profile.DefaultProfile; import com.google.gson.Gson; import java.util.*; import com.aliyuncs.emr.model.v20160408.*; public class CreateFlowProjectUser { public static void main(String[] args) { DefaultProfile profile = DefaultProfile.getProfile("cn-hangzhou", "<accessKeyId>", "<accessSecret>"); IAcsClient client = new DefaultAcsClient(profile); CreateFlowProjectUserRequest request = new CreateFlowProjectUserRequest(); request.setRegionId("cn-hangzhou"); request.setProjectId("FP-D18E9976D5A5****"); List<CreateFlowProjectUserRequest.User> userList = new ArrayList<CreateFlowProjectUserRequest.User>(); CreateFlowProjectUserRequest.User user1 = new CreateFlowProjectUserRequest.User(); user1.setUserId(" 24339807068856****"); user1.setUserName("test"); userList.add(user1); request.setUsers(userList); try { CreateFlowProjectUserResponse response = client.getAcsResponse(request); System.out.println(new Gson().toJson(response)); } catch (ServerException e) { e.printStackTrace(); } catch (ClientException e) { System.out.println("ErrCode:" + e.getErrCode()); System.out.println("ErrMsg:" + e.getErrMsg()); System.out.println("RequestId:" + e.getRequestId()); } } }
- 创建数据开发项目。在华东1(杭州)创建名为emr_openapi_demo_project的项目。
- Python
- 创建数据开发项目。在华东1(杭州)创建名为emr_openapi_demo_project的项目。
#!/usr/bin/env python #coding=utf-8 from aliyunsdkcore.client import AcsClient from aliyunsdkcore.acs_exception.exceptions import ClientException from aliyunsdkcore.acs_exception.exceptions import ServerException from aliyunsdkemr.request.v20160408.CreateFlowProjectRequest import CreateFlowProjectRequest client = AcsClient('<accessKeyId>', '<accessSecret>', 'cn-hangzhou') request = CreateFlowProjectRequest() request.set_accept_format('json') request.set_Name("emr_openapi_demo_project") request.set_Description("Create flow project via CreateFlowProject") response = client.do_action_with_exception(request) # python2: print(response) print(str(response, encoding='utf-8'))
在返回结果中找到项目ID。{ "RequestId": "BF47231B-29AE-4A4F-A1A3-10205ECD9634", "Id": "FP-D18E9976D5A5****" }
- 为了关联项目集群资源,可以先查询当前Region下数据开发可用的集群列表。
#!/usr/bin/env python #coding=utf-8 from aliyunsdkcore.client import AcsClient from aliyunsdkcore.acs_exception.exceptions import ClientException from aliyunsdkcore.acs_exception.exceptions import ServerException from aliyunsdkemr.request.v20160408.ListFlowClusterRequest import ListFlowClusterRequest client = AcsClient('<accessKeyId>', '<accessSecret>', 'cn-hangzhou') request = ListFlowClusterRequest() request.set_accept_format('json') request.set_ProjectId("FP-D18E9976D5A5****") response = client.do_action_with_exception(request) # python2: print(response) print(str(response, encoding='utf-8'))
在返回结果中找到目标集群的集群ID。返回结果以JSON为例,集群ID的查找路径如下。Clusters -> ClusterInfo -> Id -> 集群ID
- 为项目emr_openapi_demo_project添加集群设置,使项目中的工作流运行在添加的集群环境中。也可以设置提交队列、提交用户和提交机器(当前只支持Gateway和Master机器)。
#!/usr/bin/env python #coding=utf-8 from aliyunsdkcore.client import AcsClient from aliyunsdkcore.acs_exception.exceptions import ClientException from aliyunsdkcore.acs_exception.exceptions import ServerException from aliyunsdkemr.request.v20160408.CreateFlowProjectClusterSettingRequest import CreateFlowProjectClusterSettingRequest client = AcsClient('<accessKeyId>', '<accessSecret>', 'cn-hangzhou') request = CreateFlowProjectClusterSettingRequest() request.set_accept_format('json') request.set_ProjectId("FP-D18E9976D5A5****") request.set_ClusterId("C-B503DDB15B34****") response = client.do_action_with_exception(request) # python2: print(response) print(str(response, encoding='utf-8'))
- 为项目添加RAM用户。下面示例为项目emr_openapi_demo_project添加RAM用户test,使其有权限开发项目。
#!/usr/bin/env python #coding=utf-8 from aliyunsdkcore.client import AcsClient from aliyunsdkcore.acs_exception.exceptions import ClientException from aliyunsdkcore.acs_exception.exceptions import ServerException from aliyunsdkemr.request.v20160408.CreateFlowProjectUserRequest import CreateFlowProjectUserRequest client = AcsClient('<accessKeyId>', '<accessSecret>', 'cn-hangzhou') request = CreateFlowProjectUserRequest() request.set_accept_format('json') request.set_ProjectId("FP-D18E9976D5A5****") request.set_Users([ { "UserId": " 24339807068856****", "UserName": "test" } ]) response = client.do_action_with_exception(request) # python2: print(response) print(str(response, encoding='utf-8'))
- 创建数据开发项目。在华东1(杭州)创建名为emr_openapi_demo_project的项目。