DataWorks提供了丰富的OpenAPI,您可以根据需要使用DataWorks的OpenAPI等开放能力实现各种业务场景。本文以数据开发为例,为您示例如何使用OpenAPI快速进行数据开发、提交与运行。
背景信息
本实践将满足以下业务场景的需求,建议您先学习了解不同业务场景涉及的核心能力与概念。
查询与管理工作空间列表、业务流程列表、节点文件夹与节点列表;提交发布节点任务。主要需要应用数据开发的OpenAPI,如CreateBusiness、ListBusiness等。
进行冒烟测试并查看过程日志。主要需要应用运维中心的部分OpenAPI,如RunSmokeTest等。
完成本实践后,您可实现如下的一个可进行数据开发、提交、运行的本地环境。
后端代码开发
步骤1:开发ProjectService类,获取工作空间
您需要开发一个ProjectService类,类中定义了ListProjects函数来调用ListProjects来获取工作空间列表,这个函数会返回工作空间列表供前端调用以获取界面中需要的工作空间下拉列表内容。
package com.aliyun.dataworks.services;
import com.aliyuncs.dataworks_public.model.v20200518.ListProjectsRequest;
import com.aliyuncs.dataworks_public.model.v20200518.ListProjectsResponse;
import com.aliyuncs.exceptions.ClientException;
import com.aliyuncs.exceptions.ServerException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
public class ProjectService {
@Autowired
private DataWorksOpenApiClient dataWorksOpenApiClient;
/**
* @param pageNumber
* @param pageSize
* @return
*/
public ListProjectsResponse.PageResult listProjects(Integer pageNumber, Integer pageSize) {
try {
ListProjectsRequest listProjectsRequest = new ListProjectsRequest();
listProjectsRequest.setPageNumber(pageNumber);
listProjectsRequest.setPageSize(pageSize);
ListProjectsResponse listProjectsResponse = dataWorksOpenApiClient.createClient().getAcsResponse(listProjectsRequest);
System.out.println(listProjectsResponse.getRequestId());
return listProjectsResponse.getPageResult();
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return null;
}
}
步骤2:开发BusinessService类,处理业务流程
您需要开发一个BusinessService类,类中定义如下函数。
定义了CreateBusiness函数,调用CreateBusiness来创建业务流程。
定义了ListBusiness函数,调用ListBusiness来获取业务流程列表。
这两个函数会在前端界面中被使用,分别用来创建示例业务流程与拉取列表。
您也可以定义一个FolderService的函数,用于展示目录树(目录树由业务流程、节点文件夹与节点组成),下面的例省略了节点文件夹的相关步骤,只示例了核心流程,文件夹的操作相关的FolderService函数,您可在Github中的全量示例代码中获取,获取地址为Github代码示例。
package com.aliyun.dataworks.services;
import com.aliyun.dataworks.dto.CreateBusinessDTO;
import com.aliyun.dataworks.dto.DeleteBusinessDTO;
import com.aliyun.dataworks.dto.ListBusinessesDTO;
import com.aliyun.dataworks.dto.UpdateBusinessDTO;
import com.aliyuncs.dataworks_public.model.v20200518.*;
import com.aliyuncs.exceptions.ClientException;
import com.aliyuncs.exceptions.ServerException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List;
/**
* @author dataworks demo
*/
@Service
public class BusinessService {
@Autowired
private DataWorksOpenApiClient dataWorksOpenApiClient;
/**
* create a business
*
* @param createBusinessDTO
*/
public Long createBusiness(CreateBusinessDTO createBusinessDTO) {
try {
CreateBusinessRequest createBusinessRequest = new CreateBusinessRequest();
//业务流程的名称
createBusinessRequest.setBusinessName(createBusinessDTO.getBusinessName());
createBusinessRequest.setDescription(createBusinessDTO.getDescription());
createBusinessRequest.setOwner(createBusinessDTO.getOwner());
createBusinessRequest.setProjectId(createBusinessDTO.getProjectId());
//业务流程所属的功能模块 NORMAL(数据开发) MANUAL_BIZ(手动业务流程)
createBusinessRequest.setUseType(createBusinessDTO.getUseType());
CreateBusinessResponse createBusinessResponse = dataWorksOpenApiClient.createClient().getAcsResponse(createBusinessRequest);
System.out.println("create business requestId:" + createBusinessResponse.getRequestId());
System.out.println("create business successful,the businessId:" + createBusinessResponse.getBusinessId());
return createBusinessResponse.getBusinessId();
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return null;
}
/**
* @param listBusinessesDTO
* @return
*/
public List<ListBusinessResponse.Data.BusinessItem> listBusiness(ListBusinessesDTO listBusinessesDTO) {
try {
ListBusinessRequest listBusinessRequest = new ListBusinessRequest();
listBusinessRequest.setKeyword(listBusinessesDTO.getKeyword());
listBusinessRequest.setPageNumber(listBusinessesDTO.getPageNumber() < 1 ? 1 : listBusinessesDTO.getPageNumber());
listBusinessRequest.setPageSize(listBusinessesDTO.getPageSize() < 10 ? 10 : listBusinessesDTO.getPageSize());
listBusinessRequest.setProjectId(listBusinessesDTO.getProjectId());
ListBusinessResponse listBusinessResponse = dataWorksOpenApiClient.createClient().getAcsResponse(listBusinessRequest);
System.out.println("list business requestId:" + listBusinessResponse.getRequestId());
ListBusinessResponse.Data data = listBusinessResponse.getData();
System.out.println("total count:" + data.getTotalCount());
if (!CollectionUtils.isEmpty(data.getBusiness())) {
for (ListBusinessResponse.Data.BusinessItem businessItem : data.getBusiness()) {
System.out.println(businessItem.getBusinessId() + "," + businessItem.getBusinessName() + "," + businessItem.getUseType());
}
}
return data.getBusiness();
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return null;
}
/**
* update a business
* @param updateBusinessDTO
* @return
*/
public Boolean updateBusiness(UpdateBusinessDTO updateBusinessDTO) {
try {
UpdateBusinessRequest updateBusinessRequest = new UpdateBusinessRequest();
updateBusinessRequest.setBusinessId(updateBusinessDTO.getBusinessId());
updateBusinessRequest.setBusinessName(updateBusinessDTO.getBusinessName());
updateBusinessRequest.setDescription(updateBusinessDTO.getDescription());
updateBusinessRequest.setOwner(updateBusinessDTO.getOwner());
updateBusinessRequest.setProjectId(updateBusinessDTO.getProjectId());
UpdateBusinessResponse updateBusinessResponse = dataWorksOpenApiClient.createClient().getAcsResponse(updateBusinessRequest);
System.out.println(updateBusinessResponse.getRequestId());
System.out.println(updateBusinessResponse.getSuccess());
return updateBusinessResponse.getSuccess();
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return false;
}
/**
* delete a business
* @param deleteBusinessDTO
*/
public boolean deleteBusiness(DeleteBusinessDTO deleteBusinessDTO) {
try {
DeleteBusinessRequest deleteBusinessRequest = new DeleteBusinessRequest();
deleteBusinessRequest.setBusinessId(deleteBusinessDTO.getBusinessId());
deleteBusinessRequest.setProjectId(deleteBusinessDTO.getProjectId());
DeleteBusinessResponse deleteBusinessResponse = dataWorksOpenApiClient.createClient().getAcsResponse(deleteBusinessRequest);
System.out.println("delete business:" + deleteBusinessResponse.getRequestId());
System.out.println("delete business" + deleteBusinessResponse.getSuccess());
return deleteBusinessResponse.getSuccess();
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return false;
}
}
步骤3:开发FileService类,处理文件事务
您需要开发一个FileService类,这个类中包含了所有与文件事务相关的操作函数:
定义一个listFile函数,调用ListFiles来获取文件列表。
定义一个createFile函数,调用CreateFile来创建文件列表。
定义一个updateFile函数,调用UpdateFile来更新文件列表。
定义一个deployFile函数,调用DeployFile来发布文件列表。
定义一个runSmokeTest函数,调用RunSmokeTest来执行冒烟测试。
定义一个getInstanceLog函数,调用GetInstanceLog来获取冒烟测试过程日志内容。
这些方法将在界面中被调用,分别用来创建文件、拉取文件列表、保存文件、提交与执行使用。
package com.aliyun.dataworks.services;
import com.aliyun.dataworks.dto.*;
import com.aliyuncs.dataworks_public.model.v20200518.*;
import com.aliyuncs.exceptions.ClientException;
import com.aliyuncs.exceptions.ServerException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List;
/**
* the ide files manager service
*
* @author dataworks demo
*/
@Service
public class FileService {
@Autowired
private DataWorksOpenApiClient dataWorksOpenApiClient;
public static final int CYCLE_NUM = 10;
/**
* 分页查询
* @param listFilesDTO
* @return
*/
public List<ListFilesResponse.Data.File> listFiles(ListFilesDTO listFilesDTO) {
try {
ListFilesRequest listFilesRequest = new ListFilesRequest();
// 文件路径: “业务流程/” + 目标业务流程名 + 目录名 + 最新文件夹名
// 业务流程/我的第一个业务流程/MaxCompute/ods层,不要加"数据开发"
listFilesRequest.setFileFolderPath(listFilesDTO.getFileFolderPath());
// 文件的代码类型。支持多个,以逗号分隔,例子:10,23
listFilesRequest.setFileTypes(listFilesDTO.getFileTypes());
// 文件名称的关键字。支持模糊匹配
listFilesRequest.setKeyword(listFilesDTO.getKeyword());
// 调度节点的ID
listFilesRequest.setNodeId(listFilesDTO.getNodeId());
// 文件责任人
listFilesRequest.setOwner(listFilesDTO.getOwner());
// 请求的数据页数
listFilesRequest.setPageNumber(listFilesDTO.getPageNumber() <= 0 ? 1 : listFilesDTO.getPageNumber());
// 每页显示的条数
listFilesRequest.setPageSize(listFilesDTO.getPageSize() <= 10 ? 10 : listFilesDTO.getPageSize());
// DataWorks工作空间的ID
listFilesRequest.setProjectId(listFilesDTO.getProjectId());
// 文件所属的功能模块
listFilesRequest.setUseType(listFilesDTO.getUseType());
ListFilesResponse listFilesResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(listFilesRequest);
ListFilesResponse.Data fileData = listFilesResponse.getData();
if (fileData.getFiles() != null && !fileData.getFiles().isEmpty()) {
for (ListFilesResponse.Data.File file : fileData.getFiles()) {
// 业务流程ID
System.out.println(file.getBusinessId());
// 文件ID
System.out.println(file.getFileId());
// 文件名称
System.out.println(file.getFileName());
// 文件类型 10
System.out.println(file.getFileType());
// 节点ID
System.out.println(file.getNodeId());
// 文件夹ID
System.out.println(file.getFileFolderId());
}
}
return fileData.getFiles();
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return null;
}
/**
* 新增文件
* @param createFileDTO
*/
public Long createFile(CreateFileDTO createFileDTO) {
try {
CreateFileRequest createFileRequest = new CreateFileRequest();
// 任务的高级配置
createFileRequest.setAdvancedSettings(createFileDTO.getAdvancedSettings());
// 文件是否开启自动解析功能 必填
createFileRequest.setAutoParsing(createFileDTO.getAutoParsing());
// 出错自动重跑时间间隔,单位为毫秒。最大为1800000毫秒(30分钟)
createFileRequest.setAutoRerunIntervalMillis(createFileDTO.getAutoRerunIntervalMillis());
// 自动重试次数
createFileRequest.setAutoRerunTimes(createFileDTO.getAutoRerunTimes());
// 文件发布成任务后,任务执行时连接的数据源。 必填
createFileRequest.setConnectionName(createFileDTO.getConnectionName());
// 文件代码内容 必填
createFileRequest.setContent(createFileDTO.getContent());
// 周期调度的cron表达式 必填
createFileRequest.setCronExpress(createFileDTO.getCronExpress());
// 调度周期的类型 必填
createFileRequest.setCycleType(createFileDTO.getCycleType());
// 依赖上一周期的节点列表
createFileRequest.setDependentNodeIdList(createFileDTO.getDependentNodeIdList());
// 依赖上一周期的方式 必填
createFileRequest.setDependentType(createFileDTO.getDependentType());
// 停止自动调度的时间戳,单位为毫秒。
createFileRequest.setEndEffectDate(createFileDTO.getEndEffectDate());
// 文件描述
createFileRequest.setFileDescription(createFileDTO.getFileDescription());
// 文件的路径 必填
createFileRequest.setFileFolderPath(createFileDTO.getFileFolderPath());
// 文件的名称 必填
createFileRequest.setFileName(createFileDTO.getFileName());
// 文件的代码类型 必填
createFileRequest.setFileType(createFileDTO.getFileType());
// 文件依赖的上游文件的输出名称,多个输出使用英文逗号(,)分隔。必填
createFileRequest.setInputList(createFileDTO.getInputList());
// 文件责任人的阿里云用户ID。如果该参数为空,则默认使用调用者的阿里云用户ID。 必填
createFileRequest.setOwner(createFileDTO.getOwner());
// 调度参数。
createFileRequest.setParaValue(createFileDTO.getParaValue());
// 项目空间ID 必填
createFileRequest.setProjectId(createFileDTO.getProjectId());
// 重跑属性
createFileRequest.setRerunMode(createFileDTO.getRerunMode());
// 任务的资源组 必填
createFileRequest.setResourceGroupIdentifier(createFileDTO.getResourceGroupIdentifier());
// 调度的类型
createFileRequest.setSchedulerType(createFileDTO.getSchedulerType());
// 开始自动调度的毫秒时间戳
createFileRequest.setStartEffectDate(createFileDTO.getStartEffectDate());
// 是否暂停调度
createFileRequest.setStop(createFileDTO.getStop());
CreateFileResponse createFileResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(createFileRequest);
// requestId
System.out.println(createFileResponse.getRequestId());
// fileId
System.out.println(createFileResponse.getData());
return createFileResponse.getData();
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return null;
}
/**
* 更改文件
*
* @param updateFileDTO
*/
public boolean updateFile(UpdateFileDTO updateFileDTO) {
try {
UpdateFileRequest updateFileRequest = new UpdateFileRequest();
// 任务的高级配置,具体格式参考文档
updateFileRequest.setAdvancedSettings(updateFileDTO.getAdvancedSettings());
// 文件是否开启自动解析功能
updateFileRequest.setAutoParsing(updateFileDTO.getAutoParsing());
// 出错自动重跑时间间隔,单位为毫秒。最大为1800000毫秒(30分钟)。
updateFileRequest.setAutoRerunIntervalMillis(updateFileDTO.getAutoRerunIntervalMillis());
// 文件出错后,自动重跑的次数
updateFileRequest.setAutoRerunTimes(updateFileDTO.getAutoRerunTimes());
// 文件对应任务执行时,任务使用的数据源标识符
updateFileRequest.setConnectionName(updateFileDTO.getConnectionName());
// 文件代码内容
updateFileRequest.setContent(updateFileDTO.getContent());
// 周期调度的cron表达式,
updateFileRequest.setCronExpress(updateFileDTO.getCronExpress());
// 调度周期的类型,包括NOT_DAY(分钟、小时)和DAY(日、周、月)
updateFileRequest.setCycleType(updateFileDTO.getCycleType());
// 当DependentType参数配置为USER_DEFINE时,用于设置当前文件具体依赖的节点ID。依赖多个节点时,使用英文逗号(,)分隔
updateFileRequest.setDependentNodeIdList(updateFileDTO.getDependentNodeIdList());
// 依赖上一周期的方式
updateFileRequest.setDependentType(updateFileDTO.getDependentType());
// 停止自动调度的时间戳,单位为毫秒。
updateFileRequest.setEndEffectDate(updateFileDTO.getEndEffectDate());
// 文件的描述
updateFileRequest.setFileDescription(updateFileDTO.getFileDescription());
// 文件所在的路径
updateFileRequest.setFileFolderPath(updateFileDTO.getFileFolderPath());
// 文件的ID
updateFileRequest.setFileId(updateFileDTO.getFileId());
// 文件的名称
updateFileRequest.setFileName(updateFileDTO.getFileName());
// 文件依赖的上游文件的输出名称。依赖多个输出时,使用英文逗号(,)分隔
updateFileRequest.setInputList(updateFileDTO.getInputList());
// 文件的输出
updateFileRequest.setOutputList(updateFileDTO.getOutputList());
// 文件所有者的用户ID
updateFileRequest.setOwner(updateFileDTO.getOwner());
// 调度参数
updateFileRequest.setParaValue(updateFileDTO.getParaValue());
// DataWorks工作空间的ID
updateFileRequest.setProjectId(updateFileDTO.getProjectId());
// 重跑属性 ALL_ALLOWED
updateFileRequest.setRerunMode(updateFileDTO.getRerunMode());
// 文件发布成任务后,任务执行时对应的资源组
updateFileRequest.setResourceGroupIdentifier(updateFileDTO.getResourceGroupIdentifier());
// 调度的类型 NORMAL
updateFileRequest.setSchedulerType(updateFileDTO.getSchedulerType());
// 开始自动调度的毫秒时间戳
updateFileRequest.setStartEffectDate(updateFileDTO.getStartEffectDate());
// 发布后是否立即启动
updateFileRequest.setStartImmediately(updateFileDTO.getStartImmediately());
// 是否暂停调度
updateFileRequest.setStop(updateFileDTO.getStop());
UpdateFileResponse updateFileResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(updateFileRequest);
// requestId
System.out.println(updateFileResponse.getRequestId());
// 成功与否
System.out.println(updateFileResponse.getSuccess());
return updateFileResponse.getSuccess();
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return false;
}
/**
* 删除一个文件
* @param deleteFileDTO
* @return
* @throws InterruptedException
*/
public boolean deleteFile(DeleteFileDTO deleteFileDTO) throws InterruptedException {
try {
DeleteFileRequest deleteFileRequest = new DeleteFileRequest();
deleteFileRequest.setFileId(deleteFileDTO.getFileId());
deleteFileRequest.setProjectId(deleteFileDTO.getProjectId());
DeleteFileResponse deleteFileResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(deleteFileRequest);
System.out.println(deleteFileResponse.getRequestId());
System.out.println(deleteFileResponse.getDeploymentId());
GetDeploymentRequest getDeploymentRequest = new GetDeploymentRequest();
getDeploymentRequest.setProjectId(deleteFileDTO.getProjectId());
getDeploymentRequest.setDeploymentId(deleteFileResponse.getDeploymentId());
for (int i = 0; i < CYCLE_NUM; i++) {
GetDeploymentResponse getDeploymentResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(getDeploymentRequest);
// 发布包当前的状态,包括0(就绪)、1(成功)和2(失败)。
Integer deleteStatus = getDeploymentResponse.getData().getDeployment().getStatus();
// 此处可以循环判断删除状态
if (deleteStatus == 1) {
System.out.println("成功删除文件。");
break;
} else {
System.out.println("正在删除文件");
Thread.sleep(1000L);
}
}
GetProjectRequest getProjectRequest = new GetProjectRequest();
getProjectRequest.setProjectId(deleteFileDTO.getProjectId());
GetProjectResponse getProjectResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(getProjectRequest);
// 标准模式有DEV和PROD,简单模式只有PROD
Boolean standardMode = getProjectResponse.getData().getEnvTypes().size() == 2;
if (standardMode) {
// 标准模式需要把删除发布到线上
DeployFileRequest deployFileRequest = new DeployFileRequest();
deployFileRequest.setProjectId(deleteFileDTO.getProjectId());
deployFileRequest.setFileId(deleteFileDTO.getFileId());
DeployFileResponse deployFileResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(deployFileRequest);
getDeploymentRequest.setDeploymentId(deployFileResponse.getData());
for (int i = 0; i < CYCLE_NUM; i++) {
GetDeploymentResponse getDeploymentResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(getDeploymentRequest);
// 发布包当前的状态,包括0(就绪)、1(成功)和2(失败)。
Integer deleteStatus = getDeploymentResponse.getData().getDeployment().getStatus();
// 此处可以循环判断删除状态
if (deleteStatus == 1) {
System.out.println("成功删除文件。");
break;
} else {
System.out.println("正在删除文件");
Thread.sleep(1000L);
}
}
}
return true;
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return false;
}
/**
* 查询文件
* @param getFileDTO
*/
public GetFileResponse.Data.File getFile(GetFileDTO getFileDTO) {
try {
GetFileRequest getFileRequest = new GetFileRequest();
getFileRequest.setFileId(getFileDTO.getFileId());
getFileRequest.setProjectId(getFileDTO.getProjectId());
getFileRequest.setNodeId(getFileDTO.getNodeId());
GetFileResponse getFileResponse = dataWorksOpenApiClient.createClient().getAcsResponse(getFileRequest);
System.out.println(getFileResponse.getRequestId());
GetFileResponse.Data.File file = getFileResponse.getData().getFile();
System.out.println(file.getFileName());
System.out.println(file.getFileType());
System.out.println(file.getNodeId());
System.out.println(file.getCreateUser());
return file;
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return null;
}
/**
* @param deployFileDTO
* @return
* @throws InterruptedException
*/
public Boolean deployFile(DeployFileDTO deployFileDTO) throws InterruptedException {
try {
GetProjectRequest getProjectRequest = new GetProjectRequest();
getProjectRequest.setProjectId(deployFileDTO.getProjectId());
GetProjectResponse getProjectResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(getProjectRequest);
// 标准模式有DEV和PROD,简单模式只有PROD
Boolean standardMode = getProjectResponse.getData().getEnvTypes().size() == 2;
if (standardMode) {
SubmitFileRequest submitFileRequest = new SubmitFileRequest();
submitFileRequest.setFileId(deployFileDTO.getFileId());
submitFileRequest.setProjectId(deployFileDTO.getProjectId());
SubmitFileResponse submitFileResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(submitFileRequest);
System.out.println("submit file requestId:" + submitFileResponse.getRequestId());
System.out.println("submit file deploymentId:" + submitFileResponse.getData());
for (int i = 0; i < CYCLE_NUM; i++) {
GetDeploymentRequest getDeploymentRequest = new GetDeploymentRequest();
getDeploymentRequest.setProjectId(deployFileDTO.getProjectId());
getDeploymentRequest.setDeploymentId(submitFileResponse.getData());
GetDeploymentResponse getDeploymentResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(getDeploymentRequest);
// 发布包当前的状态,包括0(就绪)、1(成功)和2(失败)。
Integer deleteStatus = getDeploymentResponse.getData().getDeployment().getStatus();
// 此处可以循环判断删除状态
if (deleteStatus == 1) {
System.out.println("成功提交文件。");
break;
} else {
System.out.println("正在提交文件...");
Thread.sleep(1000L);
}
}
}
DeployFileRequest deployFileRequest = new DeployFileRequest();
deployFileRequest.setFileId(deployFileDTO.getFileId());
deployFileRequest.setProjectId(deployFileDTO.getProjectId());
DeployFileResponse deployFileResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(deployFileRequest);
System.out.println("deploy file requestId:" + deployFileResponse.getRequestId());
System.out.println("deploy file deploymentId:" + deployFileResponse.getData());
for (int i = 0; i < CYCLE_NUM; i++) {
GetDeploymentRequest getDeploymentRequest = new GetDeploymentRequest();
getDeploymentRequest.setProjectId(deployFileDTO.getProjectId());
getDeploymentRequest.setDeploymentId(deployFileResponse.getData());
GetDeploymentResponse getDeploymentResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(getDeploymentRequest);
// 发布包当前的状态,包括0(就绪)、1(成功)和2(失败)。
Integer deleteStatus = getDeploymentResponse.getData().getDeployment().getStatus();
// 此处可以循环判断删除状态
if (deleteStatus == 1) {
System.out.println("成功发布文件。");
break;
} else {
System.out.println("正在发布文件...");
Thread.sleep(1000L);
}
}
return true;
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return false;
}
public List<ListInstancesResponse.Data.Instance> runSmokeTest(RunSmokeTestDTO runSmokeTestDTO) {
try {
RunSmokeTestRequest runSmokeTestRequest = new RunSmokeTestRequest();
runSmokeTestRequest.setBizdate(runSmokeTestDTO.getBizdate());
runSmokeTestRequest.setNodeId(runSmokeTestDTO.getNodeId());
runSmokeTestRequest.setNodeParams(runSmokeTestDTO.getNodeParams());
runSmokeTestRequest.setName(runSmokeTestDTO.getName());
runSmokeTestRequest.setProjectEnv(runSmokeTestDTO.getProjectEnv());
RunSmokeTestResponse runSmokeTestResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(runSmokeTestRequest);
System.out.println(runSmokeTestResponse.getRequestId());
// DAGID
System.out.println(runSmokeTestResponse.getData());
ListInstancesRequest listInstancesRequest = new ListInstancesRequest();
listInstancesRequest.setDagId(runSmokeTestResponse.getData());
listInstancesRequest.setProjectId(runSmokeTestDTO.getProjectId());
listInstancesRequest.setProjectEnv(runSmokeTestDTO.getProjectEnv());
listInstancesRequest.setNodeId(runSmokeTestDTO.getNodeId());
listInstancesRequest.setPageNumber(1);
listInstancesRequest.setPageSize(10);
ListInstancesResponse listInstancesResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(listInstancesRequest);
System.out.println(listInstancesResponse.getRequestId());
List<ListInstancesResponse.Data.Instance> instances = listInstancesResponse.getData().getInstances();
if (CollectionUtils.isEmpty(instances)) {
return null;
}
return instances;
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return null;
}
public InstanceDetail getInstanceLog(Long instanceId, String projectEnv) {
try {
GetInstanceLogRequest getInstanceLogRequest = new GetInstanceLogRequest();
getInstanceLogRequest.setInstanceId(instanceId);
getInstanceLogRequest.setProjectEnv(projectEnv);
GetInstanceLogResponse getInstanceLogResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(getInstanceLogRequest);
System.out.println(getInstanceLogResponse.getRequestId());
GetInstanceRequest getInstanceRequest = new GetInstanceRequest();
getInstanceRequest.setInstanceId(instanceId);
getInstanceRequest.setProjectEnv(projectEnv);
GetInstanceResponse getInstanceResponse = dataWorksOpenApiClient.createClient()
.getAcsResponse(getInstanceRequest);
System.out.println(getInstanceResponse.getRequestId());
System.out.println(getInstanceResponse.getData());
InstanceDetail instanceDetail = new InstanceDetail();
instanceDetail.setInstance(getInstanceResponse.getData());
instanceDetail.setInstanceLog(getInstanceLogResponse.getData());
return instanceDetail;
} catch (ServerException e) {
e.printStackTrace();
} catch (ClientException e) {
e.printStackTrace();
// 请求ID
System.out.println(e.getRequestId());
// 错误码
System.out.println(e.getErrCode());
// 错误信息
System.out.println(e.getErrMsg());
}
return null;
}
}
步骤4:开发一个IdeController
您需要定义一个IdeController,提供前端调用的路由接口。
package com.aliyun.dataworks.demo;
import com.aliyun.dataworks.dto.*;
import com.aliyun.dataworks.services.BusinessService;
import com.aliyun.dataworks.services.FileService;
import com.aliyun.dataworks.services.FolderService;
import com.aliyun.dataworks.services.ProjectService;
import com.aliyuncs.dataworks_public.model.v20200518.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.List;
/**
* @author dataworks demo
*/
@RestController
@RequestMapping("/ide")
public class IdeController {
@Autowired
private FileService fileService;
@Autowired
private FolderService folderService;
@Autowired
private BusinessService businessService;
@Autowired
private ProjectService projectService;
/**
* for list those files
*
* @param listFilesDTO
* @return ListFilesResponse.Data.File
*/
@CrossOrigin(origins = "http://localhost:8080")
@GetMapping("/listFiles")
public List<ListFilesResponse.Data.File> listFiles(ListFilesDTO listFilesDTO) {
return fileService.listFiles(listFilesDTO);
}
/**
* for list those folders
*
* @param listFoldersDTO
* @return ListFoldersResponse.Data.FoldersItem
*/
@CrossOrigin(origins = "http://localhost:8080")
@GetMapping("/listFolders")
public List<ListFoldersResponse.Data.FoldersItem> listFolders(ListFoldersDTO listFoldersDTO) {
return folderService.listFolders(listFoldersDTO);
}
/**
* for create the folder
*
* @param createFolderDTO
* @return boolean
*/
@CrossOrigin(origins = "http://localhost:8080")
@PostMapping("/createFolder")
public boolean createFolder(@RequestBody CreateFolderDTO createFolderDTO) {
return folderService.createFolder(createFolderDTO);
}
/**
* for update the folder
*
* @param updateFolderDTO
* @return boolean
*/
@CrossOrigin(origins = "http://localhost:8080")
@PostMapping("/updateFolder")
public boolean updateFolder(@RequestBody UpdateFolderDTO updateFolderDTO) {
return folderService.updateFolder(updateFolderDTO);
}
/**
* for get the file
*
* @param getFileDTO
* @return GetFileResponse.Data.File
*/
@CrossOrigin(origins = "http://localhost:8080")
@GetMapping("/getFile")
public GetFileResponse.Data.File getFile(GetFileDTO getFileDTO) {
return fileService.getFile(getFileDTO);
}
/**
* for create the file
*
* @param createFileDTO
* @return fileId
*/
@CrossOrigin(origins = "http://localhost:8080")
@PostMapping("/createFile")
public Long createFile(@RequestBody CreateFileDTO createFileDTO) {
return fileService.createFile(createFileDTO);
}
/**
* for update the file
*
* @param updateFileDTO
* @return boolean
*/
@CrossOrigin(origins = "http://localhost:8080")
@PostMapping("/updateFile")
public boolean updateFile(@RequestBody UpdateFileDTO updateFileDTO) {
return fileService.updateFile(updateFileDTO);
}
/**
* for deploy the file
*
* @param deployFileDTO
* @return boolean
*/
@CrossOrigin(origins = "http://localhost:8080")
@PostMapping("/deployFile")
public boolean deployFile(@RequestBody DeployFileDTO deployFileDTO) {
try {
return fileService.deployFile(deployFileDTO);
} catch (Exception e) {
System.out.println(e);
}
return false;
}
/**
* for delete the file
*
* @param deleteFileDTO
* @return
*/
@CrossOrigin(origins = "http://localhost:8080")
@DeleteMapping("/deleteFile")
public boolean deleteFile(DeleteFileDTO deleteFileDTO) {
try {
return fileService.deleteFile(deleteFileDTO);
} catch (Exception e) {
System.out.println(e);
}
return false;
}
/**
* for delete the folder
*
* @param deleteFolderDTO
* @return
*/
@CrossOrigin(origins = "http://localhost:8080")
@DeleteMapping("/deleteFolder")
public boolean deleteFolder(DeleteFolderDTO deleteFolderDTO) {
return folderService.deleteFolder(deleteFolderDTO);
}
/**
* list businesses
*
* @param listBusinessesDTO
* @return
*/
@CrossOrigin(origins = "http://localhost:8080")
@GetMapping("/listBusinesses")
public List<ListBusinessResponse.Data.BusinessItem> listBusiness(ListBusinessesDTO listBusinessesDTO) {
return businessService.listBusiness(listBusinessesDTO);
}
/**
* create a business
*
* @param createBusinessDTO
* @return
*/
@CrossOrigin(origins = "http://localhost:8080")
@PostMapping("/createBusiness")
public Long createBusiness(@RequestBody CreateBusinessDTO createBusinessDTO) {
return businessService.createBusiness(createBusinessDTO);
}
/**
* update a business
*
* @param updateBusinessDTO
* @return
*/
@CrossOrigin(origins = "http://localhost:8080")
@PostMapping("/updateBusiness")
public boolean updateBusiness(@RequestBody UpdateBusinessDTO updateBusinessDTO) {
return businessService.updateBusiness(updateBusinessDTO);
}
/**
* delete a business
*
* @param deleteBusinessDTO
* @return
*/
@CrossOrigin(origins = "http://localhost:8080")
@PostMapping("/deleteBusiness")
public boolean deleteBusiness(@RequestBody DeleteBusinessDTO deleteBusinessDTO) {
return businessService.deleteBusiness(deleteBusinessDTO);
}
/**
* @param pageNumber
* @param pageSize
* @return
*/
@CrossOrigin(origins = "http://localhost:8080")
@GetMapping("/listProjects")
public ListProjectsResponse.PageResult listProjects(Integer pageNumber, Integer pageSize) {
return projectService.listProjects(pageNumber, pageSize);
}
/**
* @param runSmokeTestDTO
* @return
*/
@CrossOrigin(origins = "http://localhost:8080")
@PutMapping("/runSmokeTest")
public List<ListInstancesResponse.Data.Instance> runSmokeTest(@RequestBody RunSmokeTestDTO runSmokeTestDTO) {
return fileService.runSmokeTest(runSmokeTestDTO);
}
/**
* @param instanceId
* @param projectEnv
* @return
*/
@CrossOrigin(origins = "http://localhost:8080")
@GetMapping("/getLog")
public InstanceDetail getLog(@RequestParam Long instanceId, @RequestParam String projectEnv) {
return fileService.getInstanceLog(instanceId, projectEnv);
}
}
前端代码开发
初始化编辑器、目录树与terminal终端。
示例代码如下。
const App: FunctionComponent<Props> = () => { const editorRef = useRef<HTMLDivElement>(null); const termianlRef = useRef<HTMLDivElement>(null); const [terminal, setTerminal] = useState<NextTerminal>(); const [editor, setEditor] = useState<monaco.editor.IStandaloneCodeEditor>(); const [expnadedKeys, setExpandedKeys] = useState<any[]>(); const [workspace, setWorkspace] = useState<number>(); const [workspaces, setWorkspaces] = useState<{ label: string, value: number }[]>([]); const [dataSource, setDataSource] = useState<any[]>(); const [selectedFile, setSelectedFile] = useState<number>(); const [loading, setLoading] = useState<boolean>(false); // 创建编辑器实例 useEffect(() => { if (editorRef.current) { const nextEditor = monaco.editor.create(editorRef.current, editorOptions); setEditor(nextEditor); return () => { nextEditor.dispose(); }; } }, [editorRef.current]); // 添加保存文件按键事件 useEffect(() => { editor?.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, () => { if (!workspace) { showTips('Please select workspace first'); return; } saveFile(workspace, editor, selectedFile); }); }, [editor, workspace, selectedFile]); // 创建terminal实例 useEffect(() => { if (termianlRef.current) { const term: NextTerminal = new Terminal(terminalOptions) as any; term.pointer = -1; term.stack = []; setTerminal(term); const fitAddon = new FitAddon(); term.loadAddon(fitAddon); term.open(termianlRef.current); fitAddon.fit(); term.write('$ '); return () => { term.dispose(); }; } }, [termianlRef.current]); // 注册terminal输入事件 useEffect(() => { const event = terminal?.onKey(e => onTerminalKeyChange(e, terminal, dataSource, workspace)); return () => { event?.dispose(); }; }, [terminal, dataSource, workspace]); // 获取目录树数据源 useEffect(() => { workspace && (async () => { setLoading(true); const nextDataSource = await getTreeDataSource(workspace, workspaces); const defaultKey = nextDataSource?.[0]?.key; defaultKey && setExpandedKeys([defaultKey]); setDataSource(nextDataSource); setLoading(false); })(); }, [workspace]); // 当目录树文件被点击时,获取文件详情并展示代码 useEffect(() => { workspace && selectedFile && (async () => { setLoading(true); const file = await getFileInfo(workspace, selectedFile); editor?.setValue(file.content); editor?.getAction('editor.action.formatDocument').run(); setLoading(false); })(); }, [selectedFile]); // 获取工作空间列表 useEffect(() => { (async () => { const list = await getWorkspaceList(); setWorkspaces(list); })(); }, []); const onExapnd = useCallback((keys: number[]) => { setExpandedKeys(keys); }, []); const onWorkspaceChange = useCallback((value: number) => { setWorkspace(value) }, []); const onTreeNodeSelect = useCallback((key: number[]) => { key[0] && setSelectedFile(key[0]) }, []); return ( <div className={cn(classes.appWrapper)}> <div className={cn(classes.leftArea)}> <div className={cn(classes.workspaceWrapper)}> Workspace: <Select value={workspace} dataSource={workspaces} onChange={onWorkspaceChange} autoWidth={false} showSearch /> </div> <div className={cn(classes.treeWrapper)}> <Tree dataSource={dataSource} isNodeBlock={{ defaultPaddingLeft: 20 }} expandedKeys={expnadedKeys} selectedKeys={[selectedFile]} onExpand={onExapnd} onSelect={onTreeNodeSelect} defaultExpandAll /> </div> </div> <div className={cn(classes.rightArea)}> <div className={cn(classes.monacoEditorWrapper)} ref={editorRef} /> <div className={cn(classes.panelWrapper)} ref={termianlRef} /> </div> <div className={cn(classes.loaderLine)} style={{ display: loading ? 'block' : 'none' }} /> </div> ); };
获取示例业务流程与文件,并展示目录树。
整体流程和示例代码如下。
/** * 获取目录树数据源 * @param workspace 工作空间id * @param dataSource 工作空间列表 */ async function getTreeDataSource(workspace: number, dataSource: { label: string, value: number }[]) { try { const businesses = await services.ide.getBusinessList(workspace, openPlatformBusinessName); businesses.length === 0 && await services.ide.createBusiness(workspace, openPlatformBusinessName); } catch (e) { showError('You have no permission to access this workspace.'); return; } const fileFolderPath = `业务流程/${openPlatformBusinessName}/MaxCompute`; const files = await services.ide.getFileList(workspace, fileFolderPath); let children: { key: number, label: string }[] = []; if (files.length === 0) { try { const currentWorkspace = dataSource.find(i => i.value === workspace); const file1 = await services.ide.createFile(workspace, currentWorkspace!.label, fileFolderPath, 'simpleSQL.mc.sql', 'SELECT 1'); const file2 = await services.ide.createFile(workspace, currentWorkspace!.label, fileFolderPath, 'createTable.mc.sql', 'CREATE TABLE IF NOT EXISTS _qcc_mysql1_odps_source_20220113100903_done_ (\ncol string\n)\nCOMMENT \'全量数据同步完成标DONE表\'\nPARTITIONED BY\n(\nstatus STRING COMMENT \'标DONE分区\'\n)\nLIFECYCLE 36500;'); children = children.concat([ { key: file1, label: 'simpleSQL.mc.sql' }, { key: file2, label: 'createTable.mc.sql' }, ]); } catch (e) { showError('Create file failed. The datasource odps_source does not exist.'); return; } } else { children = files.map((i) => ({ key: i.fileId, label: i.fileName })); } return [{ key: 1, label: openPlatformBusinessName, children }]; }
当用户编辑文件并保存时,需要将编辑的内容传回后端并更新数据。
代码示例如下。
/** * 保存文件,当Ctrl+S时触发 * @param workspace 工作空间id * @param editor 编辑器实例 * @param selectedFile 已选择的文件 */ async function saveFile(workspace: number, editor: monaco.editor.IStandaloneCodeEditor, selectedFile?: number) { if (!selectedFile) { showTips('Please select a file.'); return; } const content = editor.getValue(); const result = await services.ide.updateFile(workspace, selectedFile, { content }); result ? showTips('Saved file') : showError('Failed to save file'); }
当用户在terminal终端中输入
dw run ...
时,会将文件提交到调度系统,并执行冒烟测试运行。处理流程和代码示例如下。
/** * 处理Terminal键盘事件 * @param e 事件对象 * @param term terminal实例 * @param dataSource 目录树数据源 * @param workspace 工作空间id */ function onTerminalKeyChange(e: { key: string; domEvent: KeyboardEvent; }, term: NextTerminal, dataSource: any, workspace?: number) { const ev = e.domEvent; const printable = !ev.altKey && !ev.ctrlKey && !ev.metaKey; term.inputText = typeof term.inputText === 'string' ? term.inputText : ''; switch (ev.key) { case 'ArrowUp': term.pointer = term.pointer < (term.stack.length - 1) ? term.pointer + 1 : term.pointer; term.inputText = term.stack[term.pointer]; term.write(`\x1b[2K\r$ ${term.inputText}`); break; case 'ArrowDown': term.pointer = term.pointer > -1 ? term.pointer - 1 : -1; term.inputText = term.pointer === -1 ? '' : term.stack[term.pointer]; term.write(`\x1b[2K\r$ ${term.inputText}`); break; case 'ArrowLeft': (term as any)._core.buffer.x > 2 && printable && term.write(e.key); break; case 'ArrowRight': (term as any)._core.buffer.x <= (term.inputText.length + 1) && printable && term.write(e.key); break; case 'Enter': commandHandler(term, dataSource, workspace); break; case 'Backspace': if ((term as any)._core.buffer.x > 2) { term.inputText = term.inputText.slice(0, -1); term.write('\b \b'); } break; default: if (printable) { term.inputText += e.key; term.write(e.key); } } } /** * 处理提交任务的方法,当terminal中输入dw run ...时触发 * @param term terminal实例 * @param dataSource 目录树数据源 * @param workspace 工作空间id */ async function commandHandler(term: NextTerminal, dataSource: any, workspace?: number) { term.write('\r\n$ '); const input = term.inputText; term.inputText = ''; if (['', undefined].includes(input)) { return; } term.stack = [input!, ...term.stack]; term.pointer = -1; if (!workspace) { term.write(highlight.text('[ERROR] You should select workspace first.\r\n$ ', brush)); return; } // 这里简单的处理了下输入的命令行解析,如果命令开头为dw,且执行命令为run就继续往下处理,否则报错 const words = input?.split(' '); const tag = words?.[0].toLowerCase(); const command = words?.[1]?.toLowerCase(); const fileName = words?.[2]; if (tag !== 'dw' || !validCommands.includes(command!)) { term.write(highlight.text('[ERROR] Invalid command.\r\n$ ', brush)); return; } // 获取输入文件 const source = dataSource?.[0]?.children.find((i: any) => i.label === fileName); const file = await services.ide.getFile(workspace, source.key); if (!file) { term.write(highlight.text('[ERROR] File name does not exist.\r\n$ ', brush)); return; } term.write(highlight.text('[INFO] Submiting file.\r\n$ ', brush)); // 调用部署文件接口,将文件发布到调度系统中 const response = await services.ide.deployFile(workspace, source.key); if (response) { term.write(highlight.text('[INFO] Submit file success.\r\n$ ', brush)); } else { term.write(highlight.text('[ERROR] Submit file failed.\r\n$ ', brush)); return; } // 执行冒烟测试,运行调度任务 let dag: services.ide.Dag; try { term.write(highlight.text('[INFO] Start to run task.\r\n$ ', brush)); dag = (await services.ide.runSmoke(workspace, file.nodeId, openPlatformBusinessName))[0]; term.write(highlight.text('[INFO] Trigger sql task success.\r\n$ ', brush)); } catch (e) { term.write(highlight.text('[ERROR] Trigger sql task failed.\r\n$ ', brush)); return; } // 轮询获取任务日志 const event = setInterval(async () => { try { const logInfo = await services.ide.getLog(dag.instanceId, 'DEV'); let log: string; switch (logInfo.instance.status) { case 'WAIT_TIME': log = '等待定时时间到来'; break; case 'WAIT_RESOURCE': log = '等待资源...'; break; default: log = logInfo.instanceLog; } term.write(`${highlight.text(log, brush).replace(/\n/g, '\r\n')}\r\n$ `); const finished = ['SUCCESS', 'FAILURE', 'NOT_RUN'].includes(logInfo.instance.status); finished && clearInterval(event); } catch (e) { term.write(highlight.text('[ERROR] SQL Task run failed.\r\n$ ', brush)); return; } }, 3000); }
本地部署运行
您需要按照Github代码示例中的指示信息完成环境准备工作,包含依赖的环境:java8 及以上、maven 构建工具、node 环境、pnpm 工具。并执行初始化安装。
pnpm install
您还需要在根路径下修改AK、SK相关信息。最后,在工程根目录下执行以下命令,运行示例实践代码。
npm run example:ide
完成后,您可以在浏览器中访问以下页面验证结果。
https://localhost:8080
参考:全量代码示例与源代码下载
您可在Github中下载全量示例源代码,获取地址为Github代码示例。全部流程的代码示例汇总如下。
import { useEffect, useRef, useState, useCallback } from 'react';
import type { FunctionComponent } from 'react';
import cn from 'classnames';
import * as monaco from 'monaco-editor';
import { Terminal } from 'xterm';
import { FitAddon } from 'xterm-addon-fit';
import { Tree, Select, Message } from '@alifd/next';
import * as highlight from '../helpers/highlight';
import * as services from '../services';
import classes from '../styles/app.module.css';
export interface Props {}
export interface NextTerminal extends Terminal {
inputText?: string;
stack: string[];
pointer: number;
}
const brush = {
rules: [
{ regex: /\bERROR\b/gmi, theme: 'red' },
{ regex: /\bWARN\b/gmi, theme: 'yellow' },
{ regex: /\bINFO\b/gmi, theme: 'green' },
{ regex: /^FAILED:.*$/gmi, theme: 'red' },
],
};
// 示例业务流程名称
const openPlatformBusinessName = '开放平台示例业务流程';
// 编辑器创建参数
const editorOptions = {
content: '',
language: 'sql',
theme: 'vs-dark',
automaticLayout: true,
fontSize: 16,
};
// Terminal创建参数
const terminalOptions = {
cursorBlink: true,
cursorStyle: 'underline' as const,
fontSize: 16,
};
const validCommands = [
'run',
];
/**
* 展示错误信息弹窗的方法
* @param message 错误信息
*/
function showError(message: string) {
Message.error({ title: 'Error Message', content: message });
}
/**
* 展示提示信息弹窗的方法
* @param message 提示信息
*/
function showTips(message: string) {
Message.show({ title: 'Tips', content: message });
}
/**
* 处理提交任务的方法,当terminal中输入dw run ...时触发
* @param term terminal实例
* @param dataSource 目录树数据源
* @param workspace 工作空间id
*/
async function commandHandler(term: NextTerminal, dataSource: any, workspace?: number) {
term.write('\r\n$ ');
const input = term.inputText;
term.inputText = '';
if (['', undefined].includes(input)) {
return;
}
term.stack = [input!, ...term.stack];
term.pointer = -1;
if (!workspace) {
term.write(highlight.text('[ERROR] You should select workspace first.\r\n$ ', brush));
return;
}
// 这里简单的处理了下输入的命令行解析,如果命令开头为dw,且执行命令为run就继续往下处理,否则报错
const words = input?.split(' ');
const tag = words?.[0].toLowerCase();
const command = words?.[1]?.toLowerCase();
const fileName = words?.[2];
if (tag !== 'dw' || !validCommands.includes(command!)) {
term.write(highlight.text('[ERROR] Invalid command.\r\n$ ', brush));
return;
}
// 获取输入文件
const source = dataSource?.[0]?.children.find((i: any) => i.label === fileName);
const file = await services.ide.getFile(workspace, source.key);
if (!file) {
term.write(highlight.text('[ERROR] File name does not exist.\r\n$ ', brush));
return;
}
term.write(highlight.text('[INFO] Submiting file.\r\n$ ', brush));
// 调用部署文件接口,将文件发布到调度系统中
const response = await services.ide.deployFile(workspace, source.key);
if (response) {
term.write(highlight.text('[INFO] Submit file success.\r\n$ ', brush));
} else {
term.write(highlight.text('[ERROR] Submit file failed.\r\n$ ', brush));
return;
}
// 执行冒烟测试,运行调度任务
let dag: services.ide.Dag;
try {
term.write(highlight.text('[INFO] Start to run task.\r\n$ ', brush));
dag = (await services.ide.runSmoke(workspace, file.nodeId, openPlatformBusinessName))[0];
term.write(highlight.text('[INFO] Trigger sql task success.\r\n$ ', brush));
} catch (e) {
term.write(highlight.text('[ERROR] Trigger sql task failed.\r\n$ ', brush));
return;
}
// 轮询获取任务日志
const event = setInterval(async () => {
try {
const logInfo = await services.ide.getLog(dag.instanceId, 'DEV');
let log: string;
switch (logInfo.instance.status) {
case 'WAIT_TIME':
log = '等待定时时间到来';
break;
case 'WAIT_RESOURCE':
log = '等待资源...';
break;
default:
log = logInfo.instanceLog;
}
term.write(`${highlight.text(log, brush).replace(/\n/g, '\r\n')}\r\n$ `);
const finished = ['SUCCESS', 'FAILURE', 'NOT_RUN'].includes(logInfo.instance.status);
finished && clearInterval(event);
} catch (e) {
term.write(highlight.text('[ERROR] SQL Task run failed.\r\n$ ', brush));
return;
}
}, 3000);
}
/**
* 处理Terminal键盘事件
* @param e 事件对象
* @param term terminal实例
* @param dataSource 目录树数据源
* @param workspace 工作空间id
*/
function onTerminalKeyChange(e: { key: string; domEvent: KeyboardEvent; }, term: NextTerminal, dataSource: any, workspace?: number) {
const ev = e.domEvent;
const printable = !ev.altKey && !ev.ctrlKey && !ev.metaKey;
term.inputText = typeof term.inputText === 'string' ? term.inputText : '';
switch (ev.key) {
case 'ArrowUp':
term.pointer = term.pointer < (term.stack.length - 1) ? term.pointer + 1 : term.pointer;
term.inputText = term.stack[term.pointer];
term.write(`\x1b[2K\r$ ${term.inputText}`);
break;
case 'ArrowDown':
term.pointer = term.pointer > -1 ? term.pointer - 1 : -1;
term.inputText = term.pointer === -1 ? '' : term.stack[term.pointer];
term.write(`\x1b[2K\r$ ${term.inputText}`);
break;
case 'ArrowLeft':
(term as any)._core.buffer.x > 2 && printable && term.write(e.key);
break;
case 'ArrowRight':
(term as any)._core.buffer.x <= (term.inputText.length + 1) && printable && term.write(e.key);
break;
case 'Enter':
commandHandler(term, dataSource, workspace);
break;
case 'Backspace':
if ((term as any)._core.buffer.x > 2) {
term.inputText = term.inputText.slice(0, -1);
term.write('\b \b');
}
break;
default:
if (printable) {
term.inputText += e.key;
term.write(e.key);
}
}
}
/**
* 获取工作空间列表
*/
async function getWorkspaceList() {
const response = await services.tenant.getProjectList();
const list = response.projectList.filter(i => i.projectStatusCode === 'AVAILABLE').map(i => (
{ label: i.projectName, value: i.projectId }
));
return list;
}
/**
* 获取目录树数据源
* @param workspace 工作空间id
* @param dataSource 工作空间列表
*/
async function getTreeDataSource(workspace: number, dataSource: { label: string, value: number }[]) {
try {
const businesses = await services.ide.getBusinessList(workspace, openPlatformBusinessName);
businesses.length === 0 && await services.ide.createBusiness(workspace, openPlatformBusinessName);
} catch (e) {
showError('You have no permission to access this workspace.');
return;
}
const fileFolderPath = `业务流程/${openPlatformBusinessName}/MaxCompute`;
const files = await services.ide.getFileList(workspace, fileFolderPath);
let children: { key: number, label: string }[] = [];
if (files.length === 0) {
try {
const currentWorkspace = dataSource.find(i => i.value === workspace);
const file1 = await services.ide.createFile(workspace, currentWorkspace!.label, fileFolderPath, 'simpleSQL.mc.sql', 'SELECT 1');
const file2 = await services.ide.createFile(workspace, currentWorkspace!.label, fileFolderPath, 'createTable.mc.sql', 'CREATE TABLE IF NOT EXISTS _qcc_mysql1_odps_source_20220113100903_done_ (\ncol string\n)\nCOMMENT \'全量数据同步完成标DONE表\'\nPARTITIONED BY\n(\nstatus STRING COMMENT \'标DONE分区\'\n)\nLIFECYCLE 36500;');
children = children.concat([
{ key: file1, label: 'simpleSQL.mc.sql' },
{ key: file2, label: 'createTable.mc.sql' },
]);
} catch (e) {
showError('Create file failed. The datasource odps_source does not exist.');
return;
}
} else {
children = files.map((i) => ({ key: i.fileId, label: i.fileName }));
}
return [{ key: 1, label: openPlatformBusinessName, children }];
}
/**
* 获取文件详细信息
* @param workspace 工作空间id
* @param fileId 文件id
*/
async function getFileInfo(workspace: number, fileId: number) {
const response = await services.ide.getFile(workspace, fileId);
return response;
}
/**
* 保存文件,当Ctrl+S时触发
* @param workspace 工作空间id
* @param editor 编辑器实例
* @param selectedFile 已选择的文件
*/
async function saveFile(workspace: number, editor: monaco.editor.IStandaloneCodeEditor, selectedFile?: number) {
if (!selectedFile) {
showTips('Please select a file.');
return;
}
const content = editor.getValue();
const result = await services.ide.updateFile(workspace, selectedFile, { content });
result ? showTips('Saved file') : showError('Failed to save file');
}
const App: FunctionComponent<Props> = () => {
const editorRef = useRef<HTMLDivElement>(null);
const termianlRef = useRef<HTMLDivElement>(null);
const [terminal, setTerminal] = useState<NextTerminal>();
const [editor, setEditor] = useState<monaco.editor.IStandaloneCodeEditor>();
const [expnadedKeys, setExpandedKeys] = useState<any[]>();
const [workspace, setWorkspace] = useState<number>();
const [workspaces, setWorkspaces] = useState<{ label: string, value: number }[]>([]);
const [dataSource, setDataSource] = useState<any[]>();
const [selectedFile, setSelectedFile] = useState<number>();
const [loading, setLoading] = useState<boolean>(false);
// 创建编辑器实例
useEffect(() => {
if (editorRef.current) {
const nextEditor = monaco.editor.create(editorRef.current, editorOptions);
setEditor(nextEditor);
return () => { nextEditor.dispose(); };
}
}, [editorRef.current]);
// 添加保存文件按键事件
useEffect(() => {
editor?.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, () => {
if (!workspace) {
showTips('Please select workspace first');
return;
}
saveFile(workspace, editor, selectedFile);
});
}, [editor, workspace, selectedFile]);
// 创建terminal实例
useEffect(() => {
if (termianlRef.current) {
const term: NextTerminal = new Terminal(terminalOptions) as any;
term.pointer = -1;
term.stack = [];
setTerminal(term);
const fitAddon = new FitAddon();
term.loadAddon(fitAddon);
term.open(termianlRef.current);
fitAddon.fit();
term.write('$ ');
return () => { term.dispose(); };
}
}, [termianlRef.current]);
// 注册terminal输入事件
useEffect(() => {
const event = terminal?.onKey(e => onTerminalKeyChange(e, terminal, dataSource, workspace));
return () => {
event?.dispose();
};
}, [terminal, dataSource, workspace]);
// 获取目录树数据源
useEffect(() => {
workspace && (async () => {
setLoading(true);
const nextDataSource = await getTreeDataSource(workspace, workspaces);
const defaultKey = nextDataSource?.[0]?.key;
defaultKey && setExpandedKeys([defaultKey]);
setDataSource(nextDataSource);
setLoading(false);
})();
}, [workspace]);
// 当目录树文件被点击时,获取文件详情并展示代码
useEffect(() => {
workspace && selectedFile && (async () => {
setLoading(true);
const file = await getFileInfo(workspace, selectedFile);
editor?.setValue(file.content);
editor?.getAction('editor.action.formatDocument').run();
setLoading(false);
})();
}, [selectedFile]);
// 获取工作空间列表
useEffect(() => {
(async () => {
const list = await getWorkspaceList();
setWorkspaces(list);
})();
}, []);
const onExapnd = useCallback((keys: number[]) => { setExpandedKeys(keys); }, []);
const onWorkspaceChange = useCallback((value: number) => { setWorkspace(value) }, []);
const onTreeNodeSelect = useCallback((key: number[]) => { key[0] && setSelectedFile(key[0]) }, []);
return (
<div className={cn(classes.appWrapper)}>
<div className={cn(classes.leftArea)}>
<div className={cn(classes.workspaceWrapper)}>
Workspace:
<Select
value={workspace}
dataSource={workspaces}
onChange={onWorkspaceChange}
autoWidth={false}
showSearch
/>
</div>
<div className={cn(classes.treeWrapper)}>
<Tree
dataSource={dataSource}
isNodeBlock={{ defaultPaddingLeft: 20 }}
expandedKeys={expnadedKeys}
selectedKeys={[selectedFile]}
onExpand={onExapnd}
onSelect={onTreeNodeSelect}
defaultExpandAll
/>
</div>
</div>
<div className={cn(classes.rightArea)}>
<div
className={cn(classes.monacoEditorWrapper)}
ref={editorRef}
/>
<div
className={cn(classes.panelWrapper)}
ref={termianlRef}
/>
</div>
<div className={cn(classes.loaderLine)} style={{ display: loading ? 'block' : 'none' }} />
</div>
);
};
export default App;