diff --git a/docs/dataartsstudio/api-ref/ALL_META.TXT.json b/docs/dataartsstudio/api-ref/ALL_META.TXT.json new file mode 100644 index 000000000..7f28f483b --- /dev/null +++ b/docs/dataartsstudio/api-ref/ALL_META.TXT.json @@ -0,0 +1,1382 @@ +[ + { + "uri":"dataartsstudio_02_0001.html", + "product_code":"dgc", + "code":"1", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Before You Start", + "title":"Before You Start", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0002.html", + "product_code":"dgc", + "code":"2", + "des":"Welcome to DataArts Studio. DataArts Studio is a one-stop operations platform that provides data lifecycle management and intelligent data management for enterprises' dig", + "doc_type":"api", + "kw":"Overview,Before You Start,API Reference", + "title":"Overview", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0003.html", + "product_code":"dgc", + "code":"3", + "des":"DataArts Studio supports Representational State Transfer (REST) APIs, allowing you to call APIs using HTTPS. For details on API calling, see Calling APIs.", + "doc_type":"api", + "kw":"API Calling,Before You Start,API Reference", + "title":"API Calling", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0006.html", + "product_code":"dgc", + "code":"4", + "des":"DomainA domain is created upon successful registration. The domain has full access permissions for all of its cloud services and resources. It can be used to reset user p", + "doc_type":"api", + "kw":"Concepts,Before You Start,API Reference", + "title":"Concepts", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0004.html", + "product_code":"dgc", + "code":"5", + "des":"An endpoint is the request address for calling an API. Endpoints vary depending on services and regions.You can obtain endpoints from Regions and Endpoints.", + "doc_type":"api", + "kw":"Endpoints,Before You Start,API Reference", + "title":"Endpoints", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0314.html", + "product_code":"", + "code":"6", + "des":"You can obtain the project ID and account ID by performing the following steps:Register with and log in to the management console.Hover the cursor on the username in the ", + "doc_type":"", + "kw":"Project ID and Account ID,Before You Start,API Reference", + "title":"Project ID and Account ID", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0350.html", + "product_code":"", + "code":"7", + "des":"To obtain a DataArts Studio instance ID and workspace ID, perform the following steps:On the DataArts Studio console, locate a workspace and click any module, such as Man", + "doc_type":"", + "kw":"DataArts Studio Instance ID and Workspace ID,Before You Start,API Reference", + "title":"DataArts Studio Instance ID and Workspace ID", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0316.html", + "product_code":"", + "code":"8", + "des":"A job ID (jobid) is required for some URLs when an API is called. To obtain a job ID, perform the following operations:Log in to the DataArts Studio console. Locate an in", + "doc_type":"", + "kw":"Data Development Job ID,Before You Start,API Reference", + "title":"Data Development Job ID", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0005.html", + "product_code":"dgc", + "code":"9", + "des":"The restrictions on DataArts Catalog APIs are as follows:CDM jobs carry large volumes of data, which increases the database load. You are advised to periodically delete u", + "doc_type":"api", + "kw":"Constraints,Before You Start,API Reference", + "title":"Constraints", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0007.html", + "product_code":"dgc", + "code":"10", + "des":"DataArts Studio provides self-developed APIs that comply with RESTful API design specifications. You can call those APIs to perform operations on DataArts Studio.", + "doc_type":"api", + "kw":"API Overview,API Reference", + "title":"API Overview", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0008.html", + "product_code":"dgc", + "code":"11", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Calling APIs", + "title":"Calling APIs", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0009.html", + "product_code":"dgc", + "code":"12", + "des":"This section describes the structure of a REST API, and uses the IAM API for obtaining a user token as an example to demonstrate how to call an API. The obtained token ca", + "doc_type":"api", + "kw":"Making an API Request,Calling APIs,API Reference", + "title":"Making an API Request", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0010.html", + "product_code":"dgc", + "code":"13", + "des":"Requests for calling an API can be authenticated using either of the following methods:Token-based authentication: Requests are authenticated using a token.AK/SK-based au", + "doc_type":"api", + "kw":"Authentication,Calling APIs,API Reference", + "title":"Authentication", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0011.html", + "product_code":"dgc", + "code":"14", + "des":"After sending a request, you will receive a response, including a status code, response header, and response body.A status code is a group of digits, ranging from 1xx to ", + "doc_type":"api", + "kw":"Response,Calling APIs,API Reference", + "title":"Response", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0012.html", + "product_code":"dgc", + "code":"15", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Application Cases", + "title":"Application Cases", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0013.html", + "product_code":"dgc", + "code":"16", + "des":"This section describes how to use cURL to call CDM APIs to migrate data from a local MySQL database to DWS in the cloud.Obtaining a TokenCall the API to obtain the user t", + "doc_type":"api", + "kw":"Application Example,Application Cases,API Reference", + "title":"Application Example", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0015.html", + "product_code":"dgc", + "code":"17", + "des":"DataArts Studio helps enterprises quickly build an end-to-end intelligent data system that covers the entire process from data ingestion to data analytics. The system can", + "doc_type":"api", + "kw":"Example of Using Data Development APIs,Application Cases,API Reference", + "title":"Example of Using Data Development APIs", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0016.html", + "product_code":"dgc", + "code":"18", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"DataArts Migration APIs", + "title":"DataArts Migration APIs", + "githuburl":"" + }, + { + "uri":"topic_300000000_3.html", + "product_code":"cdm", + "code":"19", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Cluster Management", + "title":"Cluster Management", + "githuburl":"" + }, + { + "uri":"ShowClusterDetail.html", + "product_code":"cdm", + "code":"20", + "des":"This API is used to query cluster details.GET /v1.1/{project_id}/clusters/{cluster_id}Status code: 200Status code: 200okSee Error Codes.", + "doc_type":"api", + "kw":"Querying Cluster Details,Cluster Management,API Reference", + "title":"Querying Cluster Details", + "githuburl":"" + }, + { + "uri":"DeleteCluster.html", + "product_code":"cdm", + "code":"21", + "des":"This API is used to delete a cluster.DELETE /v1.1/{project_id}/clusters/{cluster_id}Status code: 202Status code: 202AcceptedSee Error Codes.", + "doc_type":"api", + "kw":"Deleting a Cluster,Cluster Management,API Reference", + "title":"Deleting a Cluster", + "githuburl":"" + }, + { + "uri":"RestartCluster.html", + "product_code":"cdm", + "code":"22", + "des":"This API is used to restart a cluster.POST /v1.1/{project_id}/clusters/{cluster_id}/actionStatus code: 200Restarting a ClusterStatus code: 200okSee Error Codes.", + "doc_type":"api", + "kw":"Restarting a Cluster,Cluster Management,API Reference", + "title":"Restarting a Cluster", + "githuburl":"" + }, + { + "uri":"StartCluster.html", + "product_code":"cdm", + "code":"23", + "des":"This API is used to start a cluster.POST /v1.1/{project_id}/clusters/{cluster_id}/actionStatus code: 200Starting a ClusterStatus code: 200okSee Error Codes.", + "doc_type":"api", + "kw":"Starting a Cluster,Cluster Management,API Reference", + "title":"Starting a Cluster", + "githuburl":"" + }, + { + "uri":"StopCluster.html", + "product_code":"cdm", + "code":"24", + "des":"This API is used to stop a cluster.POST /v1.1/{project_id}/clusters/{cluster_id}/actionStatus code: 200Stopping a ClusterStatus code: 200okSee Error Codes.", + "doc_type":"api", + "kw":"Stopping a Cluster,Cluster Management,API Reference", + "title":"Stopping a Cluster", + "githuburl":"" + }, + { + "uri":"CreateCluster.html", + "product_code":"cdm", + "code":"25", + "des":"This API is used to create a cluster.POST /v1.1/{project_id}/clustersStatus code: 202Status code: 202AcceptedSee Error Codes.", + "doc_type":"api", + "kw":"Creating a Cluster,Cluster Management,API Reference", + "title":"Creating a Cluster", + "githuburl":"" + }, + { + "uri":"ListClusters.html", + "product_code":"cdm", + "code":"26", + "des":"This API is used to query the cluster list.GET /v1.1/{project_id}/clustersStatus code: 200Status code: 200okSee Error Codes.", + "doc_type":"api", + "kw":"Querying the Cluster List,Cluster Management,API Reference", + "title":"Querying the Cluster List", + "githuburl":"" + }, + { + "uri":"topic_300000001_2.html", + "product_code":"cdm", + "code":"27", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Job Management", + "title":"Job Management", + "githuburl":"" + }, + { + "uri":"ShowJobs.html", + "product_code":"cdm", + "code":"28", + "des":"This API is used to query jobs.GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}Status code: 200Status code: 200okSee Error Codes.", + "doc_type":"api", + "kw":"Querying a Job,Job Management,API Reference", + "title":"Querying a Job", + "githuburl":"" + }, + { + "uri":"DeleteJob.html", + "product_code":"cdm", + "code":"29", + "des":"This API is used to delete a job.DELETE /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}Status code: 500Status code: 500An internal service error occurred. For", + "doc_type":"api", + "kw":"Deleting a Job,Job Management,API Reference", + "title":"Deleting a Job", + "githuburl":"" + }, + { + "uri":"UpdateJob.html", + "product_code":"cdm", + "code":"30", + "des":"This API is used to modify a job.PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}Status code: 200Status code: 400Status code: 200okStatus code: 400Error co", + "doc_type":"api", + "kw":"Modifying a Job,Job Management,API Reference", + "title":"Modifying a Job", + "githuburl":"" + }, + { + "uri":"CreateAndStartRandomClusterJob.html", + "product_code":"cdm", + "code":"31", + "des":"This API is used to create and execute a job in a random cluster.POST /v1.1/{project_id}/clusters/jobStatus code: 200Status code: 200okSee Error Codes.", + "doc_type":"api", + "kw":"Creating and Executing a Job in a Random Cluster,Job Management,API Reference", + "title":"Creating and Executing a Job in a Random Cluster", + "githuburl":"" + }, + { + "uri":"StopJob.html", + "product_code":"cdm", + "code":"32", + "des":"This API is used to stop a job.PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}/stopStatus code: 200Status code: 200okSee Error Codes.", + "doc_type":"api", + "kw":"Stopping a Job,Job Management,API Reference", + "title":"Stopping a Job", + "githuburl":"" + }, + { + "uri":"CreateJob.html", + "product_code":"cdm", + "code":"33", + "des":"This API is used to create a job in a specified cluster.POST /v1.1/{project_id}/clusters/{cluster_id}/cdm/jobStatus code: 200Status code: 400Status code: 200okStatus code", + "doc_type":"api", + "kw":"Creating a Job in a Specified Cluster,Job Management,API Reference", + "title":"Creating a Job in a Specified Cluster", + "githuburl":"" + }, + { + "uri":"StartJob.html", + "product_code":"cdm", + "code":"34", + "des":"This API is used to start a job.PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}/startStatus code: 200Status code: 200okSee Error Codes.", + "doc_type":"api", + "kw":"Starting a Job,Job Management,API Reference", + "title":"Starting a Job", + "githuburl":"" + }, + { + "uri":"ShowJobStatus.html", + "product_code":"cdm", + "code":"35", + "des":"This API is used to query the job status.GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}/statusStatus code: 200Status code: 200okSee Error Codes.", + "doc_type":"api", + "kw":"Querying Job Status,Job Management,API Reference", + "title":"Querying Job Status", + "githuburl":"" + }, + { + "uri":"ShowSubmissions.html", + "product_code":"cdm", + "code":"36", + "des":"This API is used to query the job execution history.GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/submissionsStatus code: 200Status code: 200okSee Error Codes.", + "doc_type":"api", + "kw":"Querying Job Execution History,Job Management,API Reference", + "title":"Querying Job Execution History", + "githuburl":"" + }, + { + "uri":"topic_300000002_2.html", + "product_code":"cdm", + "code":"37", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Link Management", + "title":"Link Management", + "githuburl":"" + }, + { + "uri":"CreateLink.html", + "product_code":"cdm", + "code":"38", + "des":"This API is used to create a link.POST /v1.1/{project_id}/clusters/{cluster_id}/cdm/linkStatus code: 200Status code: 400Status code: 500Status code: 200OKStatus code: 400", + "doc_type":"api", + "kw":"Creating a Link,Link Management,API Reference", + "title":"Creating a Link", + "githuburl":"" + }, + { + "uri":"ShowLink.html", + "product_code":"cdm", + "code":"39", + "des":"This API is used to query a link.GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/link/{link_name}Status code: 200Status code: 200OKSee Error Codes.", + "doc_type":"api", + "kw":"Querying a Link,Link Management,API Reference", + "title":"Querying a Link", + "githuburl":"" + }, + { + "uri":"DeleteLink.html", + "product_code":"cdm", + "code":"40", + "des":"This API is used to delete a link.DELETE /v1.1/{project_id}/clusters/{cluster_id}/cdm/link/{link_name}Status code: 500Status code: 500An internal service error occurred. ", + "doc_type":"api", + "kw":"Deleting a Link,Link Management,API Reference", + "title":"Deleting a Link", + "githuburl":"" + }, + { + "uri":"UpdateLink.html", + "product_code":"cdm", + "code":"41", + "des":"This API is used to modify a link.PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/link/{link_name}Status code: 200Status code: 500Status code: 200OKStatus code: 500An in", + "doc_type":"api", + "kw":"Modifying a Link,Link Management,API Reference", + "title":"Modifying a Link", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0260.html", + "product_code":"dgc", + "code":"42", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Public Data Structures", + "title":"Public Data Structures", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0261.html", + "product_code":"dgc", + "code":"43", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Link Parameter Description", + "title":"Link Parameter Description", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0262.html", + "product_code":"dgc", + "code":"44", + "des":"By creating a JDBC link, you can extract data from or load data to the following relational databases:Data Warehouse ServiceRDS for MySQLRDS for PostgreSQLRDS for SQL Ser", + "doc_type":"api", + "kw":"Link to a Relational Database,Link Parameter Description,API Reference", + "title":"Link to a Relational Database", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0263.html", + "product_code":"dgc", + "code":"45", + "des":"By creating an OBS link, you can extract files from or load files to OBS. Files in CSV, JSON, and binary format are supported.", + "doc_type":"api", + "kw":"Link to OBS,Link Parameter Description,API Reference", + "title":"Link to OBS", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0266.html", + "product_code":"dgc", + "code":"46", + "des":"By creating an HDFS link, you can extract files from or load files to MRS, FusionInsight HD, or Apache Hadoop. Files in CSV, Parquet, and binary formats are supported.", + "doc_type":"api", + "kw":"Link to HDFS,Link Parameter Description,API Reference", + "title":"Link to HDFS", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0267.html", + "product_code":"dgc", + "code":"47", + "des":"By creating an HBase link, you can extract data from or load data to HBase of MRS, FusionInsight HD, and Apache Hadoop.", + "doc_type":"api", + "kw":"Link to HBase,Link Parameter Description,API Reference", + "title":"Link to HBase", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0268.html", + "product_code":"dgc", + "code":"48", + "des":"By creating a CloudTable link, you can extract data from or load data to CloudTable.", + "doc_type":"api", + "kw":"Link to CloudTable,Link Parameter Description,API Reference", + "title":"Link to CloudTable", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0269.html", + "product_code":"dgc", + "code":"49", + "des":"By creating a Hive link, you can extract data from or load data to Hive of MRS.", + "doc_type":"api", + "kw":"Link to Hive,Link Parameter Description,API Reference", + "title":"Link to Hive", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0270.html", + "product_code":"dgc", + "code":"50", + "des":"By creating an FTP or SFTP link, you are able to extract files from or load files to the FTP or SFTP server. Files in CSV, JSON, and binary format are supported.Parameter", + "doc_type":"api", + "kw":"Link to an FTP or SFTP Server,Link Parameter Description,API Reference", + "title":"Link to an FTP or SFTP Server", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0271.html", + "product_code":"dgc", + "code":"51", + "des":"By creating a MongoDB link, you can extract data from or load data to MongoDB.", + "doc_type":"api", + "kw":"Link to MongoDB,Link Parameter Description,API Reference", + "title":"Link to MongoDB", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0272.html", + "product_code":"dgc", + "code":"52", + "des":"By creating a Redis link, you can extract data from or load data to the Redis server. By creating a DCS link, you can load data to Data Cache Service (DCS), but not extra", + "doc_type":"api", + "kw":"Link to Redis/DCS (to Be Brought Offline),Link Parameter Description,API Reference", + "title":"Link to Redis/DCS (to Be Brought Offline)", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0274.html", + "product_code":"dgc", + "code":"53", + "des":"By creating a Kafka link, you are able to access open source Kafka and migrate data from Kafka to other data sources as required. Currently, only data export from Kafka i", + "doc_type":"api", + "kw":"Link to Kafka,Link Parameter Description,API Reference", + "title":"Link to Kafka", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0276.html", + "product_code":"dgc", + "code":"54", + "des":"By creating an Elasticsearch link, you can extract data from or load data to the Elasticsearch server or Cloud Search Service.", + "doc_type":"api", + "kw":"Link to Elasticsearch/Cloud Search Service,Link Parameter Description,API Reference", + "title":"Link to Elasticsearch/Cloud Search Service", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0277.html", + "product_code":"dgc", + "code":"55", + "des":"By creating a DLI link, you can import data to DLI. Currently, you cannot export data from DLI using CDM.", + "doc_type":"api", + "kw":"Link to DLI,Link Parameter Description,API Reference", + "title":"Link to DLI", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0278.html", + "product_code":"dgc", + "code":"56", + "des":"By creating an OpenTSDB link, you can extract data from and load data to CloudTable OpenTSDB.", + "doc_type":"api", + "kw":"Link to CloudTable OpenTSDB,Link Parameter Description,API Reference", + "title":"Link to CloudTable OpenTSDB", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0280.html", + "product_code":"dgc", + "code":"57", + "des":"By creating a DMS Kafka link, you can connect to Kafka Basic or Kafka Platinum on DMS. Currently, you can only export data from DMS Kafka to Cloud Search Service.", + "doc_type":"api", + "kw":"Link to DMS Kafka,Link Parameter Description,API Reference", + "title":"Link to DMS Kafka", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0281.html", + "product_code":"dgc", + "code":"58", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Source Job Parameters", + "title":"Source Job Parameters", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0282.html", + "product_code":"dgc", + "code":"59", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"From a Relational Database,Source Job Parameters,API Reference", + "title":"From a Relational Database", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0283.html", + "product_code":"dgc", + "code":"60", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"From Object Storage,Source Job Parameters,API Reference", + "title":"From Object Storage", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0284.html", + "product_code":"dgc", + "code":"61", + "des":"HDFS job parameter descriptionParameterMandatoryTypeDescriptionfromJobConfig.inputDirectoryYesStringPath for storing data to be extracted. For example, /data_dir.fromJobC", + "doc_type":"api", + "kw":"From HDFS,Source Job Parameters,API Reference", + "title":"From HDFS", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0285.html", + "product_code":"dgc", + "code":"62", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"From Hive,Source Job Parameters,API Reference", + "title":"From Hive", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0286.html", + "product_code":"dgc", + "code":"63", + "des":"HBase/CloudTable job parameter descriptionParameterMandatoryTypeDescriptionfromJobConfig.tableYesStringName of the table from which data is extracted. For example, cdm.fr", + "doc_type":"api", + "kw":"From HBase/CloudTable,Source Job Parameters,API Reference", + "title":"From HBase/CloudTable", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0287.html", + "product_code":"dgc", + "code":"64", + "des":"Source link job parameters of FTP and SFTP are the same. Table 1 describes the parameters.Source link job parameters of file systemsParameterMandatoryTypeDescriptionfromJ", + "doc_type":"api", + "kw":"From FTP/SFTP/NAS (to Be Brought Offline)/SFS (to Be Brought Offline),Source Job Parameters,API Refe", + "title":"From FTP/SFTP/NAS (to Be Brought Offline)/SFS (to Be Brought Offline)", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0288.html", + "product_code":"dgc", + "code":"65", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"From HTTP/HTTPS,Source Job Parameters,API Reference", + "title":"From HTTP/HTTPS", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0289.html", + "product_code":"dgc", + "code":"66", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"From MongoDB/DDS,Source Job Parameters,API Reference", + "title":"From MongoDB/DDS", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0290.html", + "product_code":"dgc", + "code":"67", + "des":"Redis job parameter descriptionParameterMandatoryTypeDescriptionfromJobConfig.isBatchMigrationNoBooleanWhether to migrate all data in the databasefromJobConfig.keyPrefixY", + "doc_type":"api", + "kw":"From Redis/DCS (to Be Brought Offline),Source Job Parameters,API Reference", + "title":"From Redis/DCS (to Be Brought Offline)", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0291.html", + "product_code":"dgc", + "code":"68", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"From DIS,Source Job Parameters,API Reference", + "title":"From DIS", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0292.html", + "product_code":"dgc", + "code":"69", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"From Kafka,Source Job Parameters,API Reference", + "title":"From Kafka", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0293.html", + "product_code":"dgc", + "code":"70", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"From Elasticsearch/Cloud Search Service,Source Job Parameters,API Reference", + "title":"From Elasticsearch/Cloud Search Service", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0294.html", + "product_code":"dgc", + "code":"71", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"From OpenTSDB,Source Job Parameters,API Reference", + "title":"From OpenTSDB", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0295.html", + "product_code":"dgc", + "code":"72", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Destination Job Parameters", + "title":"Destination Job Parameters", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0296.html", + "product_code":"dgc", + "code":"73", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"To a Relational Database,Destination Job Parameters,API Reference", + "title":"To a Relational Database", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0297.html", + "product_code":"dgc", + "code":"74", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"To OBS,Destination Job Parameters,API Reference", + "title":"To OBS", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0298.html", + "product_code":"dgc", + "code":"75", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"To HDFS,Destination Job Parameters,API Reference", + "title":"To HDFS", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0299.html", + "product_code":"dgc", + "code":"76", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"To Hive,Destination Job Parameters,API Reference", + "title":"To Hive", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0300.html", + "product_code":"dgc", + "code":"77", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"To HBase/CloudTable,Destination Job Parameters,API Reference", + "title":"To HBase/CloudTable", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0301.html", + "product_code":"dgc", + "code":"78", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"To FTP/SFTP/NAS (to Be Brought Offline)/SFS (to Be Brought Offline),Destination Job Parameters,API R", + "title":"To FTP/SFTP/NAS (to Be Brought Offline)/SFS (to Be Brought Offline)", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0302.html", + "product_code":"dgc", + "code":"79", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"To DDS,Destination Job Parameters,API Reference", + "title":"To DDS", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0303.html", + "product_code":"dgc", + "code":"80", + "des":"Parameter descriptionParameterMandatoryTypeDescriptiontoJobConfig.isBatchMigrationNoBooleanWhether to migrate all data in the databasetoJobConfig.shouldClearDatabaseNoBoo", + "doc_type":"api", + "kw":"To DCS,Destination Job Parameters,API Reference", + "title":"To DCS", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0304.html", + "product_code":"dgc", + "code":"81", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"To Elasticsearch/Cloud Search Service,Destination Job Parameters,API Reference", + "title":"To Elasticsearch/Cloud Search Service", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0305.html", + "product_code":"dgc", + "code":"82", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"To DLI,Destination Job Parameters,API Reference", + "title":"To DLI", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0306.html", + "product_code":"dgc", + "code":"83", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"To DIS,Destination Job Parameters,API Reference", + "title":"To DIS", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0307.html", + "product_code":"dgc", + "code":"84", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"To OpenTSDB,Destination Job Parameters,API Reference", + "title":"To OpenTSDB", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0308.html", + "product_code":"dgc", + "code":"85", + "des":"When Creating a Job in a Specified Cluster or Creating and Executing a Job in a Random Cluster, the driver-config-values parameter specifies the job configuration, which ", + "doc_type":"api", + "kw":"Job Parameter Description,Public Data Structures,API Reference", + "title":"Job Parameter Description", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0127.html", + "product_code":"dgc", + "code":"86", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"DataArts Factory APIs", + "title":"DataArts Factory APIs", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0049.html", + "product_code":"dgc", + "code":"87", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Connection Management APIs", + "title":"Connection Management APIs", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0050.html", + "product_code":"dgc", + "code":"88", + "des":"This API is used to create a connection. The supported connection types include DWS, DLI, Spark SQL, RDS, CloudTable, and Hive.URI formatPOST /v1/{project_id}/connections", + "doc_type":"api", + "kw":"Creating a Connection,Connection Management APIs,API Reference", + "title":"Creating a Connection", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0051.html", + "product_code":"dgc", + "code":"89", + "des":"This API is used to query a connection list.URI formatGET /v1/{project_id}/connections?offset={offset}&limit={limit}&connectionName={connectionName}GET /v1/{project_id}/c", + "doc_type":"api", + "kw":"Querying a Connection List,Connection Management APIs,API Reference", + "title":"Querying a Connection List", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0052.html", + "product_code":"dgc", + "code":"90", + "des":"This API is used to query configuration details of a specific connection.URI formatGET /v1/{project_id}/connections/{connection_name}GET /v1/{project_id}/connections/{con", + "doc_type":"api", + "kw":"Viewing Connection Details,Connection Management APIs,API Reference", + "title":"Viewing Connection Details", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0053.html", + "product_code":"dgc", + "code":"91", + "des":"This API is used to edit a connection.URI formatPUT /v1/{project_id}/connections/{connection_name}?ischeck=truePUT /v1/{project_id}/connections/{connection_name}?ischeck=", + "doc_type":"api", + "kw":"Editing a Connection,Connection Management APIs,API Reference", + "title":"Editing a Connection", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0054.html", + "product_code":"dgc", + "code":"92", + "des":"This API is used to delete a connection.URI formatDELETE /v1/{project_id}/connections/{connection_name}DELETE /v1/{project_id}/connections/{connection_name}Parameter desc", + "doc_type":"api", + "kw":"Deleting a Connection,Connection Management APIs,API Reference", + "title":"Deleting a Connection", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0055.html", + "product_code":"dgc", + "code":"93", + "des":"This API is used to export all connection information that is compressed in ZIP format.URI formatPOST /v1/{project_id}/connections/exportPOST /v1/{project_id}/connections", + "doc_type":"api", + "kw":"Exporting a Connection,Connection Management APIs,API Reference", + "title":"Exporting a Connection", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0056.html", + "product_code":"dgc", + "code":"94", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Script Development APIs", + "title":"Script Development APIs", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0057.html", + "product_code":"dgc", + "code":"95", + "des":"This API is used to delete a specific script.URI formatDELETE /v1/{project_id}/scripts/{script_name}DELETE /v1/{project_id}/scripts/{script_name}Parameter descriptionURI ", + "doc_type":"api", + "kw":"Deleting a Script,Script Development APIs,API Reference", + "title":"Deleting a Script", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0058.html", + "product_code":"dgc", + "code":"96", + "des":"This API is used to execute a specific script, which can be a DWS SQL, DLI SQL, RDS SQL, Flink SQL, Hive SQL, Presto SQL, or Spark SQL script. A script instance is genera", + "doc_type":"api", + "kw":"Executing a Script,Script Development APIs,API Reference", + "title":"Executing a Script", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0059.html", + "product_code":"dgc", + "code":"97", + "des":"This API is used to stop executing a script instance.URI formatPOST /v1/{project_id}/scripts/{script_name}/instances/{instance_id}/stopPOST /v1/{project_id}/scripts/{scri", + "doc_type":"api", + "kw":"Stopping Executing a Script Instance,Script Development APIs,API Reference", + "title":"Stopping Executing a Script Instance", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0060.html", + "product_code":"dgc", + "code":"98", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Resource Management APIs", + "title":"Resource Management APIs", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0061.html", + "product_code":"dgc", + "code":"99", + "des":"This API is used to delete a resource.URI formatDELETE /v1/{project_id}/resources/{resource_id}DELETE /v1/{project_id}/resources/{resource_id}Parameter descriptionURI par", + "doc_type":"api", + "kw":"Deleting a Resource,Resource Management APIs,API Reference", + "title":"Deleting a Resource", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0062.html", + "product_code":"dgc", + "code":"100", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Job Development APIs", + "title":"Job Development APIs", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0063.html", + "product_code":"dgc", + "code":"101", + "des":"This API is used to check whether there are jobs and scripts in the job file to be imported from OBS to DLF.URI formatPOST /v1/{project_id}/jobs/check-filePOST /v1/{proje", + "doc_type":"api", + "kw":"Viewing a Job File,Job Development APIs,API Reference", + "title":"Viewing a Job File", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0064.html", + "product_code":"dgc", + "code":"102", + "des":"This API is used to stop a job.URI formatPOST /v1/{project_id}/jobs/{job_name}/stopPOST /v1/{project_id}/jobs/{job_name}/stopParameter descriptionURI parametersParamete", + "doc_type":"api", + "kw":"Stopping a Job,Job Development APIs,API Reference", + "title":"Stopping a Job", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0065.html", + "product_code":"dgc", + "code":"103", + "des":"This API is used to delete a job.URI formatDELETE /v1/{project_id}/jobs/{job_name}DELETE /v1/{project_id}/jobs/{job_name}Parameter descriptionURI parametersParameterManda", + "doc_type":"api", + "kw":"Deleting a Job,Job Development APIs,API Reference", + "title":"Deleting a Job", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0066.html", + "product_code":"dgc", + "code":"104", + "des":"This API is used to stop a specific job instance. A job instance can be stopped only when it is in the running state.URI formatPOST /v1/{project_id}/jobs/{job_name}/insta", + "doc_type":"api", + "kw":"Stopping a Job Instance,Job Development APIs,API Reference", + "title":"Stopping a Job Instance", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0067.html", + "product_code":"dgc", + "code":"105", + "des":"This API is used to restart a specific job instance. A job instance can be restarted only when it is in the successful, failed, or canceled state.URI formatPOST /v1/{proj", + "doc_type":"api", + "kw":"Restarting a Job Instance,Job Development APIs,API Reference", + "title":"Restarting a Job Instance", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0082.html", + "product_code":"dgc", + "code":"106", + "des":"Table 1 describes common request headers.Table 2 describes common response headers.", + "doc_type":"api", + "kw":"Data Structure,DataArts Factory APIs,API Reference", + "title":"Data Structure", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0083.html", + "product_code":"dgc", + "code":"107", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"APIs to Be Taken Offline", + "title":"APIs to Be Taken Offline", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0084.html", + "product_code":"dgc", + "code":"108", + "des":"This API is used to create a job. A job consists of one or more nodes, such as Hive SQL and CDM Job nodes. DLF supports two types of jobs: batch jobs and real-time jobs.U", + "doc_type":"api", + "kw":"Creating a Job,APIs to Be Taken Offline,API Reference", + "title":"Creating a Job", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0085.html", + "product_code":"dgc", + "code":"109", + "des":"This API is used to edit a job.URI formatPUT /v1/{project_id}/jobs/{job_name}PUT /v1/{project_id}/jobs/{job_name}Parameter descriptionURI parametersParameterMandatoryType", + "doc_type":"api", + "kw":"Editing a Job,APIs to Be Taken Offline,API Reference", + "title":"Editing a Job", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0086.html", + "product_code":"dgc", + "code":"110", + "des":"This API is used to query a list of batch or real-time jobs. A maximum of 1000 jobs can be returned for each query.URI formatGET /v1/{project_id}/jobs?jobType={jobType}&o", + "doc_type":"api", + "kw":"Viewing a Job List,APIs to Be Taken Offline,API Reference", + "title":"Viewing a Job List", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0087.html", + "product_code":"dgc", + "code":"111", + "des":"This API is used to view job details.URI formatGET /v1/{project_id}/jobs/{name}GET /v1/{project_id}/jobs/{name}Parameter descriptionURI parameterParameterMandatoryTypeDes", + "doc_type":"api", + "kw":"Viewing Job Details,APIs to Be Taken Offline,API Reference", + "title":"Viewing Job Details", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0088.html", + "product_code":"dgc", + "code":"112", + "des":"This API is used to export a job, including job definitions, job dependency scripts, and CDM job definitions.URI formatPOST /v1/{project_id}/jobs/{name}/exportPOST /v1/{p", + "doc_type":"api", + "kw":"Exporting a Job,APIs to Be Taken Offline,API Reference", + "title":"Exporting a Job", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0089.html", + "product_code":"dgc", + "code":"113", + "des":"This API is used to batch export jobs, including job dependency scripts and CDM job definitions.URI formatPOST /v1/{project_id}/jobs/batch-exportPOST /v1/{project_id}/job", + "doc_type":"api", + "kw":"Batch Exporting Jobs,APIs to Be Taken Offline,API Reference", + "title":"Batch Exporting Jobs", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0090.html", + "product_code":"dgc", + "code":"114", + "des":"This API is used to import one or more job files from OBS to DLF.Before using this API, store job files in OBS buckets.URI formatPOST /v1/{project_id}/jobs/importPOST /", + "doc_type":"api", + "kw":"Importing a Job,APIs to Be Taken Offline,API Reference", + "title":"Importing a Job", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0091.html", + "product_code":"dgc", + "code":"115", + "des":"This API is used to execute a job immediately and check whether the job can be executed successfully.URI formatPOST /v1/{project_id}/jobs/{job_name}/run-immediatePOST /", + "doc_type":"api", + "kw":"Executing a Job Immediately,APIs to Be Taken Offline,API Reference", + "title":"Executing a Job Immediately", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0092.html", + "product_code":"dgc", + "code":"116", + "des":"This API is used to start a job.URI formatPOST /v1/{project_id}/jobs/{name}/startPOST /v1/{project_id}/jobs/{name}/startParameter descriptionURI parametersParameterMand", + "doc_type":"api", + "kw":"Starting a Job,APIs to Be Taken Offline,API Reference", + "title":"Starting a Job", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0093.html", + "product_code":"dgc", + "code":"117", + "des":"This API is used to view running status of a real-time job.URI formatGET /v1/{project_id}/jobs/{job_name}/statusGET /v1/{project_id}/jobs/{job_name}/statusParameter descr", + "doc_type":"api", + "kw":"Viewing Running Status of a Real-Time Job,APIs to Be Taken Offline,API Reference", + "title":"Viewing Running Status of a Real-Time Job", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0094.html", + "product_code":"dgc", + "code":"118", + "des":"This API is used to view a job instance list.A job instance is generated each time you run a batch job for which periodic scheduling or event-based scheduling is configur", + "doc_type":"api", + "kw":"Viewing a Job Instance List,APIs to Be Taken Offline,API Reference", + "title":"Viewing a Job Instance List", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0095.html", + "product_code":"dgc", + "code":"119", + "des":"This API is used to view job instance details, including the execution information about each node in a job instance.URI formatGET /v1/{project_id}/jobs/{job_name}/instan", + "doc_type":"api", + "kw":"Viewing Job Instance Details,APIs to Be Taken Offline,API Reference", + "title":"Viewing Job Instance Details", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0096.html", + "product_code":"dgc", + "code":"120", + "des":"This API is used to query details about asynchronous tasks.URI formatGET /v1/{project_id}/system-tasks/{task_id}GET /v1/{project_id}/system-tasks/{task_id}Parameter descr", + "doc_type":"api", + "kw":"Querying a System Task,APIs to Be Taken Offline,API Reference", + "title":"Querying a System Task", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0097.html", + "product_code":"dgc", + "code":"121", + "des":"This API is used to create a script. Currently, the following script types are supported: DLI SQL, Flink SQL, RDS SQL, Spark SQL, Hive SQL, DWS SQL, Shell, and Presto.URI", + "doc_type":"api", + "kw":"Creating a Script,APIs to Be Taken Offline,API Reference", + "title":"Creating a Script", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0098.html", + "product_code":"dgc", + "code":"122", + "des":"This API is used to modify the configuration items or script contents of a script.When modifying a script, specify the name of the script to be modified.The script name a", + "doc_type":"api", + "kw":"Modifying a Script,APIs to Be Taken Offline,API Reference", + "title":"Modifying a Script", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0099.html", + "product_code":"dgc", + "code":"123", + "des":"This API is used to query a script, including the script type and script content.URI formatGET /v1/{project_id}/script/{script_name}GET /v1/{project_id}/script/{script_na", + "doc_type":"api", + "kw":"Querying a Script,APIs to Be Taken Offline,API Reference", + "title":"Querying a Script", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0100.html", + "product_code":"dgc", + "code":"124", + "des":"This API is used to query the script list. A maximum of 1000 scripts can be returned for each query.URI formatGET /v1/{project_id}/scripts?offset={offset}&limit={limit}&s", + "doc_type":"api", + "kw":"Querying a Script List,APIs to Be Taken Offline,API Reference", + "title":"Querying a Script List", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0101.html", + "product_code":"dgc", + "code":"125", + "des":"This API is used to obtain the execution status and result of a script instance.During the query, specify the script name and script instance ID.URI formatGET /v1/{projec", + "doc_type":"api", + "kw":"Querying the Execution Result of a Script Instance,APIs to Be Taken Offline,API Reference", + "title":"Querying the Execution Result of a Script Instance", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0102.html", + "product_code":"dgc", + "code":"126", + "des":"This API is used to create a resource. Types of nodes, including DLI Spark, MRS Spark, and MRS MapReduce, can reference files such as JAR and properties through resources", + "doc_type":"api", + "kw":"Creating a Resource,APIs to Be Taken Offline,API Reference", + "title":"Creating a Resource", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0103.html", + "product_code":"dgc", + "code":"127", + "des":"This API is used to modify a specific resource. When modifying the resource, specify the resource ID.The resource type and directory cannot be modified.URI formatPUT /v1/", + "doc_type":"api", + "kw":"Modifying a Resource,APIs to Be Taken Offline,API Reference", + "title":"Modifying a Resource", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0104.html", + "product_code":"dgc", + "code":"128", + "des":"This API is used to query resource details. A resource contains various files such as JAR, ZIP, and properties files. A created resource can be used in job nodes such as ", + "doc_type":"api", + "kw":"Querying a Resource,APIs to Be Taken Offline,API Reference", + "title":"Querying a Resource", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0105.html", + "product_code":"dgc", + "code":"129", + "des":"This API is used to query a resource list. During the query, you can specify the page number and the maximum number of records on each page.URI formatGET /v1/{project_id}", + "doc_type":"api", + "kw":"Querying a Resource List,APIs to Be Taken Offline,API Reference", + "title":"Querying a Resource List", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0106.html", + "product_code":"dgc", + "code":"130", + "des":"This API is used to import one or more connection files from OBS to the Data Development module. Before using this API, store connection files in OBS buckets.URI formatPO", + "doc_type":"api", + "kw":"Importing a Connection,APIs to Be Taken Offline,API Reference", + "title":"Importing a Connection", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0309.html", + "product_code":"dgc", + "code":"131", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"api", + "kw":"Appendix", + "title":"Appendix", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0318.html", + "product_code":"dgc", + "code":"132", + "des":"Table 1 describes common request headers.Table 2 describes common response headers.", + "doc_type":"api", + "kw":"Common Message Headers,Appendix,API Reference", + "title":"Common Message Headers", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0310.html", + "product_code":"dgc", + "code":"133", + "des":"A status code consists of three digits. The first digit defines the class of a response. There are five values for the first digit:1xx: indication information, indicating", + "doc_type":"api", + "kw":"Status Codes,Appendix,API Reference", + "title":"Status Codes", + "githuburl":"" + }, + { + "uri":"ErrorCode.html", + "product_code":"", + "code":"134", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"", + "kw":"Error Codes", + "title":"Error Codes", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0311.html", + "product_code":"dgc", + "code":"135", + "des":"If an error occurs in API calling, no result is returned. Identify the error cause based on the error codes of each API. If an error occurs in API calling, HTTP status co", + "doc_type":"api", + "kw":"CDM Error Codes,Error Codes,API Reference", + "title":"CDM Error Codes", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0313.html", + "product_code":"dgc", + "code":"136", + "des":"The error response is in the following format:{\"error_code\":\"DLF.0100\",\"error_msg\":\"The job not found\"}Table 1 describes the error codes.", + "doc_type":"api", + "kw":"DLF Error Codes,Error Codes,API Reference", + "title":"DLF Error Codes", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0317.html", + "product_code":"dgc", + "code":"137", + "des":"The response messages of the job export API and connection export API are streams that need to be converted to files. For details, see the following sample code:", + "doc_type":"api", + "kw":"Parsing a Stream in a Response Message,Appendix,API Reference", + "title":"Parsing a Stream in a Response Message", + "githuburl":"" + }, + { + "uri":"dataartsstudio_02_0330.html", + "product_code":"", + "code":"138", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"", + "kw":"Change History,API Reference", + "title":"Change History", + "githuburl":"" + } +] \ No newline at end of file diff --git a/docs/dataartsstudio/api-ref/CLASS.TXT.json b/docs/dataartsstudio/api-ref/CLASS.TXT.json new file mode 100644 index 000000000..172ceb845 --- /dev/null +++ b/docs/dataartsstudio/api-ref/CLASS.TXT.json @@ -0,0 +1,1244 @@ +[ + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Before You Start", + "uri":"dataartsstudio_02_0001.html", + "doc_type":"api", + "p_code":"", + "code":"1" + }, + { + "desc":"Welcome to DataArts Studio. DataArts Studio is a one-stop operations platform that provides data lifecycle management and intelligent data management for enterprises' dig", + "product_code":"dataartsstudio", + "title":"Overview", + "uri":"dataartsstudio_02_0002.html", + "doc_type":"api", + "p_code":"1", + "code":"2" + }, + { + "desc":"DataArts Studio supports Representational State Transfer (REST) APIs, allowing you to call APIs using HTTPS. For details on API calling, see Calling APIs.", + "product_code":"dataartsstudio", + "title":"API Calling", + "uri":"dataartsstudio_02_0003.html", + "doc_type":"api", + "p_code":"1", + "code":"3" + }, + { + "desc":"DomainA domain is created upon successful registration. The domain has full access permissions for all of its cloud services and resources. It can be used to reset user p", + "product_code":"dataartsstudio", + "title":"Concepts", + "uri":"dataartsstudio_02_0006.html", + "doc_type":"api", + "p_code":"1", + "code":"4" + }, + { + "desc":"An endpoint is the request address for calling an API. Endpoints vary depending on services and regions.You can obtain endpoints from Regions and Endpoints.", + "product_code":"dataartsstudio", + "title":"Endpoints", + "uri":"dataartsstudio_02_0004.html", + "doc_type":"api", + "p_code":"1", + "code":"5" + }, + { + "desc":"You can obtain the project ID and account ID by performing the following steps:Register with and log in to the management console.Hover the cursor on the username in the ", + "product_code":"dataartsstudio", + "title":"Project ID and Account ID", + "uri":"dataartsstudio_02_0314.html", + "doc_type":"api", + "p_code":"1", + "code":"6" + }, + { + "desc":"To obtain a DataArts Studio instance ID and workspace ID, perform the following steps:On the DataArts Studio console, locate a workspace and click any module, such as Man", + "product_code":"dataartsstudio", + "title":"DataArts Studio Instance ID and Workspace ID", + "uri":"dataartsstudio_02_0350.html", + "doc_type":"api", + "p_code":"1", + "code":"7" + }, + { + "desc":"A job ID (jobid) is required for some URLs when an API is called. To obtain a job ID, perform the following operations:Log in to the DataArts Studio console. Locate an in", + "product_code":"dataartsstudio", + "title":"Data Development Job ID", + "uri":"dataartsstudio_02_0316.html", + "doc_type":"api", + "p_code":"1", + "code":"8" + }, + { + "desc":"The restrictions on DataArts Catalog APIs are as follows:CDM jobs carry large volumes of data, which increases the database load. You are advised to periodically delete u", + "product_code":"dataartsstudio", + "title":"Constraints", + "uri":"dataartsstudio_02_0005.html", + "doc_type":"api", + "p_code":"1", + "code":"9" + }, + { + "desc":"DataArts Studio provides self-developed APIs that comply with RESTful API design specifications. You can call those APIs to perform operations on DataArts Studio.", + "product_code":"dataartsstudio", + "title":"API Overview", + "uri":"dataartsstudio_02_0007.html", + "doc_type":"api", + "p_code":"", + "code":"10" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Calling APIs", + "uri":"dataartsstudio_02_0008.html", + "doc_type":"api", + "p_code":"", + "code":"11" + }, + { + "desc":"This section describes the structure of a REST API, and uses the IAM API for obtaining a user token as an example to demonstrate how to call an API. The obtained token ca", + "product_code":"dataartsstudio", + "title":"Making an API Request", + "uri":"dataartsstudio_02_0009.html", + "doc_type":"api", + "p_code":"11", + "code":"12" + }, + { + "desc":"Requests for calling an API can be authenticated using either of the following methods:Token-based authentication: Requests are authenticated using a token.AK/SK-based au", + "product_code":"dataartsstudio", + "title":"Authentication", + "uri":"dataartsstudio_02_0010.html", + "doc_type":"api", + "p_code":"11", + "code":"13" + }, + { + "desc":"After sending a request, you will receive a response, including a status code, response header, and response body.A status code is a group of digits, ranging from 1xx to ", + "product_code":"dataartsstudio", + "title":"Response", + "uri":"dataartsstudio_02_0011.html", + "doc_type":"api", + "p_code":"11", + "code":"14" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Application Cases", + "uri":"dataartsstudio_02_0012.html", + "doc_type":"api", + "p_code":"", + "code":"15" + }, + { + "desc":"This section describes how to use cURL to call CDM APIs to migrate data from a local MySQL database to DWS in the cloud.Obtaining a TokenCall the API to obtain the user t", + "product_code":"dataartsstudio", + "title":"Application Example", + "uri":"dataartsstudio_02_0013.html", + "doc_type":"api", + "p_code":"15", + "code":"16" + }, + { + "desc":"DataArts Studio helps enterprises quickly build an end-to-end intelligent data system that covers the entire process from data ingestion to data analytics. The system can", + "product_code":"dataartsstudio", + "title":"Example of Using Data Development APIs", + "uri":"dataartsstudio_02_0015.html", + "doc_type":"api", + "p_code":"15", + "code":"17" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"DataArts Migration APIs", + "uri":"dataartsstudio_02_0016.html", + "doc_type":"api", + "p_code":"", + "code":"18" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Cluster Management", + "uri":"topic_300000000_3.html", + "doc_type":"api", + "p_code":"18", + "code":"19" + }, + { + "desc":"This API is used to query cluster details.GET /v1.1/{project_id}/clusters/{cluster_id}Status code: 200Status code: 200okSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Querying Cluster Details", + "uri":"ShowClusterDetail.html", + "doc_type":"api", + "p_code":"19", + "code":"20" + }, + { + "desc":"This API is used to delete a cluster.DELETE /v1.1/{project_id}/clusters/{cluster_id}Status code: 202Status code: 202AcceptedSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Deleting a Cluster", + "uri":"DeleteCluster.html", + "doc_type":"api", + "p_code":"19", + "code":"21" + }, + { + "desc":"This API is used to restart a cluster.POST /v1.1/{project_id}/clusters/{cluster_id}/actionStatus code: 200Restarting a ClusterStatus code: 200okSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Restarting a Cluster", + "uri":"RestartCluster.html", + "doc_type":"api", + "p_code":"19", + "code":"22" + }, + { + "desc":"This API is used to start a cluster.POST /v1.1/{project_id}/clusters/{cluster_id}/actionStatus code: 200Starting a ClusterStatus code: 200okSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Starting a Cluster", + "uri":"StartCluster.html", + "doc_type":"api", + "p_code":"19", + "code":"23" + }, + { + "desc":"This API is used to stop a cluster.POST /v1.1/{project_id}/clusters/{cluster_id}/actionStatus code: 200Stopping a ClusterStatus code: 200okSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Stopping a Cluster", + "uri":"StopCluster.html", + "doc_type":"api", + "p_code":"19", + "code":"24" + }, + { + "desc":"This API is used to create a cluster.POST /v1.1/{project_id}/clustersStatus code: 202Status code: 202AcceptedSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Creating a Cluster", + "uri":"CreateCluster.html", + "doc_type":"api", + "p_code":"19", + "code":"25" + }, + { + "desc":"This API is used to query the cluster list.GET /v1.1/{project_id}/clustersStatus code: 200Status code: 200okSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Querying the Cluster List", + "uri":"ListClusters.html", + "doc_type":"api", + "p_code":"19", + "code":"26" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Job Management", + "uri":"topic_300000001_2.html", + "doc_type":"api", + "p_code":"18", + "code":"27" + }, + { + "desc":"This API is used to query jobs.GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}Status code: 200Status code: 200okSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Querying a Job", + "uri":"ShowJobs.html", + "doc_type":"api", + "p_code":"27", + "code":"28" + }, + { + "desc":"This API is used to delete a job.DELETE /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}Status code: 500Status code: 500An internal service error occurred. For", + "product_code":"dataartsstudio", + "title":"Deleting a Job", + "uri":"DeleteJob.html", + "doc_type":"api", + "p_code":"27", + "code":"29" + }, + { + "desc":"This API is used to modify a job.PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}Status code: 200Status code: 400Status code: 200okStatus code: 400Error co", + "product_code":"dataartsstudio", + "title":"Modifying a Job", + "uri":"UpdateJob.html", + "doc_type":"api", + "p_code":"27", + "code":"30" + }, + { + "desc":"This API is used to create and execute a job in a random cluster.POST /v1.1/{project_id}/clusters/jobStatus code: 200Status code: 200okSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Creating and Executing a Job in a Random Cluster", + "uri":"CreateAndStartRandomClusterJob.html", + "doc_type":"api", + "p_code":"27", + "code":"31" + }, + { + "desc":"This API is used to stop a job.PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}/stopStatus code: 200Status code: 200okSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Stopping a Job", + "uri":"StopJob.html", + "doc_type":"api", + "p_code":"27", + "code":"32" + }, + { + "desc":"This API is used to create a job in a specified cluster.POST /v1.1/{project_id}/clusters/{cluster_id}/cdm/jobStatus code: 200Status code: 400Status code: 200okStatus code", + "product_code":"dataartsstudio", + "title":"Creating a Job in a Specified Cluster", + "uri":"CreateJob.html", + "doc_type":"api", + "p_code":"27", + "code":"33" + }, + { + "desc":"This API is used to start a job.PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}/startStatus code: 200Status code: 200okSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Starting a Job", + "uri":"StartJob.html", + "doc_type":"api", + "p_code":"27", + "code":"34" + }, + { + "desc":"This API is used to query the job status.GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}/statusStatus code: 200Status code: 200okSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Querying Job Status", + "uri":"ShowJobStatus.html", + "doc_type":"api", + "p_code":"27", + "code":"35" + }, + { + "desc":"This API is used to query the job execution history.GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/submissionsStatus code: 200Status code: 200okSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Querying Job Execution History", + "uri":"ShowSubmissions.html", + "doc_type":"api", + "p_code":"27", + "code":"36" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Link Management", + "uri":"topic_300000002_2.html", + "doc_type":"api", + "p_code":"18", + "code":"37" + }, + { + "desc":"This API is used to create a link.POST /v1.1/{project_id}/clusters/{cluster_id}/cdm/linkStatus code: 200Status code: 400Status code: 500Status code: 200OKStatus code: 400", + "product_code":"dataartsstudio", + "title":"Creating a Link", + "uri":"CreateLink.html", + "doc_type":"api", + "p_code":"37", + "code":"38" + }, + { + "desc":"This API is used to query a link.GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/link/{link_name}Status code: 200Status code: 200OKSee Error Codes.", + "product_code":"dataartsstudio", + "title":"Querying a Link", + "uri":"ShowLink.html", + "doc_type":"api", + "p_code":"37", + "code":"39" + }, + { + "desc":"This API is used to delete a link.DELETE /v1.1/{project_id}/clusters/{cluster_id}/cdm/link/{link_name}Status code: 500Status code: 500An internal service error occurred. ", + "product_code":"dataartsstudio", + "title":"Deleting a Link", + "uri":"DeleteLink.html", + "doc_type":"api", + "p_code":"37", + "code":"40" + }, + { + "desc":"This API is used to modify a link.PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/link/{link_name}Status code: 200Status code: 500Status code: 200OKStatus code: 500An in", + "product_code":"dataartsstudio", + "title":"Modifying a Link", + "uri":"UpdateLink.html", + "doc_type":"api", + "p_code":"37", + "code":"41" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Public Data Structures", + "uri":"dataartsstudio_02_0260.html", + "doc_type":"api", + "p_code":"18", + "code":"42" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Link Parameter Description", + "uri":"dataartsstudio_02_0261.html", + "doc_type":"api", + "p_code":"42", + "code":"43" + }, + { + "desc":"By creating a JDBC link, you can extract data from or load data to the following relational databases:Data Warehouse ServiceRDS for MySQLRDS for PostgreSQLRDS for SQL Ser", + "product_code":"dataartsstudio", + "title":"Link to a Relational Database", + "uri":"dataartsstudio_02_0262.html", + "doc_type":"api", + "p_code":"43", + "code":"44" + }, + { + "desc":"By creating an OBS link, you can extract files from or load files to OBS. Files in CSV, JSON, and binary format are supported.", + "product_code":"dataartsstudio", + "title":"Link to OBS", + "uri":"dataartsstudio_02_0263.html", + "doc_type":"api", + "p_code":"43", + "code":"45" + }, + { + "desc":"By creating an HDFS link, you can extract files from or load files to MRS, FusionInsight HD, or Apache Hadoop. Files in CSV, Parquet, and binary formats are supported.", + "product_code":"dataartsstudio", + "title":"Link to HDFS", + "uri":"dataartsstudio_02_0266.html", + "doc_type":"api", + "p_code":"43", + "code":"46" + }, + { + "desc":"By creating an HBase link, you can extract data from or load data to HBase of MRS, FusionInsight HD, and Apache Hadoop.", + "product_code":"dataartsstudio", + "title":"Link to HBase", + "uri":"dataartsstudio_02_0267.html", + "doc_type":"api", + "p_code":"43", + "code":"47" + }, + { + "desc":"By creating a CloudTable link, you can extract data from or load data to CloudTable.", + "product_code":"dataartsstudio", + "title":"Link to CloudTable", + "uri":"dataartsstudio_02_0268.html", + "doc_type":"api", + "p_code":"43", + "code":"48" + }, + { + "desc":"By creating a Hive link, you can extract data from or load data to Hive of MRS.", + "product_code":"dataartsstudio", + "title":"Link to Hive", + "uri":"dataartsstudio_02_0269.html", + "doc_type":"api", + "p_code":"43", + "code":"49" + }, + { + "desc":"By creating an FTP or SFTP link, you are able to extract files from or load files to the FTP or SFTP server. Files in CSV, JSON, and binary format are supported.Parameter", + "product_code":"dataartsstudio", + "title":"Link to an FTP or SFTP Server", + "uri":"dataartsstudio_02_0270.html", + "doc_type":"api", + "p_code":"43", + "code":"50" + }, + { + "desc":"By creating a MongoDB link, you can extract data from or load data to MongoDB.", + "product_code":"dataartsstudio", + "title":"Link to MongoDB", + "uri":"dataartsstudio_02_0271.html", + "doc_type":"api", + "p_code":"43", + "code":"51" + }, + { + "desc":"By creating a Redis link, you can extract data from or load data to the Redis server. By creating a DCS link, you can load data to Data Cache Service (DCS), but not extra", + "product_code":"dataartsstudio", + "title":"Link to Redis/DCS (to Be Brought Offline)", + "uri":"dataartsstudio_02_0272.html", + "doc_type":"api", + "p_code":"43", + "code":"52" + }, + { + "desc":"By creating a Kafka link, you are able to access open source Kafka and migrate data from Kafka to other data sources as required. Currently, only data export from Kafka i", + "product_code":"dataartsstudio", + "title":"Link to Kafka", + "uri":"dataartsstudio_02_0274.html", + "doc_type":"api", + "p_code":"43", + "code":"53" + }, + { + "desc":"By creating an Elasticsearch link, you can extract data from or load data to the Elasticsearch server or Cloud Search Service.", + "product_code":"dataartsstudio", + "title":"Link to Elasticsearch/Cloud Search Service", + "uri":"dataartsstudio_02_0276.html", + "doc_type":"api", + "p_code":"43", + "code":"54" + }, + { + "desc":"By creating a DLI link, you can import data to DLI. Currently, you cannot export data from DLI using CDM.", + "product_code":"dataartsstudio", + "title":"Link to DLI", + "uri":"dataartsstudio_02_0277.html", + "doc_type":"api", + "p_code":"43", + "code":"55" + }, + { + "desc":"By creating an OpenTSDB link, you can extract data from and load data to CloudTable OpenTSDB.", + "product_code":"dataartsstudio", + "title":"Link to CloudTable OpenTSDB", + "uri":"dataartsstudio_02_0278.html", + "doc_type":"api", + "p_code":"43", + "code":"56" + }, + { + "desc":"By creating a DMS Kafka link, you can connect to Kafka Basic or Kafka Platinum on DMS. Currently, you can only export data from DMS Kafka to Cloud Search Service.", + "product_code":"dataartsstudio", + "title":"Link to DMS Kafka", + "uri":"dataartsstudio_02_0280.html", + "doc_type":"api", + "p_code":"43", + "code":"57" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Source Job Parameters", + "uri":"dataartsstudio_02_0281.html", + "doc_type":"api", + "p_code":"42", + "code":"58" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"From a Relational Database", + "uri":"dataartsstudio_02_0282.html", + "doc_type":"api", + "p_code":"58", + "code":"59" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"From Object Storage", + "uri":"dataartsstudio_02_0283.html", + "doc_type":"api", + "p_code":"58", + "code":"60" + }, + { + "desc":"HDFS job parameter descriptionParameterMandatoryTypeDescriptionfromJobConfig.inputDirectoryYesStringPath for storing data to be extracted. For example, /data_dir.fromJobC", + "product_code":"dataartsstudio", + "title":"From HDFS", + "uri":"dataartsstudio_02_0284.html", + "doc_type":"api", + "p_code":"58", + "code":"61" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"From Hive", + "uri":"dataartsstudio_02_0285.html", + "doc_type":"api", + "p_code":"58", + "code":"62" + }, + { + "desc":"HBase/CloudTable job parameter descriptionParameterMandatoryTypeDescriptionfromJobConfig.tableYesStringName of the table from which data is extracted. For example, cdm.fr", + "product_code":"dataartsstudio", + "title":"From HBase/CloudTable", + "uri":"dataartsstudio_02_0286.html", + "doc_type":"api", + "p_code":"58", + "code":"63" + }, + { + "desc":"Source link job parameters of FTP and SFTP are the same. Table 1 describes the parameters.Source link job parameters of file systemsParameterMandatoryTypeDescriptionfromJ", + "product_code":"dataartsstudio", + "title":"From FTP/SFTP/NAS (to Be Brought Offline)/SFS (to Be Brought Offline)", + "uri":"dataartsstudio_02_0287.html", + "doc_type":"api", + "p_code":"58", + "code":"64" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"From HTTP/HTTPS", + "uri":"dataartsstudio_02_0288.html", + "doc_type":"api", + "p_code":"58", + "code":"65" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"From MongoDB/DDS", + "uri":"dataartsstudio_02_0289.html", + "doc_type":"api", + "p_code":"58", + "code":"66" + }, + { + "desc":"Redis job parameter descriptionParameterMandatoryTypeDescriptionfromJobConfig.isBatchMigrationNoBooleanWhether to migrate all data in the databasefromJobConfig.keyPrefixY", + "product_code":"dataartsstudio", + "title":"From Redis/DCS (to Be Brought Offline)", + "uri":"dataartsstudio_02_0290.html", + "doc_type":"api", + "p_code":"58", + "code":"67" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"From DIS", + "uri":"dataartsstudio_02_0291.html", + "doc_type":"api", + "p_code":"58", + "code":"68" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"From Kafka", + "uri":"dataartsstudio_02_0292.html", + "doc_type":"api", + "p_code":"58", + "code":"69" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"From Elasticsearch/Cloud Search Service", + "uri":"dataartsstudio_02_0293.html", + "doc_type":"api", + "p_code":"58", + "code":"70" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"From OpenTSDB", + "uri":"dataartsstudio_02_0294.html", + "doc_type":"api", + "p_code":"58", + "code":"71" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Destination Job Parameters", + "uri":"dataartsstudio_02_0295.html", + "doc_type":"api", + "p_code":"42", + "code":"72" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"To a Relational Database", + "uri":"dataartsstudio_02_0296.html", + "doc_type":"api", + "p_code":"72", + "code":"73" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"To OBS", + "uri":"dataartsstudio_02_0297.html", + "doc_type":"api", + "p_code":"72", + "code":"74" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"To HDFS", + "uri":"dataartsstudio_02_0298.html", + "doc_type":"api", + "p_code":"72", + "code":"75" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"To Hive", + "uri":"dataartsstudio_02_0299.html", + "doc_type":"api", + "p_code":"72", + "code":"76" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"To HBase/CloudTable", + "uri":"dataartsstudio_02_0300.html", + "doc_type":"api", + "p_code":"72", + "code":"77" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"To FTP/SFTP/NAS (to Be Brought Offline)/SFS (to Be Brought Offline)", + "uri":"dataartsstudio_02_0301.html", + "doc_type":"api", + "p_code":"72", + "code":"78" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"To DDS", + "uri":"dataartsstudio_02_0302.html", + "doc_type":"api", + "p_code":"72", + "code":"79" + }, + { + "desc":"Parameter descriptionParameterMandatoryTypeDescriptiontoJobConfig.isBatchMigrationNoBooleanWhether to migrate all data in the databasetoJobConfig.shouldClearDatabaseNoBoo", + "product_code":"dataartsstudio", + "title":"To DCS", + "uri":"dataartsstudio_02_0303.html", + "doc_type":"api", + "p_code":"72", + "code":"80" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"To Elasticsearch/Cloud Search Service", + "uri":"dataartsstudio_02_0304.html", + "doc_type":"api", + "p_code":"72", + "code":"81" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"To DLI", + "uri":"dataartsstudio_02_0305.html", + "doc_type":"api", + "p_code":"72", + "code":"82" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"To DIS", + "uri":"dataartsstudio_02_0306.html", + "doc_type":"api", + "p_code":"72", + "code":"83" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"To OpenTSDB", + "uri":"dataartsstudio_02_0307.html", + "doc_type":"api", + "p_code":"72", + "code":"84" + }, + { + "desc":"When Creating a Job in a Specified Cluster or Creating and Executing a Job in a Random Cluster, the driver-config-values parameter specifies the job configuration, which ", + "product_code":"dataartsstudio", + "title":"Job Parameter Description", + "uri":"dataartsstudio_02_0308.html", + "doc_type":"api", + "p_code":"42", + "code":"85" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"DataArts Factory APIs", + "uri":"dataartsstudio_02_0127.html", + "doc_type":"api", + "p_code":"", + "code":"86" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Connection Management APIs", + "uri":"dataartsstudio_02_0049.html", + "doc_type":"api", + "p_code":"86", + "code":"87" + }, + { + "desc":"This API is used to create a connection. The supported connection types include DWS, DLI, Spark SQL, RDS, CloudTable, and Hive.URI formatPOST /v1/{project_id}/connections", + "product_code":"dataartsstudio", + "title":"Creating a Connection", + "uri":"dataartsstudio_02_0050.html", + "doc_type":"api", + "p_code":"87", + "code":"88" + }, + { + "desc":"This API is used to query a connection list.URI formatGET /v1/{project_id}/connections?offset={offset}&limit={limit}&connectionName={connectionName}GET /v1/{project_id}/c", + "product_code":"dataartsstudio", + "title":"Querying a Connection List", + "uri":"dataartsstudio_02_0051.html", + "doc_type":"api", + "p_code":"87", + "code":"89" + }, + { + "desc":"This API is used to query configuration details of a specific connection.URI formatGET /v1/{project_id}/connections/{connection_name}GET /v1/{project_id}/connections/{con", + "product_code":"dataartsstudio", + "title":"Viewing Connection Details", + "uri":"dataartsstudio_02_0052.html", + "doc_type":"api", + "p_code":"87", + "code":"90" + }, + { + "desc":"This API is used to edit a connection.URI formatPUT /v1/{project_id}/connections/{connection_name}?ischeck=truePUT /v1/{project_id}/connections/{connection_name}?ischeck=", + "product_code":"dataartsstudio", + "title":"Editing a Connection", + "uri":"dataartsstudio_02_0053.html", + "doc_type":"api", + "p_code":"87", + "code":"91" + }, + { + "desc":"This API is used to delete a connection.URI formatDELETE /v1/{project_id}/connections/{connection_name}DELETE /v1/{project_id}/connections/{connection_name}Parameter desc", + "product_code":"dataartsstudio", + "title":"Deleting a Connection", + "uri":"dataartsstudio_02_0054.html", + "doc_type":"api", + "p_code":"87", + "code":"92" + }, + { + "desc":"This API is used to export all connection information that is compressed in ZIP format.URI formatPOST /v1/{project_id}/connections/exportPOST /v1/{project_id}/connections", + "product_code":"dataartsstudio", + "title":"Exporting a Connection", + "uri":"dataartsstudio_02_0055.html", + "doc_type":"api", + "p_code":"87", + "code":"93" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Script Development APIs", + "uri":"dataartsstudio_02_0056.html", + "doc_type":"api", + "p_code":"86", + "code":"94" + }, + { + "desc":"This API is used to delete a specific script.URI formatDELETE /v1/{project_id}/scripts/{script_name}DELETE /v1/{project_id}/scripts/{script_name}Parameter descriptionURI ", + "product_code":"dataartsstudio", + "title":"Deleting a Script", + "uri":"dataartsstudio_02_0057.html", + "doc_type":"api", + "p_code":"94", + "code":"95" + }, + { + "desc":"This API is used to execute a specific script, which can be a DWS SQL, DLI SQL, RDS SQL, Flink SQL, Hive SQL, Presto SQL, or Spark SQL script. A script instance is genera", + "product_code":"dataartsstudio", + "title":"Executing a Script", + "uri":"dataartsstudio_02_0058.html", + "doc_type":"api", + "p_code":"94", + "code":"96" + }, + { + "desc":"This API is used to stop executing a script instance.URI formatPOST /v1/{project_id}/scripts/{script_name}/instances/{instance_id}/stopPOST /v1/{project_id}/scripts/{scri", + "product_code":"dataartsstudio", + "title":"Stopping Executing a Script Instance", + "uri":"dataartsstudio_02_0059.html", + "doc_type":"api", + "p_code":"94", + "code":"97" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Resource Management APIs", + "uri":"dataartsstudio_02_0060.html", + "doc_type":"api", + "p_code":"86", + "code":"98" + }, + { + "desc":"This API is used to delete a resource.URI formatDELETE /v1/{project_id}/resources/{resource_id}DELETE /v1/{project_id}/resources/{resource_id}Parameter descriptionURI par", + "product_code":"dataartsstudio", + "title":"Deleting a Resource", + "uri":"dataartsstudio_02_0061.html", + "doc_type":"api", + "p_code":"98", + "code":"99" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Job Development APIs", + "uri":"dataartsstudio_02_0062.html", + "doc_type":"api", + "p_code":"86", + "code":"100" + }, + { + "desc":"This API is used to check whether there are jobs and scripts in the job file to be imported from OBS to DLF.URI formatPOST /v1/{project_id}/jobs/check-filePOST /v1/{proje", + "product_code":"dataartsstudio", + "title":"Viewing a Job File", + "uri":"dataartsstudio_02_0063.html", + "doc_type":"api", + "p_code":"100", + "code":"101" + }, + { + "desc":"This API is used to stop a job.URI formatPOST /v1/{project_id}/jobs/{job_name}/stopPOST /v1/{project_id}/jobs/{job_name}/stopParameter descriptionURI parametersParamete", + "product_code":"dataartsstudio", + "title":"Stopping a Job", + "uri":"dataartsstudio_02_0064.html", + "doc_type":"api", + "p_code":"100", + "code":"102" + }, + { + "desc":"This API is used to delete a job.URI formatDELETE /v1/{project_id}/jobs/{job_name}DELETE /v1/{project_id}/jobs/{job_name}Parameter descriptionURI parametersParameterManda", + "product_code":"dataartsstudio", + "title":"Deleting a Job", + "uri":"dataartsstudio_02_0065.html", + "doc_type":"api", + "p_code":"100", + "code":"103" + }, + { + "desc":"This API is used to stop a specific job instance. A job instance can be stopped only when it is in the running state.URI formatPOST /v1/{project_id}/jobs/{job_name}/insta", + "product_code":"dataartsstudio", + "title":"Stopping a Job Instance", + "uri":"dataartsstudio_02_0066.html", + "doc_type":"api", + "p_code":"100", + "code":"104" + }, + { + "desc":"This API is used to restart a specific job instance. A job instance can be restarted only when it is in the successful, failed, or canceled state.URI formatPOST /v1/{proj", + "product_code":"dataartsstudio", + "title":"Restarting a Job Instance", + "uri":"dataartsstudio_02_0067.html", + "doc_type":"api", + "p_code":"100", + "code":"105" + }, + { + "desc":"Table 1 describes common request headers.Table 2 describes common response headers.", + "product_code":"dataartsstudio", + "title":"Data Structure", + "uri":"dataartsstudio_02_0082.html", + "doc_type":"api", + "p_code":"86", + "code":"106" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"APIs to Be Taken Offline", + "uri":"dataartsstudio_02_0083.html", + "doc_type":"api", + "p_code":"86", + "code":"107" + }, + { + "desc":"This API is used to create a job. A job consists of one or more nodes, such as Hive SQL and CDM Job nodes. DLF supports two types of jobs: batch jobs and real-time jobs.U", + "product_code":"dataartsstudio", + "title":"Creating a Job", + "uri":"dataartsstudio_02_0084.html", + "doc_type":"api", + "p_code":"107", + "code":"108" + }, + { + "desc":"This API is used to edit a job.URI formatPUT /v1/{project_id}/jobs/{job_name}PUT /v1/{project_id}/jobs/{job_name}Parameter descriptionURI parametersParameterMandatoryType", + "product_code":"dataartsstudio", + "title":"Editing a Job", + "uri":"dataartsstudio_02_0085.html", + "doc_type":"api", + "p_code":"107", + "code":"109" + }, + { + "desc":"This API is used to query a list of batch or real-time jobs. A maximum of 1000 jobs can be returned for each query.URI formatGET /v1/{project_id}/jobs?jobType={jobType}&o", + "product_code":"dataartsstudio", + "title":"Viewing a Job List", + "uri":"dataartsstudio_02_0086.html", + "doc_type":"api", + "p_code":"107", + "code":"110" + }, + { + "desc":"This API is used to view job details.URI formatGET /v1/{project_id}/jobs/{name}GET /v1/{project_id}/jobs/{name}Parameter descriptionURI parameterParameterMandatoryTypeDes", + "product_code":"dataartsstudio", + "title":"Viewing Job Details", + "uri":"dataartsstudio_02_0087.html", + "doc_type":"api", + "p_code":"107", + "code":"111" + }, + { + "desc":"This API is used to export a job, including job definitions, job dependency scripts, and CDM job definitions.URI formatPOST /v1/{project_id}/jobs/{name}/exportPOST /v1/{p", + "product_code":"dataartsstudio", + "title":"Exporting a Job", + "uri":"dataartsstudio_02_0088.html", + "doc_type":"api", + "p_code":"107", + "code":"112" + }, + { + "desc":"This API is used to batch export jobs, including job dependency scripts and CDM job definitions.URI formatPOST /v1/{project_id}/jobs/batch-exportPOST /v1/{project_id}/job", + "product_code":"dataartsstudio", + "title":"Batch Exporting Jobs", + "uri":"dataartsstudio_02_0089.html", + "doc_type":"api", + "p_code":"107", + "code":"113" + }, + { + "desc":"This API is used to import one or more job files from OBS to DLF.Before using this API, store job files in OBS buckets.URI formatPOST /v1/{project_id}/jobs/importPOST /", + "product_code":"dataartsstudio", + "title":"Importing a Job", + "uri":"dataartsstudio_02_0090.html", + "doc_type":"api", + "p_code":"107", + "code":"114" + }, + { + "desc":"This API is used to execute a job immediately and check whether the job can be executed successfully.URI formatPOST /v1/{project_id}/jobs/{job_name}/run-immediatePOST /", + "product_code":"dataartsstudio", + "title":"Executing a Job Immediately", + "uri":"dataartsstudio_02_0091.html", + "doc_type":"api", + "p_code":"107", + "code":"115" + }, + { + "desc":"This API is used to start a job.URI formatPOST /v1/{project_id}/jobs/{name}/startPOST /v1/{project_id}/jobs/{name}/startParameter descriptionURI parametersParameterMand", + "product_code":"dataartsstudio", + "title":"Starting a Job", + "uri":"dataartsstudio_02_0092.html", + "doc_type":"api", + "p_code":"107", + "code":"116" + }, + { + "desc":"This API is used to view running status of a real-time job.URI formatGET /v1/{project_id}/jobs/{job_name}/statusGET /v1/{project_id}/jobs/{job_name}/statusParameter descr", + "product_code":"dataartsstudio", + "title":"Viewing Running Status of a Real-Time Job", + "uri":"dataartsstudio_02_0093.html", + "doc_type":"api", + "p_code":"107", + "code":"117" + }, + { + "desc":"This API is used to view a job instance list.A job instance is generated each time you run a batch job for which periodic scheduling or event-based scheduling is configur", + "product_code":"dataartsstudio", + "title":"Viewing a Job Instance List", + "uri":"dataartsstudio_02_0094.html", + "doc_type":"api", + "p_code":"107", + "code":"118" + }, + { + "desc":"This API is used to view job instance details, including the execution information about each node in a job instance.URI formatGET /v1/{project_id}/jobs/{job_name}/instan", + "product_code":"dataartsstudio", + "title":"Viewing Job Instance Details", + "uri":"dataartsstudio_02_0095.html", + "doc_type":"api", + "p_code":"107", + "code":"119" + }, + { + "desc":"This API is used to query details about asynchronous tasks.URI formatGET /v1/{project_id}/system-tasks/{task_id}GET /v1/{project_id}/system-tasks/{task_id}Parameter descr", + "product_code":"dataartsstudio", + "title":"Querying a System Task", + "uri":"dataartsstudio_02_0096.html", + "doc_type":"api", + "p_code":"107", + "code":"120" + }, + { + "desc":"This API is used to create a script. Currently, the following script types are supported: DLI SQL, Flink SQL, RDS SQL, Spark SQL, Hive SQL, DWS SQL, Shell, and Presto.URI", + "product_code":"dataartsstudio", + "title":"Creating a Script", + "uri":"dataartsstudio_02_0097.html", + "doc_type":"api", + "p_code":"107", + "code":"121" + }, + { + "desc":"This API is used to modify the configuration items or script contents of a script.When modifying a script, specify the name of the script to be modified.The script name a", + "product_code":"dataartsstudio", + "title":"Modifying a Script", + "uri":"dataartsstudio_02_0098.html", + "doc_type":"api", + "p_code":"107", + "code":"122" + }, + { + "desc":"This API is used to query a script, including the script type and script content.URI formatGET /v1/{project_id}/script/{script_name}GET /v1/{project_id}/script/{script_na", + "product_code":"dataartsstudio", + "title":"Querying a Script", + "uri":"dataartsstudio_02_0099.html", + "doc_type":"api", + "p_code":"107", + "code":"123" + }, + { + "desc":"This API is used to query the script list. A maximum of 1000 scripts can be returned for each query.URI formatGET /v1/{project_id}/scripts?offset={offset}&limit={limit}&s", + "product_code":"dataartsstudio", + "title":"Querying a Script List", + "uri":"dataartsstudio_02_0100.html", + "doc_type":"api", + "p_code":"107", + "code":"124" + }, + { + "desc":"This API is used to obtain the execution status and result of a script instance.During the query, specify the script name and script instance ID.URI formatGET /v1/{projec", + "product_code":"dataartsstudio", + "title":"Querying the Execution Result of a Script Instance", + "uri":"dataartsstudio_02_0101.html", + "doc_type":"api", + "p_code":"107", + "code":"125" + }, + { + "desc":"This API is used to create a resource. Types of nodes, including DLI Spark, MRS Spark, and MRS MapReduce, can reference files such as JAR and properties through resources", + "product_code":"dataartsstudio", + "title":"Creating a Resource", + "uri":"dataartsstudio_02_0102.html", + "doc_type":"api", + "p_code":"107", + "code":"126" + }, + { + "desc":"This API is used to modify a specific resource. When modifying the resource, specify the resource ID.The resource type and directory cannot be modified.URI formatPUT /v1/", + "product_code":"dataartsstudio", + "title":"Modifying a Resource", + "uri":"dataartsstudio_02_0103.html", + "doc_type":"api", + "p_code":"107", + "code":"127" + }, + { + "desc":"This API is used to query resource details. A resource contains various files such as JAR, ZIP, and properties files. A created resource can be used in job nodes such as ", + "product_code":"dataartsstudio", + "title":"Querying a Resource", + "uri":"dataartsstudio_02_0104.html", + "doc_type":"api", + "p_code":"107", + "code":"128" + }, + { + "desc":"This API is used to query a resource list. During the query, you can specify the page number and the maximum number of records on each page.URI formatGET /v1/{project_id}", + "product_code":"dataartsstudio", + "title":"Querying a Resource List", + "uri":"dataartsstudio_02_0105.html", + "doc_type":"api", + "p_code":"107", + "code":"129" + }, + { + "desc":"This API is used to import one or more connection files from OBS to the Data Development module. Before using this API, store connection files in OBS buckets.URI formatPO", + "product_code":"dataartsstudio", + "title":"Importing a Connection", + "uri":"dataartsstudio_02_0106.html", + "doc_type":"api", + "p_code":"107", + "code":"130" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Appendix", + "uri":"dataartsstudio_02_0309.html", + "doc_type":"api", + "p_code":"", + "code":"131" + }, + { + "desc":"Table 1 describes common request headers.Table 2 describes common response headers.", + "product_code":"dataartsstudio", + "title":"Common Message Headers", + "uri":"dataartsstudio_02_0318.html", + "doc_type":"api", + "p_code":"131", + "code":"132" + }, + { + "desc":"A status code consists of three digits. The first digit defines the class of a response. There are five values for the first digit:1xx: indication information, indicating", + "product_code":"dataartsstudio", + "title":"Status Codes", + "uri":"dataartsstudio_02_0310.html", + "doc_type":"api", + "p_code":"131", + "code":"133" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Error Codes", + "uri":"ErrorCode.html", + "doc_type":"api", + "p_code":"131", + "code":"134" + }, + { + "desc":"If an error occurs in API calling, no result is returned. Identify the error cause based on the error codes of each API. If an error occurs in API calling, HTTP status co", + "product_code":"dataartsstudio", + "title":"CDM Error Codes", + "uri":"dataartsstudio_02_0311.html", + "doc_type":"api", + "p_code":"134", + "code":"135" + }, + { + "desc":"The error response is in the following format:{\"error_code\":\"DLF.0100\",\"error_msg\":\"The job not found\"}Table 1 describes the error codes.", + "product_code":"dataartsstudio", + "title":"DLF Error Codes", + "uri":"dataartsstudio_02_0313.html", + "doc_type":"api", + "p_code":"134", + "code":"136" + }, + { + "desc":"The response messages of the job export API and connection export API are streams that need to be converted to files. For details, see the following sample code:", + "product_code":"dataartsstudio", + "title":"Parsing a Stream in a Response Message", + "uri":"dataartsstudio_02_0317.html", + "doc_type":"api", + "p_code":"131", + "code":"137" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dataartsstudio", + "title":"Change History", + "uri":"dataartsstudio_02_0330.html", + "doc_type":"api", + "p_code":"", + "code":"138" + } +] \ No newline at end of file diff --git a/docs/dataartsstudio/api-ref/CreateAndStartRandomClusterJob.html b/docs/dataartsstudio/api-ref/CreateAndStartRandomClusterJob.html new file mode 100644 index 000000000..9115eb749 --- /dev/null +++ b/docs/dataartsstudio/api-ref/CreateAndStartRandomClusterJob.html @@ -0,0 +1,882 @@ + + +

Creating and Executing a Job in a Random Cluster

+

Function

This API is used to create and execute a job in a random cluster.

+
+

URI

POST /v1.1/{project_id}/clusters/job

+ +
+ + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+

X-Language

+

Yes

+

String

+

Request language

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 3 Request body parameters

Parameter

+

Mandatory

+

Type

+

Description

+

jobs

+

Yes

+

Array of Job objects

+

Job list. For details, see the descriptions of jobs parameters.

+

clusters

+

Yes

+

Array of strings

+

IDs of CDM clusters. The system selects a random cluster in running state from the specified clusters and creates and executes a migration job in the cluster.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 Job

Parameter

+

Mandatory

+

Type

+

Description

+

job_type

+

No

+

String

+

Job type

+

  • NORMAL_JOB: table/file migration

    +

  • BATCH_JOB: entire DB migration

    +

  • SCENARIO_JOB: scenario migration

    +
+

Enumeration values:

+

  • NORMAL_JOB

    +

  • BATCH_JOB

    +

  • SCENARIO_JOB

    +
+

from-connector-name

+

Yes

+

String

+

Source link type

+

to-config-values

+

Yes

+

ConfigValues object

+

Destination link parameter configuration

+

to-link-name

+

Yes

+

String

+

Destination link name

+

driver-config-values

+

Yes

+

ConfigValues object

+

Job parameter configuration

+

from-config-values

+

Yes

+

ConfigValues object

+

Source link parameter configuration

+

to-connector-name

+

No

+

String

+

Destination link type

+

name

+

No

+

String

+

Job name, which contains 1 to 240 characters

+

Minimum: 1

+

Maximum: 240

+

from-link-name

+

No

+

String

+

Source link name

+

creation-user

+

No

+

String

+

User who created the job

+

creation-date

+

No

+

Long

+

Time when the job was created, accurate to millisecond

+

update-date

+

No

+

Long

+

Time when the job was last updated, accurate to millisecond

+

is_incre_job

+

No

+

Boolean

+

Incremental or not

+

flag

+

No

+

Integer

+

Flag

+

files_read

+

No

+

Integer

+

Number of read files

+

update-user

+

No

+

String

+

User who last updated the job

+

external_id

+

No

+

String

+

External ID

+

type

+

No

+

String

+

Task type

+

execute_start_date

+

No

+

Long

+

Execution start date

+

delete_rows

+

No

+

Integer

+

Number of deleted rows

+

enabled

+

No

+

Boolean

+

Whether to activate the link

+

bytes_written

+

No

+

Long

+

Number of written bytes

+

id

+

No

+

Integer

+

Job ID

+

is_use_sql

+

No

+

Boolean

+

Whether to use SQL statements

+

update_rows

+

No

+

Integer

+

Number of updated rows

+

group_name

+

No

+

String

+

Group name

+

bytes_read

+

No

+

Long

+

Number of read bytes

+

execute_update_date

+

No

+

Long

+

Execution update date

+

write_rows

+

No

+

Integer

+

Number of written rows

+

rows_written

+

No

+

Integer

+

Number of written rows

+

rows_read

+

No

+

Long

+

Number of rows that are read

+

files_written

+

No

+

Integer

+

Number of written files

+

is_incrementing

+

No

+

Boolean

+

Incremental or not

+

execute_create_date

+

No

+

Long

+

Execution creation date

+

status

+

No

+

String

+

Job execution status

+

  • BOOTING: The job is starting.

    +

  • RUNNING: The job is running.

    +

  • SUCCEEDED: The job was successfully executed.

    +

  • FAILED: The job execution failed.

    +

  • NEW: The job was not executed.

    +
+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 5 ConfigValues

Parameter

+

Mandatory

+

Type

+

Description

+

configs

+

Yes

+

Array of configs objects

+

The data structures of source link parameters, destination link parameters, and job parameters are the same. However, the inputs parameter varies. For details, see the descriptions of configs parameters.

+

extended-configs

+

No

+

extended-configs object

+

Extended configuration. For details, see the descriptions of extended-configs parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 6 configs

Parameter

+

Mandatory

+

Type

+

Description

+

inputs

+

Yes

+

Array of Input objects

+

Input parameter list. Each element in the list is in name,value format. For details, see the descriptions of inputs parameters. In the from-config-values data structure, the value of this parameter varies with the source link type. For details, see section "Source Job Parameters" in the Cloud Data Migration User Guide. In the to-cofig-values data structure, the value of this parameter varies with the destination link type. For details, see section "Destination Job Parameters" in the Cloud Data Migration User Guide. For details about the inputs parameter in the driver-config-values data structure, see the job parameter descriptions.

+

name

+

Yes

+

String

+

Configuration name. The value is fromJobConfig for a source job, toJobConfig for a destination job, and linkConfig for a link.

+

id

+

No

+

Integer

+

Configuration ID

+

type

+

No

+

String

+

Configuration type

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 7 Input

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Parameter name

+

value

+

Yes

+

String

+

Parameter value

+

type

+

No

+

String

+

Value type

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 8 extended-configs

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

No

+

String

+

Name

+

value

+

No

+

String

+

Value

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + +
Table 9 Response body parameters

Parameter

+

Type

+

Description

+

submissions

+

Array of StartJobSubmission objects

+

Job running information. For details, see the descriptions of submission parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 10 StartJobSubmission

Parameter

+

Type

+

Description

+

isIncrementing

+

Boolean

+

Whether the job migrates incremental data

+

delete_rows

+

Integer

+

Number of deleted rows

+

update_rows

+

Integer

+

Number of updated rows

+

write_rows

+

Integer

+

Number of written rows

+

submission-id

+

Integer

+

ID of the submitted job

+

job-name

+

String

+

Job name

+

creation-user

+

String

+

User who created the job

+

creation-date

+

Long

+

Job creation time, accurate to millisecond

+

execute-date

+

Long

+

Job execution time

+

progress

+

Float

+

Job progress. If a job fails, the value is -1. Otherwise, the value ranges from 0 to 100.

+

status

+

String

+

Job status

+

  • BOOTING: The job is starting.

    +

  • FAILURE_ON_SUBMIT: The job failed to be submitted.

    +

  • RUNNING: The job is running.

    +

  • SUCCEEDED: The job was successfully executed.

    +

  • FAILED: The job execution failed.

    +

  • UNKNOWN: The job status is unknown.

    +

  • NEVER_EXECUTED: The job was not executed.

    +
+

isStopingIncrement

+

String

+

Whether to stop incremental data migration

+

is-execute-auto

+

Boolean

+

Whether to execute the job as scheduled

+

last-update-date

+

Long

+

Time when the job was last updated

+

last-udpate-user

+

String

+

User who last updated the job status

+

isDeleteJob

+

Boolean

+

Whether to delete the job after it is executed

+
+
+
+

Example Requests

POST /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/job
+
+{
+  "jobs" : [ {
+    "job_type" : "NORMAL_JOB",
+    "from-connector-name" : "elasticsearch-connector",
+    "to-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "toJobConfig.streamName",
+          "value" : "dis-lkGm"
+        }, {
+          "name" : "toJobConfig.separator",
+          "value" : "|"
+        }, {
+          "name" : "toJobConfig.columnList",
+          "value" : "1&2&3"
+        } ],
+        "name" : "toJobConfig"
+      } ]
+    },
+    "to-link-name" : "dis",
+    "driver-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "throttlingConfig.numExtractors",
+          "value" : "1"
+        }, {
+          "name" : "throttlingConfig.submitToCluster",
+          "value" : "false"
+        }, {
+          "name" : "throttlingConfig.numLoaders",
+          "value" : "1"
+        }, {
+          "name" : "throttlingConfig.recordDirtyData",
+          "value" : "false"
+        } ],
+        "name" : "throttlingConfig"
+      }, {
+        "inputs" : { },
+        "name" : "jarConfig"
+      }, {
+        "inputs" : [ {
+          "name" : "schedulerConfig.isSchedulerJob",
+          "value" : "false"
+        }, {
+          "name" : "schedulerConfig.disposableType",
+          "value" : "NONE"
+        } ],
+        "name" : "schedulerConfig"
+      }, {
+        "inputs" : { },
+        "name" : "transformConfig"
+      }, {
+        "inputs" : [ {
+          "name" : "retryJobConfig.retryJobType",
+          "value" : "NONE"
+        } ],
+        "name" : "retryJobConfig"
+      } ]
+    },
+    "from-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "fromJobConfig.index",
+          "value" : "52est"
+        }, {
+          "name" : "fromJobConfig.type",
+          "value" : "est_array"
+        }, {
+          "name" : "fromJobConfig.columnList",
+          "value" : "array_f1_int:long&array_f2_text:string&array_f3_object:nested"
+        }, {
+          "name" : "fromJobConfig.splitNestedField",
+          "value" : "false"
+        } ],
+        "name" : "fromJobConfig"
+      } ]
+    },
+    "to-connector-name" : "dis-connector",
+    "name" : "es_css",
+    "from-link-name" : "css"
+  } ],
+  "clusters" : [ "b0791496-e111-4e75-b7ca-9277aeab9297", "c2db1191-eb6c-464a-a0d3-b434e6c6df26", "c2db1191-eb6c-464a-a0d3-b434e6c6df26" ]
+}
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "submissions" : [ {
+    "isIncrementing" : false,
+    "job-name" : "obs2obs-03",
+    "submisson-id" : 13,
+    "isStopingIncrement" : "",
+    "last-update-date" : 1635909057030,
+    "is-execute-auto" : false,
+    "delete_rows" : 0,
+    "write_rows" : 0,
+    "last-update-user" : "mwx5316849",
+    "isDeleteJob" : false,
+    "creation-user" : "mwx5316849",
+    "progress" : 0,
+    "creation-date" : 1635909057030,
+    "update_rows" : 0,
+    "status" : "PENDING"
+  } ]
+}
+
+

Status Codes

+
+ + + + + + + +

Status Code

+

Description

+

200

+

ok

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/CreateCluster.html b/docs/dataartsstudio/api-ref/CreateCluster.html new file mode 100644 index 000000000..d7ef3b5c8 --- /dev/null +++ b/docs/dataartsstudio/api-ref/CreateCluster.html @@ -0,0 +1,655 @@ + + +

Creating a Cluster

+

Function

This API is used to create a cluster.

+
+

URI

POST /v1.1/{project_id}/clusters

+ +
+ + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+

X-Language

+

Yes

+

String

+

Request language

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Request body parameters

Parameter

+

Mandatory

+

Type

+

Description

+

cluster

+

Yes

+

cluster object

+

Cluster object. For details, see the descriptions of cluster parameters.

+

auto_remind

+

No

+

Boolean

+

Whether to enable message notification. If you enable this function, you can configure a maximum of five mobile numbers or email addresses. You will be notified of table/file migration job failures and EIP exceptions by SMS message or email.

+

phone_num

+

No

+

String

+

Mobile number for receiving notifications

+

email

+

No

+

String

+

Email address for receiving notifications

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 cluster

Parameter

+

Mandatory

+

Type

+

Description

+

scheduleBootTime

+

No

+

String

+

Time for scheduled startup of a CDM cluster. The CDM cluster starts at this time every day.

+

isScheduleBootOff

+

No

+

Boolean

+

Whether to enable scheduled startup/shutdown. The scheduled startup/shutdown and auto shutdown functions cannot be enabled at the same time.

+

instances

+

No

+

Array of instance objects

+

Node list. For details, see the descriptions of instances parameters.

+

datastore

+

No

+

Datastore object

+

Cluster information. For details, see the descriptions of datastore parameters.

+

extended_properties

+

No

+

ExtendedProperties object

+

Extended attribute. For details, see parameters in the extended_properties field.

+

scheduleOffTime

+

No

+

String

+

Time for scheduled shutdown of a CDM cluster. The CDM cluster shuts down directly at this time every day without waiting for unfinished jobs to complete.

+

vpcId

+

No

+

String

+

VPC ID, which is used for configuring a network for the cluster.

+

name

+

No

+

String

+

Cluster name

+

sys_tags

+

No

+

Array of sys_tags objects

+

Enterprise project information. For details, see the descriptions of sys_tags parameters.

+

isAutoOff

+

No

+

Boolean

+

Whether to enable auto shutdown. The auto shutdown and scheduled startup/shutdown functions cannot be enabled at the same time. When auto shutdown is enabled, if no job is running in the cluster and no scheduled job is available, a cluster will be automatically shut down 15 minutes after it starts running, which reduces costs for you.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 5 instance

Parameter

+

Mandatory

+

Type

+

Description

+

availability_zone

+

Yes

+

String

+

AZ where a cluster is located

+

nics

+

Yes

+

Array of nics objects

+

NIC list. A maximum of two NICs are supported. For details, see the descriptions of nics parameters.

+

flavorRef

+

Yes

+

String

+

Instance flavor

+
  • a79fd5ae-1833-448a-88e8-3ea2b913e1f6: cdm.small with 2 vCPUs and 4 GB memory applicable to Proof of Concept (PoC) verification and development and tests
  • fb8fe666-6734-4b11-bc6c-43d11db3c745: cdm.medium with 4 vCPUs and 8 GB memory applicable to the migration of a single database table with fewer than 10 million pieces of data
  • 5ddb1071-c5d7-40e0-a874-8a032e81a697: cdm.large with 8 vCPUs and 16 GB memory applicable to the migration of a single database table with 10 million or more pieces of data
  • 6ddb1072-c5d7-40e0-a874-8a032e81a698: cdm.xlarge with 16 vCPUs and 32 GB memory applicable to the migration of terabytes of data requiring 10GE bandwidth
+

type

+

Yes

+

String

+

Node type. Currently, only cdm is available.

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 6 nics

Parameter

+

Mandatory

+

Type

+

Description

+

securityGroupId

+

Yes

+

String

+

Security group ID

+

net-id

+

Yes

+

String

+

Subnet ID

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 7 Datastore

Parameter

+

Mandatory

+

Type

+

Description

+

type

+

No

+

String

+

Type. Generally, the value is cdm.

+

version

+

No

+

String

+

Cluster version

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 8 ExtendedProperties

Parameter

+

Mandatory

+

Type

+

Description

+

workSpaceId

+

No

+

String

+

Workspace ID

+

resourceId

+

No

+

String

+

Resource ID

+

trial

+

No

+

String

+

Whether the cluster is a trial cluster

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 9 sys_tags

Parameter

+

Mandatory

+

Type

+

Description

+

value

+

Yes

+

String

+

Enterprise project ID

+

key

+

Yes

+

String

+

The value is fixed at _sys_enterprise_project_id.

+
+
+
+

Response Parameters

Status code: 202

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Table 10 Response body parameters

Parameter

+

Type

+

Description

+

name

+

String

+

Cluster name

+

id

+

String

+

Cluster ID

+

task

+

Task object

+

Task information

+

datastore

+

Datastore object

+

Cluster information

+

instances

+

Array of ClusterInstance objects

+

Cluster node information

+
+
+ +
+ + + + + + + + + + + + + +
Table 11 Task

Parameter

+

Type

+

Description

+

id

+

String

+

Task ID

+

name

+

String

+

Task name

+
+
+ +
+ + + + + + + + + + + + + +
Table 12 Datastore

Parameter

+

Type

+

Description

+

type

+

String

+

Type. Generally, the value is cdm.

+

version

+

String

+

Cluster version

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 13 ClusterInstance

Parameter

+

Type

+

Description

+

id

+

String

+

Node VM ID

+

name

+

String

+

Name of the VM on the node

+

type

+

String

+

Node type. Currently, only cdm is available.

+

shard_id

+

String

+

Shard ID

+
+
+
+

Example Requests

POST /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters
+
+{
+  "cluster" : {
+    "scheduleBootTime" : "",
+    "isScheduleBootOff" : false,
+    "instances" : [ {
+      "availability_zone" : "eu-de",
+      "nics" : [ {
+        "securityGroupId" : "c37852d2-2d12-41cb-af47-65c80e995c80",
+        "net-id" : "2d120298-6130-44d4-a438-454912fff901"
+      } ],
+      "flavorRef" : "5ddb1071-c5d7-40e0-a874-8a032e81a697",
+      "type" : "cdm"
+    } ],
+    "datastore" : {
+      "type" : "cdm",
+      "version" : "1.8.10"
+    },
+    "scheduleOffTime" : "",
+    "vpcId" : "67c06084-2212-4242-bcd4-d2144c2385a9",
+    "name" : "cdm-ab82",
+    "sys_tags" : [ {
+      "value" : "1ce45885-4033-40d2-bdde-d4dbaceb387d",
+      "key" : "_sys_enterprise_project_id"
+    } ],
+    "isAutoOff" : false
+  },
+  "auto_remind" : false,
+  "phone_num" : "",
+  "email" : ""
+}
+
+

Example Responses

Status code: 202

+

Accepted

+
{
+  "id" : "befc862c-9286-46a0-a1d6-300d98b63aad",
+  "name" : "cdm-4ef213",
+  "task" : {
+    "id" : "2c9080047f1b1185017f1ef6ad0500ac",
+    "name" : "rdsCreateBackupJob"
+  },
+  "datastore" : {
+    "type" : "cdm",
+    "version" : "2.9.1.100"
+  },
+  "instances" : [ {
+    "id" : "b2672e7d-2faf-423f-96bb-0664cd743cfd",
+    "name" : "cdm-4ef213-cdm-dn-1-1",
+    "type" : "cdm",
+    "shard_id" : "dn-1"
+  } ]
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

202

+

Accepted

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

Internal service error.

+

503

+

Service unavailable.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Cluster Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/CreateJob.html b/docs/dataartsstudio/api-ref/CreateJob.html new file mode 100644 index 000000000..68297bb69 --- /dev/null +++ b/docs/dataartsstudio/api-ref/CreateJob.html @@ -0,0 +1,803 @@ + + +

Creating a Job in a Specified Cluster

+

Function

This API is used to create a job in a specified cluster.

+
+

URI

POST /v1.1/{project_id}/clusters/{cluster_id}/cdm/job

+ +
+ + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+ +
+ + + + + + + + + + + +
Table 3 Request body parameters

Parameter

+

Mandatory

+

Type

+

Description

+

jobs

+

Yes

+

Array of Job objects

+

Job list. For details, see the descriptions of jobs parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 Job

Parameter

+

Mandatory

+

Type

+

Description

+

job_type

+

No

+

String

+

Job type

+
  • NORMAL_JOB: table/file migration
  • BATCH_JOB: entire DB migration
  • SCENARIO_JOB: scenario migration
+

Enumeration values:

+
  • NORMAL_JOB
  • BATCH_JOB
  • SCENARIO_JOB
+

from-connector-name

+

Yes

+

String

+

Source link type

+

to-config-values

+

Yes

+

ConfigValues object

+

Destination link parameter configuration

+

to-link-name

+

Yes

+

String

+

Destination link name

+

driver-config-values

+

Yes

+

ConfigValues object

+

Job parameter configuration

+

from-config-values

+

Yes

+

ConfigValues object

+

Source link parameter configuration

+

to-connector-name

+

No

+

String

+

Destination link type

+

name

+

No

+

String

+

Job name, which contains 1 to 240 characters

+

Minimum: 1

+

Maximum: 240

+

from-link-name

+

No

+

String

+

Source link name

+

creation-user

+

No

+

String

+

User who created the job

+

creation-date

+

No

+

Long

+

Time when the job was created, accurate to millisecond

+

update-date

+

No

+

Long

+

Time when the job was last updated, accurate to millisecond

+

is_incre_job

+

No

+

Boolean

+

Incremental or not

+

flag

+

No

+

Integer

+

Flag

+

files_read

+

No

+

Integer

+

Number of read files

+

update-user

+

No

+

String

+

User who last updated the job

+

external_id

+

No

+

String

+

External ID

+

type

+

No

+

String

+

Task type

+

execute_start_date

+

No

+

Long

+

Execution start date

+

delete_rows

+

No

+

Integer

+

Number of deleted rows

+

enabled

+

No

+

Boolean

+

Whether to activate the link

+

bytes_written

+

No

+

Long

+

Number of written bytes

+

id

+

No

+

Integer

+

Job ID

+

is_use_sql

+

No

+

Boolean

+

Whether to use SQL statements

+

update_rows

+

No

+

Integer

+

Number of updated rows

+

group_name

+

No

+

String

+

Group name

+

bytes_read

+

No

+

Long

+

Number of read bytes

+

execute_update_date

+

No

+

Long

+

Execution update date

+

write_rows

+

No

+

Integer

+

Number of written rows

+

rows_written

+

No

+

Integer

+

Number of written rows

+

rows_read

+

No

+

Long

+

Number of rows that are read

+

files_written

+

No

+

Integer

+

Number of written files

+

is_incrementing

+

No

+

Boolean

+

Incremental or not

+

execute_create_date

+

No

+

Long

+

Execution creation date

+

status

+

No

+

String

+

Job execution status

+
  • BOOTING: The job is starting.
  • RUNNING: The job is running.
  • SUCCEEDED: The job was successfully executed.
  • FAILED: The job execution failed.
  • NEW: The job was not executed.
+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 5 ConfigValues

Parameter

+

Mandatory

+

Type

+

Description

+

configs

+

Yes

+

Array of configs objects

+

The data structures of source link parameters, destination link parameters, and job parameters are the same. However, the inputs parameter varies. For details, see the descriptions of configs parameters.

+

extended-configs

+

No

+

extended-configs object

+

Extended configuration. For details, see the descriptions of extended-configs parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 6 configs

Parameter

+

Mandatory

+

Type

+

Description

+

inputs

+

Yes

+

Array of Input objects

+

Input parameter list. Each element in the list is in name,value format. For details, see the descriptions of inputs parameters. In the from-config-values data structure, the value of this parameter varies with the source link type. For details, see section "Source Job Parameters" in the Cloud Data Migration User Guide. In the to-cofig-values data structure, the value of this parameter varies with the destination link type. For details, see section "Destination Job Parameters" in the Cloud Data Migration User Guide. For details about the inputs parameter in the driver-config-values data structure, see the job parameter descriptions.

+

name

+

Yes

+

String

+

Configuration name. The value is fromJobConfig for a source job, toJobConfig for a destination job, and linkConfig for a link.

+

id

+

No

+

Integer

+

Configuration ID

+

type

+

No

+

String

+

Configuration type

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 7 Input

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Parameter name

+

value

+

Yes

+

String

+

Parameter value

+

type

+

No

+

String

+

Value type

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 8 extended-configs

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

No

+

String

+

Name

+

value

+

No

+

String

+

Value

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + + + + + +
Table 9 Response body parameters

Parameter

+

Type

+

Description

+

name

+

String

+

Job name

+

validation-result

+

Array of JobValidationResult objects

+

Check result

+
  • If the modification fails, the failure cause is returned.
  • If the modification is successful, an empty list is returned.
+
+
+ +
+ + + + + + + + + + + + + +
Table 10 JobValidationResult

Parameter

+

Type

+

Description

+

message

+

String

+

Error message

+

status

+

String

+

ERROR,WARNING

+

Enumeration values:

+
  • ERROR
  • WARNING
+
+
+

Status code: 400

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 11 Response body parameters

Parameter

+

Type

+

Description

+

code

+

String

+

Return code

+

errCode

+

String

+

Error code

+

message

+

String

+

Error message

+

externalMessage

+

String

+

Additional information

+
+
+
+

Example Requests

POST /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/job
+
+{
+  "jobs" : [ {
+    "job_type" : "NORMAL_JOB",
+    "from-connector-name" : "elasticsearch-connector",
+    "to-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "toJobConfig.streamName",
+          "value" : "dis-lkGm"
+        }, {
+          "name" : "toJobConfig.separator",
+          "value" : "|"
+        }, {
+          "name" : "toJobConfig.columnList",
+          "value" : "1&2&3"
+        } ],
+        "name" : "toJobConfig"
+      } ]
+    },
+    "to-link-name" : "dis",
+    "driver-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "throttlingConfig.numExtractors",
+          "value" : "1"
+        }, {
+          "name" : "throttlingConfig.submitToCluster",
+          "value" : "false"
+        }, {
+          "name" : "throttlingConfig.numLoaders",
+          "value" : "1"
+        }, {
+          "name" : "throttlingConfig.recordDirtyData",
+          "value" : "false"
+        } ],
+        "name" : "throttlingConfig"
+      }, {
+        "inputs" : { },
+        "name" : "jarConfig"
+      }, {
+        "inputs" : [ {
+          "name" : "schedulerConfig.isSchedulerJob",
+          "value" : "false"
+        }, {
+          "name" : "schedulerConfig.disposableType",
+          "value" : "NONE"
+        } ],
+        "name" : "schedulerConfig"
+      }, {
+        "inputs" : { },
+        "name" : "transformConfig"
+      }, {
+        "inputs" : [ {
+          "name" : "retryJobConfig.retryJobType",
+          "value" : "NONE"
+        } ],
+        "name" : "retryJobConfig"
+      } ]
+    },
+    "from-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "fromJobConfig.index",
+          "value" : "52est"
+        }, {
+          "name" : "fromJobConfig.type",
+          "value" : "est_array"
+        }, {
+          "name" : "fromJobConfig.columnList",
+          "value" : "array_f1_int:long&array_f2_text:string&array_f3_object:nested"
+        }, {
+          "name" : "fromJobConfig.splitNestedField",
+          "value" : "false"
+        } ],
+        "name" : "fromJobConfig"
+      } ]
+    },
+    "to-connector-name" : "dis-connector",
+    "name" : "es_css",
+    "from-link-name" : "css"
+  } ]
+}
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "name" : "mysql2hive"
+}
+

Status code: 400

+

Request error

+
{
+  "code" : "Cdm.0104",
+  "errCode" : "Cdm.0104",
+  "message" : "Job name already exist or created by other.",
+  "externalMessage" : "Job name already exist or created by other."
+}
+
+

Status Codes

+
+ + + + + + + + + + +

Status Code

+

Description

+

200

+

ok

+

400

+

Request error

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/CreateLink.html b/docs/dataartsstudio/api-ref/CreateLink.html new file mode 100644 index 000000000..efd7f3dd9 --- /dev/null +++ b/docs/dataartsstudio/api-ref/CreateLink.html @@ -0,0 +1,639 @@ + + +

Creating a Link

+

Function

This API is used to create a link.

+
+

URI

POST /v1.1/{project_id}/clusters/{cluster_id}/cdm/link

+ +
+ + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+
+
+ +
+ + + + + + + + + + + +
Table 2 Query Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

validate

+

No

+

String

+

When the parameter is set to true, the API only validates whether the parameters are correctly configured, but does not create any link.

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 3 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+ +
+ + + + + + + + + + + +
Table 4 Request body parameters

Parameter

+

Mandatory

+

Type

+

Description

+

links

+

Yes

+

Array of links objects

+

Link list. For details, see the descriptions of links parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 5 links

Parameter

+

Mandatory

+

Type

+

Description

+

link-config-values

+

Yes

+

link-config-values object

+

Link parameters. For details, see the descriptions of link-config-values parameters.

+

creation-user

+

No

+

String

+

User who created the link

+

name

+

Yes

+

String

+

Link name

+

id

+

No

+

Integer

+

Link ID

+

creation-date

+

No

+

Long

+

Time when the link was created

+

connector-name

+

Yes

+

String

+

Connector name. The mappings between the connectors and links are as follows:generic-jdbc-connector: link to a relational databaseobs-connector: OBS linkhdfs-connector: HDFS linkhbase-connector: HBase or CloudTable linkhive-connector: Hive linkftp-connector/sftp-connector: link to an FTP or SFTP servermongodb-connector: MongoDB linkredis-connector: Redis or DCS linkkafka-connector: Kafka linkdis-connector: DIS linkelasticsearch-connector: Elasticsearch or CSS linkdli-connector: DLI linkhttp-connector: HTTP or HTTPS link. This type of link does not have parameters.dms-kafka-connector: DMS Kafka link

+

update-date

+

No

+

Long

+

Time when the link was updated

+

enabled

+

No

+

Boolean

+

Whether to activate the link. The default value is true.

+

update-user

+

No

+

String

+

User who updated the link

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 6 link-config-values

Parameter

+

Mandatory

+

Type

+

Description

+

configs

+

Yes

+

Array of configs objects

+

Data structure of link parameters. For details, see the descriptions of configs parameters.

+

extended-configs

+

No

+

extended-configs object

+

Extended configuration. For details, see the descriptions of extended-configs parameters.

+

validators

+

No

+

Array of strings

+

Validator

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 7 configs

Parameter

+

Mandatory

+

Type

+

Description

+

inputs

+

Yes

+

Array of Input objects

+

Input parameter list. Each element in the list is in name,value format. For details, see the descriptions of inputs parameters. In the from-config-values data structure, the value of this parameter varies with the source link type. For details, see section "Source Job Parameters" in the Cloud Data Migration User Guide. In the to-cofig-values data structure, the value of this parameter varies with the destination link type. For details, see section "Destination Job Parameters" in the Cloud Data Migration User Guide. For details about the inputs parameter in the driver-config-values data structure, see the job parameter descriptions.

+

name

+

Yes

+

String

+

Configuration name. The value is fromJobConfig for a source job, toJobConfig for a destination job, and linkConfig for a link.

+

id

+

No

+

Integer

+

Configuration ID

+

type

+

No

+

String

+

Configuration type

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 8 Input

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Parameter name

+

value

+

Yes

+

String

+

Parameter value

+

type

+

No

+

String

+

Value type

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 9 extended-configs

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

No

+

String

+

Name

+

value

+

No

+

String

+

Value

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + + + + + +
Table 10 Response body parameters

Parameter

+

Type

+

Description

+

name

+

String

+

Link name

+

validation-result

+

Array of validationResult objects

+

Validation structure. If a link fails to be created, the failure cause is returned. For details, see the descriptions of validation-result parameters. If a link is successfully created, an empty list is returned.

+
+
+ +
+ + + + + + + + + +
Table 11 validationResult

Parameter

+

Type

+

Description

+

linkConfig

+

Array of validationLinkConfig objects

+

Validation result of link creation or update. For details, see the descriptions of validationLinkConfig parameters.

+
+
+ +
+ + + + + + + + + + + + + +
Table 12 validationLinkConfig

Parameter

+

Type

+

Description

+

message

+

String

+

Error message

+

status

+

String

+

ERROR,WARNING

+

Enumeration values:

+

  • ERROR

    +

  • WARNING

    +
+
+
+

Status code: 400

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 13 Response body parameters

Parameter

+

Type

+

Description

+

code

+

String

+

Return code

+

errCode

+

String

+

Error code

+

message

+

String

+

Error message

+

externalMessage

+

String

+

Additional information

+
+
+

Status code: 500

+ +
+ + + + + + + + + + + + + +
Table 14 Response body parameters

Parameter

+

Type

+

Description

+

message

+

String

+

Error message

+

status

+

String

+

ERROR,WARNING

+

Enumeration values:

+

  • ERROR

    +

  • WARNING

    +
+
+
+
+

Example Requests

POST /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/link
+
+{
+  "links" : [ {
+    "link-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "linkConfig.databaseType",
+          "value" : "MYSQL"
+        }, {
+          "name" : "linkConfig.host",
+          "value" : "100.94.8.163"
+        }, {
+          "name" : "linkConfig.port",
+          "value" : "3306"
+        }, {
+          "name" : "linkConfig.database",
+          "value" : "DB_name"
+        }, {
+          "name" : "linkConfig.username",
+          "value" : "username"
+        }, {
+          "name" : "linkConfig.password",
+          "value" : "DB_password"
+        }, {
+          "name" : "linkConfig.fetchSize",
+          "value" : "100000"
+        }, {
+          "name" : "linkConfig.usingNative",
+          "value" : false
+        } ],
+        "name" : "linkConfig"
+      } ]
+    },
+    "name" : "mysql_link",
+    "creation-date" : 1496654788622,
+    "connector-name" : "generic-jdbc-connector",
+    "update-date" : 1496654788622,
+    "enabled" : true
+  } ]
+}
+
+

Example Responses

Status code: 200

+

OK

+
{
+  "name" : "rdb_link",
+  "validation-result" : [ { } ]
+}
+

Status code: 400

+

Request error.

+
{
+  "code" : "Cdm.0315",
+  "errCode" : "Cdm.0315",
+  "message" : "Link name [ftp_link] already exist or created by other user.",
+  "externalMessage" : "Link name [ftp_link] already exist or created by other user."
+}
+

Status code: 500

+

An internal service error occurred. For details, see error codes.

+
{
+  "validation-result" : [ {
+    "linkConfig" : [ {
+      "message" : "Can't connect to the database with given credentials: The authentication type 12 is not supported. Check that you have configured the pg_hba.conf file to include the client's IP address or subnet, and that it is using an authentication scheme supported by the driver.",
+      "status" : "ERROR"
+    } ]
+  } ]
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

200

+

OK

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

An internal service error occurred. For details, see error codes.

+

503

+

Service unavailable.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Link Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/DeleteCluster.html b/docs/dataartsstudio/api-ref/DeleteCluster.html new file mode 100644 index 000000000..82be3d35b --- /dev/null +++ b/docs/dataartsstudio/api-ref/DeleteCluster.html @@ -0,0 +1,175 @@ + + +

Deleting a Cluster

+

Function

This API is used to delete a cluster.

+
+

URI

DELETE /v1.1/{project_id}/clusters/{cluster_id}

+ +
+ + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+ +
+ + + + + + + + + + + +
Table 3 Request body parameters

Parameter

+

Mandatory

+

Type

+

Description

+

keep_last_manual_backup

+

Yes

+

Integer

+

Number of backup log files. Retain the default value 0.

+

Default: 0

+
+
+
+

Response Parameters

Status code: 202

+ +
+ + + + + + + + + +
Table 4 Response body parameters

Parameter

+

Type

+

Description

+

jobId

+

String

+

Job ID

+
+
+
+

Example Requests

DELETE /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920
+
+{
+  "keep_last_manual_backup" : 0
+}
+
+

Example Responses

Status code: 202

+

Accepted

+
{
+  "jobId" : "ff8080815e55125a015e552eddba001a"
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

202

+

Accepted

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

An internal service error occurred. For details, see error codes.

+

503

+

Service unavailable.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Cluster Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/DeleteJob.html b/docs/dataartsstudio/api-ref/DeleteJob.html new file mode 100644 index 000000000..875b7586c --- /dev/null +++ b/docs/dataartsstudio/api-ref/DeleteJob.html @@ -0,0 +1,159 @@ + + +

Deleting a Job

+

Function

This API is used to delete a job.

+
+

URI

DELETE /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+

job_name

+

Yes

+

String

+

Job name

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+
+

Response Parameters

Status code: 500

+ +
+ + + + + + + + + + + + + +
Table 3 Response body parameters

Parameter

+

Type

+

Description

+

errCode

+

String

+

Error code

+

externalMessage

+

String

+

Error message

+
+
+
+

Example Requests

DELETE /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/job/jdbc2hive
+
+

Example Responses

Status code: 500

+

An internal service error occurred. For details, see error codes.

+
{
+  "errCode" : "Cdm.0100",
+  "externalMessage" : "Job[jdbc2hive] doesn't exist."
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

200

+

OK

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

An internal service error occurred. For details, see error codes.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/DeleteLink.html b/docs/dataartsstudio/api-ref/DeleteLink.html new file mode 100644 index 000000000..78421bcc3 --- /dev/null +++ b/docs/dataartsstudio/api-ref/DeleteLink.html @@ -0,0 +1,164 @@ + + +

Deleting a Link

+

Function

This API is used to delete a link.

+
+

URI

DELETE /v1.1/{project_id}/clusters/{cluster_id}/cdm/link/{link_name}

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+

link_name

+

Yes

+

String

+

Name of the link to be deleted

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+
+

Response Parameters

Status code: 500

+ +
+ + + + + + + + + + + + + +
Table 3 Response body parameters

Parameter

+

Type

+

Description

+

errCode

+

String

+

Error code

+

externalMessage

+

String

+

Error message

+
+
+
+

Example Requests

DELETE /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/link/jdbclink
+
+

Example Responses

Status code: 500

+

An internal service error occurred. For details, see error codes.

+
{
+  "errCode" : "Cdm.0021",
+  "externalMessage" : "Given link name is in use"
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

200

+

OK

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

An internal service error occurred. For details, see error codes.

+

503

+

Service unavailable.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Link Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/ErrorCode.html b/docs/dataartsstudio/api-ref/ErrorCode.html new file mode 100644 index 000000000..c063fbd1d --- /dev/null +++ b/docs/dataartsstudio/api-ref/ErrorCode.html @@ -0,0 +1,21 @@ + + + +

Error Codes

+ +

+
+ +
+ + +
+
Parent topic: Appendix
+
+
+ diff --git a/docs/dataartsstudio/api-ref/ListClusters.html b/docs/dataartsstudio/api-ref/ListClusters.html new file mode 100644 index 000000000..1320ba7ac --- /dev/null +++ b/docs/dataartsstudio/api-ref/ListClusters.html @@ -0,0 +1,856 @@ + + +

Querying the Cluster List

+

Function

This API is used to query the cluster list.

+
+

URI

GET /v1.1/{project_id}/clusters

+ +
+ + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + +
Table 3 Response body parameters

Parameter

+

Type

+

Description

+

clusters

+

Array of clusters objects

+

Cluster list. For details, see the descriptions of clusters parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 clusters

Parameter

+

Type

+

Description

+

customerConfig

+

CustomerConfig object

+

User configuration

+

datastore

+

Datastore object

+

CDM information

+

instances

+

Array of ClusterDetailInstance objects

+

Cluster node information. For details, see the descriptions of instances parameters.

+

azName

+

String

+

AZ name

+

dbuser

+

String

+

Database user

+

flavorName

+

String

+

Flavor name

+

recentEvent

+

Integer

+

Number of events

+

isAutoOff

+

Boolean

+

Auto shutdown

+

isScheduleBootOff

+

Boolean

+

Whether to enable scheduled startup/shutdown. The scheduled startup/shutdown and auto shutdown functions cannot be enabled at the same time.

+

clusterMode

+

String

+

Cluster mode: sharding

+

namespace

+

String

+

Namespace

+

task

+

ClusterTask object

+

Task information

+

publicEndpoint

+

String

+

EIP bound to the cluster

+

actionProgress

+

ActionProgress object

+

Cluster operation progress, which consists of a key and a value. The key indicates an ongoing task, and the value indicates the progress of the ongoing task. An example is "action_progress":{"SNAPSHOTTING":"16%"}.

+

created

+

String

+

Cluster creation time in ISO 8601 format: YYYY-MM-DDThh:mm:ssZ

+

bakExpectedStartTime

+

String

+

Start time

+

bakKeepDay

+

Integer

+

Retention duration

+

name

+

String

+

Cluster name

+

statusDetail

+

String

+

Cluster status: normal

+

id

+

String

+

Cluster ID

+

isFrozen

+

String

+

Whether the cluster is frozen. The value can be 0 (not frozen) or 1 (frozen).

+

updated

+

String

+

Cluster update time in ISO 8601 format: YYYY-MM-DDThh:mm:ssZ

+

status

+

String

+

Cluster status

+

  • 100: creating

    +

  • 200: normal

    +

  • 300: failed

    +

  • 303: failed to be created

    +

  • 500: restarting

    +

  • 800: frozen

    +

  • 900: stopped

    +

  • 910: stopping

    +

  • 920: starting

    +
+

failedReasons

+

FailedReasons object

+

Failure cause. If this parameter is left empty, the cluster is in normal state.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 5 CustomerConfig

Parameter

+

Type

+

Description

+

failureRemind

+

String

+

Failure notification

+

clusterName

+

String

+

Cluster type

+

serviceProvider

+

String

+

Service provisioning

+

localDisk

+

String

+

Whether the disk is a local disk

+

ssl

+

String

+

Whether to enable SSL

+

createFrom

+

String

+

Source

+

resourceId

+

String

+

Resource ID

+

flavorType

+

String

+

Flavor type

+

workSpaceId

+

String

+

Workspace ID

+

trial

+

String

+

Trial

+
+
+ +
+ + + + + + + + + + + + + +
Table 6 Datastore

Parameter

+

Type

+

Description

+

type

+

String

+

Type. Generally, the value is cdm.

+

version

+

String

+

Cluster version

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 7 ClusterDetailInstance

Parameter

+

Type

+

Description

+

flavor

+

flavor object

+

VM flavor of a node. For details, see the descriptions of flavor parameters.

+

volume

+

volume object

+

Disk information of a node. For details, see the descriptions of volume parameters.

+

status

+

String

+

Node status

+

  • 100: creating

    +

  • 200: normal

    +

  • 300: failed

    +

  • 303: failed to be created

    +

  • 400: deleted

    +

  • 800: frozen

    +
+

actions

+

Array of strings

+

Node operation status

+

  • REBOOTING: restarting

    +

  • RESTORING: restoring

    +

  • REBOOT_FAILURE: failed to restart

    +
+

type

+

String

+

Node type. Currently, only cdm is available.

+

id

+

String

+

Node VM ID

+

name

+

String

+

Name of the VM on the node

+

isFrozen

+

String

+

Whether the node is frozen. The value can be 0 (not frozen) or 1 (frozen).

+

components

+

String

+

Component

+

config_status

+

String

+

Node configuration status. The value is null when the cluster list is queried.

+

  • In-Sync: The configuration has been synchronized.

    +

  • Applying: The configuration is in progress.

    +

  • Sync-Failure: The configuration fails.

    +
+

role

+

String

+

Instance role

+

group

+

String

+

Group

+

links

+

Array of ClusterLinks objects

+

Link information

+

paramsGroupId

+

String

+

Group ID

+

publicIp

+

String

+

Public IP address

+

manageIp

+

String

+

Management IP address

+

trafficIp

+

String

+

Traffic IP address

+

shard_id

+

String

+

Slice ID

+

manage_fix_ip

+

String

+

Management fix IP address

+

private_ip

+

String

+

Private IP address

+

internal_ip

+

String

+

Internal IP address

+

resource

+

Array of Resource objects

+

Resource information (null is returned for querying the cluster list)

+
+
+ +
+ + + + + + + + + + + + + +
Table 8 flavor

Parameter

+

Type

+

Description

+

id

+

String

+

VM flavor ID

+

links

+

Array of ClusterLinks objects

+

Link information

+
+
+ +
+ + + + + + + + + + + + + +
Table 9 volume

Parameter

+

Type

+

Description

+

type

+

String

+

Type of disks on the node. Only local disks are supported.

+

size

+

Long

+

Size of the disk on the node (GB)

+
+
+ +
+ + + + + + + + + + + + + +
Table 10 ClusterLinks

Parameter

+

Type

+

Description

+

rel

+

String

+

Relationship

+

href

+

String

+

Link address

+
+
+ +
+ + + + + + + + + + + + + +
Table 11 Resource

Parameter

+

Type

+

Description

+

resource_id

+

String

+

Resource ID

+

resource_type

+

String

+

Resource type: server

+
+
+ +
+ + + + + + + + + + + + + + + + + +
Table 12 ClusterTask

Parameter

+

Type

+

Description

+

description

+

String

+

Task description

+

id

+

String

+

Task ID

+

name

+

String

+

Task name

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Table 13 ActionProgress

Parameter

+

Type

+

Description

+

CREATING

+

String

+

Cluster creation progress, for example, 29%

+

GROWING

+

String

+

Cluster expansion progress, for example, 29%

+

RESTORING

+

String

+

Cluster restoration progress, for example, 29%

+

SNAPSHOTTING

+

String

+

Cluster snapshotting progress, for example, 29%

+

REPAIRING

+

String

+

Cluster repairing progress, for example, 29%

+
+
+ +
+ + + + + + + + + +
Table 14 FailedReasons

Parameter

+

Type

+

Description

+

CREATE_FAILED

+

CREATE_FAILED object

+

Cause of the cluster creation failure

+
+
+ +
+ + + + + + + + + + + + + +
Table 15 CREATE_FAILED

Parameter

+

Type

+

Description

+

errorCode

+

String

+

Error code

+

errorMsg

+

String

+

Failure cause

+
+
+
+

Example Requests

GET /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "clusters" : [ {
+    "publicEndpoint" : "49.xx.xx.10",
+    "actionProgress" : { },
+    "created" : "2018-09-05T08:38:25",
+    "name" : "cdm-c018",
+    "statusDetail" : "Normal",
+    "id" : "bae65496-643e-47ca-84af-948672de7eeb",
+    "isFrozen" : "0",
+    "updated" : "2018-09-05T08:38:25",
+    "status" : "200"
+  } ]
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

200

+

ok

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

An internal service error occurred. For details, see error codes.

+

503

+

Service unavailable.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Cluster Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/PARAMETERS.txt b/docs/dataartsstudio/api-ref/PARAMETERS.txt new file mode 100644 index 000000000..6da8d5f07 --- /dev/null +++ b/docs/dataartsstudio/api-ref/PARAMETERS.txt @@ -0,0 +1,3 @@ +version="" +language="en-us" +type="" \ No newline at end of file diff --git a/docs/dataartsstudio/api-ref/RestartCluster.html b/docs/dataartsstudio/api-ref/RestartCluster.html new file mode 100644 index 000000000..8cf9b3d31 --- /dev/null +++ b/docs/dataartsstudio/api-ref/RestartCluster.html @@ -0,0 +1,256 @@ + + +

Restarting a Cluster

+

Function

This API is used to restart a cluster.

+
+

URI

POST /v1.1/{project_id}/clusters/{cluster_id}/action

+ +
+ + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+ +
+ + + + + + + + + + + +
Table 3 Request body parameters

Parameter

+

Mandatory

+

Type

+

Description

+

restart

+

Yes

+

restart object

+

Cluster restart. For details about how to define the cluster to restart, see the descriptions of restart parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 restart

Parameter

+

Mandatory

+

Type

+

Description

+

restartDelayTime

+

No

+

Integer

+

Restart delay, in seconds

+

restartMode

+

No

+

String

+

Restart mode

+

  • IMMEDIATELY: immediate restart

    +

  • FORCELY: forcible restart

    +

  • SOFTLY: common restart

    +
+

The default value is IMMEDIATELY. Forcibly restarting the service process will interrupt the service process and restart the VMs in the cluster.

+

restartLevel

+

No

+

String

+

Restart level

+

  • SERVICE: service restart

    +

  • VM: VM restart

    +
+

The default value is SERVICE.

+

type

+

No

+

String

+

Type of the cluster to be restarted. The value can be set to cdm only.

+

instance

+

No

+

String

+

Reserved field. When restartLevel is set to SERVICE, this parameter is mandatory and an empty string should be entered.

+

group

+

No

+

String

+

Reserved field. When restartLevel is set to SERVICE, this parameter is mandatory and an empty string should be entered.

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + +
Table 5 Response body parameters

Parameter

+

Type

+

Description

+

jobId

+

String

+

Job ID

+
+
+
+

Example Requests

Restarting a Cluster

+
POST /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/bae65496-643e-47ca-84af-948672de7eeb/action
+
+{
+  "restart" : {
+    "instance" : "",
+    "type" : "cdm",
+    "group" : ""
+  }
+}
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "jobId" : "ff8080815e59d92d015e5b27ccb0004d"
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

200

+

ok

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

An internal service error occurred. For details, see error codes.

+

503

+

Service unavailable.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Cluster Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/ShowClusterDetail.html b/docs/dataartsstudio/api-ref/ShowClusterDetail.html new file mode 100644 index 000000000..f11f5d125 --- /dev/null +++ b/docs/dataartsstudio/api-ref/ShowClusterDetail.html @@ -0,0 +1,1000 @@ + + +

Querying Cluster Details

+

Function

This API is used to query cluster details.

+
+

URI

GET /v1.1/{project_id}/clusters/{cluster_id}

+ +
+ + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Response body parameters

Parameter

+

Type

+

Description

+

publicEndpoint

+

String

+

EIP bound to the cluster

+

instances

+

Array of ClusterDetailInstance objects

+

Cluster node information. For details, see the descriptions of instances parameters.

+

security_group_id

+

String

+

Security group ID

+

subnet_id

+

String

+

Subnet ID

+

vpc_id

+

String

+

VPC ID

+

customerConfig

+

CustomerConfig object

+

User configuration

+

datastore

+

Datastore object

+

CDM information

+

isAutoOff

+

Boolean

+

Auto shutdown

+

publicEndpointDomainName

+

String

+

Domain name for the EIP bound to the cluster

+

bakExpectedStartTime

+

String

+

Start time

+

bakKeepDay

+

Integer

+

Retention duration

+

maintainWindow

+

maintainWindow object

+

Maintenance window

+

recentEvent

+

Integer

+

Number of events

+

flavorName

+

String

+

Flavor name

+

azName

+

String

+

AZ name

+

endpointDomainName

+

String

+

Peer domain name

+

publicEndpointStatus

+

publicEndpointStatus object

+

EIP status

+

isScheduleBootOff

+

Boolean

+

Whether to enable scheduled startup/shutdown. The scheduled startup/shutdown and auto shutdown functions cannot be enabled at the same time.

+

namespace

+

String

+

Namespace

+

eipId

+

String

+

EIP ID

+

failedReasons

+

FailedReasons object

+

Failure cause. If this parameter is left empty, the cluster is in normal state.

+

dbuser

+

String

+

Database user

+

links

+

Array of ClusterLinks objects

+

Cluster link information

+

clusterMode

+

String

+

Cluster mode: sharding

+

task

+

ClusterTask object

+

Task information

+

created

+

String

+

Cluster creation time in ISO 8601 format: YYYY-MM-DDThh:mm:ssZ

+

statusDetail

+

String

+

Cluster status: normal

+

config_status

+

String

+

Cluster configuration status

+

  • In-Sync: The configuration has been synchronized.

    +

  • Applying: The configuration is in progress.

    +

  • Sync-Failure: The configuration fails.

    +
+

actionProgress

+

ActionProgress object

+

Cluster operation progress, which consists of a key and a value. The key indicates an ongoing task, and the value indicates the progress of the ongoing task. An example is "action_progress":{"SNAPSHOTTING":"16%"}.

+

name

+

String

+

Cluster name

+

id

+

String

+

Cluster ID

+

isFrozen

+

String

+

Whether the cluster is frozen. The value can be 0 (not frozen) or 1 (frozen).

+

actions

+

Array of strings

+

Cluster configuration status. Options: - In-Sync: The cluster configuration has been synchronized. - Applying: The cluster is being configured. - Sync-Failure: The cluster configuration failed.

+

updated

+

String

+

Cluster update time in ISO 8601 format: YYYY-MM-DDThh:mm:ssZ

+

status

+

String

+

Cluster status

+

  • 100: creating

    +

  • 200: normal

    +

  • 300: failed

    +

  • 303: failed to be created

    +

  • 800: frozen

    +

  • 900: stopped

    +

  • 910: stopping

    +

  • 920: starting

    +
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 ClusterDetailInstance

Parameter

+

Type

+

Description

+

flavor

+

flavor object

+

VM flavor of a node. For details, see the descriptions of flavor parameters.

+

volume

+

volume object

+

Disk information of a node. For details, see the descriptions of volume parameters.

+

status

+

String

+

Node status

+

  • 100: creating

    +

  • 200: normal

    +

  • 300: failed

    +

  • 303: failed to be created

    +

  • 400: deleted

    +

  • 800: frozen

    +
+

actions

+

Array of strings

+

Node operation status

+

  • REBOOTING: restarting

    +

  • RESTORING: restoring

    +

  • REBOOT_FAILURE: failed to restart

    +
+

type

+

String

+

Node type. Currently, only cdm is available.

+

id

+

String

+

Node VM ID

+

name

+

String

+

Name of the VM on the node

+

isFrozen

+

String

+

Whether the node is frozen. The value can be 0 (not frozen) or 1 (frozen).

+

components

+

String

+

Component

+

config_status

+

String

+

Node configuration status. The value is null when the cluster list is queried.

+

  • In-Sync: The configuration has been synchronized.

    +

  • Applying: The configuration is in progress.

    +

  • Sync-Failure: The configuration fails.

    +
+

role

+

String

+

Instance role

+

group

+

String

+

Group

+

links

+

Array of ClusterLinks objects

+

Link information

+

paramsGroupId

+

String

+

Group ID

+

publicIp

+

String

+

Public IP address

+

manageIp

+

String

+

Management IP address

+

trafficIp

+

String

+

Traffic IP address

+

shard_id

+

String

+

Slice ID

+

manage_fix_ip

+

String

+

Management fix IP address

+

private_ip

+

String

+

Private IP address

+

internal_ip

+

String

+

Internal IP address

+

resource

+

Array of Resource objects

+

Resource information (null is returned for querying the cluster list)

+
+
+ +
+ + + + + + + + + + + + + +
Table 5 flavor

Parameter

+

Type

+

Description

+

id

+

String

+

VM flavor ID

+

links

+

Array of ClusterLinks objects

+

Link information

+
+
+ +
+ + + + + + + + + + + + + +
Table 6 volume

Parameter

+

Type

+

Description

+

type

+

String

+

Type of disks on the node. Only local disks are supported.

+

size

+

Long

+

Size of the disk on the node (GB)

+
+
+ +
+ + + + + + + + + + + + + +
Table 7 Resource

Parameter

+

Type

+

Description

+

resource_id

+

String

+

Resource ID

+

resource_type

+

String

+

Resource type: server

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 8 CustomerConfig

Parameter

+

Type

+

Description

+

failureRemind

+

String

+

Failure notification

+

clusterName

+

String

+

Cluster type

+

serviceProvider

+

String

+

Service provisioning

+

localDisk

+

String

+

Whether the disk is a local disk

+

ssl

+

String

+

Whether to enable SSL

+

createFrom

+

String

+

Source

+

resourceId

+

String

+

Resource ID

+

flavorType

+

String

+

Flavor type

+

workSpaceId

+

String

+

Workspace ID

+

trial

+

String

+

Trial

+
+
+ +
+ + + + + + + + + + + + + +
Table 9 Datastore

Parameter

+

Type

+

Description

+

type

+

String

+

Type. Generally, the value is cdm.

+

version

+

String

+

Cluster version

+
+
+ +
+ + + + + + + + + + + + + + + + + +
Table 10 maintainWindow

Parameter

+

Type

+

Description

+

day

+

String

+

Day of a week

+

startTime

+

String

+

Start time

+

endTime

+

String

+

End time

+
+
+ +
+ + + + + + + + + + + + + +
Table 11 publicEndpointStatus

Parameter

+

Type

+

Description

+

status

+

String

+

Status

+

errorMessage

+

String

+

Error message

+
+
+ +
+ + + + + + + + + +
Table 12 FailedReasons

Parameter

+

Type

+

Description

+

CREATE_FAILED

+

CREATE_FAILED object

+

Cause of the cluster creation failure

+
+
+ +
+ + + + + + + + + + + + + +
Table 13 CREATE_FAILED

Parameter

+

Type

+

Description

+

errorCode

+

String

+

Error code

+

errorMsg

+

String

+

Failure cause

+
+
+ +
+ + + + + + + + + + + + + +
Table 14 ClusterLinks

Parameter

+

Type

+

Description

+

rel

+

String

+

Relationship

+

href

+

String

+

Link address

+
+
+ +
+ + + + + + + + + + + + + + + + + +
Table 15 ClusterTask

Parameter

+

Type

+

Description

+

description

+

String

+

Task description

+

id

+

String

+

Task ID

+

name

+

String

+

Task name

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Table 16 ActionProgress

Parameter

+

Type

+

Description

+

CREATING

+

String

+

Cluster creation progress, for example, 29%

+

GROWING

+

String

+

Cluster expansion progress, for example, 29%

+

RESTORING

+

String

+

Cluster restoration progress, for example, 29%

+

SNAPSHOTTING

+

String

+

Cluster snapshotting progress, for example, 29%

+

REPAIRING

+

String

+

Cluster repairing progress, for example, 29%

+
+
+
+

Example Requests

GET /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/bae65496-643e-47ca-84af-948672de7eeb
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "publicEndpoint" : "49.xx.xx.10",
+  "instances" : [ {
+    "flavor" : {
+      "id" : "fb8fe666-6734-4b11-bc6c-43d11db3c745"
+    },
+    "volume" : {
+      "size" : "100",
+      "type" : "LOCAL_DISK"
+    },
+    "name" : "cdm-c018",
+    "id" : "635dce67-3df8-4756-b4c7-90e45e687367",
+    "isFrozen" : "0",
+    "type" : "cdm",
+    "actions" : "REBOOTING",
+    "config_status" : "In-Sync",
+    "status" : "200"
+  } ],
+  "created" : "2018-09-05T08:38:25",
+  "statusDetail" : "Normal",
+  "actionProgress" : { },
+  "name" : "cdm-c018",
+  "id" : "bae65496-643e-47ca-84af-948672de7eeb",
+  "isFrozen" : "0",
+  "actions" : "REBOOTING",
+  "updated" : "2018-09-05T08:38:25",
+  "status" : "200"
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

200

+

ok

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

An internal service error occurred. For details, see error codes.

+

503

+

Service unavailable.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Cluster Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/ShowJobStatus.html b/docs/dataartsstudio/api-ref/ShowJobStatus.html new file mode 100644 index 000000000..ccf841a99 --- /dev/null +++ b/docs/dataartsstudio/api-ref/ShowJobStatus.html @@ -0,0 +1,393 @@ + + +

Querying Job Status

+

Function

This API is used to query the job status.

+
+

URI

GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}/status

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+

job_name

+

Yes

+

String

+

Job name

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + +
Table 3 Response body parameters

Parameter

+

Type

+

Description

+

submissions

+

Array of Submission objects

+

Job running information. For details, see the descriptions of submission parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 Submission

Parameter

+

Type

+

Description

+

isIncrementing

+

Boolean

+

Whether the job migrates incremental data

+

job-name

+

String

+

Job name

+

counters

+

counters object

+

Job running result statistics. This parameter is available only when status is SUCCEEDED. For details, see the description of the counters parameter.

+

isStopingIncrement

+

String

+

Whether to stop incremental data migration

+

is-execute-auto

+

Boolean

+

Whether to execute the job as scheduled

+

last-update-date

+

Long

+

Time when the job was last updated

+

last-udpate-user

+

String

+

User who last updated the job status

+

isDeleteJob

+

Boolean

+

Whether to delete the job after it is executed

+

creation-user

+

String

+

User who created the job

+

creation-date

+

Long

+

Creation time

+

external-id

+

String

+

Job ID

+

progress

+

Float

+

Job progress. If a job fails, the value is -1. Otherwise, the value ranges from 0 to 100.

+

submission-id

+

Integer

+

Job submission ID

+

delete_rows

+

Integer

+

Number of deleted rows

+

update_rows

+

Integer

+

Number of updated rows

+

write_rows

+

Integer

+

Number of written rows

+

execute-date

+

Long

+

Execution time

+

status

+

String

+

Job status

+

  • BOOTING: The job is starting.

    +

  • FAILURE_ON_SUBMIT: The job failed to be submitted.

    +

  • RUNNING: The job is running.

    +

  • SUCCEEDED: The job was successfully executed.

    +

  • FAILED: The job execution failed.

    +

  • UNKNOWN: The job status is unknown.

    +

  • NEVER_EXECUTED: The job was not executed.

    +
+

error-details

+

String

+

Error details. This parameter is available only when status is FAILED.

+

error-summary

+

String

+

Error summary. This parameter is available only when status is FAILED.

+
+
+ +
+ + + + + + + + + +
Table 5 counters

Parameter

+

Type

+

Description

+

org.apache.sqoop.submission.counter.SqoopCounters

+

counter object

+

Job running result statistics. For details, see the descriptions of counter parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 6 counter

Parameter

+

Type

+

Description

+

BYTES_WRITTEN

+

Long

+

Number of bytes that are written

+

TOTAL_FILES

+

Integer

+

Total number of files

+

ROWS_READ

+

Long

+

Number of rows that are read

+

BYTES_READ

+

Long

+

Number of bytes that are read

+

ROWS_WRITTEN

+

Long

+

Number of rows that are written

+

FILES_WRITTEN

+

Integer

+

Number of files that are written

+

FILES_READ

+

Integer

+

Number of files that are read

+

TOTAL_SIZE

+

Long

+

Total number of bytes

+

FILES_SKIPPED

+

Integer

+

Number of files that are skipped

+

ROWS_WRITTEN_SKIPPED

+

Long

+

Number of rows that are skipped

+
+
+
+

Example Requests

GET /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/job/jdbc2hive/status
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "submissions" : [ {
+    "job-name" : "jdbc2hive",
+    "creation-user" : "cdm",
+    "creation-date" : "1536905778725",
+    "progress" : 1,
+    "status" : "BOOTING"
+  } ]
+}
+
+

Status Codes

+
+ + + + + + + +

Status Code

+

Description

+

200

+

ok

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/ShowJobs.html b/docs/dataartsstudio/api-ref/ShowJobs.html new file mode 100644 index 000000000..3c2fcf7c2 --- /dev/null +++ b/docs/dataartsstudio/api-ref/ShowJobs.html @@ -0,0 +1,690 @@ + + +

Querying a Job

+

Function

This API is used to query jobs.

+
+

URI

GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+

job_name

+

Yes

+

String

+

Job name. When this parameter is set to all, all jobs are to be queried.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 2 Query Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

filter

+

No

+

String

+

When job_name is all, this parameter is used for fuzzy job filtering.

+

page_no

+

No

+

Integer

+

Page number

+

Minimum: 1

+

page_size

+

No

+

Integer

+

Number of jobs on each page. The value ranges from 10 to 100.

+

Minimum: 10

+

Maximum: 100

+

jobType

+

No

+

String

+

Type of the jobs to be queried

+

  • jobType=NORMAL_JOB: table/file migration job

    +

  • jobType=BATCH_JOB: entire DB migration job

    +

  • jobType=SCENARIO_JOB: scenario migration job If this parameter is not specified, only table/file migration jobs are queried by default.

    +
+

Enumeration values:

+

  • NORMAL_JOB

    +

  • BATCH_JOB

    +

  • SCENARIO_JOB

    +
+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 3 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 4 Response body parameters

Parameter

+

Type

+

Description

+

total

+

Integer

+

Number of jobs

+

jobs

+

Array of Job objects

+

Job list. For details, see the descriptions of jobs parameters.

+

page_no

+

Integer

+

Page number. Jobs on the specified page will be returned.

+

page_size

+

Integer

+

Number of jobs on each page.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 5 Job

Parameter

+

Type

+

Description

+

job_type

+

String

+

Job type

+

  • NORMAL_JOB: table/file migration

    +

  • BATCH_JOB: entire DB migration

    +

  • SCENARIO_JOB: scenario migration

    +
+

Enumeration values:

+

  • NORMAL_JOB

    +

  • BATCH_JOB

    +

  • SCENARIO_JOB

    +
+

from-connector-name

+

String

+

Source link type

+

to-config-values

+

ConfigValues object

+

Destination link parameter configuration

+

to-link-name

+

String

+

Destination link name

+

driver-config-values

+

ConfigValues object

+

Job parameter configuration

+

from-config-values

+

ConfigValues object

+

Source link parameter configuration

+

to-connector-name

+

String

+

Destination link type

+

name

+

String

+

Job name, which contains 1 to 240 characters

+

Minimum: 1

+

Maximum: 240

+

from-link-name

+

String

+

Source link name

+

creation-user

+

String

+

User who created the job

+

creation-date

+

Long

+

Time when the job was created, accurate to millisecond

+

update-date

+

Long

+

Time when the job was last updated, accurate to millisecond

+

is_incre_job

+

Boolean

+

Incremental or not

+

flag

+

Integer

+

Flag

+

files_read

+

Integer

+

Number of read files

+

update-user

+

String

+

User who last updated the job

+

external_id

+

String

+

External ID

+

type

+

String

+

Task type

+

execute_start_date

+

Long

+

Execution start date

+

delete_rows

+

Integer

+

Number of deleted rows

+

enabled

+

Boolean

+

Whether to activate the link

+

bytes_written

+

Long

+

Number of written bytes

+

id

+

Integer

+

Job ID

+

is_use_sql

+

Boolean

+

Whether to use SQL statements

+

update_rows

+

Integer

+

Number of updated rows

+

group_name

+

String

+

Group name

+

bytes_read

+

Long

+

Number of read bytes

+

execute_update_date

+

Long

+

Execution update date

+

write_rows

+

Integer

+

Number of written rows

+

rows_written

+

Integer

+

Number of written rows

+

rows_read

+

Long

+

Number of rows that are read

+

files_written

+

Integer

+

Number of written files

+

is_incrementing

+

Boolean

+

Incremental or not

+

execute_create_date

+

Long

+

Execution creation date

+

status

+

String

+

Job execution status

+

  • BOOTING: The job is starting.

    +

  • RUNNING: The job is running.

    +

  • SUCCEEDED: The job was successfully executed.

    +

  • FAILED: The job execution failed.

    +

  • NEW: The job was not executed.

    +
+
+
+ +
+ + + + + + + + + + + + + +
Table 6 ConfigValues

Parameter

+

Type

+

Description

+

configs

+

Array of configs objects

+

The data structures of source link parameters, destination link parameters, and job parameters are the same. However, the inputs parameter varies. For details, see the descriptions of configs parameters.

+

extended-configs

+

extended-configs object

+

Extended configuration. For details, see the descriptions of extended-configs parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 7 configs

Parameter

+

Type

+

Description

+

inputs

+

Array of Input objects

+

Input parameter list. Each element in the list is in name,value format. For details, see the descriptions of inputs parameters. In the from-config-values data structure, the value of this parameter varies with the source link type. For details, see section "Source Job Parameters" in the Cloud Data Migration User Guide. In the to-cofig-values data structure, the value of this parameter varies with the destination link type. For details, see section "Destination Job Parameters" in the Cloud Data Migration User Guide. For details about the inputs parameter in the driver-config-values data structure, see the job parameter descriptions.

+

name

+

String

+

Configuration name. The value is fromJobConfig for a source job, toJobConfig for a destination job, and linkConfig for a link.

+

id

+

Integer

+

Configuration ID

+

type

+

String

+

Configuration type

+
+
+ +
+ + + + + + + + + + + + + + + + + +
Table 8 Input

Parameter

+

Type

+

Description

+

name

+

String

+

Parameter name

+

value

+

String

+

Parameter value

+

type

+

String

+

Value type

+
+
+ +
+ + + + + + + + + + + + + +
Table 9 extended-configs

Parameter

+

Type

+

Description

+

name

+

String

+

Name

+

value

+

String

+

Value

+
+
+
+

Example Requests

GET /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/job/all?jobType=NORMAL_JOB
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "total" : 1,
+  "jobs" : [ {
+    "job_type" : "NORMAL_JOB",
+    "from-connector-name" : "elasticsearch-connector",
+    "to-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "toJobConfig.streamName",
+          "value" : "dis-lkGm"
+        }, {
+          "name" : "toJobConfig.separator",
+          "value" : "|"
+        }, {
+          "name" : "toJobConfig.columnList",
+          "value" : "1&2&3"
+        } ],
+        "name" : "toJobConfig"
+      } ]
+    },
+    "to-link-name" : "dis",
+    "driver-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "throttlingConfig.numExtractors",
+          "value" : "1"
+        }, {
+          "name" : "throttlingConfig.submitToCluster",
+          "value" : "false"
+        }, {
+          "name" : "throttlingConfig.numLoaders",
+          "value" : "1"
+        }, {
+          "name" : "throttlingConfig.recordDirtyData",
+          "value" : "false"
+        } ],
+        "name" : "throttlingConfig"
+      }, {
+        "inputs" : { },
+        "name" : "jarConfig"
+      }, {
+        "inputs" : [ {
+          "name" : "schedulerConfig.isSchedulerJob",
+          "value" : "false"
+        }, {
+          "name" : "schedulerConfig.disposableType",
+          "value" : "NONE"
+        } ],
+        "name" : "schedulerConfig"
+      }, {
+        "inputs" : { },
+        "name" : "transformConfig"
+      }, {
+        "inputs" : [ {
+          "name" : "retryJobConfig.retryJobType",
+          "value" : "NONE"
+        } ],
+        "name" : "retryJobConfig"
+      } ]
+    },
+    "from-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "fromJobConfig.index",
+          "value" : "52est"
+        }, {
+          "name" : "fromJobConfig.type",
+          "value" : "est_array"
+        }, {
+          "name" : "fromJobConfig.columnList",
+          "value" : "array_f1_int:long&array_f2_text:string&array_f3_object:nested"
+        }, {
+          "name" : "fromJobConfig.splitNestedField",
+          "value" : "false"
+        } ],
+        "name" : "fromJobConfig"
+      } ]
+    },
+    "to-connector-name" : "dis-connector",
+    "name" : "es_css",
+    "from-link-name" : "css"
+  } ],
+  "page_no" : 1,
+  "page_size" : 10
+}
+
+

Status Codes

+
+ + + + + + + +

Status Code

+

Description

+

200

+

ok

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/ShowLink.html b/docs/dataartsstudio/api-ref/ShowLink.html new file mode 100644 index 000000000..1b1ad1b63 --- /dev/null +++ b/docs/dataartsstudio/api-ref/ShowLink.html @@ -0,0 +1,407 @@ + + +

Querying a Link

+

Function

This API is used to query a link.

+
+

URI

GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/link/{link_name}

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+

link_name

+

Yes

+

String

+

Link name

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + + + + + + + + + +
Table 3 Response body parameters

Parameter

+

Type

+

Description

+

links

+

Array of links objects

+

Link list. For details, see the descriptions of links parameters.

+

fromTo-unMapping

+

String

+

Source and destination data sources not supported by table/file migration

+

batchFromTo-mapping

+

String

+

Source and destination data sources supported by entire DB migration

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 links

Parameter

+

Type

+

Description

+

link-config-values

+

link-config-values object

+

Link parameters. For details, see the descriptions of link-config-values parameters.

+

creation-user

+

String

+

User who created the link

+

name

+

String

+

Link name

+

id

+

Integer

+

Link ID

+

creation-date

+

Long

+

Time when the link was created

+

connector-name

+

String

+

Connector name. The mappings between the connectors and links are as follows:generic-jdbc-connector: link to a relational databaseobs-connector: OBS linkhdfs-connector: HDFS linkhbase-connector: HBase or CloudTable linkhive-connector: Hive linkftp-connector/sftp-connector: link to an FTP or SFTP servermongodb-connector: MongoDB linkredis-connector: Redis or DCS linkkafka-connector: Kafka linkdis-connector: DIS linkelasticsearch-connector: Elasticsearch or CSS linkdli-connector: DLI linkhttp-connector: HTTP or HTTPS link. This type of link does not have parameters.dms-kafka-connector: DMS Kafka link

+

update-date

+

Long

+

Time when the link was updated

+

enabled

+

Boolean

+

Whether to activate the link. The default value is true.

+

update-user

+

String

+

User who updated the link

+
+
+ +
+ + + + + + + + + + + + + + + + + +
Table 5 link-config-values

Parameter

+

Type

+

Description

+

configs

+

Array of configs objects

+

Data structure of link parameters. For details, see the descriptions of configs parameters.

+

extended-configs

+

extended-configs object

+

Extended configuration. For details, see the descriptions of extended-configs parameters.

+

validators

+

Array of strings

+

Validator

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 6 configs

Parameter

+

Type

+

Description

+

inputs

+

Array of Input objects

+

Input parameter list. Each element in the list is in name,value format. For details, see the descriptions of inputs parameters. In the from-config-values data structure, the value of this parameter varies with the source link type. For details, see section "Source Job Parameters" in the Cloud Data Migration User Guide. In the to-cofig-values data structure, the value of this parameter varies with the destination link type. For details, see section "Destination Job Parameters" in the Cloud Data Migration User Guide. For details about the inputs parameter in the driver-config-values data structure, see the job parameter descriptions.

+

name

+

String

+

Configuration name. The value is fromJobConfig for a source job, toJobConfig for a destination job, and linkConfig for a link.

+

id

+

Integer

+

Configuration ID

+

type

+

String

+

Configuration type

+
+
+ +
+ + + + + + + + + + + + + + + + + +
Table 7 Input

Parameter

+

Type

+

Description

+

name

+

String

+

Parameter name

+

value

+

String

+

Parameter value

+

type

+

String

+

Value type

+
+
+ +
+ + + + + + + + + + + + + +
Table 8 extended-configs

Parameter

+

Type

+

Description

+

name

+

String

+

Name

+

value

+

String

+

Value

+
+
+
+

Example Requests

GET /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/link/sftplink
+
+

Example Responses

Status code: 200

+

OK

+
{
+  "links" : [ {
+    "link-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "linkConfig.server",
+          "type" : "STRING",
+          "value" : "100.94.8.163"
+        }, {
+          "name" : "linkConfig.port",
+          "type" : "INTEGER",
+          "value" : 22
+        }, {
+          "name" : "linkConfig.username",
+          "type" : "STRING",
+          "value" : "root"
+        }, {
+          "name" : "linkConfig.password",
+          "type" : "STRING",
+          "value" : "Add password here"
+        } ],
+        "name" : "linkConfig"
+      } ]
+    },
+    "creation-user" : "cdm",
+    "name" : "sftp_link",
+    "creation-date" : 1516674482640,
+    "connector-name" : "sftp-connector",
+    "update-date" : 1516674476022,
+    "enabled" : true,
+    "update-user" : "cdm"
+  } ]
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

200

+

OK

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

An internal service error occurred. For details, see error codes.

+

503

+

Service unavailable.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Link Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/ShowSubmissions.html b/docs/dataartsstudio/api-ref/ShowSubmissions.html new file mode 100644 index 000000000..6bd614250 --- /dev/null +++ b/docs/dataartsstudio/api-ref/ShowSubmissions.html @@ -0,0 +1,428 @@ + + +

Querying Job Execution History

+

Function

This API is used to query the job execution history.

+
+

URI

GET /v1.1/{project_id}/clusters/{cluster_id}/cdm/submissions

+ +
+ + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+
+
+ +
+ + + + + + + + + + + +
Table 2 Query Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

jname

+

Yes

+

String

+

Job name

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 3 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 4 Response body parameters

Parameter

+

Type

+

Description

+

submissions

+

Array of Submission objects

+

Job running information. For details, see the descriptions of submission parameters.

+

total

+

Integer

+

Total number of historical records for a job

+

page_no

+

Integer

+

Page number

+

page_size

+

Integer

+

Number of records on each page. The default value is 10.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 5 Submission

Parameter

+

Type

+

Description

+

isIncrementing

+

Boolean

+

Whether the job migrates incremental data

+

job-name

+

String

+

Job name

+

counters

+

counters object

+

Job running result statistics. This parameter is available only when status is SUCCEEDED. For details, see the description of the counters parameter.

+

isStopingIncrement

+

String

+

Whether to stop incremental data migration

+

is-execute-auto

+

Boolean

+

Whether to execute the job as scheduled

+

last-update-date

+

Long

+

Time when the job was last updated

+

last-udpate-user

+

String

+

User who last updated the job status

+

isDeleteJob

+

Boolean

+

Whether to delete the job after it is executed

+

creation-user

+

String

+

User who created the job

+

creation-date

+

Long

+

Creation time

+

external-id

+

String

+

Job ID

+

progress

+

Float

+

Job progress. If a job fails, the value is -1. Otherwise, the value ranges from 0 to 100.

+

submission-id

+

Integer

+

Job submission ID

+

delete_rows

+

Integer

+

Number of deleted rows

+

update_rows

+

Integer

+

Number of updated rows

+

write_rows

+

Integer

+

Number of written rows

+

execute-date

+

Long

+

Execution time

+

status

+

String

+

Job status

+

  • BOOTING: The job is starting.

    +

  • FAILURE_ON_SUBMIT: The job failed to be submitted.

    +

  • RUNNING: The job is running.

    +

  • SUCCEEDED: The job was successfully executed.

    +

  • FAILED: The job execution failed.

    +

  • UNKNOWN: The job status is unknown.

    +

  • NEVER_EXECUTED: The job was not executed.

    +
+

error-details

+

String

+

Error details. This parameter is available only when status is FAILED.

+

error-summary

+

String

+

Error summary. This parameter is available only when status is FAILED.

+
+
+ +
+ + + + + + + + + +
Table 6 counters

Parameter

+

Type

+

Description

+

org.apache.sqoop.submission.counter.SqoopCounters

+

counter object

+

Job running result statistics. For details, see the descriptions of counter parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 7 counter

Parameter

+

Type

+

Description

+

BYTES_WRITTEN

+

Long

+

Number of bytes that are written

+

TOTAL_FILES

+

Integer

+

Total number of files

+

ROWS_READ

+

Long

+

Number of rows that are read

+

BYTES_READ

+

Long

+

Number of bytes that are read

+

ROWS_WRITTEN

+

Long

+

Number of rows that are written

+

FILES_WRITTEN

+

Integer

+

Number of files that are written

+

FILES_READ

+

Integer

+

Number of files that are read

+

TOTAL_SIZE

+

Long

+

Total number of bytes

+

FILES_SKIPPED

+

Integer

+

Number of files that are skipped

+

ROWS_WRITTEN_SKIPPED

+

Long

+

Number of rows that are skipped

+
+
+
+

Example Requests

GET /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/submissions?jname=jdbc2hive
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "submissions" : [ {
+    "job-name" : "jdbc2hive",
+    "creation-user" : "cdm",
+    "creation-date" : "1536905778725",
+    "progress" : 1,
+    "status" : "BOOTING"
+  } ]
+}
+
+

Status Codes

+
+ + + + + + + +

Status Code

+

Description

+

200

+

ok

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/StartCluster.html b/docs/dataartsstudio/api-ref/StartCluster.html new file mode 100644 index 000000000..a7f308b5f --- /dev/null +++ b/docs/dataartsstudio/api-ref/StartCluster.html @@ -0,0 +1,175 @@ + + +

Starting a Cluster

+

Function

This API is used to start a cluster.

+
+

URI

POST /v1.1/{project_id}/clusters/{cluster_id}/action

+ +
+ + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+ +
+ + + + + + + + + + + +
Table 3 Request body parameters

Parameter

+

Mandatory

+

Type

+

Description

+

start

+

Yes

+

Object

+

Starting a cluster. This parameter is an empty object.

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + +
Table 4 Response body parameters

Parameter

+

Type

+

Description

+

jobId

+

Array of strings

+

Job ID

+
+
+
+

Example Requests

Starting a Cluster

+
POST /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/bae65496-643e-47ca-84af-948672de7eeb/action
+
+{
+  "start" : { }
+}
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "jobId" : [ "ff8080815e59d92d015e5b27ccb0004d" ]
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

200

+

ok

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

An internal service error occurred. For details, see error codes.

+

503

+

Service unavailable.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Cluster Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/StartJob.html b/docs/dataartsstudio/api-ref/StartJob.html new file mode 100644 index 000000000..05892d6fc --- /dev/null +++ b/docs/dataartsstudio/api-ref/StartJob.html @@ -0,0 +1,264 @@ + + +

Starting a Job

+

Function

This API is used to start a job.

+
+

URI

PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}/start

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+

job_name

+

Yes

+

String

+

Job name

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + +
Table 3 Response body parameters

Parameter

+

Type

+

Description

+

submissions

+

Array of StartJobSubmission objects

+

Job running information. For details, see the descriptions of submission parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 StartJobSubmission

Parameter

+

Type

+

Description

+

isIncrementing

+

Boolean

+

Whether the job migrates incremental data

+

delete_rows

+

Integer

+

Number of deleted rows

+

update_rows

+

Integer

+

Number of updated rows

+

write_rows

+

Integer

+

Number of written rows

+

submission-id

+

Integer

+

ID of the submitted job

+

job-name

+

String

+

Job name

+

creation-user

+

String

+

User who created the job

+

creation-date

+

Long

+

Job creation time, accurate to millisecond

+

execute-date

+

Long

+

Job execution time

+

progress

+

Float

+

Job progress. If a job fails, the value is -1. Otherwise, the value ranges from 0 to 100.

+

status

+

String

+

Job status

+

  • BOOTING: The job is starting.

    +

  • FAILURE_ON_SUBMIT: The job failed to be submitted.

    +

  • RUNNING: The job is running.

    +

  • SUCCEEDED: The job was successfully executed.

    +

  • FAILED: The job execution failed.

    +

  • UNKNOWN: The job status is unknown.

    +

  • NEVER_EXECUTED: The job was not executed.

    +
+

isStopingIncrement

+

String

+

Whether to stop incremental data migration

+

is-execute-auto

+

Boolean

+

Whether to execute the job as scheduled

+

last-update-date

+

Long

+

Time when the job was last updated

+

last-udpate-user

+

String

+

User who last updated the job status

+

isDeleteJob

+

Boolean

+

Whether to delete the job after it is executed

+
+
+
+

Example Requests

PUT /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/job/jdbc2hive/start
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "submissions" : [ {
+    "job-name" : "jdbc2hive",
+    "creation-user" : "cdm",
+    "creation-date" : "1536905778725",
+    "progress" : 1,
+    "status" : "BOOTING"
+  } ]
+}
+
+

Status Codes

+
+ + + + + + + +

Status Code

+

Description

+

200

+

ok

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/StopCluster.html b/docs/dataartsstudio/api-ref/StopCluster.html new file mode 100644 index 000000000..02ed550c7 --- /dev/null +++ b/docs/dataartsstudio/api-ref/StopCluster.html @@ -0,0 +1,213 @@ + + +

Stopping a Cluster

+

Function

This API is used to stop a cluster.

+
+

URI

POST /v1.1/{project_id}/clusters/{cluster_id}/action

+ +
+ + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+ +
+ + + + + + + + + + + +
Table 3 Request body parameters

Parameter

+

Mandatory

+

Type

+

Description

+

stop

+

Yes

+

stop object

+

Cluster stop operation, which defines the parameters for stopping a cluster.

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 4 stop

Parameter

+

Mandatory

+

Type

+

Description

+

stopMode

+

No

+

String

+

Stop mode

+
  • IMMEDIATELY: immediate stop
  • GRACEFULLY: graceful stop
+

Enumeration values:

+
  • IMMEDIATELY
  • GRACEFULLY
+

delayTime

+

No

+

Integer

+

Stop delay, in seconds. This parameter is valid only when stopMode is set to GRACEFULLY. If the value of this parameter is set to -1, the system waits for all jobs to complete and stops accepting new jobs. If the value of this parameter is greater than 0, the system stops the cluster after the specified time and stops accepting new jobs.

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + +
Table 5 Response body parameters

Parameter

+

Type

+

Description

+

jobId

+

Array of strings

+

Job ID

+
+
+
+

Example Requests

Stopping a Cluster

+
POST /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/bae65496-643e-47ca-84af-948672de7eeb/action
+
+{
+  "stop" : {
+    "stopMode" : "GRACEFULLY",
+    "delayTime" : -1
+  }
+}
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "jobId" : [ "ff8080815e59d92d015e5b27ccb0004d" ]
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

200

+

ok

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

An internal service error occurred. For details, see error codes.

+

503

+

Service unavailable.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Cluster Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/StopJob.html b/docs/dataartsstudio/api-ref/StopJob.html new file mode 100644 index 000000000..1fa2d7f25 --- /dev/null +++ b/docs/dataartsstudio/api-ref/StopJob.html @@ -0,0 +1,154 @@ + + +

Stopping a Job

+

Function

This API is used to stop a job.

+
+

URI

PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}/stop

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+

job_name

+

Yes

+

String

+

Job name

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + +
Table 3 Response body parameters

Parameter

+

Type

+

Description

+

validation-result

+

Array of JobValidationResult objects

+

Validation result. If the job fails to be stopped, the failure cause is returned. For details, see the validation-result parameter. If the job is stopped, an empty list is returned.

+
+
+ +
+ + + + + + + + + + + + + +
Table 4 JobValidationResult

Parameter

+

Type

+

Description

+

message

+

String

+

Error message

+

status

+

String

+

ERROR,WARNING

+

Enumeration values:

+

  • ERROR

    +

  • WARNING

    +
+
+
+
+

Example Requests

PUT /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/job/jdbc2hive/stop
+
+

Example Responses

Status code: 200

+

ok

+
{ }
+
+

Status Codes

+
+ + + + + + + +

Status Code

+

Description

+

200

+

ok

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/UpdateJob.html b/docs/dataartsstudio/api-ref/UpdateJob.html new file mode 100644 index 000000000..a60fafe99 --- /dev/null +++ b/docs/dataartsstudio/api-ref/UpdateJob.html @@ -0,0 +1,817 @@ + + +

Modifying a Job

+

Function

This API is used to modify a job.

+
+

URI

PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/job/{job_name}

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+

job_name

+

Yes

+

String

+

Job name

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+ +
+ + + + + + + + + + + +
Table 3 Request body parameters

Parameter

+

Mandatory

+

Type

+

Description

+

jobs

+

Yes

+

Array of Job objects

+

Job list. For details, see the descriptions of jobs parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 Job

Parameter

+

Mandatory

+

Type

+

Description

+

job_type

+

No

+

String

+

Job type

+

  • NORMAL_JOB: table/file migration

    +

  • BATCH_JOB: entire DB migration

    +

  • SCENARIO_JOB: scenario migration

    +
+

Enumeration values:

+

  • NORMAL_JOB

    +

  • BATCH_JOB

    +

  • SCENARIO_JOB

    +
+

from-connector-name

+

Yes

+

String

+

Source link type

+

to-config-values

+

Yes

+

ConfigValues object

+

Destination link parameter configuration

+

to-link-name

+

Yes

+

String

+

Destination link name

+

driver-config-values

+

Yes

+

ConfigValues object

+

Job parameter configuration

+

from-config-values

+

Yes

+

ConfigValues object

+

Source link parameter configuration

+

to-connector-name

+

No

+

String

+

Destination link type

+

name

+

No

+

String

+

Job name, which contains 1 to 240 characters

+

Minimum: 1

+

Maximum: 240

+

from-link-name

+

No

+

String

+

Source link name

+

creation-user

+

No

+

String

+

User who created the job

+

creation-date

+

No

+

Long

+

Time when the job was created, accurate to millisecond

+

update-date

+

No

+

Long

+

Time when the job was last updated, accurate to millisecond

+

is_incre_job

+

No

+

Boolean

+

Incremental or not

+

flag

+

No

+

Integer

+

Flag

+

files_read

+

No

+

Integer

+

Number of read files

+

update-user

+

No

+

String

+

User who last updated the job

+

external_id

+

No

+

String

+

External ID

+

type

+

No

+

String

+

Task type

+

execute_start_date

+

No

+

Long

+

Execution start date

+

delete_rows

+

No

+

Integer

+

Number of deleted rows

+

enabled

+

No

+

Boolean

+

Whether to activate the link

+

bytes_written

+

No

+

Long

+

Number of written bytes

+

id

+

No

+

Integer

+

Job ID

+

is_use_sql

+

No

+

Boolean

+

Whether to use SQL statements

+

update_rows

+

No

+

Integer

+

Number of updated rows

+

group_name

+

No

+

String

+

Group name

+

bytes_read

+

No

+

Long

+

Number of read bytes

+

execute_update_date

+

No

+

Long

+

Execution update date

+

write_rows

+

No

+

Integer

+

Number of written rows

+

rows_written

+

No

+

Integer

+

Number of written rows

+

rows_read

+

No

+

Long

+

Number of rows that are read

+

files_written

+

No

+

Integer

+

Number of written files

+

is_incrementing

+

No

+

Boolean

+

Incremental or not

+

execute_create_date

+

No

+

Long

+

Execution creation date

+

status

+

No

+

String

+

Job execution status

+

  • BOOTING: The job is starting.

    +

  • RUNNING: The job is running.

    +

  • SUCCEEDED: The job was successfully executed.

    +

  • FAILED: The job execution failed.

    +

  • NEW: The job was not executed.

    +
+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 5 ConfigValues

Parameter

+

Mandatory

+

Type

+

Description

+

configs

+

Yes

+

Array of configs objects

+

The data structures of source link parameters, destination link parameters, and job parameters are the same. However, the inputs parameter varies. For details, see the descriptions of configs parameters.

+

extended-configs

+

No

+

extended-configs object

+

Extended configuration. For details, see the descriptions of extended-configs parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 6 configs

Parameter

+

Mandatory

+

Type

+

Description

+

inputs

+

Yes

+

Array of Input objects

+

Input parameter list. Each element in the list is in name,value format. For details, see the descriptions of inputs parameters. In the from-config-values data structure, the value of this parameter varies with the source link type. For details, see section "Source Job Parameters" in the Cloud Data Migration User Guide. In the to-cofig-values data structure, the value of this parameter varies with the destination link type. For details, see section "Destination Job Parameters" in the Cloud Data Migration User Guide. For details about the inputs parameter in the driver-config-values data structure, see the job parameter descriptions.

+

name

+

Yes

+

String

+

Configuration name. The value is fromJobConfig for a source job, toJobConfig for a destination job, and linkConfig for a link.

+

id

+

No

+

Integer

+

Configuration ID

+

type

+

No

+

String

+

Configuration type

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 7 Input

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Parameter name

+

value

+

Yes

+

String

+

Parameter value

+

type

+

No

+

String

+

Value type

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 8 extended-configs

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

No

+

String

+

Name

+

value

+

No

+

String

+

Value

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + +
Table 9 Response body parameters

Parameter

+

Type

+

Description

+

validation-result

+

Array of JobValidationResult objects

+

Validation result. - If a job fails to be modified, the failure cause is returned. - If a job is successfully modified, an empty list is returned.

+
+
+ +
+ + + + + + + + + + + + + +
Table 10 JobValidationResult

Parameter

+

Type

+

Description

+

message

+

String

+

Error message

+

status

+

String

+

ERROR,WARNING

+

Enumeration values:

+

  • ERROR

    +

  • WARNING

    +
+
+
+

Status code: 400

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 11 Response body parameters

Parameter

+

Type

+

Description

+

code

+

String

+

Return code

+

errCode

+

String

+

Error code

+

message

+

String

+

Error message

+

externalMessage

+

String

+

Additional information

+
+
+
+

Example Requests

PUT /v1.1/1551c7f6c808414d8e9f3c514a170f2e/cluster/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/job/es_css
+
+{
+  "jobs" : [ {
+    "job_type" : "NORMAL_JOB",
+    "from-connector-name" : "elasticsearch-connector",
+    "to-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "toJobConfig.streamName",
+          "value" : "dis-lkGm"
+        }, {
+          "name" : "toJobConfig.separator",
+          "value" : "|"
+        }, {
+          "name" : "toJobConfig.columnList",
+          "value" : "1&2&3"
+        } ],
+        "name" : "toJobConfig"
+      } ]
+    },
+    "to-link-name" : "dis",
+    "driver-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "throttlingConfig.numExtractors",
+          "value" : "1"
+        }, {
+          "name" : "throttlingConfig.submitToCluster",
+          "value" : "false"
+        }, {
+          "name" : "throttlingConfig.numLoaders",
+          "value" : "1"
+        }, {
+          "name" : "throttlingConfig.recordDirtyData",
+          "value" : "false"
+        } ],
+        "name" : "throttlingConfig"
+      }, {
+        "inputs" : { },
+        "name" : "jarConfig"
+      }, {
+        "inputs" : [ {
+          "name" : "schedulerConfig.isSchedulerJob",
+          "value" : "false"
+        }, {
+          "name" : "schedulerConfig.disposableType",
+          "value" : "NONE"
+        } ],
+        "name" : "schedulerConfig"
+      }, {
+        "inputs" : { },
+        "name" : "transformConfig"
+      }, {
+        "inputs" : [ {
+          "name" : "retryJobConfig.retryJobType",
+          "value" : "NONE"
+        } ],
+        "name" : "retryJobConfig"
+      } ]
+    },
+    "from-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "fromJobConfig.index",
+          "value" : "52est"
+        }, {
+          "name" : "fromJobConfig.type",
+          "value" : "est_array"
+        }, {
+          "name" : "fromJobConfig.columnList",
+          "value" : "array_f1_int:long&array_f2_text:string&array_f3_object:nested"
+        }, {
+          "name" : "fromJobConfig.splitNestedField",
+          "value" : "false"
+        } ],
+        "name" : "fromJobConfig"
+      } ]
+    },
+    "to-connector-name" : "dis-connector",
+    "name" : "es_css",
+    "from-link-name" : "css"
+  } ]
+}
+
+

Example Responses

Status code: 200

+

ok

+
{
+  "validation-result" : [ { }, { }, { } ]
+}
+

Status code: 400

+

Error code

+
{
+  "code" : "Cdm.0095",
+  "errCode" : "Cdm.00095",
+  "message" : "A job with the name obs-obs does not exist.",
+  "externalMessage" : "A job with the name obs-obs does not exist."
+}
+
+

Status Codes

+
+ + + + + + + + + + +

Status Code

+

Description

+

200

+

ok

+

400

+

Error code

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Job Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/UpdateLink.html b/docs/dataartsstudio/api-ref/UpdateLink.html new file mode 100644 index 000000000..45f8fa4ba --- /dev/null +++ b/docs/dataartsstudio/api-ref/UpdateLink.html @@ -0,0 +1,569 @@ + + +

Modifying a Link

+

Function

This API is used to modify a link.

+
+

URI

PUT /v1.1/{project_id}/clusters/{cluster_id}/cdm/link/{link_name}

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 1 Path Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

project_id

+

Yes

+

String

+

Project ID

+

cluster_id

+

Yes

+

String

+

Cluster ID

+

link_name

+

Yes

+

String

+

Link name

+
+
+
+

Request Parameters

+
+ + + + + + + + + + + +
Table 2 Request header parameters

Parameter

+

Mandatory

+

Type

+

Description

+

X-Auth-Token

+

Yes

+

String

+

User token. It can be obtained by calling the IAM API (value of X-Subject-Token in the response header).

+
+
+ +
+ + + + + + + + + + + +
Table 3 Request body parameters

Parameter

+

Mandatory

+

Type

+

Description

+

links

+

Yes

+

Array of links objects

+

Link list. For details, see the descriptions of links parameters.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 links

Parameter

+

Mandatory

+

Type

+

Description

+

link-config-values

+

Yes

+

link-config-values object

+

Link parameters. For details, see the descriptions of link-config-values parameters.

+

creation-user

+

No

+

String

+

User who created the link

+

name

+

Yes

+

String

+

Link name

+

id

+

No

+

Integer

+

Link ID

+

creation-date

+

No

+

Long

+

Time when the link was created

+

connector-name

+

Yes

+

String

+

Connector name. The mappings between the connectors and links are as follows:generic-jdbc-connector: link to a relational databaseobs-connector: OBS linkhdfs-connector: HDFS linkhbase-connector: HBase or CloudTable linkhive-connector: Hive linkftp-connector/sftp-connector: link to an FTP or SFTP servermongodb-connector: MongoDB linkredis-connector: Redis or DCS linkkafka-connector: Kafka linkdis-connector: DIS linkelasticsearch-connector: Elasticsearch or CSS linkdli-connector: DLI linkhttp-connector: HTTP or HTTPS link. This type of link does not have parameters.dms-kafka-connector: DMS Kafka link

+

update-date

+

No

+

Long

+

Time when the link was updated

+

enabled

+

No

+

Boolean

+

Whether to activate the link. The default value is true.

+

update-user

+

No

+

String

+

User who updated the link

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 5 link-config-values

Parameter

+

Mandatory

+

Type

+

Description

+

configs

+

Yes

+

Array of configs objects

+

Data structure of link parameters. For details, see the descriptions of configs parameters.

+

extended-configs

+

No

+

extended-configs object

+

Extended configuration. For details, see the descriptions of extended-configs parameters.

+

validators

+

No

+

Array of strings

+

Validator

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 6 configs

Parameter

+

Mandatory

+

Type

+

Description

+

inputs

+

Yes

+

Array of Input objects

+

Input parameter list. Each element in the list is in name,value format. For details, see the descriptions of inputs parameters. In the from-config-values data structure, the value of this parameter varies with the source link type. For details, see section "Source Job Parameters" in the Cloud Data Migration User Guide. In the to-cofig-values data structure, the value of this parameter varies with the destination link type. For details, see section "Destination Job Parameters" in the Cloud Data Migration User Guide. For details about the inputs parameter in the driver-config-values data structure, see the job parameter descriptions.

+

name

+

Yes

+

String

+

Configuration name. The value is fromJobConfig for a source job, toJobConfig for a destination job, and linkConfig for a link.

+

id

+

No

+

Integer

+

Configuration ID

+

type

+

No

+

String

+

Configuration type

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 7 Input

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Parameter name

+

value

+

Yes

+

String

+

Parameter value

+

type

+

No

+

String

+

Value type

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 8 extended-configs

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

No

+

String

+

Name

+

value

+

No

+

String

+

Value

+
+
+
+

Response Parameters

Status code: 200

+ +
+ + + + + + + + + +
Table 9 Response body parameters

Parameter

+

Type

+

Description

+

validation-result

+

Array of validationResult objects

+

Validation structure. If a link fails to be created, the failure cause is returned. For details, see the descriptions of validation-result parameters. If a link is successfully created, an empty list is returned.

+
+
+ +
+ + + + + + + + + +
Table 10 validationResult

Parameter

+

Type

+

Description

+

linkConfig

+

Array of validationLinkConfig objects

+

Validation result of link creation or update. For details, see the descriptions of validationLinkConfig parameters.

+
+
+ +
+ + + + + + + + + + + + + +
Table 11 validationLinkConfig

Parameter

+

Type

+

Description

+

message

+

String

+

Error message

+

status

+

String

+

ERROR,WARNING

+

Enumeration values:

+

  • ERROR

    +

  • WARNING

    +
+
+
+

Status code: 500

+ +
+ + + + + + + + + + + + + +
Table 12 Response body parameters

Parameter

+

Type

+

Description

+

message

+

String

+

Error message

+

status

+

String

+

ERROR,WARNING

+

Enumeration values:

+

  • ERROR

    +

  • WARNING

    +
+
+
+
+

Example Requests

PUT /v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/link/rdb_link
+
+{
+  "links" : [ {
+    "link-config-values" : {
+      "configs" : [ {
+        "inputs" : [ {
+          "name" : "linkConfig.databaseType",
+          "value" : "MYSQL"
+        }, {
+          "name" : "linkConfig.host",
+          "value" : "100.94.8.163"
+        }, {
+          "name" : "linkConfig.port",
+          "value" : "3306"
+        }, {
+          "name" : "linkConfig.database",
+          "value" : "DB_name"
+        }, {
+          "name" : "linkConfig.username",
+          "value" : "username"
+        }, {
+          "name" : "linkConfig.password",
+          "value" : "DB_password"
+        }, {
+          "name" : "linkConfig.fetchSize",
+          "value" : "100000"
+        }, {
+          "name" : "linkConfig.usingNative",
+          "value" : false
+        } ],
+        "name" : "linkConfig"
+      } ]
+    },
+    "name" : "mysql_link",
+    "creation-date" : 1496654788622,
+    "connector-name" : "generic-jdbc-connector",
+    "update-date" : 1496654788622,
+    "enabled" : true
+  } ]
+}
+
+

Example Responses

Status code: 200

+

OK

+
{
+  "name" : "rdb_link",
+  "validation-result" : { }
+}
+

Status code: 500

+

An internal service error occurred. For details, see error codes.

+
{
+  "validation-result" : [ {
+    "linkConfig" : [ {
+      "message" : "Can't connect to the database with given credentials: The authentication type 12 is not supported. Check that you have configured the pg_hba.conf file to include the client's IP address or subnet, and that it is using an authentication scheme supported by the driver.",
+      "status" : "ERROR"
+    } ]
+  } ]
+}
+
+

Status Codes

+
+ + + + + + + + + + + + + + + + + + + + + + + + + +

Status Code

+

Description

+

200

+

OK

+

400

+

Request error.

+

401

+

Authentication failed.

+

403

+

You do not have required permissions to perform this operation.

+

404

+

The requested resource was not found.

+

500

+

An internal service error occurred. For details, see error codes.

+

503

+

Service unavailable.

+
+
+
+

Error Codes

See Error Codes.

+
+
+
+
+
Parent topic: Link Management
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0001.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0001.html new file mode 100644 index 000000000..f32b1f5fd --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0001.html @@ -0,0 +1,25 @@ + + +

Before You Start

+
+
+ +
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0002.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0002.html new file mode 100644 index 000000000..96e72f078 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0002.html @@ -0,0 +1,13 @@ + + +

Overview

+

Welcome to DataArts Studio. DataArts Studio is a one-stop operations platform that provides data lifecycle management and intelligent data management for enterprises' digital operations. It provides functions such as DataArts Migration, DataArts Architecture, DataArts Factory, DataArts Quality, DataArts Catalog, and DataArts DataService. DataArts Studio supports intelligent construction of industrial knowledge libraries as well as data foundations such as big data storage and big data computing and analysis engines. DataArts Studio helps you quickly construct intelligent end-to-end data systems from data ingestion to data analysis. It helps eliminate data silos, unify standards, accelerate monetization, and generally assist with digital transformation.

+

This document describes how to use APIs to perform operations on DataArts Studio components, such as creating clusters and connections.

+

Before calling a DataArts Studio API, ensure that you have fully understood the concepts related to DataArts Studio. For details, see section "Service Overview" in DataArts Studio User Manual.

+
+
+
+
Parent topic: Before You Start
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0003.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0003.html new file mode 100644 index 000000000..dce9de7fc --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0003.html @@ -0,0 +1,11 @@ + + +

API Calling

+

DataArts Studio supports Representational State Transfer (REST) APIs, allowing you to call APIs using HTTPS. For details on API calling, see Calling APIs.

+
+
+
+
Parent topic: Before You Start
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0004.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0004.html new file mode 100644 index 000000000..ca1156621 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0004.html @@ -0,0 +1,13 @@ + + +

Endpoints

+

Obtaining an Endpoint

An endpoint is the request address for calling an API. Endpoints vary depending on services and regions.

+

You can obtain endpoints from Regions and Endpoints.

+
+
+
+
+
Parent topic: Before You Start
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0005.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0005.html new file mode 100644 index 000000000..47619fcfc --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0005.html @@ -0,0 +1,12 @@ + + +

Constraints

+
+
+
+
+
Parent topic: Before You Start
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0006.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0006.html new file mode 100644 index 000000000..66642422f --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0006.html @@ -0,0 +1,16 @@ + + +

Concepts

+
+
+
+
+
Parent topic: Before You Start
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0007.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0007.html new file mode 100644 index 000000000..676a0e7eb --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0007.html @@ -0,0 +1,196 @@ + + +

API Overview

+

DataArts Studio provides self-developed APIs that comply with RESTful API design specifications. You can call those APIs to perform operations on DataArts Studio.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 DataArts Migration APIs

Type

+

API

+

Description

+

Maximum Number of Requests Made Per User Per Minute

+

Cluster Management

+

Creating a Cluster

+

Used to create a CDM cluster.

+

5

+

Querying the Cluster List

+

Used to query the list of CDM clusters.

+

120

+

Querying Cluster Details

+

Used to query the details of a CDM cluster.

+

120

+

Restarting a Cluster

+

Used to restart a CDM cluster.

+

20

+

Deleting a Cluster

+

Used to delete a CDM cluster.

+

20

+

Stopping a Cluster

+

Used to stop a CDM cluster.

+

20

+

Starting a Cluster

+

Used to start a CDM cluster.

+

20

+

Link Management

+

Creating a Link

+

Used to connect to a data source.

+

120

+

Querying a Link

+

Used to query the link list.

+

120

+

Modifying a Link

+

Used to modify the link configuration.

+

120

+

Deleting a Link

+

Used to delete a link.

+

120

+

Job Management

+

Creating a Job in a Specific Cluster

+

Used to create a data migration job in a specific CDM cluster. The job will not automatically start.

+

1200

+

Creating and Executing a Job in a Random Cluster

+

Used to select a random cluster in the running state from the specified clusters and create and execute a data migration job.

+

120

+

Querying a Job

+

Used to query the job list.

+

120

+

Modifying a Job

+

Used to modify the job configuration.

+

120

+

Starting a Job

+

Used to start a data migration job.

+

1200

+

Stopping a Job

+

Used to stop a running job.

+

1200

+

Querying Job Status

+

Used to query the running status of a job.

+

120

+

Querying Job Execution History

+

Used to query the historical status of job execution.

+

120

+

Deleting a job

+

Used to delete a job.

+

120

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 2 DataArts Factory APIs

Type

+

Description

+

Connection Management APIs

+

Used to create, edit, delete, import, and export data connections, query the data connection list, and query data connection details. For details, see DataArts Factory > Data Management > Creating a Data Connection in the User Guide.

+

Script Development APIs

+

Used to create, query, modify, delete, and execute scripts. For details, see DataArts Factory > Script Development in the User Guide.

+

Resource Management APIs

+

Used to create, query, and modify resources. For details, see DataArts Factory > Configuration and Management > Managing a Resource in the User Guide.

+

Job Development APIs

+

Used to create, edit, delete, import, and export jobs, query the job list, and query job details. For details, see DataArts Factory > Job Development in the User Guide.

+
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0008.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0008.html new file mode 100644 index 000000000..4bf35242c --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0008.html @@ -0,0 +1,15 @@ + + +

Calling APIs

+
+
+ +
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0009.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0009.html new file mode 100644 index 000000000..9b9a5a834 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0009.html @@ -0,0 +1,247 @@ + + +

Making an API Request

+

This section describes the structure of a REST API, and uses the IAM API for obtaining a user token as an example to demonstrate how to call an API. The obtained token can then be used to authenticate the calling of other APIs.

+

Request URI

A request URI is in the following format:

+

{URI-scheme}://{Endpoint}/{resource-path}?{query-string}

+

Although a request URI is included in the request header, most programming languages or frameworks require the request URI to be transmitted separately.

+ +
+ + + + + + + + + + + + + + + + +
Table 1 URI parameter description

Parameter

+

Description

+

URI-scheme

+

Protocol used to transmit requests. All APIs use HTTPS.

+

Endpoint

+

Domain name or IP address of the server bearing the REST service endpoint. The endpoint varies between services in different regions.

+

An endpoint is the request address for calling an API. Endpoints vary depending on services and regions. You can obtain endpoints from Regions and Endpoints.

+

resource-path

+

Access path of an API for performing a specified operation. Obtain the value from the URI of an API. For example, the resource-path of the API used to obtain a user token is /v3/auth/tokens.

+

query-string

+

Query parameter, which is optional. Ensure that a question mark (?) is included before each query parameter that is in the format of "Parameter name=Parameter value", for example, ?limit=10, it indicates that a maximum of 10 data records are allowed.

+
+
+

To simplify the URI display in this document, each API is provided only with a resource-path and a request method. The URI-scheme value of all APIs is HTTPS, and the endpoints of all APIs in the same region are identical.

+
+
+

Request Method

The HTTP protocol defines the following request methods that can be used to send a request to the server: +
+ + + + + + + + + + + + + + + + + + + + + + +
Table 2 HTTP methods

Method

+

Description

+

GET

+

Requests the server to return specified resources.

+

PUT

+

Requests the server to update specified resources.

+

POST

+

Requests the server to add resources or perform special operations.

+

DELETE

+

Requests the server to delete specified resources, for example, an object.

+

HEAD

+

Same as GET except that the server must return only the response header.

+

PATCH

+

Requests the server to update partial content of a specified resource.

+

If the resource does not exist, a new resource will be created.

+
+
+
+

For example, in the case of the API used to obtain a user token, the request method is POST. The request is as follows:

+
POST https://{{endpoint}}/v3/auth/tokens
+
+

Request Header

You can also add additional header fields to a request, such as the fields required by a specified URI or HTTP method. For example, to request the authentication information, add Content-Type, which specifies the request body type.

+
Table 3 lists common request header fields. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Common request header fields

Field

+

Description

+

Mandatory

+

Example

+

Content-Type

+

Specifies the request body MIME type. This field is mandatory and its default value is application/json. Other values of this field will be provided for specific APIs if any.

+

Yes

+

The default value is application/json.

+

Content-Length

+

Specifies the length of the request body. The unit is byte.

+

No

+

3495

+

X-Language

+

Specifies the request language.

+

No

+

en_us

+

X-Project-Id

+

Specifies a project ID. Obtain the project ID by following the instructions in Project ID and Account ID.

+

No

+

e9993fc787d94b6c886cbaa340f9c0f4

+

X-Auth-Token

+

Specifies a user token.

+

The user token is a response to the API used to obtain a user token. This API is the only one that does not require authentication.

+

The token is the value of X-Subject-Token in the response header.

+

This field is mandatory for token authentication.

+

The following is part of an example token:

+

MIIPAgYJKoZIhvcNAQcCo...ggg1BBIINPXsidG9rZ

+

X-Sdk-Date

+

Time when the request is sent. The time is in YYYYMMDD'T'HHMMSS'Z' format.

+

The value is the current GMT time of the system.

+

This field is mandatory for AK/SK-based authentication.

+

20150907T101459Z

+

Host

+

Specifies the server domain name and port number of the resources being requested. The value can be obtained from the URL of the service API. The value is in the format of hostname[:port]. If the port number is not specified, the default port is used. The default port number for HTTPS is 443.

+

This field is mandatory for AK/SK-based authentication.

+

code.test.com

+

or

+

code.test.com:443

+

Authorization

+

Authentication information.

+

The value can be obtained from the request signing result.

+

This field is mandatory for AK/SK-based authentication.

+

SDK-HMAC-SHA256 Credential=QRUP2R3QFNAOVAWMYHZW/20160202/northchina/test/sdk_request, SignedHeaders=host;x-sdk-date, Signature=9d8b56b055c0e1f7a9498d881a7cb726be91b4f0cde1773b0b1557e987a480ce

+

workspace

+

Workspace ID. For details about how to obtain it, see DataArts Studio Instance ID and Workspace ID.

+
  • You do not need to set this field when calling a CDM API because in that case, only the default workspace can be used.
  • When a DataArts Factory API is called, data in the default workspace is queried by default. To query data in other workspaces, include this header in the query request.
+

d1cd7861478748a6925bc02f47c69279

+

X-Dlm-Type

+

DataArts DataService edition, which is Exclusive

+

This parameter is mandatory for calling DataArts DataService APIs.

+

EXCLUSIVE: exclusive edition

+
+
+
+

In addition to supporting token-based authentication, APIs also support authentication using access key ID/secret access key (AK/SK). During AK/SK-based authentication, an SDK is used to sign the request, and the Authorization (signature authentication) and X-Sdk-Date (time when the request is sent) header fields are automatically added to the request.

+

For more information, see "AK/SK-based Authentication" in Authentication.

+
+

The API used to obtain a user token does not require authentication. Therefore, only the Content-Type field needs to be added to requests for calling the API. An example of such requests is as follows:

+
POST https://{{endpoint}}/v3/auth/tokens
+Content-Type: application/json
+
+

Request Body

The body of a request is often sent in a structured format as specified in the Content-Type header field. The request body transfers content except the request header.

+

The request body varies between APIs. Some APIs do not require the request body, such as the APIs requested using the GET and DELETE methods.

+

In the case of the API used to obtain a user token, the request parameters and parameter description can be obtained from the API request. The following provides an example request with a body included. Replace username, domainname, ******** (login password), and xxxxxxxxxxxxxxxxxx (project ID) with the actual values. To learn how to obtain a project ID, see Project ID and Account ID.

+

The scope parameter specifies where a token takes effect. In the following example, the token takes effect only for the resources in a specified project.

+
+
+POST https://{{endpoint}}/v3/auth/tokens
+Content-Type: application/json
+
+{
+    "auth": {
+        "identity": {
+            "methods": [
+                "password"
+            ],
+            "password": {
+                "user": {
+                    "name": "username",
+                    "password": "********",
+                    "domain": {
+                        "name": "domainname"
+                    }
+                }
+            }
+        },
+        "scope": {
+            "project": {
+                "id": "xxxxxxxxxxxxxxxxxx"
+            }
+        }
+    }
+}
+

If all data required for the API request is available, you can send the request to call the API through curl, Postman, or coding. In the response to the API used to obtain a user token, x-subject-token is the desired user token. This token can then be used to authenticate the calling of other APIs.

+
+
+
+
+
Parent topic: Calling APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0010.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0010.html new file mode 100644 index 000000000..1711cef7c --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0010.html @@ -0,0 +1,58 @@ + + +

Authentication

+
Requests for calling an API can be authenticated using either of the following methods: +
+

Token-based Authentication

The validity period of a token is 24 hours. When using a token for authentication, cache it to prevent frequently calling the IAM API used to obtain a user token.

+
+

A token specifies temporary permissions in a computer system. During API authentication using a token, the token is added to a request to get permissions for calling the API.

+

When calling the API to obtain a user token, you must set auth.scope in the request body to project.

+
{ 
+    "auth": { 
+        "identity": { 
+            "methods": [ 
+                "password" 
+            ], 
+            "password": { 
+                "user": { 
+                    "name": "username", 
+                    "password": "********", 
+                    "domain": { 
+                        "name": "domainname" 
+                    } 
+                } 
+            } 
+        }, 
+        "scope": { 
+            "project": { 
+                "id": "xxxxxxxxxxxxxxxxxx" 
+            } 
+        } 
+    } 
+}
+

After a token is obtained, the X-Auth-Token header field must be added to requests to specify the token when calling other APIs, for example, the API used to query a connection list. For example, if the token is ABCDEFJ...., X-Auth-Token: ABCDEFJ.... can be added to a request as follows:

+
+GET https://{{endpoint}}/v1/{project_id}/connections
+Content-Type: application/json
+X-Auth-Token: ABCDEFJ....
+
+

AK/SK-based Authentication

AK/SK-based authentication supports API requests with a body not larger than 12 MB. For API requests with a larger body, token-based authentication is recommended.

+
+

In AK/SK-based authentication, AK/SK is used to sign requests and the signature is then added to the requests for authentication.

+ +
To obtain an access key, perform the following steps:
  1. Log in to the management console, move the cursor to the username in the upper right corner, and select My Credentials from the drop-down list.
  2. On the My Credentials page, choose Access Keys, and click Create Access Key. See Figure 1.
    Figure 1 Clicking Create Access Key
    +
  3. Click OK and save the access key file as prompted. The access key file will be saved to your browser's configured download location. Open the credentials.csv file to view Access Key Id and Secret Access Key.
    • Only two access keys can be added for each user.
    • To ensure access key security, the access key is automatically downloaded only when it is generated for the first time and cannot be obtained from the management console later. Keep them properly.
    +
    +
+
+
In AK/SK-based authentication, you can use an AK/SK to sign requests based on the signature algorithm or use the signing SDK to sign requests. For details about how to sign requests and use the signing SDK, see API Request Signing Guide.

The signing SDKs are only used for signing requests and different from the SDKs provided by services.

+
+
+
+
+
+
+
Parent topic: Calling APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0011.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0011.html new file mode 100644 index 000000000..97e94b1a0 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0011.html @@ -0,0 +1,39 @@ + + +

Response

+

Status Code

After sending a request, you will receive a response, including a status code, response header, and response body.

+

A status code is a group of digits, ranging from 1xx to 5xx. It indicates the status of a request. For more information, see Status Codes.

+

For example, if status code 201 is returned for calling the API used to obtain a user token, the request is successful.

+
+

Response Header

Similar to a request, a response also has a header, for example, Content-Type.

+

Figure 1 shows the response header fields for the API used to obtain a user token. The x-subject-token header field is the desired user token. This token can then be used to authenticate the calling of other APIs.

+
Figure 1 Header fields of the response to the request for obtaining a user token
+
+

(Optional) Response Body

The body of a response is often returned in structured format (such as JSON or XML) as specified in the Content-Type header field. The response body transfers content except the response header.

+
+

The following is part of the response body for the API used to obtain a user token. The following describes part of the request body.

+
{
+    "token": {
+        "expires_at": "2019-02-13T06:52:13.855000Z",
+        "methods": [
+            "password"
+        ],
+        "catalog": [
+            {
+                "endpoints": [
+                    {
+                        "region_id": "xxxxxx",
+...
+

If an error occurs during API calling, an error code and a message will be displayed. The following shows an error response body.

+
{
+    "error_msg": "The format of message is error",
+    "error_code": "AS.0001"
+}
+

In the response body, error_code is an error code, and error_msg provides information about the error.

+
+
+
+
Parent topic: Calling APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0012.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0012.html new file mode 100644 index 000000000..af6a9059a --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0012.html @@ -0,0 +1,14 @@ + + +

Application Cases

+

+
+
+ +
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0013.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0013.html new file mode 100644 index 000000000..5146f9892 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0013.html @@ -0,0 +1,489 @@ + + +

Application Example

+
This section describes how to use cURL to call CDM APIs to migrate data from a local MySQL database to DWS in the cloud.
  1. Obtaining a Token

    Call the API to obtain the user token, which will be put into the request header for authentication in a subsequent request.

    +
  2. Creating a CDM Cluster
    • If you have created a CDM cluster, skip this step and directly use the ID of the created cluster.
    • If you want to use a new cluster for migration, call the API in Creating a Cluster to create a CDM cluster.
    +
  3. Creating Links

    Call the API in Creating a Link to create the MySQL and DWS links.

    +
  4. Creating a Migration Job

    Call the API in Creating a Job in a Specified Cluster to create a job for migrating data from MySQL to DWS.

    +
  5. Viewing Job Result

    Call the API in Querying Job Execution History to execute the job.

    +
+
+

Preparing Data

Before calling an API, prepare the following data.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Preparing data

Item

+

Name

+

Description

+

Example

+

Account information

+

Project name

+

Name of the project where CDM resides

+

Project Name

+

Project ID

+

ID of the project where CDM resides

+

1551c7f6c808414d8e9f3c514a170f2e

+

Account name

+

Name of an enterprise account to which a user belongs

+

Account Name

+

Username

+

Username for using a cloud service. The user must have operation permissions on CDM.

+

Username

+

Password

+

User password

+

password

+

VPC information

+

VPC ID

+

The VPC where CDM resides must be the same as that of DWS.

+

6b47302a-bf79-4b20-bf7a-80987408e196

+

Subnet ID

+

The subnet where CDM resides must be the same as that of DWS.

+

63bdc3cb-a4e7-486f-82ee-d9bf208c8f8c

+

Security group ID

+

The security group where CDM resides must be the same as that of DWS.

+

005af77a-cce5-45ac-99c7-2ea50ea8addf

+

Endpoint

+

IAM endpoint

+

An endpoint is the request address for calling an API. Endpoints vary depending on services and regions. You can obtain endpoints from Regions and Endpoints.

+

iam_endpoint

+

CDM endpoint

+

cdm_endpoint

+

MySQL database

+

IP address

+

IP address of the local MySQL database, which allows CDM to access the MySQL database using a public IP address

+

1xx.120.85.24

+

Port

+

MySQL database port

+

3306

+

Database name

+

Name of the MySQL database from which data is to be exported

+

DB_name

+

Username

+

Username for accessing the MySQL database. The user must have the read, write, and delete permissions on the MySQL database.

+

username

+

Password

+

Password for accessing the MySQL database

+

DB_password

+

DWS database

+

IP address

+

IP address of the DWS database. CDM can access the IP address through the internal network.

+

10.120.85.24

+

Port

+

DWS database port

+

3306

+

Database name

+

Name of the DWS database to which data is written

+

DWS

+

Username

+

Username for accessing the DWS database. The user must have the read, write, and delete permissions on the DWS database.

+

user_dws

+

Password

+

Password for accessing the DWS database

+

dws_password

+
+
+
+

Obtaining a Token

  1. Before calling other APIs, obtain the token and set it as an environment variable.
    curl -H "Content-Type:application/json" https://{iam_endpoint}/v3/auth/tokens -X POST -d '
+{
+    "auth": {
+        "identity": {
+            "methods": [
+                "password"
+            ],
+            "password": {
+                "user": {
+                    "name": "Username",
+                    "password": "password",
+                    "domain": {
+                        "name": "Account Name"
+                    }
+                }
+            }
+        },
+        "scope": {
+            "project": {
+                "id": "1551c7f6c808414d8e9f3c514a170f2e"
+            }
+        }
+    }
+}
+' -v -k
    +

    The value of X-Subject-Token in the response header is the token.

    +
    X-Subject-Token:MIIDkgYJKoZIhvcNAQcCoIIDgzCCA38CAQExDTALBglghkgBZQMEAgEwgXXXXX...
    +
  2. Run the following command to set the token as an environment variable for future use:
    export X-Auth-Token = MIIDkgYJKoZIhvcNAQcCoIIDgzCCA38CAQExDTALBglghkgBZQMEAgEwgXXXXX...
    +
+
+

Creating a CDM Cluster

  1. Call the API in Creating a Cluster to create a cluster. The following values are examples:
    • Cluster name: cdm-ab82
    • Cluster flavor: cdm.medium
    • The VPC, subnet, and security group are the same as those of DWS, and the EIP is automatically bound.
    +

    If status code 200 is returned, the cluster is successfully created.

    +
    curl -X POST -H 'Content-Type:application/json;charset=utf-8' -H "X-Auth-Token:$Token" -d '
+{
+  "cluster": {
+    "name": "cdm-ab82",
+    "vpcId": "6b47302a-bf79-4b20-bf7a-80987408e196", 
+    "instances": [{
+            "flavorRef": "fb8fe666-6734-4b11-bc6c-43d11db3c745",
+            "nics": [{
+                "net-id": "63bdc3cb-a4e7-486f-82ee-d9bf208c8f8c",
+                "securityGroupId": "005af77a-cce5-45ac-99c7-2ea50ea8addf"
+            }],
+            "availability_zone": "Project Name",
+            "type": "cdm"
+        }],
+    "datastore": {
+            "version": "1.8.5",
+            "type": "cdm"
+        },
+    "isScheduleBootOff": false,
+    "scheduleBootTime": "null",
+    "scheduleOffTime": "null",
+    "isAutoOff": false,
+    "sys_tags": [{
+            "key": "_sys_enterprise_project_id",
+            "value": "1ce45885-4033-40d2-bdde-d4dbaceb387d"
+        }]
+    },
+  "autoRemind": false,
+  "phoneNum": "null",
+  "email": "null"
+}' 
+https://{cdm_endpoint}/v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters -v -k
    +
  2. Call the API in Querying Cluster Details to query cluster information, obtain the cluster ID, and set the cluster ID to a global variable.
    curl -X GET -H 'Content-Type:application/json;charset=utf-8' -H "X-Auth-Token:$Token" https://{cdm_endpoint}/v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters -k -v
    +

    The response is as follows:

    +
    {
+  "clusters": [{
+    "version": "x.x.x",
+    "updated": "2018-09-05T08:38:25",
+    "name": "cdm-ab82",
+    "created": "2018-09-05T08:38:25",
+    "id": "bae65496-643e-47ca-84af-948672de7eeb",
+    "status": "200",
+    "isFrozen": "0",
+    "statusDetail": "Normal",
+    "actionProgress": {},
+    "config_status": "In-Sync"
+  }]
+}
    +

    If the value of status is 200, the cluster is successfully created. The cluster ID is bae65496-643e-47ca-84af-948672de7eeb.

    +
  3. Run the following command to set the cluster ID to a global variable for future use:
    export ID = bae65496-643e-47ca-84af-948672de7eeb
    +
+
+

Creating Links

  1. Call the API in Creating a Link to create the MySQL link mysql_link. The following values are examples:
    • IP address: 1xx.120.85.24
    • Port number: 3306
    • Database name: DB_name
    • Login username: username
    • Password: DB_password
    +

    If status code 200 is returned, the link is successfully created.

    +
    curl -X POST -H "Content-Type:application/json" -H "X-Auth-Token:$Token" -d '{
+  "links": [{
+    "enabled": true,
+    "update-user": null,
+    "name": "mysql_link",
+    "link-config-values": {
+      "configs": [
+                    {
+                        "name": "linkConfig",
+                        "inputs": [
+                            {
+                                "name": "linkConfig.databaseType",
+                                "value": "MYSQL"
+                            },
+                            {
+                                "name": "linkConfig.host",
+                                "value": "1xx.120.85.24"
+                            },
+                            {
+                                "name": "linkConfig.port",
+                                "value": "3306"
+                            },
+                            {
+                                "name": "linkConfig.database",
+                                "value": "DB_name"
+                            },
+                            {
+                                "name": "linkConfig.username",
+                                "value": "username"
+                            },
+                            {
+                                "name": "linkConfig.password",
+                                "value": "DB_password"
+                            },
+                            {
+                                "name": "linkConfig.fetchSize",
+                                "value": "100000"
+                            },
+                            {
+                                "name": "linkConfig.usingNative",
+                                "value": "true"
+                            }
+                        ]
+                    }
+                ]
+    },
+    "connector-name": "generic-jdbc-connector",
+    "creation-date": 1536654788622,
+    "update-date": 1536654788622,
+    "creation-user": null
+  }]
+}' 
+https://{cdm_endpoint}/v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/bae65496-643e-47ca-84af-948672de7eeb/cdm/link -k -v
    +
  2. Call the API in Creating a Link to create the DWS link dws_link. The following values are examples:
    • IP address of the database: 10.120.85.24
    • Port number: 3306
    • Database name: DWS
    • Login username: user_dws
    • Password: dws_password
    +
    curl -X POST -H "Content-Type:application/json" -H "X-Auth-Token:$Token" -d '{
+  "links": [{
+    "enabled": true,
+    "update-user": null,
+    "name": "dws_link",
+    "link-config-values": {
+      "configs": [
+                    {
+                        "name": "linkConfig",
+                        "inputs": [
+                            {
+                                "name": "linkConfig.databaseType",
+                                "value": "DWS"
+                            },
+                            {
+                                "name": "linkConfig.host",
+                                "value": "10.120.85.24"
+                            },
+                            {
+                                "name": "linkConfig.port",
+                                "value": "3306"
+                            },
+                            {
+                                "name": "linkConfig.database",
+                                "value": "DWS"
+                            },
+                            {
+                                "name": "linkConfig.username",
+                                "value": "user_dws"
+                            },
+                            {
+                                "name": "linkConfig.password",
+                                "value": "dws_password"
+                            },
+                            {
+                                "name": "linkConfig.fetchSize",
+                                "value": "100000"
+                            },
+                            {
+                                "name": "linkConfig.usingNative",
+                                "value": "true"
+                            }
+                        ]
+                    }
+                ]
+    },
+    "connector-name": "generic-jdbc-connector",
+    "creation-date": 1536654788622,
+    "update-date": 1536654788622,
+    "creation-user": null
+  }]
+}' 
+https://{cdm_endpoint}/v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/bae65496-643e-47ca-84af-948672de7eeb/cdm/link -k -v
    +
+
+

Creating a Migration Job

  1. After the links are created, call the API in Creating a Job in a Specified Cluster to create a migration job. The following is a sample job:
    • The job name is mysql2dws.
    • The name of the MySQL database from which data is exported is default, and the name of the exported table is mysql_tbl. The job is split into multiple tasks by id and the tasks are executed concurrently.
    • The name of the database on DWS to which the data is imported is public, and the table name is cdm_all_type. Do not clear the data in the table before import.
    • If no table in the local MySQL database exists in the database on DWS, CDM automatically creates the table on DWS.
    • The field list loaded to DWS is id&gid&name.
    • When the job extracts data, three extractors are concurrently executed.
    +

    If status code 200 is returned, the job is successfully created.

    +
    curl -X POST -H "Content-Type:application/json" -H "X-Cluster-ID:$ID" -H "X-Auth-Token:$Token" -d '{
+  "jobs": [{
+    "job_type": "NORMAL_JOB",
+    "name": "mysql2dws",
+    "from-link-name": "mysql_link",
+    "from-connector-name": "generic-jdbc-connector",
+    "to-link-name": "dws_link",
+    "to-connector-name": "generic-jdbc-connector",
+    "from-config-values": {
+      "configs": [{
+        "name": "fromJobConfig",
+        "inputs": [{
+          "name": "fromJobConfig.schemaName",
+          "value": "default"
+        },
+        {
+          "name": "fromJobConfig.tableName",
+          "value": "mysql_tbl"
+        },
+        {
+          "name": "fromJobConfig.partitionColumn",
+          "value": "id"
+        }]
+      }]
+    },
+"to-config-values": {
+             "configs": [
+                 {
+                     "inputs": [
+                         {
+                             "name": "toJobConfig.schemaName",
+                             "value": "public"
+                         },
+                         {
+                             "name": "toJobConfig.tablePreparation",
+                             "value": "CREATE_WHEN_NOT_EXIST"
+                         },
+                         {
+                             "name": "toJobConfig.tableName",
+                             "value": "cdm_all_type"
+                         },
+                         {
+                             "name": "toJobConfig.columnList",
+                             "value": "id&gid&name"
+                         },
+                         {
+                             "name": "toJobConfig.shouldClearTable",
+                             "value": "false"
+                         }
+                     ],
+                     "name": "toJobConfig"
+                 }
+             ]
+         },
+    "driver-config-values": {
+      "configs": [{
+        "name": "throttlingConfig",
+        "inputs": [{
+          "name": "throttlingConfig.numExtractors",
+          "value": "3"
+        }]
+      }]
+    }
+  }]
+}' https://{cdm_endpoint}/v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/bae65496-643e-47ca-84af-948672de7eeb/cdm/job -k -v
    +
  2. Call the API in Starting a Job to execute the job.
    curl -X GET -H 'Content-Type:application/json;charset=utf-8' -H "X-Cluster-ID:$ID" -H "X-Auth-Token:$Token" https://{cdm_endpoint}/v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/bae65496-643e-47ca-84af-948672de7eeb/cdm/job/mysql2dws/start -k -v
    +

    The response is as follows:

    +
    {
+  "submissions": [{
+    "progress": 1,
+    "job-name": "mysql2dws",
+    "status": "BOOTING",
+    "creation-date": 1536654788622,
+    "creation-user": "cdm"
+  }]
+}
    +
+
+

Viewing Job Result

  1. Call the API in Querying Job Status to query the job status.
    curl -X GET -H 'Content-Type:application/json;charset=utf-8' -H "X-Cluster-ID:$ID" -H "X-Auth-Token:$Token" https://{cdm_endpoint}/v1.1/1551c7f6c808414d8e9f3c514a170f2e/clusters/6ec9a0a4-76be-4262-8697-e7af1fac7920/cdm/job/mysql2dws/status -k -v
    +
  2. View the job execution result. The response to successful job execution is as follows:
    {
+  "submissions": [{
+    "progress": 0,
+    "job-name": "mysql2dws",
+    "status": "SUCCEEDED",
+    "creation-date": 1536654788622,
+    "creation-user": "cdm",
+    "isStopingIncrement": "",
+    "last-update-date": 1536654888622,
+    "is-execute-auto": false,
+    "last-udpate-user": "cdm",
+    "isDeleteJob": false,
+    "isIncrementing": false,
+    "external-id": "job_local1127970451_0009",
+    "counters": {
+      "org.apache.sqoop.submission.counter.SqoopCounters": {
+        "BYTES_WRITTEN": -1,
+        "TOTAL_FILES": -1,
+        "BYTES_READ": -1,
+        "FILES_WRITTEN": -1,
+        "TOTAL_SIZE": -1,
+        "FILES_READ": -1,
+        "ROWS_WRITTEN": 80,
+        "ROWS_READ": 80
+      }
+    }
+  }]
+}
    +
    • BYTES_WRITTEN: number of written bytes
    • BYTES_READ: number of read bytes
    • TOTAL_FILES: total number of files
    • FILES_WRITTEN: number of written files
    • FILES_READ: number of read files
    • ROWS_WRITTEN: number of rows that are successfully written
    • ROWS_READ: number of rows that are successfully read
    +
    +
+
+
+
+
+
Parent topic: Application Cases
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0015.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0015.html new file mode 100644 index 000000000..6781266c4 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0015.html @@ -0,0 +1,36 @@ + + +

Example of Using Data Development APIs

+

Scenarios

DataArts Studio helps enterprises quickly build an end-to-end intelligent data system that covers the entire process from data ingestion to data analytics. The system can eliminate data silos, unify data standards, accelerate data monetization, and promote digital transformation.

+

The following describes how to create a script by calling the Creating a Script API. For details, see Calling APIs.

+
+

Prerequisites

You have planned the region where Data Development is located and determined the endpoint for calling an API based on the region.

+

An endpoint is the request address for calling an API. Endpoints vary depending on services and regions. You can obtain endpoints from Regions and Endpoints.

+
+

Creating a Shell Script

The following is an example of creating a shell script:
{
+"name":"echoTimeShell",
+"type":"Shell",
+"content":"echo a",
+"connectionName":"con"
+}
+
+ +
+

Creating a DLI SQL Script

You can also create a DLI SQL script. The following is an example:
{
+"name":"dlisql1",
+"type":"DLISQL",
+"content":"show tables",
+"connectionName":"dliCon1",
+"database":"testDatabase1",
+"queueName":"queue1"
+}
+
+ +
+
+
+
+
Parent topic: Application Cases
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0016.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0016.html new file mode 100644 index 000000000..bcd0214a4 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0016.html @@ -0,0 +1,17 @@ + + +

DataArts Migration APIs

+
+
+ +
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0049.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0049.html new file mode 100644 index 000000000..66c79604a --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0049.html @@ -0,0 +1,25 @@ + + +

Connection Management APIs

+
+
+ + +
+
Parent topic: DataArts Factory APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0050.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0050.html new file mode 100644 index 000000000..118cbc943 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0050.html @@ -0,0 +1,589 @@ + + +

Creating a Connection

+

Function

This API is used to create a connection. The supported connection types include DWS, DLI, Spark SQL, RDS, CloudTable, and Hive.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Connection parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Connection name. The name contains a maximum of 100 characters, including only letters, numbers, hyphens (-), and underscores (_). The connection name must be unique.

+

type

+

Yes

+

String

+

Connection type.

+
  • DWS
  • DLI
  • SparkSQL
  • HIVE
  • RDS
  • CloudTable
  • HOST
+

config

+

No

+

Map<String,String>

+

Connection configuration item. The configuration item varies with the connection type. You do not need to set the config parameter for DLI connections. For other types of connections, see the description of connection configuration items.

+

description

+

No

+

String

+

Description of the connection. The description contains a maximum of 255 characters.

+
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 DWS connection parameters

Parameter

+

Mandatory

+

Type

+

Description

+

clusterName

+

No

+

String

+

Name of a DWS cluster.

+

Perform the following operations to obtain the DWS cluster name:

+
  1. Log in to the management console.
  2. Click Data Warehouse Service and select Cluster Management from the list on the left.
+

You can obtain the cluster name from the cluster management list.

+

By default, this parameter is left blank.

+

ip

+

No

+

String

+

IP address for accessing the DWS cluster.

+

Perform the following operations to obtain the DWS access address:

+
  1. Log in to the management console.
  2. Click Data Warehouse Service and select Cluster Management from the list on the left.
  3. Click the cluster name. The basic information page of the cluster is displayed.
+

You can obtain the private network IP address on the Database Attribute tab page. If there are multiple IP addresses, select the first IP address.

+

By default, this parameter is left blank.

+

port

+

No

+

String

+

Port for accessing the DWS cluster.

+

Perform the following operations to obtain the DWS access port:

+
  1. Log in to the management console.
  2. Click Data Warehouse Service and select Cluster Management from the list on the left.
  3. Click the cluster name. The basic information page of the cluster is displayed.
+

You can obtain the port information on the Database Attribute tab page.

+

For example, set port to 8000. By default, this parameter is left blank.

+

userName

+

Yes

+

String

+

Username of the database. This username is the username entered during the creation of the DWS cluster.

+

password

+

Yes

+

String

+

Password for accessing the database. This password is the password entered during the creation of the DWS cluster.

+

sslEnable

+

Yes

+

boolean

+

Specifies whether to enable the SSL connection.

+

kmsKey

+

Yes

+

String

+

Name of a KMS key.

+

Perform the following operations to obtain the key:

+
  1. Log in to the management console.
  2. Click Key Management Service and select Key Management Service from the list on the left.
+

You can obtain the key name from the key list.

+

agentName

+

Yes

+

String

+

Name of a CDM cluster.

+

You can obtain the cluster name from the CDM cluster list on the DataArts Migration page of the DataArts Studio console.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 5 Spark SQL connection parameters

Parameter

+

Mandatory

+

Type

+

Description

+

clusterName

+

Yes

+

String

+

Name of an MRS cluster.

+

Perform the following operations to obtain the MRS cluster name:

+
  1. Log in to the management console.
  2. Click MapReduce Service and select Active Clusters from the list on the left.
+

You can obtain the cluster name from the active clusters, such as mrsCluster1.

+

connectionMethod

+

Yes

+

String

+

Method to connect.

+
  • agent: connected through an agent.
  • direct: connected directly.
+

userName

+

No

+

String

+

Username of the MRS cluster. This parameter is mandatory when connectionMethod is set to agent.

+

password

+

No

+

String

+

Password for accessing the MRS cluster. This parameter is mandatory when connectionMethod is set to agent.

+

agentName

+

No

+

String

+

Name of a CDM cluster. This parameter is mandatory when connectionMethod is set to agent.

+

You can obtain the cluster name from the CDM cluster list on the DataArts Migration page of the DataArts Studio console.

+

kmsKey

+

No

+

String

+

Name of a KMS key. This parameter is mandatory when connectionMethod is set to agent.

+

Perform the following operations to obtain the key:

+
  1. Log in to the management console.
  2. Click Key Management Service and select Key Management Service from the list on the left.
+

You can obtain the key name from the key list.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 6 Hive connection parameters

Parameter

+

Mandatory

+

Type

+

Description

+

clusterName

+

Yes

+

String

+

Name of an MRS cluster, for example, mrsCluster1.

+

connectionMethod

+

Yes

+

String

+

Method to connect.

+
  • agent: connected through an agent.
  • direct: connected directly.
+

userName

+

No

+

String

+

Username of the MRS cluster. This parameter is mandatory when connectionMethod is set to agent.

+

password

+

No

+

String

+

Password for accessing the MRS cluster. This parameter is mandatory when connectionMethod is set to agent.

+

agentName

+

No

+

String

+

Name of a CDM cluster. This parameter is mandatory when connectionMethod is set to agent.

+

You can obtain the cluster name from the CDM cluster list on the DataArts Migration page of the DataArts Studio console.

+

kmsKey

+

No

+

String

+

Name of a KMS key. This parameter is mandatory when connectionMethod is set to agent.

+

Perform the following operations to obtain the key:

+
  1. Log in to the management console.
  2. Click Key Management Service and select Key Management Service from the list on the left.
+

You can obtain the key name from the key list.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 7 RDS connection parameters

Parameter

+

Mandatory

+

Type

+

Description

+

ip

+

Yes

+

String

+

Address for accessing RDS.

+

Perform the following operations to obtain the RDS access address:

+
  1. Log in to the management console.
  2. Click Relational Database Service and select Instance Management from the list on the left.
  3. Click the name of an instance. The basic information page of the instance is displayed.
+

You can obtain the IP address on the Connection Information tab.

+

port

+

Yes

+

String

+

Port for accessing RDS.

+

Perform the following operations to obtain the RDS access port:

+
  1. Log in to the management console.
  2. Click Relational Database Service and select Instance Management from the list on the left.
  3. Click the name of an instance. The basic information page of the instance is displayed.
+

You can obtain the database port on the Connection Information tab page.

+

userName

+

Yes

+

String

+

Username of the database. This username is the username entered during the creation of the cluster.

+

password

+

Yes

+

String

+

Password for accessing the database. This password is the password entered during the creation of the cluster.

+

kmsKey

+

Yes

+

String

+

Name of a KMS key.

+

Perform the following operations to obtain the key:

+
  1. Log in to the management console.
  2. Click Key Management Service and select Key Management Service from the list on the left.
+

You can obtain the key name from the key list.

+

agentName

+

Yes

+

String

+

Name of a CDM cluster.

+

You can obtain the cluster name from the CDM cluster list on the DataArts Migration page of the DataArts Studio console.

+

driverName

+

Yes

+

String

+

Name of the driver.

+
  • com.mysql.jdbc.Driver
  • org.postgresql.Driver
+

driverPath

+

Yes

+

String

+

Path of the driver on OBS.

+
+
+ +
+ + + + + + + + + + + +
Table 8 CloudTable connection parameters

Parameter

+

Mandatory

+

Type

+

Description

+

clusterName

+

Yes

+

String

+

Name of a CloudTable cluster.

+

Perform the following operations to obtain the cluster name:

+
  1. Log in to the management console.
  2. Click CloudTable Service and select Cluster Mode from the list on the left.
+

You can obtain the cluster name from the cluster list.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 9 HOST connection parameters

Parameter

+

Mandatory

+

Type

+

Description

+

ip

+

Yes

+

String

+

IP address of the host

+

port

+

Yes

+

String

+

SSH port number of the host

+

userName

+

Yes

+

String

+

Username for logging in to the host

+

password

+

Yes

+

String

+

Password for logging in to the host

+

agentName

+

Yes

+

String

+

Name of a CDM cluster.

+

You can obtain the cluster name from the CDM cluster list on the DataArts Migration page of the DataArts Studio console.

+

kmsKey

+

Yes

+

String

+

Name of a KMS key.

+

Perform the following operations to obtain the key:

+
  1. Log in to the management console.
  2. Click Key Management Service and select Key Management Service from the list on the left.
+

You can obtain the key name from the key list.

+
+
+

Response

None.

+
+

Example

Create a connection.

+ +
+ +

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: Connection Management APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0051.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0051.html new file mode 100644 index 000000000..52ec1fda4 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0051.html @@ -0,0 +1,204 @@ + + +

Querying a Connection List

+

Function

This API is used to query a connection list.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + +
Table 3 Response parameter

Parameter

+

Mandatory

+

Type

+

Description

+

total

+

Yes

+

Integer

+

The total number of connections.

+

connections

+

Yes

+

List<Connections>

+

Connection list.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 connections parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Connection name. The name contains a maximum of 100 characters, including only letters, numbers, hyphens (-), and underscores (_). The connection name must be unique.

+

type

+

Yes

+

String

+

Connection type.

+
  • DWS
  • DLI
  • SparkSQL
  • HIVE
  • RDS
  • CloudTable
  • HOST
+

config

+

No

+

Map<String,String>

+

Connection configuration item. The configuration item varies with the connection type. You do not need to set the config parameter for DLI connections. For other types of connections, see the description of connection configuration items.

+

description

+

No

+

String

+

Description of the connection. The description contains a maximum of 255 characters.

+
+
+
+

Example

Query a connection list.

+ +
+ +
+
+
+
Parent topic: Connection Management APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0052.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0052.html new file mode 100644 index 000000000..7e085050a --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0052.html @@ -0,0 +1,150 @@ + + +

Viewing Connection Details

+

Function

This API is used to query configuration details of a specific connection.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Connection parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Connection name. The name contains a maximum of 100 characters, including only letters, numbers, hyphens (-), and underscores (_). The connection name must be unique.

+

type

+

Yes

+

String

+

Connection type.

+
  • DWS
  • DLI
  • SparkSQL
  • HIVE
  • RDS
  • CloudTable
  • HOST
+

config

+

No

+

Map<String,String>

+

Connection configuration item. The configuration item varies with the connection type. You do not need to set the config parameter for DLI connections. For other types of connections, see the description of connection configuration items.

+

description

+

No

+

String

+

Description of the connection. The description contains a maximum of 255 characters.

+
+
+
+

Example

Query connection details.

+ +
+ +

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: Connection Management APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0053.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0053.html new file mode 100644 index 000000000..7753b3c03 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0053.html @@ -0,0 +1,160 @@ + + +

Editing a Connection

+

Function

This API is used to edit a connection.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Connection parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Connection name. The name contains a maximum of 100 characters, including only letters, numbers, hyphens (-), and underscores (_). The connection name must be unique.

+

type

+

Yes

+

String

+

Connection type.

+
  • DWS
  • DLI
  • SparkSQL
  • HIVE
  • RDS
  • CloudTable
  • HOST
+

config

+

No

+

Map<String,String>

+

Connection configuration item. The configuration item varies with the connection type. You do not need to set the config parameter for DLI connections. For other types of connections, see the description of connection configuration items.

+

description

+

No

+

String

+

Description of the connection. The description contains a maximum of 255 characters.

+
+
+
+

Response

None.

+
+

Example

Modify a connection.

+ +
+ +

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: Connection Management APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0054.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0054.html new file mode 100644 index 000000000..6c21e9d9c --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0054.html @@ -0,0 +1,81 @@ + + +

Deleting a Connection

+

Function

This API is used to delete a connection.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

None.

+
+

Example

Delete the connection.

+ +
+ +
+
+
+
Parent topic: Connection Management APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0055.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0055.html new file mode 100644 index 000000000..222fb540b --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0055.html @@ -0,0 +1,85 @@ + + +

Exporting a Connection

+

Function

This API is used to export all connection information that is compressed in ZIP format.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

The value of Content-Type in the response message is application/octet-stream that needs to be converted into a file. For details, see Parsing a Stream in a Response Message. Response messages are compressed as a file. The file name format is DLF_All_DataConnections.zip. The file directory is as follows:

+
connections
+├─{dwsConnection}.conn
+

+
+

Example

Export a connection.

+ +
+ +

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: Connection Management APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0056.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0056.html new file mode 100644 index 000000000..2b756167b --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0056.html @@ -0,0 +1,19 @@ + + +

Script Development APIs

+
+
+ + +
+
Parent topic: DataArts Factory APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0057.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0057.html new file mode 100644 index 000000000..4138d2edd --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0057.html @@ -0,0 +1,87 @@ + + +

Deleting a Script

+

Function

This API is used to delete a specific script.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

None.

+
+

Example

+ +
+

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: Script Development APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0058.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0058.html new file mode 100644 index 000000000..5e55e0ce3 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0058.html @@ -0,0 +1,142 @@ + + +

Executing a Script

+

Function

This API is used to execute a specific script, which can be a DWS SQL, DLI SQL, RDS SQL, Flink SQL, Hive SQL, Presto SQL, or Spark SQL script. A script instance is generated each time the script is executed. You can call the API Querying the Execution Result of a Script Instance to obtain script execution results.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + +
Table 3 Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

params

+

No

+

Object

+

Script parameters of the Map<String,String> type. If a parameter is defined in the script, the parameter value is carried in the params. By default, this parameter is left blank.

+
+
+

Script parameters refer to the parameters in the script content, as shown in the following figure.

+

+
+

Response

+
+ + + + + + + + + + + +
Table 4 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

instanceId

+

Yes

+

String

+

ID of the instance that executes the script. You can obtain the execution result by using the instance ID in Querying the Execution Result of a Script Instance.

+
+
+
+

Example

+
+ + +

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: Script Development APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0059.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0059.html new file mode 100644 index 000000000..319e8cd45 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0059.html @@ -0,0 +1,96 @@ + + +

Stopping Executing a Script Instance

+

Function

This API is used to stop executing a script instance.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

None.

+
+

Example

+
+ +

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: Script Development APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0060.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0060.html new file mode 100644 index 000000000..bb0abcd85 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0060.html @@ -0,0 +1,15 @@ + + +

Resource Management APIs

+
+
+ + +
+
Parent topic: DataArts Factory APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0061.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0061.html new file mode 100644 index 000000000..2a66c4ac2 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0061.html @@ -0,0 +1,88 @@ + + +

Deleting a Resource

+

Function

This API is used to delete a resource.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

None.

+
+

Example

+ +
+

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: Resource Management APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0062.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0062.html new file mode 100644 index 000000000..edb23be02 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0062.html @@ -0,0 +1,23 @@ + + +

Job Development APIs

+
+
+ + +
+
Parent topic: DataArts Factory APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0063.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0063.html new file mode 100644 index 000000000..06498ff77 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0063.html @@ -0,0 +1,213 @@ + + +

Viewing a Job File

+

Function

This API is used to check whether there are jobs and scripts in the job file to be imported from OBS to DLF.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request parameters

Parameter

+

Mandatory

+

Type

+

Description

+

path

+

No

+

String

+

If OBS is deployed, the job definition file is stored on OBS, for example, obs://myBucket/jobs.zip.

+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + +
Table 3 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

jobs

+

No

+

List<Job>

+

Job information. For details, see Table 4.

+

scripts

+

No

+

List<Script>

+

Script information. For details, see Table 5.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 4 job data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

params

+

No

+

Map<String,String>

+

Job parameter.

+

name

+

Yes

+

String

+

Job name.

+

path

+

Yes

+

String

+

Path of the job.

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 5 Data structure description of Script

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Script name.

+

path

+

Yes

+

String

+

Path of the script.

+
+
+
+

Example

View the parameter definitions in the job file on OBS.

+ + + +
+

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: Job Development APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0064.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0064.html new file mode 100644 index 000000000..e01852a91 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0064.html @@ -0,0 +1,86 @@ + + +

Stopping a Job

+

Function

This API is used to stop a job.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

None.

+
+

Example

Stops job myJob.

+ + +
+
+
+
+
Parent topic: Job Development APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0065.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0065.html new file mode 100644 index 000000000..83c43ad3a --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0065.html @@ -0,0 +1,87 @@ + + +

Deleting a Job

+

Function

This API is used to delete a job.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

None.

+
+

Example

+ +
+

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: Job Development APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0066.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0066.html new file mode 100644 index 000000000..bf6f20ff3 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0066.html @@ -0,0 +1,95 @@ + + +

Stopping a Job Instance

+

Function

This API is used to stop a specific job instance. A job instance can be stopped only when it is in the running state.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

None.

+
+

Example

+ +
+
+
+
+
Parent topic: Job Development APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0067.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0067.html new file mode 100644 index 000000000..6b9761524 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0067.html @@ -0,0 +1,95 @@ + + +

Restarting a Job Instance

+

Function

This API is used to restart a specific job instance. A job instance can be restarted only when it is in the successful, failed, or canceled state.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

None.

+
+

Example

+ +
+
+
+
+
Parent topic: Job Development APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0082.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0082.html new file mode 100644 index 000000000..dd3302eb5 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0082.html @@ -0,0 +1,127 @@ + + +

Data Structure

+

Table 1 describes common request headers.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Common request header fields

Parameter

+

Mandatory

+

Type

+

Description

+

X-Sdk-Date

+

Yes

+

String

+

Time when the request is sent. The time is in YYYYMMDD'T'HHMMSS'Z' format.

+

The value is the current GMT time of the system.

+

Authorization

+

Yes

+

String

+

Authentication information.

+

The value can be obtained from the request signing result.

+

For details, see AK/SK-based Authentication.

+

Host

+

Yes

+

String

+

Server information of the resource being requested. The value can be obtained from the URL of the service API. The value is in the format of hostname:port. If the port number is not specified, the default port is used. The default port number for HTTPS is 443.

+

Content-type

+

Yes

+

String

+

Request body MIME type.

+

Content-Length

+

No

+

Int

+

Size of the request body, measured in bytes.

+

Mandatory for POST and PUT requests, but must be left blank for GET requests.

+

X-Project-Id

+

No

+

String

+

Project ID used to obtain a token for each project.

+

X-Auth-Token

+

No

+

String

+

User token.

+

This parameter is mandatory for authentication using tokens.

+
+
+

Table 2 describes common response headers.

+ +
+ + + + + + + + + + + + + + + + + +
Table 2 Common response headers

Parameter

+

Type

+

Description

+

Content-Length

+

Int

+

Length of the response body. The unit is byte.

+

Date

+

String

+

Time when a response is returned.

+

Content-type

+

String

+

Request body MIME type.

+
+
+
+
+
+
Parent topic: DataArts Factory APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0083.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0083.html new file mode 100644 index 000000000..2a3665ef6 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0083.html @@ -0,0 +1,63 @@ + + + +

APIs to Be Taken Offline

+ +

+
+ +
+ + +
+
Parent topic: DataArts Factory APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0084.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0084.html new file mode 100644 index 000000000..19c0edd6b --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0084.html @@ -0,0 +1,1436 @@ + + +

Creating a Job

+

Function

This API is used to create a job. A job consists of one or more nodes, such as Hive SQL and CDM Job nodes. DLF supports two types of jobs: batch jobs and real-time jobs.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Job name. The name contains a maximum of 128 characters, including only letters, numbers, hyphens (-), underscores (_), and periods (.). The job name must be unique.

+

nodes

+

Yes

+

List<Node>

+

Node definition. For details, see Table 4.

+

schedule

+

Yes

+

Schedule data structure

+

Scheduling configuration. For details, see Table 5.

+

params

+

No

+

List<Param>

+

Job parameter definition. For details, see Table 6.

+

directory

+

No

+

String

+

Directory for saving the job. The value must be an existing directory, for example, /dir/a/. The default value is the root directory.

+

processType

+

Yes

+

String

+

Job type.

+
  • REAL_TIME: real-time processing
  • BATCH: batch processing
+

basicConfig

+

No

+

BasicConfig data structure

+

Basic job information. For details, see Table 26.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 Node data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Node name. The name contains a maximum of 128 characters, including only letters, numbers, hyphens (-), underscores (_), and periods (.). Names of the nodes in a job must be unique.

+

type

+

Yes

+

String

+

Node type. The options are as follows:

+
  • Hive SQL: Runs Hive SQL scripts.
  • Spark SQL: Runs Spark SQL scripts.
  • DWS SQL: Runs DWS SQL scripts.
  • DLISQL: Runs DLI SQL scripts.
  • Shell: Runs shell SQL scripts.
  • CDM Job: Runs CDM jobs.
  • CloudTable Manager: Manages CloudTable tables, including creating and deleting tables.
  • OBS Manager: Manages OBS paths, including creating and deleting paths.
  • RESTAPI: Sends REST API requests.
  • SMN: Sends short messages or emails.
  • MRS Spark: Runs Spark jobs of MRS.
  • MapReduce: Runs MapReduce jobs of MRS.
  • DLI Spark: Runs Spark jobs of DLF.
  • RDS SQL: Transfers SQL statements to RDS for execution.
+

location

+

Yes

+

Location data structure

+

Location of a node on the job canvas. For details, see Table 7.

+

preNodeName

+

No

+

List<String>

+

Name of the previous node on which the current node depends.

+

conditions

+

No

+

List<Condition>

+

Node execution condition. Whether the node is executed or not depends on the calculation result of the EL expression saved in the expression field of condition. For details, see Table 8.

+

properties

+

Yes

+

List

+

Node property. Each type of node has its own property definition.

+ +

pollingInterval

+

No

+

Int

+

Interval at which node running results are checked.

+

Unit: second; value range: 1 to 60

+

Default value: 10

+

maxExecutionTime

+

No

+

Int

+

Maximum execution time of a node. If a node is not executed within the maximum execution time, the node is set to the failed state.

+

Unit: minute; value range: 5 to 1440

+

Default value: 60

+

retryTimes

+

No

+

Int

+

Number of the node retries. The value ranges from 0 to 5. 0 indicates no retry.

+

Default value: 0

+

retryInterval

+

No

+

Int

+

Interval at which a retry is performed upon a failure. The value ranges from 5 to 120.

+

Unit: second

+

Default value: 120

+

failPolicy

+

No

+

String

+

Node failure policy.

+
  • FAIL: Terminate the execution of the current job.
  • IGNORE: Continue to execute the next node.
  • SUSPEND: Suspend the execution of the current job.
  • FAIL_CHILD: Terminate the execution of the subsequent node.

    The default value is FAIL.

    +
+

eventTrigger

+

No

+

Event data structure

+

Node event triggering configuration. For details, see Table 10.

+

cronTrigger

+

No

+

Cron data structure

+

Node Cron triggering configuration. For details, see Table 9.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 5 Schedule data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

type

+

Yes

+

String

+

Scheduling type.

+
  • EXECUTE_ONCE: The job runs immediately and runs only once.
  • CRON: The job runs periodically.
  • EVENT: The job is triggered by events.
+

cron

+

No

+

Data structure

+

When type is set to CRON, configure the scheduling frequency and start time. For details, see Table 9.

+

event

+

No

+

Data structure

+

When type is set to EVENT, configure information such as the event source. For details, see Table 10.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 6 Param data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Name of a parameter. The name contains a maximum of 64 characters, including only letters, numbers, hyphens (-), and underscores (_).

+

value

+

Yes

+

String

+

Value of the parameter. It cannot exceed 1024 characters.

+

type

+

No

+

String

+

Parameter type.

+
  • variable
  • constants

    Default value: variable

    +
+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 7 Location data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

x

+

Yes

+

Int

+

Position of the node on the horizontal axis of the job canvas.

+

y

+

Yes

+

Int

+

Position of the node on the vertical axis of the job canvas.

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 8 condition data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

preNodeName

+

Yes

+

String

+

Name of the previous node on which the current node depends.

+

expression

+

Yes

+

String

+

EL expression. If the calculation result of the EL expression is true, this node is executed.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 9 Cron data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

startTime

+

Yes

+

String

+

Scheduling start time in the format of yyyy-MM-dd'T'HH:mm:ssZ, which is an ISO 8601 time format. For example, 2018-10-22T23:59:59+08, which indicates that a job starts to be scheduled at 23:59:59 on October 22nd, 2018.

+

endTime

+

No

+

String

+

Scheduling end time in the format of yyyy-MM-dd'T'HH:mm:ssZ, which is an ISO 8601 time format. For example, 2018-10-22T23:59:59+08, which indicates that a job stops to be scheduled at 23:59:59 on October 22nd, 2018. If the end time is not set, the job will continuously be executed based on the scheduling period.

+

expression

+

Yes

+

String

+

Cron expression in the format of <second><minute><hour><day><month><week>. For details about the value input in each field, see Table 11.

+

expressionTimeZone

+

No

+

String

+

Time zone corresponding to the Cron expression, for example, GMT+8.

+

Default value: time zone where DataArts Studio is located

+

dependPrePeriod

+

No

+

Boolean

+

Indicates whether to depend on the execution result of the current job's dependent job in the previous scheduling period.

+

Default value: false

+

dependJobs

+

No

+

DependJobs data structure

+

Job dependency configuration. For details, see Table 12.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 10 Event data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

failPolicy

+

No

+

String

+

Job failure policy.

+
  • SUSPEND: Suspend the event.
  • IGNORE: Ignore the failure and process with the next event.
+

Default value: SUSPEND

+

concurrent

+

No

+

int

+

Number of the concurrently scheduled jobs.

+

Value range: 1 to 128

+

Default value: 1

+

readPolicy

+

No

+

String

+

Access policy.

+
  • LAST: Access data from the last location.
  • NEW: Access data from a new location.
+

Default value: LAST

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 11 Values in the Cron expression fields

Field

+

Value Range

+

Allowed Special Character

+

Description

+

Second

+

0-59

+

, - * /

+

In the current version, only 0 is allowed.

+

Minute

+

0-59

+

, - * /

+

-

+

Hour

+

0-23

+

, - * /

+

-

+

Day

+

1-31

+

, - * ? / L W C

+

-

+

Month

+

1-12

+

, - * /

+

In the current version, only * is allowed.

+

Week

+

1-7

+

, - * ? / L C #

+

Starting from Sunday.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 12 DependJobs data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

jobs

+

Yes

+

List<String>

+

A list of dependent jobs. Only the existing jobs can be depended on.

+

dependPeriod

+

No

+

String

+

Dependency period.

+
  • SAME_PERIOD: To run a job or not depends on the execution result of its depended job in the current scheduling period.
  • PRE_PERIOD: To run a job or not depends on the execution result of its depended job in the previous scheduling period.
+

Default value: SAME_PERIOD

+

dependFailPolicy

+

No

+

String

+

Dependency job failure policy.

+
  • FAIL: Stop the job and set the job to the failed state.
  • IGNORE: Continue to run the job.
  • SUSPEND: Suspend the job.
+

Default value: FAIL

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 13 Parameters of the Hive SQL node

Parameter

+

Mandatory

+

Type

+

Description

+

scriptName

+

Yes

+

String

+

Script name.

+

database

+

No

+

String

+

Database name.

+

Database in the MRS Hive. The default value is default.

+

connectionName

+

No

+

String

+

Name of a connection.

+

scriptArgs

+

No

+

String

+

Script parameter in format of key and value. Multiple parameters are separated by newlines (\n), for example, key1=value1\nkey2=value2.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 14 Parameters of the Spark SQL node

Parameter

+

Mandatory

+

Type

+

Description

+

scriptName

+

Yes

+

String

+

Script name.

+

database

+

No

+

String

+

Database name.

+

Database in the MRS Spark SQL. The default value is default.

+

connectionName

+

No

+

String

+

Name of a connection.

+

scriptArgs

+

No

+

String

+

Script parameter in format of key and value. Multiple parameters are separated by newlines (\n), for example, key1=value1\nkey2=value2.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 15 Parameters of the DWS SQL node

Parameter

+

Mandatory

+

Type

+

Description

+

scriptName

+

Yes

+

String

+

Script name.

+

database

+

No

+

String

+

Database name.

+

Database in DWS. The default value is postgres.

+

connectionName

+

No

+

String

+

Name of a connection.

+

scriptArgs

+

No

+

String

+

Script parameter in format of key and value. Multiple parameters are separated by newlines (\n), for example, key1=value1\nkey2=value2.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 16 Parameters of the DLI SQL node

Parameter

+

Mandatory

+

Type

+

Description

+

scriptName

+

Yes

+

String

+

Script name.

+

database

+

No

+

String

+

Database name.

+

Database in DLI.

+

connectionName

+

No

+

String

+

Name of a connection.

+

scriptArgs

+

No

+

String

+

Script parameter in format of key and value. Multiple parameters are separated by newlines (\n), for example, key1=value1\nkey2=value2.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 17 Parameters of the shell node

Parameter

+

Mandatory

+

Type

+

Description

+

scriptName

+

Yes

+

String

+

Script name.

+

connectionName

+

Yes

+

String

+

Name of a connection.

+

arguments

+

No

+

String

+

Shell script parameter.

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 18 Parameters of the CDM Job node

Parameter

+

Mandatory

+

Type

+

Description

+

clusterName

+

Yes

+

String

+

Cluster name.

+

You can obtain the cluster name from the CDM cluster list on the DataArts Migration page of the DataArts Studio console.

+

jobName

+

Yes

+

String

+

Job name.

+

To obtain the job name, access the DataArts Studio console, choose DataArts Migration, click a cluster name on the Cluster Management page, and click Job Management on the displayed page.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 19 Parameters of the CloudTableManager node

Parameter

+

Mandatory

+

Type

+

Description

+

namespace

+

No

+

String

+

Namespace.

+

Default value: default

+

action

+

Yes

+

String

+

Action type.

+
  • CREATE_TABLE: Create a table.
  • DELETE_TABLE: Delete a table.
+

table

+

No

+

String

+

Table name.

+

columnFamily

+

No

+

String

+

Column family.

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 20 Parameters of the OBSManager node

Parameter

+

Mandatory

+

Type

+

Description

+

action

+

Yes

+

String

+

Action type.

+
  • CREATE_PATH: Create an OBS path.
  • DELETE_PATH: Delete an OBS path.
+

path

+

Yes

+

String

+

OBS path.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 21 Parameters of the RESTAPI node

Parameter

+

Mandatory

+

Type

+

Description

+

url

+

Yes

+

String

+

URL address.

+

URL of the cloud service.

+

method

+

Yes

+

String

+

HTTP method.

+
  • GET
  • POST
  • PUT
  • DELETE
+

headers

+

No

+

String

+

HTTP message header in the format of <message header name>=<value>. Multiple message headers are separated by newlines.

+

body

+

No

+

String

+

Message body.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 22 Parameters of the SMN node

Parameter

+

Mandatory

+

Type

+

Description

+

topic

+

Yes

+

String

+

SMN topic URN.

+

Perform the following operations to obtain an SMN topic URN:

+
  1. Log in to the management console.
  2. Click Simple Message Notification and choose Topic Management > Topics from the list on the left.
+

You can obtain the SMN topic URN in the topic list.

+

subject

+

Yes

+

String

+

Message title, which is used as the subject of an email sent to a subscriber.

+

messageType

+

Yes

+

String

+

Message type.

+
  • NORMAL
  • STRUCTURE
  • TEMPLATE
+

message

+

Yes

+

String

+

Message to be sent.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 23 Parameters of the MRS Spark node

Parameter

+

Mandatory

+

Type

+

Description

+

clusterName

+

Yes

+

String

+

MRS cluster name.

+

Perform the following operations to obtain the MRS cluster name:

+
  1. Log in to the management console.
  2. Click MapReduce Service and choose Clusters > Active Clusters from the left navigation pane.
+

You can obtain the cluster name from the active clusters.

+

jobName

+

Yes

+

String

+

MRS job name.

+

The job name is user-defined.

+

resourcePath

+

Yes

+

String

+

OBS resource path of the custom Spark JAR package

+

parameters

+

Yes

+

String

+

Custom parameters of the Spark JAR package

+

You can specify parameters for a custom JAR package.

+

input

+

No

+

String

+

Input path.

+

Input data path of the MRS Spark job. The path can be an HDFS or OBS path.

+

output

+

No

+

String

+

Output path.

+

Output data path of the MRS Spark job. The path can be an HDFS or OBS path.

+

programParameter

+

No

+

String

+

Program parameter

+

Multiple key-value pairs are allowed and separated by vertical bars (|).

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 24 Parameters of the MapReduce node

Parameter

+

Mandatory

+

Type

+

Description

+

clusterName

+

Yes

+

String

+

MRS cluster name.

+

Perform the following operations to obtain the MRS cluster name:

+
  1. Log in to the management console.
  2. Click MapReduce Service and choose Clusters > Active Clusters from the left navigation pane.
+

You can obtain the cluster name from the active clusters.

+

jobName

+

Yes

+

String

+

MRS job name.

+

The job name is user-defined.

+

resourcePath

+

Yes

+

String

+

Resource path.

+

parameters

+

Yes

+

String

+

Job parameter.

+

input

+

Yes

+

String

+

Input path.

+

Input data path of the MapReduce job. The path can be an HDFS or OBS path.

+

output

+

Yes

+

String

+

Output path.

+

Output data path of the MapReduce job. The path can be an HDFS or OBS path.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 25 Parameters of the DLI Spark node

Parameter

+

Mandatory

+

Type

+

Description

+

clusterName

+

Yes

+

String

+

DLI queue name

+

Perform the following operations to obtain the DLI queue name:

+
  1. Log in to the management console.
  2. Click Data Lake Insight and then Queue Management.
+

You can obtain the queue name from the queue management list.

+

jobName

+

Yes

+

String

+

DLI job name.

+

Perform the following operations to obtain the job name:

+
  1. Log in to the management console.
  2. Click Data Lake Insight and then Spark Jobs.
  3. Choose Job Management.
+

You can obtain the job name from the job management list.

+

resourceType

+

No

+

String

+

Resource type of the DLI job. CUSTOMIZED is returned when the parameter is customized.

+

jobClass

+

No

+

String

+

Main class name. When the application type is .jar, the main class name cannot be empty.

+

resourcePath

+

Yes

+

String

+

JAR package resource path.

+

jarArgs

+

No

+

String

+

Main-class entry parameter.

+

sparkConfig

+

No

+

String

+

Running parameter of the Spark job.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 26 BasicConfig job information

Parameter

+

Mandatory

+

Type

+

Description

+

owner

+

No

+

String

+

Job owner. The length cannot exceed 128 characters.

+

priority

+

No

+

int

+

Job priority. The value ranges from 0 to 2. The default value is 0. 0 indicates a top priority, 1 indicates a medium priority, and 2 indicates a low priority.

+

executeUser

+

No

+

String

+

Job execution user. The value must be an existing username.

+

instanceTimeout

+

No

+

int

+

Instance timeout interval. The unit is minute. The value ranges from 5 to 1440. The default value is 60.

+

customFields

+

No

+

Map<String,String>

+

User-defined field. The length cannot exceed 2048 characters.

+
+
+
+

Response

None.

+
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0085.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0085.html new file mode 100644 index 000000000..f2ec56531 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0085.html @@ -0,0 +1,199 @@ + + +

Editing a Job

+

Function

This API is used to edit a job.

+
+

URI

+ +
+

Request

See Request.

+
+

Example

Modify the properties of job dliJob1.

+ + +
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0086.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0086.html new file mode 100644 index 000000000..acc6f2d61 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0086.html @@ -0,0 +1,265 @@ + + +

Viewing a Job List

+

Function

This API is used to query a list of batch or real-time jobs. A maximum of 1000 jobs can be returned for each query.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 1 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + +
Table 2 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

total

+

Yes

+

Integer

+

Number of jobs.

+

jobs

+

Yes

+

List<Job>

+

Job list. For details, see Table 3.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Job data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Job name.

+

jobType

+

Yes

+

String

+

Job type.

+
  • REAL_TIME: real-time processing
  • BATCH: batch processing
+

status

+

Yes

+

String

+

Job status.

+

When jobType is set to REAL_TIME, the status is as follows:

+
  • STARTING
  • NORMAL
  • EXCEPTION
  • STOPPING
  • STOPPED
+

When jobType is set to BATCH, the status is as follows:

+
  • SCHEDULING
  • STOPPED
  • PAUSED
+

createUser

+

Yes

+

String

+

Job creator.

+

createTime

+

Yes

+

Long

+

Time when the job is created.

+

startTime

+

No

+

Long

+

Time when the job starts to run.

+

endTime

+

No

+

Long

+

Time when the job stops to run.

+

lastInstanceStatus

+

No

+

String

+

Most recent running status of the job instance. This parameter is available only when jobType is set to BATCH.

+

lastInstanceEndTime

+

No

+

Long

+

Time when the most recent job instance stops to run. This parameter is available only when jobType is set to BATCH.

+
+
+
+

Example

View a batch or real-time job list.

+ +
GET /v1/ff6b627b9d7b45b48f773be511c1a2b8/jobs
+ + +
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0087.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0087.html new file mode 100644 index 000000000..544845373 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0087.html @@ -0,0 +1,216 @@ + + +

Viewing Job Details

+

Function

This API is used to view job details.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

See the request parameters in Creating a Job.

+
+

Example

View details about job myJob.

+ + + +
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0088.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0088.html new file mode 100644 index 000000000..a8777ec2b --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0088.html @@ -0,0 +1,171 @@ + + +

Exporting a Job

+

Function

This API is used to export a job, including job definitions, job dependency scripts, and CDM job definitions.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + +
Table 3 Parameter

Parameter

+

Mandatory

+

Type

+

Description

+

exportDepend

+

No

+

boolean

+

Specifies whether to export the scripts and resources that the job depends on.

+

Default value: true

+
+
+
+

Response

The value of Content-Type in the response message is application/octet-stream that needs to be converted into a file. For details, see Parsing a Stream in a Response Message. Response messages are compressed as a file. The file name format is DLF_job_name.zip. The file directory is as follows:

+
jobs
+├─{job_name}.job
+scripts
+├─{script_name}.script
+resources
+├─{resource_name}.resource
+

Table 4 describes the file directory parameters.

+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 4 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

job_name

+

Yes

+

String

+

Job name.

+

script_name

+

No

+

String

+

Name of the script that the job depends on.

+

resource_name

+

No

+

String

+

Name of the resource that the job depends on.

+
+
+ +
+

Example

Export job myJob.

+ + + +
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0089.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0089.html new file mode 100644 index 000000000..f57083a53 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0089.html @@ -0,0 +1,123 @@ + + +

Batch Exporting Jobs

+

Function

This API is used to batch export jobs, including job dependency scripts and CDM job definitions.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 3 Parameters

Parameter

+

Mandatory

+

Type

+

Description

+

jobList

+

Yes

+

List

+

A list of jobs to be exported. A maximum of 100 jobs can be exported at a time.

+

exportDepend

+

No

+

boolean

+

Specifies whether to export the scripts and resources that the job depends on.

+

Default value: true

+
+
+
+

Response

The response message of batch exporting jobs is the same as that of exporting jobs. For details, see Response.

+
+

Example

Export two jobs named job_batch and job_stream.

+ + + +
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0090.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0090.html new file mode 100644 index 000000000..d8a7047a5 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0090.html @@ -0,0 +1,184 @@ + + +

Importing a Job

+

Function

This API is used to import one or more job files from OBS to DLF.

+ +

Before using this API, store job files in OBS buckets.

+
+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 2 Request parameters

Parameter

+

Mandatory

+

Type

+

Description

+

path

+

Yes

+

String

+

If OBS is deployed, it refers to the OBS path for storing the job definition file. For the format of the job definition file, see the response message of the exported job, for example, obs://myBucket/jobs.zip.

+

params

+

No

+

Map<String,String>

+

Public job parameter.

+

sameNamePolicy

+

No

+

String

+

Policy for specifying how to handle duplicate names. The options are as follows:

+
  • SKIP
  • OVERWRITE
+

Default value: SKIP

+

jobsParam

+

No

+

List<JobParam>

+

Job parameter. For details, see Table 3.

+

executeUser

+

No

+

String

+

User that executes the job.

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 3 JobParam parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Job name.

+

params

+

No

+

Map<String,String>

+

Job parameter.

+
+
+
+

Response

+
+ + + + + + + + + + + +
Table 4 Response parameter

Parameter

+

Mandatory

+

Type

+

Description

+

taskId

+

Yes

+

String

+

ID of the task. Used to call the API for querying system tasks to obtain the import status.

+
+
+
+

Example

Import jobs from OBS to DLF. If there are jobs and scripts with the same name, overwrite them.

+ + +
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0091.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0091.html new file mode 100644 index 000000000..e11a60608 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0091.html @@ -0,0 +1,177 @@ + + +

Executing a Job Immediately

+

Function

This API is used to execute a job immediately and check whether the job can be executed successfully.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + +
Table 3 Parameter

Parameter

+

Mandatory

+

Type

+

Description

+

jobParams

+

No

+

List<JobParam>

+

Parameter for starting the job.

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 4 JobParam data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Name of the parameter. It cannot exceed 64 characters.

+

value

+

Yes

+

String

+

Value of the parameter. It cannot exceed 1024 characters.

+
+
+
+

Response

+
+ + + + + + + + + + + +
Table 5 Response parameter

Parameter

+

Mandatory

+

Type

+

Description

+

instanceId

+

Yes

+

Long

+

Job instance ID.

+
+
+
+

Example

Execute the myJob job once.

+ + +
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0092.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0092.html new file mode 100644 index 000000000..87fb2d079 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0092.html @@ -0,0 +1,141 @@ + + +

Starting a Job

+

Function

This API is used to start a job.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + +
Table 3 Parameter

Parameter

+

Mandatory

+

Type

+

Description

+

jobParams

+

No

+

List<JobParam>

+

Parameter for starting the job.

+
+
+ +
+ + + + + + + + + + + + + + + + +
Table 4 JobParam data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Name of the parameter. It cannot exceed 64 characters.

+

value

+

Yes

+

String

+

Value of the parameter. It cannot exceed 1024 characters.

+
+
+
+

Response

None.

+
+

Example

Start job myJob.

+ + +
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0093.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0093.html new file mode 100644 index 000000000..4c411cc58 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0093.html @@ -0,0 +1,220 @@ + + +

Viewing Running Status of a Real-Time Job

+

Function

This API is used to view running status of a real-time job.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Name of a solution.

+

nodes

+

No

+

List

+

Node status list.

+

status

+

No

+

String

+

Job status.

+
  • STARTING
  • NORMAL
  • EXCEPTION
  • STOPPING
  • STOPPED
+

startTime

+

Yes

+

Date

+

Start time.

+

endTime

+

No

+

Date

+

End time.

+

lastUpdateTime

+

No

+

Date

+

Last update time.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 Data structure description of nodes

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Node name.

+

status

+

No

+

String

+

Node status.

+
  • STARTING
  • NORMAL
  • EXCEPTION
  • STOPPING
  • STOPPED
+

logPath

+

No

+

String

+

Path for storing node run logs.

+

type

+

Yes

+

String

+

Node type.

+
  • Hive SQL: Runs Hive SQL scripts.
  • Spark SQL: Runs Spark SQL scripts.
  • DWS SQL: Runs DWS SQL scripts.
  • DLI SQL: Runs DLI SQL scripts.
  • Shell: Runs shell SQL scripts.
  • CDM Job: Runs CDM jobs.
  • DIS Transfer Task: Creates DIS dump tasks.
  • CS Job: Creates and starts CloudStream jobs.
  • CloudTable Manager: Manages CloudTable tables, including creating and deleting tables.
  • OBS Manager: Manages OBS paths, including creating and deleting paths.
  • RESTAPI: Sends REST API requests.
  • SMN: Sends short messages or emails.
  • MRS Spark: Runs Spark jobs of MRS.
  • MapReduce: Runs MapReduce jobs of MRS.
+
+
+
+

Example

View running status of real-time job job_sms and the running status of each node in the job. job_sms has seven nodes, as shown in the following figure:

+
Figure 1 job_sms
+ + +
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0094.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0094.html new file mode 100644 index 000000000..007c42ff3 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0094.html @@ -0,0 +1,284 @@ + + +

Viewing a Job Instance List

+

Function

This API is used to view a job instance list.

+

A job instance is generated each time you run a batch job for which periodic scheduling or event-based scheduling is configured. If a real-time job contains a node for which periodic scheduling or event-based scheduling is configured, you can call this API to view the instance list of the subjobs associated with the node. The format of the jobName parameter is real-time job name_node name.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + +
Table 3 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

total

+

Yes

+

int

+

Total number of records.

+

instances

+

Yes

+

List<Instance>

+

Job instance status. For details, see Table 4.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 instances parameters

Parameter

+

Mandatory

+

Type

+

Description

+

jobName

+

Yes

+

String

+

Job name. When you view the instance list of a specified batch job, jobName is the name of the batch job. When you view the subjobs associated with a node in a real-time job, jobName is in format of real-time job name_node name.

+

jobInstanceName

+

Yes

+

String

+

Name of a job instance recorded by the log, rather than the name defined during job creation

+

status

+

Yes

+

String

+

Job instance status.

+
  • waiting
  • running
  • success
  • fail
  • running-exception
  • pause
  • manual-stop
+

planTime

+

Yes

+

Long

+

Planned execution time of the job instance.

+

startTime

+

Yes

+

Long

+

Actual execution start time of the job instance.

+

endTime

+

No

+

Long

+

Actual execution end time of the job instance.

+

executeTime

+

No

+

Long

+

Execution duration in milliseconds.

+

instanceId

+

Yes

+

Long

+

Job instance ID.

+

submitTime

+

Yes

+

Long

+

Time when a job is submitted.

+
+
+
+

Example 1

View the instance list of batch job job_batch.

+ + +
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0095.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0095.html new file mode 100644 index 000000000..099263fbd --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0095.html @@ -0,0 +1,343 @@ + + +

Viewing Job Instance Details

+

Function

This API is used to view job instance details, including the execution information about each node in a job instance.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

jobName

+

Yes

+

String

+

Job name.

+

instanceId

+

Yes

+

Long

+

Job instance ID.

+

status

+

Yes

+

String

+

Job instance status.

+
  • waiting
  • running
  • success
  • fail
  • running-exception
  • pause
  • manual-stop
+

planTime

+

Yes

+

Long

+

Planned execution time of the job instance.

+

startTime

+

Yes

+

Long

+

Actual execution start time of the job instance.

+

endTime

+

No

+

Long

+

Actual execution end time of the job instance.

+

executeTime

+

No

+

Long

+

Execution duration in milliseconds.

+

total

+

Yes

+

int

+

Total number of node records.

+

nodes

+

Yes

+

List<Node>

+

Node instance status. For details, see Table 4.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 Node parameters

Parameter

+

Mandatory

+

Type

+

Description

+

nodeName

+

Yes

+

String

+

Node name.

+

status

+

Yes

+

String

+

Node status.

+
  • waiting
  • running
  • success
  • fail
  • skip
  • pause
  • manual-stop
+

planTime

+

Yes

+

Long

+

Planned execution time of the job instance.

+

startTime

+

Yes

+

Long

+

Actual execution start time of the node.

+

endTime

+

No

+

Long

+

Actual execution end time of the node.

+

type

+

Yes

+

String

+

Node type.

+

retryTimes

+

No

+

Int

+

Number of attempts upon a failure.

+

instanceId

+

Yes

+

Long

+

Job instance ID.

+

inputRowCount

+

No

+

Long

+

Rows of input data.

+

speed

+

No

+

double

+

Write speed (row/second)

+

logPath

+

No

+

String

+

Path for storing node execution logs.

+
+
+
+

Example

View details about the instance whose ID is 34765 in job job_batch.

+ + +
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0096.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0096.html new file mode 100644 index 000000000..99834c313 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0096.html @@ -0,0 +1,257 @@ + + +

Querying a System Task

+

Function

This API is used to query details about asynchronous tasks.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

id

+

Yes

+

String

+

Task ID.

+

name

+

Yes

+

String

+

Name of the task.

+

startTime

+

Yes

+

Long

+

Start time.

+

endTime

+

No

+

Long

+

End time.

+

lastUpdate

+

Yes

+

Long

+

Time when the task was last updated.

+

status

+

Yes

+

String

+

Task status.

+
  • RUNNING
  • SUCCESSFUL
  • FAILED
+

message

+

No

+

String

+

Task information.

+

subtasks

+

No

+

List<SubTask>

+

Subtask. For details, see Table 4.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 subtasks parameters

Parameter

+

Mandatory

+

Type

+

Description

+

id

+

Yes

+

String

+

Subtask ID.

+

name

+

Yes

+

String

+

Name of the subtask.

+

startTime

+

Yes

+

Long

+

Start time.

+

endTime

+

No

+

Long

+

End time.

+

lastUpdate

+

Yes

+

Long

+

Time when the task was last updated.

+

status

+

Yes

+

String

+

Task status.

+
  • RUNNING
  • SUCCESSFUL
  • FAILED
+
+
+
+

Example

After the solution is imported successfully, query the task information.

+ + +
+ +
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0097.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0097.html new file mode 100644 index 000000000..db78f0978 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0097.html @@ -0,0 +1,184 @@ + + +

Creating a Script

+

Function

This API is used to create a script. Currently, the following script types are supported: DLI SQL, Flink SQL, RDS SQL, Spark SQL, Hive SQL, DWS SQL, Shell, and Presto.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Script parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Script name. The name contains a maximum of 128 characters, including only letters, numbers, hyphens (-), and periods (.). The script name must be unique.

+

type

+

Yes

+

String

+

Script type.

+
  • FlinkSQL
  • DLISQL
  • SparkSQL
  • HiveSQL
  • DWSSQL
  • RDSSQL
  • Shell
  • PRESTO
+

content

+

Yes

+

String

+

Script content. A maximum of 64 KB is supported.

+

directory

+

No

+

String

+

Directory for storing the script.

+

Access the DataArts Studio console and choose Data Development. In the left navigation pane, choose Development > Develop Script. In the directory tree of the script, you can view the created directories. The default directory is the root directory.

+

connectionName

+

No

+

String

+

Name of the connection associated with the script. This parameter is mandatory when type is set to DLISQL, SparkSQL, HiveSQL, DWSSQL, Shell, or PRESTO. To obtain the existing connections, refer to the instructions in Querying a Connection List. By default, this parameter is left blank.

+

database

+

No

+

String

+

Database associated with an SQL statement. This parameter is available only when type is set to DLISQL, SparkSQL, HiveSQL, DWSSQL, or PRESTO.

+
  • If type is set to DLI SQL, obtain database information by calling the API for querying all databases in the Data Lake Insight API Reference.
  • Connect to the cluster in JDBC mode to query database information if type is not set to DLISQL. By default, this parameter is left blank.
+

queueName

+

No

+

String

+

Queue name of the DLI resource. This parameter is available only when type is set to DLISQL. You can obtain the queue information by calling the API for "Querying All Queues" in the Data Lake Insight API Reference. By default, this parameter is left blank.

+

configuration

+

No

+

map <String,Object>

+

Configuration defined by a user for the job. This parameter is available only when type is set to DLISQL. For details about the supported configuration items, see conf parameter description in the "Submitting a SQL Job" section of the Data Lake Insight API Reference. By default, this parameter is left blank.

+

description

+

No

+

String

+

The description contains a maximum of 255 characters.

+
+
+
+

Response

None.

+
+

Example

Create a script.

+ +
+ +

Status Code

See Status Codes.

+
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0098.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0098.html new file mode 100644 index 000000000..f75d97ff4 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0098.html @@ -0,0 +1,191 @@ + + +

Modifying a Script

+

Function

This API is used to modify the configuration items or script contents of a script.

+ +
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Script parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Script name. The name contains a maximum of 128 characters, including only letters, numbers, hyphens (-), and periods (.). The script name must be unique.

+

type

+

Yes

+

String

+

Script type.

+
  • FlinkSQL
  • DLISQL
  • SparkSQL
  • HiveSQL
  • DWSSQL
  • RDSSQL
  • Shell
  • PRESTO
+

content

+

Yes

+

String

+

Script content. A maximum of 64 KB is supported.

+

directory

+

No

+

String

+

Directory for storing the script.

+

Access the DataArts Studio console and choose Data Development. In the left navigation pane, choose Development > Develop Script. In the directory tree of the script, you can view the created directories. The default directory is the root directory.

+

connectionName

+

No

+

String

+

Name of the connection associated with the script. This parameter is mandatory when type is set to DLISQL, SparkSQL, HiveSQL, DWSSQL, Shell, or PRESTO. To obtain the existing connections, refer to the instructions in Querying a Connection List. By default, this parameter is left blank.

+

database

+

No

+

String

+

Database associated with an SQL statement. This parameter is available only when type is set to DLISQL, SparkSQL, HiveSQL, DWSSQL, or PRESTO.

+
  • If type is set to DLI SQL, obtain database information by calling the API for querying all databases in the Data Lake Insight API Reference.
  • Connect to the cluster in JDBC mode to query database information if type is not set to DLISQL. By default, this parameter is left blank.
+

queueName

+

No

+

String

+

Queue name of the DLI resource. This parameter is available only when type is set to DLISQL. You can obtain the queue information by calling the API for "Querying All Queues" in the Data Lake Insight API Reference. By default, this parameter is left blank.

+

configuration

+

No

+

map <String,Object>

+

Configuration defined by a user for the job. This parameter is available only when type is set to DLISQL. For details about the supported configuration items, see conf parameter description in the "Submitting a SQL Job" section of the Data Lake Insight API Reference. By default, this parameter is left blank.

+

description

+

No

+

String

+

Description of the script. The description contains a maximum of 255 characters.

+
+
+
+

Response

None.

+
+

Example

Modify the script content.

+ +
+ +
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0099.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0099.html new file mode 100644 index 000000000..447273150 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0099.html @@ -0,0 +1,206 @@ + + +

Querying a Script

+

Function

This API is used to query a script, including the script type and script content.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Script parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Script name. The name contains a maximum of 128 characters, including only letters, numbers, hyphens (-), and periods (.). The script name must be unique.

+

type

+

Yes

+

String

+

Script type.

+
  • FlinkSQL
  • DLISQL
  • SparkSQL
  • HiveSQL
  • DWSSQL
  • RDSSQL
  • Shell
  • PRESTO
+

content

+

Yes

+

String

+

Script content. A maximum of 64 KB is supported.

+

directory

+

No

+

String

+

Directory for storing the script.

+

Access the DataArts Studio console and choose DataArts Factory. In the left navigation pane, choose Development > Develop Script. In the directory tree of the script, you can view the created directories. The default directory is the root directory.

+

connectionName

+

No

+

String

+

Name of the connection associated with the script. This parameter is mandatory when type is set to DLISQL, SparkSQL, HiveSQL, DWSSQL, Shell, or PRESTO. To obtain the existing connections, refer to the instructions in Querying a Connection List. By default, this parameter is left blank.

+

database

+

No

+

String

+

Database associated with an SQL statement. This parameter is available only when type is set to DLISQL, SparkSQL, HiveSQL, or DWSSQL.

+
  • If type is set to DLISQL, obtain database information by calling the API for "Querying All Databases" in the Data Lake Insight API Reference.
  • Connect to the cluster in JDBC mode to query database information if type is not set to DLISQL. By default, this parameter is left blank.
+

queueName

+

No

+

String

+

Queue name of the DLI resource. This parameter is available only when type is set to DLISQL. You can obtain the queue information by calling the API for "Querying All Queues" in the Data Lake Insight API Reference. By default, this parameter is left blank.

+

configuration

+

No

+

map <String,Object>

+

Configuration defined by a user for the job. This parameter is available only when type is set to DLISQL. For details about the supported configuration items, see conf parameter description in the "Submitting a SQL Job" section of the Data Lake Insight API Reference. By default, this parameter is left blank.

+

description

+

No

+

String

+

Description of the script. The description contains a maximum of 255 characters.

+

autoAcquireLock

+

No

+

boolean

+

Whether to automatically obtain the edit lock parameters. The default value is true.

+
+
+
+

Example

+ + +
+

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0100.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0100.html new file mode 100644 index 000000000..8b442b8c2 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0100.html @@ -0,0 +1,249 @@ + + +

Querying a Script List

+

Function

This API is used to query the script list. A maximum of 1000 scripts can be returned for each query.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + +
Table 3 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

total

+

Yes

+

Integer

+

The total number of scripts.

+

scripts

+

Yes

+

List<Script>

+

A list of scripts.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 Script parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Script name. The name contains a maximum of 128 characters, including only letters, numbers, hyphens (-), and periods (.). The script name must be unique.

+

type

+

Yes

+

String

+

Script type.

+
  • FlinkSQL
  • DLISQL
  • SparkSQL
  • HiveSQL
  • DWSSQL
  • RDSSQL
  • Shell
  • PRESTO
+

content

+

Yes

+

String

+

Script content. A maximum of 64 KB is supported.

+

directory

+

No

+

String

+

Directory for storing the script.

+

Access the DataArts Studio console and choose DataArts Factory. In the left navigation pane, choose Development > Develop Script. In the directory tree of the script, you can view the created directories. The default directory is the root directory.

+

connectionName

+

No

+

String

+

Name of the connection associated with the script. This parameter is mandatory when type is set to DLISQL, SparkSQL, HiveSQL, DWSSQL, Shell, or PRESTO. To obtain the existing connections, refer to the instructions in Querying a Connection List. By default, this parameter is left blank.

+

database

+

No

+

String

+

Database associated with an SQL statement. This parameter is available only when type is set to DLISQL, SparkSQL, HiveSQL, DWSSQL, or PRESTO.

+
  • If type is set to DLISQL, obtain database information by calling the API for "Querying All Databases" in the Data Lake Insight API Reference.
  • If type is not set to DLISQL, connect to the cluster in JDBC mode to query database information. By default, this parameter is left blank.
+

queueName

+

No

+

String

+

Queue name of the DLI resource. This parameter is available only when type is set to DLISQL. You can obtain the queue information by calling the API for "Querying All Queues" in the Data Lake Insight API Reference. By default, this parameter is left blank.

+

configuration

+

No

+

map <String,Object>

+

Configuration defined by a user for the job. This parameter is available only when type is set to DLISQL. For details about the supported configuration items, see conf parameter description in the "Submitting a SQL Job" section of the Data Lake Insight API Reference. By default, this parameter is left blank.

+

description

+

No

+

String

+

Description of the script. The description contains a maximum of 255 characters.

+
+
+
+

Example

Query a script list.

+ +
+ +
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0101.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0101.html new file mode 100644 index 000000000..187407bcc --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0101.html @@ -0,0 +1,244 @@ + + +

Querying the Execution Result of a Script Instance

+

Function

This API is used to obtain the execution status and result of a script instance.

+

During the query, specify the script name and script instance ID.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + +
Table 3 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

status

+

Yes

+

String

+

Execution status.

+

LAUNCHING: The script instance is being submitted.

+

RUNNING: The script instance is running.

+

FINISHED: The script instance is successfully run.

+

FAILED: The script instance fails to be run.

+

results

+

Yes

+

List<Result>

+

Execution result of the script instance.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 Result data structure description

Parameter

+

Mandatory

+

Type

+

Description

+

message

+

No

+

String

+

Execution failure message.

+

duration

+

No

+

float

+

Duration of executing the script instance. Unit: second

+

rowCount

+

No

+

Int

+

Number of the result rows.

+

rows

+

No

+

List<List<Object>>

+

Result data.

+

schema

+

No

+

List<Map<String, String>>

+

Format definition of the result data.

+
+
+
+

Example

+
+ + +

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0102.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0102.html new file mode 100644 index 000000000..ece0cae17 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0102.html @@ -0,0 +1,180 @@ + + +

Creating a Resource

+

Function

This API is used to create a resource. Types of nodes, including DLI Spark, MRS Spark, and MRS MapReduce, can reference files such as JAR and properties through resources.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Resource parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Name of the resource. The name contains a maximum of 32 characters, including only letters, numbers, underscores (_), and hyphens (-).

+

type

+

Yes

+

String

+

Resource type.

+
  • archive
  • file
  • jar
+

location

+

Yes

+

String

+

OBS path for storing the resource file. When type is set to jar, location is the path for storing the main JAR package. The path contains a maximum of 256 characters. For example, obs://myBucket/test.jar

+

dependFiles

+

No

+

List<String>

+

JAR package and properties file that the main JAR package depends on. The description contains a maximum of 10,240 characters.

+

desc

+

No

+

String

+

Description of the resource. The description contains a maximum of 255 characters.

+

directory

+

No

+

String

+

Directory for storing the resource.

+

Access the DataArts Studio console and choose Data Development. In the left navigation pane, choose Development > Develop Script. In the directory tree of the script, you can view the created directories. The default directory is the root directory.

+
+
+
+

Response

+
+ + + + + + + + + + + +
Table 4 Response parameter

Parameter

+

Mandatory

+

Type

+

Description

+

resourceId

+

Yes

+

String

+

Resource ID.

+
+
+
+

Example

+ +
+

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0103.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0103.html new file mode 100644 index 000000000..04405fb45 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0103.html @@ -0,0 +1,164 @@ + + +

Modifying a Resource

+

Function

This API is used to modify a specific resource. When modifying the resource, specify the resource ID.

+ +
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Resource parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Name of the resource. The name contains a maximum of 32 characters, including only letters, numbers, underscores (_), and hyphens (-).

+

type

+

Yes

+

String

+

Resource type.

+
  • archive
  • file
  • jar
+

location

+

No

+

String

+

OBS path for storing the resource file. When type is set to jar, location is the path for storing the main JAR package. The path contains a maximum of 256 characters. For example, obs://myBucket/test.jar

+

dependFiles

+

No

+

List<String>

+

JAR package and properties file that the main JAR package depends on. The description contains a maximum of 10,240 characters.

+

desc

+

No

+

String

+

Description of the resource. The description contains a maximum of 255 characters.

+

directory

+

Yes

+

String

+

Directory for storing the resource.

+

Access the DataArts Studio console and choose Data Development. In the left navigation pane, choose Development > Develop Script. In the directory tree of the script, you can view the created directories. The default directory is the root directory.

+
+
+
+

Response

None.

+
+

Request Example

PUT /v1/b384b9e9ab9b4ee8994c8633aabc9505/resources/3624d1c3-5df5-4f20-9af9-98eadad6c5f9
+{
+	"name": "test",
+	"type": "jar",
+	"location": "obs://dlf-test/hadoop-mapreduce-examples-2.4.1.jar",
+	"dependFiles": ["obs://dlf-test/depend1.jar","obs://dlf-test/depend2.jar"],
+	"desc": "test",
+	"directory":"/resource"
+}
+
+

Response Example

+
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0104.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0104.html new file mode 100644 index 000000000..b9c3e6de2 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0104.html @@ -0,0 +1,164 @@ + + +

Querying a Resource

+

Function

This API is used to query resource details. A resource contains various files such as JAR, ZIP, and properties files. A created resource can be used in job nodes such as DLI Spark and MRS Spark.

+
+

URI

+ +
+

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 3 Resource parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Name of the resource. The name contains a maximum of 32 characters, including only letters, numbers, underscores (_), and hyphens (-).

+

type

+

Yes

+

String

+

Resource type.

+
  • archive
  • file
  • jar
+

location

+

Yes

+

String

+

OBS path for storing the resource file. When type is set to jar, location is the path for storing the main JAR package. The path contains a maximum of 256 characters. For example, obs://myBucket/test.jar

+

dependFiles

+

No

+

List<String>

+

JAR package and properties file that the main JAR package depends on. The description contains a maximum of 10,240 characters.

+

desc

+

No

+

String

+

Description of the resource. The description contains a maximum of 255 characters.

+

directory

+

No

+

String

+

Directory for storing the resource.

+

Access the DataArts Studio console and choose Data Development. In the left navigation pane, choose Development > Develop Script. In the directory tree of the script, you can view the created directories. The default directory is the root directory.

+
+
+
+

Example

View resource details.

+ +
+ +

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0105.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0105.html new file mode 100644 index 000000000..63c2b5c9f --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0105.html @@ -0,0 +1,232 @@ + + +

Querying a Resource List

+

Function

This API is used to query a resource list. During the query, you can specify the page number and the maximum number of records on each page.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + +
Table 2 Request header parameter

Parameter

+

Mandatory

+

Type

+

Description

+

workspace

+

No

+

String

+

Workspace ID.

+
  • If this parameter is not set, data in the default workspace is queried by default.
  • To query data in other workspaces, this header must be carried.
+
+
+
+

Response

+
+ + + + + + + + + + + + + + + + +
Table 3 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

total

+

Yes

+

Integer

+

The total number of resources.

+

resources

+

Yes

+

List<Resource>

+

A list of resources.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 4 Resource parameters

Parameter

+

Mandatory

+

Type

+

Description

+

id

+

Yes

+

String

+

ID of the resource. The resource ID is used to query the resource.

+

name

+

Yes

+

String

+

Name of the resource. The name contains a maximum of 32 characters, including only letters, numbers, underscores (_), and hyphens (-).

+

type

+

Yes

+

String

+

Resource type.

+
  • archive
  • file
  • jar
+

location

+

Yes

+

String

+

OBS path for storing the resource file. When type is set to jar, location is the path for storing the main JAR package. The path contains a maximum of 256 characters. For example, obs://myBucket/test.jar

+

dependFiles

+

No

+

List<String>

+

JAR package and properties file that the main JAR package depends on. The description contains a maximum of 10,240 characters.

+

desc

+

No

+

String

+

The description contains a maximum of 255 characters.

+

directory

+

No

+

String

+

Directory for storing the resource.

+

Access the DataArts Studio console and choose Data Development. In the left navigation pane, choose Development > Develop Script. In the directory tree of the script, you can view the created directories. The default directory is the root directory.

+
+
+
+

Example

Query a list of resources.

+ +
+ +
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0106.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0106.html new file mode 100644 index 000000000..1679b0092 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0106.html @@ -0,0 +1,190 @@ + + +

Importing a Connection

+

Function

This API is used to import one or more connection files from OBS to the Data Development module. Before using this API, store connection files in OBS buckets.

+
+

URI

+
+ +

Request

+
+ + + + + + + + + + + + + + + + + + + + + +
Table 2 Request parameters

Parameter

+

Mandatory

+

Type

+

Description

+

path

+

Yes

+

String

+

With OBS deployed: OBS path for storing the connection definition file. For details about the format of the job definition file, see the response message of the exported connection.

+

Without OBS deployed: local path for storing the connection definition file.

+

params

+

No

+

List<Params>

+

Connection parameter. By default, this parameter is left blank.

+

sameNamePolicy

+

No

+

String

+

Policy for specifying how to handle duplicate names. The options are as follows:

+
  • SKIP
  • OVERWRITE

    Default value: SKIP

    +
+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + +
Table 3 Params connection parameters

Parameter

+

Mandatory

+

Type

+

Description

+

name

+

Yes

+

String

+

Name of a connection.

+

type

+

Yes

+

String

+

Connection type.

+

params

+

No

+

Map<String,Stri

+

ng>

+

Connection parameter. For details about parameter names, see the description of each type of connection configuration item. By default, this parameter is left blank.

+
+
+
+

Response

+
+ + + + + + + + + + + +
Table 4 Response parameters

Parameter

+

Mandatory

+

Type

+

Description

+

taskId

+

Yes

+

String

+

ID of the task. Used to call the API for querying system tasks to obtain the import status.

+
+
+
+

Example

Import a connection.

+ +
+ +

Status Codes

See Status Codes.

+
+
+
+
+
Parent topic: APIs to Be Taken Offline
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0127.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0127.html new file mode 100644 index 000000000..8de19cfd2 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0127.html @@ -0,0 +1,22 @@ + + +

DataArts Factory APIs

+

+
+
+ +
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0260.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0260.html new file mode 100644 index 000000000..fa0dfb8ea --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0260.html @@ -0,0 +1,21 @@ + + +

Public Data Structures

+
+
+ + +
+
Parent topic: DataArts Migration APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0261.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0261.html new file mode 100644 index 000000000..6864c9105 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0261.html @@ -0,0 +1,41 @@ + + +

Link Parameter Description

+
+
+ + +
+
Parent topic: Public Data Structures
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0262.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0262.html new file mode 100644 index 000000000..04ee92e90 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0262.html @@ -0,0 +1,248 @@ + + +

Link to a Relational Database

+

Description

By creating a JDBC link, you can extract data from or load data to the following relational databases:
  • Data Warehouse Service
  • RDS for MySQL
  • RDS for PostgreSQL
  • RDS for SQL Server
  • PostgreSQL
  • Microsoft SQL Server
  • IBM Db2
  • SAP HANA
+
+
+

Sample Link

{
+    "links": [
+        {
+            "link-config-values": {
+                "configs": [
+                    {
+                        "inputs": [
+                            {
+                                "name": "linkConfig.databaseType",
+                                "value": "MYSQL"
+                            },
+                            {
+                                "name": "linkConfig.host",
+                                "value": "10.120.205.30"
+                            },
+                            {
+                                "name": "linkConfig.port",
+                                "value": "3306"
+                            },
+                            {
+                                "name": "linkConfig.database",
+                                "value": "DB_name"
+                            },
+                            {
+                                "name": "linkConfig.username",
+                                "value": "username"
+                            },
+                            {
+                                "name": "linkConfig.password",
+                                "value": "Add password here"
+                            },
+                            {
+                                "name": "linkConfig.useAgent",
+                                "value": "false"
+                            },
+                            {
+                                "name": "linkConfig.fetchSize",
+                                "value": "100000"
+                            },
+                            {
+                                "name": "linkConfig.commitSize",
+                                "value": "10000"
+                            },
+                            {
+                                "name": "linkConfig.usingNative",
+                                "value": "false"
+                            },
+                            {
+                                "name": "linkConfig.useSSL",
+                                "value": "false"
+                            }
+                        ],
+                        "name": "linkConfig"
+                    }
+                ]
+            },
+            "name": "mysql_link",
+            "connector-name": "generic-jdbc-connector"
+        }
+    ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.databaseType

+

Yes

+

Enumeration

+

Database type. The options are as follows:

+
  • ORACLE
  • MYSQL
  • SQLSERVER
  • DB2
  • POSTGRESQL
  • DWS
  • DDM
  • SAP HANA
+

linkConfig.host

+

Yes

+

String

+

IP address of the database server

+

linkConfig.port

+

Yes

+

String

+

Port number of the database server

+

linkConfig.databaseconfig

+

No

+

Enumeration

+
Oracle database link type. This parameter is available only when an Oracle link is created. The options are as follows:
  • SERVICENAME: Use SERVICE_NAME to connect to the Oracle database.
  • SID: Use SID to connect to the Oracle database.
+
+

linkConfig.sidname

+

No

+

String

+

Oracle instance ID, which is used to differentiate databases by instances. This parameter is available only when an Oracle link is created and the database link type linkConfig.databaseconfig is set to SID.

+

linkConfig.database

+

No

+

String

+

Database name

+

linkConfig.username

+

Yes

+

String

+

Username

+

linkConfig.password

+

Yes

+

String

+

Password

+

linkConfig.useAgent

+

Yes

+

Boolean

+

Whether to obtain data from the data source through an agent

+

linkConfig.fetchSize

+

No

+

String

+

Number of data rows obtained each time

+

linkConfig.commitSize

+

No

+

String

+

Number of data rows submitted in each request

+

linkConfig.usingNative

+

No

+

Boolean

+

Whether to use the local API acceleration function of the database

+

When creating a MySQL link, you can use the LOAD DATA function of MySQL to accelerate data import and improve the performance of importing data to the MySQL database.

+

linkConfig.isRds

+

No

+

Boolean

+

Whether RDS is supported. The default value true is used for cloud databases and false is used for other databases.

+

linkConfig.useSSL

+

No

+

Boolean

+

Whether to enable SSL encrypted transmission for RDS. This parameter is available only when you create a DWS connection.

+

linkConfig.jdbcProperties

+

No

+

Map

+

Link attribute, which specifies the JDBC connector attributes of the data source. For details about how to configure the link attributes, see the JDBC connector description of the corresponding database.

+

linkConfig.version

+

No

+

Enumeration

+

Oracle database version. This parameter is available only when you create an Oracle link. The options are as follows:

+
  • HIGH_VERSION: Select this value if the Oracle database version is later than 12.1.
  • MED_VERSION: Select this value if the Oracle database version is 12.1.
  • LOW_VERSION: Select this value if the Oracle database version is earlier than 12.1.
+

If error message "java.sql.SQLException: Protocol violation" is displayed, select another option.

+

dialect.identifierEnclose

+

No

+

String

+

Reference identifier, which is the delimiter between the referenced table names or column names. For details, see the product documentation of the corresponding database.

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0263.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0263.html new file mode 100644 index 000000000..c9861d053 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0263.html @@ -0,0 +1,110 @@ + + +

Link to OBS

+

Description

By creating an OBS link, you can extract files from or load files to OBS. Files in CSV, JSON, and binary format are supported.

+
+

Sample Link

{
+    "links": [
+        {
+            "link-config-values": {
+                "configs": [
+                    {
+                        "inputs": [
+                            {
+                                "name": "linkConfig.storageType",
+                                "value": "OBS"
+                            },
+                            {
+                                "name": "linkConfig.server",
+                                "value": "10.121.16.183"
+                            },
+                            {
+                                "name": "linkConfig.port",
+                                "value": "443"
+                            },
+                            {
+                                "name":"linkConfig.accessKey",
+                                "value": "RSO6TTEZMJ6TTFBBAACE"
+                            },
+                            {
+                                "name":"linkConfig.securityKey",
+                                "value":"Add password here"
+                            }
+                        ],
+                        "name": "linkConfig"
+                    }
+                ]
+            },
+            "name": "obs_link",
+            "connector-name": "obs-connector"
+        }
+    ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.storageType

+

Yes

+

String

+

Storage class of an object

+

linkConfig.server

+

Yes

+

String

+

Endpoint of the OBS server. You can enter a bucket-level domain name.

+

linkConfig.port

+

Yes

+

String

+

Data transmission port. The HTTPS port number is 443 and the HTTP port number is 80.

+

linkConfig.accessKey

+

Yes

+

String

+

AK

+

linkConfig.securityKey

+

Yes

+

String

+

SK

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0266.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0266.html new file mode 100644 index 000000000..4e7fd3ae6 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0266.html @@ -0,0 +1,191 @@ + + +

Link to HDFS

+

Description

By creating an HDFS link, you can extract files from or load files to MRS, FusionInsight HD, or Apache Hadoop. Files in CSV, Parquet, and binary formats are supported.

+
+

Sample Link

{
+    "links": [
+        {
+            "link-config-values": {
+                "configs": [
+                    {
+                        "inputs": [
+                            {
+                                "name": "linkConfig.hadoopType",
+                                "value": "FusionInsight HD"
+                            },
+                            {
+                                "name": "linkConfig.host",
+                                "value": "10.120.205.143"
+                            },
+                            {
+                                "name": "linkConfig.casPort",
+                                "value": "20009"
+                            },
+                            {
+                                "name": "linkConfig.port",
+                                "value": "28443"
+                            },
+                            {
+                                "name": "linkConfig.authType",
+                                "value": "KERBEROS"
+                            },
+                            {
+                                "name": "linkConfig.user",
+                                "value": "admin"
+                            },
+                            {
+                                "name": "linkConfig.password",
+                                "value": "Add password here"
+                            },
+                            {
+                                "name": "linkConfig.runMode",
+                                "value": "STANDALONE"
+                            }
+                        ],
+                        "name": "linkConfig"
+                    }
+                ]
+            },
+            "name": "hdfslink",
+            "connector-name": "hdfs-connector"
+        }
+    ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.hadoopType

+

Yes

+

Enumeration

+

Hadoop type. The options are as follows:

+
  • MRS: link to HDFS of MRS
  • FusionInsight HD: link to HDFS of FusionInsight HD
  • Apache Hadoop: link to HDFS of Apache Hadoop
+

linkConfig.uri

+

No

+

String

+

NameNode URI required for the link to Apache Hadoop. The format is ip:port.

+

linkConfig.host

+

No

+

String

+

IP address of Manager required for the link to MRS or FusionInsight HD

+

linkConfig.port

+

No

+

String

+

Port number of Manager required for the link to FusionInsight HD

+

linkConfig.casPort

+

No

+

String

+

Port number of CAS Server that connects to FusionInsight HD required for the link to FusionInsight HD

+

linkConfig.user

+

No

+

String

+

Username for logging in to Manager. This parameter is not required if the cluster configuration is used.

+

linkConfig.password

+

No

+

String

+

Password for logging in to Manager. This parameter is not required if the cluster configuration is used.

+

linkConfig.authType

+

No

+

Enumeration

+
Authentication method. The options are as follows:
  • Simple: for non-security mode
  • Kerberos: for security mode
+
+

linkConfig.principal

+

No

+

String

+

Account principal required for Kerveros authentication. You can contact the administrator to obtain the account. Before using the cluster configuration, you must set this parameter in cluster configuration management.

+

linkConfig.keytab

+

No

+

FileContent

+

Local absolute path of the keytab file required for Kerveros authentication. You can also contact the administrator to obtain the file. Before using the cluster configuration, you must set this parameter in cluster configuration management.

+

linkConfig.runMode

+

No

+

Enumeration

+
Running mode of the HDFS link. The options are as follows:
  • EMBEDDED: The link instance runs with CDM. This mode delivers better performance.
  • STANDALONE: The link instance runs in an independent process. If CDM needs to connect to multiple Hadoop data sources (MRS, Hadoop, or CloudTable) with both Kerberos and Simple authentication methods, STANDALONE prevails.

    If STANDALONE is selected, CDM can migrate data between HDFSs of multiple MRS clusters.

    +
  • Agent: The link instance runs on an agent.

    If Agent is not used, and the CDM cluster connects to two or more clusters with Kerberos authentication enabled and the same realm, only one cluster can be connected in EMBEDDED mode, and the other clusters must be in STANDALONE mode.

    +
+
+

+

linkConfig.properties

+

No

+

Map

+

Properties. You can add configuration properties of the client. Each property must contain a name and a value.

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0267.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0267.html new file mode 100644 index 000000000..38f475fa7 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0267.html @@ -0,0 +1,213 @@ + + +

Link to HBase

+

Description

By creating an HBase link, you can extract data from or load data to HBase of MRS, FusionInsight HD, and Apache Hadoop.

+
+

Sample Link

{
+  "links": [
+    {
+      "link-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "linkConfig.hbaseType",
+                "value": "MRS"
+              },
+              {
+                "name": "linkConfig.host",
+                "value": "192.168.0.34"
+              },
+              {
+                "name": "linkConfig.user",
+                "value": "zephyr"
+              },
+              {
+                "name": "linkConfig.password",
+                "value": "Add password here."
+              },
+              {
+                "name": "linkConfig.authType",
+                "value": "KERBEROS"
+              },
+              {
+                "name": "linkConfig.serviceType",
+                "value": "HDFS"
+              },
+              {
+                "name": "linkConfig.hBaseVersion",
+                "value": "HBASE_2_X"
+              },
+              {
+                "name": "linkConfig.runMode",
+                "value": "EMBEDDED"
+              }
+            ],
+            "name": "linkConfig"
+          }
+        ],
+        "extended-configs": {
+          "name": "linkConfig.extendedFields",
+          "value": "eyL1c2VDbHVzdGVyQ29uZmlnIjoiZmFsc2UiJCLjbHVzdGVyQ29uZmlnUHLpbmNpcGFsIjoiemVwaHlyIn0="
+        }
+      },
+      "name": "mrs_hbase_dlf",
+      "connector-name": "hbase-connector"
+    }
+  ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.hbaseType

+

Yes

+

Enumeration

+

HBase type. The options are as follows:

+
  • CloudTable: link to CloudTable Service (CloudTable)
  • MRS: link to HBase of MRS
  • FusionInsight HD: link to HBase of FusionInsight HD
  • Apache Hadoop: link to HBase of Apache Hadoop
+

linkConfig.uri

+

No

+

String

+

NameNode URI required for the link to Apache Hadoop. The format is ip:port.

+

llinkConfig.host

+

No

+

String

+

IP address of Manager required for the link to MRS or FusionInsight HD

+

linkConfig.port

+

No

+

String

+

Port number of Manager required for the link to FusionInsight HD

+

linkConfig.casPort

+

No

+

String

+

Port number of CAS Server that connects to FusionInsight HD required for the link to FusionInsight HD

+

linkConfig.hBaseVersion

+

Yes

+

Enumeration

+

HBase version

+
  • HBASE_1_X
  • HBASE_2_X
+

linkConfig.user

+

No

+

String

+

Username for logging in to Manager. This parameter is not required if the cluster configuration is used.

+

linkConfig.password

+

No

+

String

+

Password for logging in to Manager. This parameter is not required if the cluster configuration is used.

+

linkConfig.authType

+

No

+

Enumeration

+
Authentication method. The options are as follows:
  • Simple: for non-security mode
  • Kerberos: for security mode
+
+

linkConfig.principal

+

No

+

String

+

Account principal required for Kerveros authentication. You can contact the administrator to obtain the account.

+

linkConfig.keytab

+

No

+

FileContent

+

Local absolute path of the keytab file required for Kerveros authentication. You can also contact the administrator to obtain the file.

+

linkConfig.serviceType

+

No

+

String

+

Service type Currently, HDFS and HBase are supported.

+

linkConfig.runMode

+

No

+

Enumeration

+
This parameter is used only when the HBase version is HBase_2_X. Possible values are:
  • EMBEDDED: The link instance runs with CDM. This mode delivers better performance.
  • STANDALONE: The link instance runs in an independent process. If CDM needs to connect to multiple Hadoop data sources (MRS, Hadoop, or CloudTable) with both Kerberos and Simple authentication methods, STANDALONE prevails.

    If STANDALONE is selected, CDM can migrate data between HDFSs of multiple MRS clusters.

    +
  • Agent: The link instance runs on an agent.

    If Agent is not used, and the CDM cluster connects to two or more clusters with Kerberos authentication enabled and the same realm, only one cluster can be connected in EMBEDDED mode, and the other clusters must be in STANDALONE mode.

    +
+
+

linkConfig.properties

+

No

+

Map

+

Properties. You can add configuration properties of the client. Each property must contain a name and a value.

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0268.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0268.html new file mode 100644 index 000000000..62f806b4d --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0268.html @@ -0,0 +1,138 @@ + + +

Link to CloudTable

+

Description

By creating a CloudTable link, you can extract data from or load data to CloudTable.

+
+

Sample Link

{
+  "links": [
+    {
+      "link-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "linkConfig.hbaseType",
+                "value": "CloudTable"
+              },
+              {
+                "name": "linkConfig.zookeeperQuorum",
+                "value": "cloudtable-pass-zk2-bae54VGN.cloudtable.com:2181,cloudtable-pass-zk1-Fu828so2.cloudtable.com:2181"
+              },
+              {
+                "name": "linkConfig.iamAuth",
+                "value": "true"
+              },
+              {
+                "name": "linkConfig.cloudtableUser",
+                "value": "zane"
+              },
+              {
+                "name": "linkConfig.accessKey",
+                "value": "GRC2WR0IxxxxxxxYLWU2"
+              },
+              {
+                "name": "linkConfig.securityKey",
+                "value": "Add password here"
+              },
+              {
+                "name": "linkConfig.runMode",
+                "value": "EMBEDDED"
+              }
+            ],
+            "name": "linkConfig"
+          }
+        ]
+      },
+      "name": "cloudtablelink",
+      "connector-name": "hbase-connector"
+    }
+  ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.hbaseType

+

Yes

+

Enumeration

+

HBase type. The options are as follows:

+
  • CloudTable: link to CloudTable
  • MRS: link to MRS
  • FusionInsight HD: link to FusionInsight HD
  • Apache Hadoop: link to Apache Hadoop
+

linkConfig.zookeeperQuorum

+

Yes

+

String

+

ZooKeeper link of CloudTable. This parameter is mandatory for the CloudTable link.

+

linkConfig.iamAuth

+

Yes

+

Boolean

+

If you choose IAM for identity authentication, enter the username, AK, and SK.

+

linkConfig.runMode

+

Yes

+

Enumeration

+

Running mode of the HBase link. The options are as follows:

+
  • EMBEDDED: The link instance runs with CDM. This mode delivers better performance.
  • STANDALONE: The link instance runs in an independent process. If CDM needs to connect to multiple Hadoop data sources (MRS, Hadoop, or CloudTable) with both Kerberos and Simple authentication methods, STANDALONE prevails.
+

linkConfig.cloudtableUser

+

Yes

+

String

+

Username for accessing the CloudTable cluster

+

linkConfig.accessKey

+

Yes

+

String

+

AK for accessing the CloudTable cluster

+

linkConfig.securityKey

+

Yes

+

String

+

SK for accessing the CloudTable cluster

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0269.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0269.html new file mode 100644 index 000000000..957445202 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0269.html @@ -0,0 +1,191 @@ + + +

Link to Hive

+

Description

By creating a Hive link, you can extract data from or load data to Hive of MRS.

+
+

Sample Link

{
+  "links": [
+    {
+      "link-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "linkConfig.host",
+                "value": "10.120.205.230"
+              },
+			  {
+                "name": "linkConfig.authType",
+                "value": "KERBEROS"
+              },
+              {
+                "name": "linkConfig.user",
+                "value": "cdm"
+              },
+              {
+                "name": "linkConfig.password",
+                "value": "Add password here"
+              } 
+            ],
+            "name": "linkConfig"
+          }
+        ]
+      },
+      "name": "hive_link",
+      "connector-name": "hive-connector"
+    }
+  ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

llinkConfig.host

+

Yes

+

String

+

IP address of MRS Manager

+

linkConfig.authType

+

Yes

+

Enumeration

+
Authentication method of MRS. The options are as follows:
  • SIMPLE: for non-security mode
  • KERBEROS: for security mode
+
+

linkConfig.principal

+

No

+

String

+

Account principal required for Kerveros authentication. You can contact the administrator to obtain the account. Before using the cluster configuration, you must set this parameter in cluster configuration management.

+

linkConfig.keytab

+

No

+

FileContent

+

Local absolute path of the keytab file required for Kerveros authentication. You can also contact the administrator to obtain the file. Before using the cluster configuration, you must set this parameter in cluster configuration management.

+

linkConfig.hiveVersion

+

Yes

+

Enumeration

+

Hive version

+
  • HIVE_1_X
  • HIVE_3_X
+

linkConfig.user

+

No

+

String

+

Username for logging in to Manager. This parameter is not required if the cluster configuration is used.

+

linkConfig.password

+

No

+

String

+

Password for logging in to Manager. This parameter is not required if the cluster configuration is used.

+

linkConfig.uri

+

No

+

String

+

NameNode URI required for the link to Apache Hadoop. The format is ip:port.

+

linkConfig.hiveMsUris

+

No

+

String

+

Hive metadata address for connecting to Apache Hadoop. For details, see the hive.metastore.uris configuration item. Example: thrift://host-192-168-1-212:9083

+

linkConfig.obsSupport

+

Yes

+

Boolean

+

The server must support OBS storage. When creating a Hive table, you can store the table in OBS.

+

linkConfig.runMode

+

Yes

+

Enumeration

+

This parameter is used only when the Hive version is HIVE_3_X. Possible values are:

+
  • EMBEDDED: The link instance runs with CDM. This mode delivers better performance.
  • STANDALONE: The link instance runs in an independent process. If CDM needs to connect to multiple Hadoop data sources (MRS, Hadoop, or CloudTable) with both Kerberos and Simple authentication modes, select STANDALONE or configure different agents.

    Note: The STANDALONE mode is used to solve the version conflict problem. If the connector versions of the source and destination ends of the same link are different, a JAR file conflict occurs. In this case, you need to place the source or destination end in the STANDALONE process to prevent the migration failure caused by the conflict.

    +
+

linkConfig.accessKey

+

No

+

String

+

AK This parameter is mandatory when OBS is supported.

+

linkConfig.securityKey

+

No

+

String

+

SK This parameter is mandatory when OBS is supported.

+

linkConfig.properties

+

No

+

Map

+

Properties. You can add configuration properties of the client. Each property must contain a name and a value.

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0270.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0270.html new file mode 100644 index 000000000..cc8965a51 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0270.html @@ -0,0 +1,98 @@ + + +

Link to an FTP or SFTP Server

+

Description

By creating an FTP or SFTP link, you are able to extract files from or load files to the FTP or SFTP server. Files in CSV, JSON, and binary format are supported.

+
+

Sample Link

{
+    "links": [
+        {
+            "link-config-values": {
+                "configs": [
+                    {
+                        "inputs": [
+                            {
+                                "name": "linkConfig.server",
+                                "value": "10.120.85.167"
+                            },
+                            {
+                                "name": "linkConfig.port",
+                                "value": "22"
+                            },
+                            {
+                                "name": "linkConfig.username",
+                                "value": "username"
+                            },
+                            {
+                                "name": "linkConfig.password",
+                                "value": "Add password here"
+                            }
+                        ],
+                        "name": "linkConfig"
+                    }
+                ]
+            },
+            "name": "sftp_link",
+            "connector-name": "sftp-connector"
+        }
+    ]
+}
+
+

Link Parameters

Parameters for creating the FTP or SFTP link are the same.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.server

+

Yes

+

String

+

IP address of the FTP or SFTP server

+

linkConfig.port

+

Yes

+

String

+

Port number of the FTP or SFTP server

+

linkConfig.username

+

Yes

+

String

+

Username for logging in to the FTP or SFTP server

+

linkConfig.password

+

Yes

+

String

+

Password for logging in to the FTP or SFTP server

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0271.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0271.html new file mode 100644 index 000000000..a53040a11 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0271.html @@ -0,0 +1,97 @@ + + +

Link to MongoDB

+

Description

By creating a MongoDB link, you can extract data from or load data to MongoDB.

+
+

Sample Link

{
+    "links": [
+        {
+            "link-config-values": {
+                "configs": [
+                    {
+                        "inputs": [
+                            {
+                                "name": "linkConfig.serverList",
+                                "value": "10.120.84.149:27017"
+                            },
+                            {
+                                "name": "linkConfig.database",
+                                "value": "DB_name"
+                            },
+                            {
+                                "name": "linkConfig.userName",
+                                "value": "username"
+                            },
+                            {
+                                "name": "linkConfig.password",
+                                "value": "Add password here"
+                            }
+                        ],
+                        "name": "linkConfig"
+                    }
+                ]
+            },
+            "name": "mongo_link",
+            "connector-name": "mongodb-connector"
+        }
+    ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.serverList

+

Yes

+

String

+

Server IP address list in host1:port1;host2:port2 format

+

linkConfig.database

+

Yes

+

String

+

Name of the MongoDB database

+

linkConfig.userName

+

Yes

+

String

+

Username for logging in to the MongoDB server

+

linkConfig.password

+

Yes

+

String

+

Password for logging in to the MongoDB server

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0272.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0272.html new file mode 100644 index 000000000..9479d9df4 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0272.html @@ -0,0 +1,98 @@ + + +

Link to Redis/DCS (to Be Brought Offline)

+

Description

By creating a Redis link, you can extract data from or load data to the Redis server. By creating a DCS link, you can load data to Data Cache Service (DCS), but not extract data for DCS. The data can be stored in string or hash format.

+
+

Sample Link

{
+    "links": [
+        {
+            "link-config-values": {
+                "configs": [
+                    {
+                        "inputs": [
+                            {
+                                "name": "linkConfig.deploymentMode",
+                                "value": "Cluster"
+                            },
+                            {
+                                "name": "linkConfig.serverlist",
+                                "value": "10.120.84.149:7300"
+                            },
+                            {
+                                "name": "linkConfig.password",
+                                "value": "Add password here"
+                            },
+                            {
+                                "name": "linkConfig.dbIndex",
+                                "value": "0"
+                            }
+                        ],
+                        "name": "linkConfig"
+                    }
+                ]
+            },
+            "name": "redis_link",
+            "connector-name": "redis-connector"
+        }
+    ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.deploymentMode

+

Yes

+

Enumeration

+

Redis deployment mode. The options are as follows:

+
  • Single: single-node deployment
  • Cluster: cluster deployment
+

linkConfig.serverlist

+

Yes

+

String

+

Server IP address list in host1:port1;host2:port2 format

+

linkConfig.password

+

Yes

+

String

+

Password for logging in to the Redis server

+

linkConfig.dbIndex

+

Yes

+

String

+

Redis database index

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0274.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0274.html new file mode 100644 index 000000000..d61fcb4b4 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0274.html @@ -0,0 +1,134 @@ + + +

Link to Kafka

+

Description

By creating a Kafka link, you are able to access open source Kafka and migrate data from Kafka to other data sources as required. Currently, only data export from Kafka is supported.

+
+

Sample Link

  {
+  "links": [
+    {
+      "link-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "linkConfig.hadoopType",
+                "value": "MRS"
+              },
+              {
+                "name": "linkConfig.host",
+                "value": "192.168.1.147"
+              },
+              {
+                "name": "linkConfig.user",
+                "value": "liuhuan1"
+              },
+              {
+                "name": "linkConfig.password",
+                "value": "Add password here."
+              },
+              {
+                "name": "linkConfig.authType",
+                "value": "KERBEROS"
+              }
+            ],
+            "name": "linkConfig"
+          }
+        ],
+        "extended-configs": {
+          "name": "linkConfig.extendedFields",
+          "value": "e30="
+        }
+      },
+      "name": "mrs_kafka_link",
+      "connector-name": "kafka-connector"
+    }
+  ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.hadoopType

+

Yes

+

Enumeration

+

Hadoop type. The options are as follows:

+
  • MRS: link to Kafka of MRS
  • Apache Kafka: link to Kafka of Apache
+

linkConfig.brokerList

+

Yes

+

String

+

This parameter is mandatory for Apache Kafka links. Kafka broker list in host1:port1,host2:port2 format

+

llinkConfig.host

+

Yes

+

String

+

Floating IP address of MRS Manager. Click Select next to the Manager IP text box to select an MRS cluster. CDM automatically fills in the authentication information.

+

linkConfig.user

+

Yes

+

String

+

Username used for logging in to MRS Manager.

+

linkConfig.password

+

Yes

+

String

+

Password used for logging in to MRS Manager.

+

linkConfig.authType

+

Yes

+

Enumeration

+
Authentication method. The options are as follows:
  • Simple: for non-security mode
  • Kerberos: for security mode
+
+

linkConfig.properties

+

No

+

Map

+

Properties. You can add configuration properties of the client. Each property must contain a name and a value.

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0276.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0276.html new file mode 100644 index 000000000..b96f5da1e --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0276.html @@ -0,0 +1,114 @@ + + +

Link to Elasticsearch/Cloud Search Service

+

Description

By creating an Elasticsearch link, you can extract data from or load data to the Elasticsearch server or Cloud Search Service.

+
+

Sample Link

{
+  "links": [
+    {
+      "link-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "linkConfig.host",
+                "value": "192.168.0.50:9200;192.168.0.62:9200"
+              },
+              {
+                "name": "linkConfig.safemode",
+                "value": "true"
+              },
+              {
+                "name": "linkConfig.user",
+                "value": "admin"
+              },
+              {
+                "name": "linkConfig.password",
+                "value": "Add password here."
+              },
+              {
+                "name": "linkConfig.linkType",
+                "value": "CSS"
+              }
+            ],
+            "name": "linkConfig"
+          }
+        ],
+        "extended-configs": {
+          "name": "linkConfig.extendedFields",
+          "value": "eyLodHRwc0FjY2VzcyI6InRydWUifQ=="
+        }
+      },
+      "name": "css-cdm-autotest-nodel",
+      "connector-name": "elasticsearch-connector"
+    }
+  ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.host

+

Yes

+

String

+

List of one or more Elasticsearch servers, including the port number. The format is ip:port. Use semicolons (;) to separate multiple IP addresses. For example, 192.168.0.1:9200;192.168.0.2:9200.

+

linkConfig.safemode

+

No

+

Boolean

+

If you select the security authentication mode, you must enter the username and password, and choose whether to enable HTTPS access.

+

linkConfig.user

+

No

+

String

+

For Elasticsearch that supports username and password authentication, configure the username and password when creating a link.

+

linkConfig.password

+

No

+

String

+

Password for accessing the Elasticsearch server

+

linkConfig.linkType

+

Yes

+

String

+

Link type, which is used to distinguish the Elasticsearch link from the Cloud Search Service link

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0277.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0277.html new file mode 100644 index 000000000..5b7269de9 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0277.html @@ -0,0 +1,97 @@ + + +

Link to DLI

+

Description

By creating a DLI link, you can import data to DLI. Currently, you cannot export data from DLI using CDM.

+
+

Sample Link

{
+    "links": [
+        {
+            "link-config-values": {
+                "configs": [
+                    {
+                        "inputs": [
+                            {
+                                "name": "linkConfig.ak",
+                                "value": "GRC2WR0IDC6NGROYLWU2"
+                            },
+                            {
+                                "name": "linkConfig.sk",
+                                "value": "Add password here"
+                            },
+                            {
+                                "name": "linkConfig.region",
+                                "value": ""
+                            },
+                            {
+                                "name": "linkConfig.projectId",
+                                "value": "c48475ce8e174a7a9f775706a3d5ebe2"
+                            }
+                        ],
+                        "name": "linkConfig"
+                    }
+                ]
+            },
+            "name": "dli",
+            "connector-name": "dli-connector"
+        }
+    ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.ak

+

Yes

+

String

+

AK for accessing the DLI database

+

linkConfig.sk

+

Yes

+

String

+

SK for accessing the DLI database

+

linkConfig.region

+

Yes

+

String

+

Region where DLI resides

+

linkConfig.projectId

+

Yes

+

String

+

Project ID of the DLI service

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0278.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0278.html new file mode 100644 index 000000000..588db442b --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0278.html @@ -0,0 +1,108 @@ + + +

Link to CloudTable OpenTSDB

+

Description

By creating an OpenTSDB link, you can extract data from and load data to CloudTable OpenTSDB.

+
+

Sample Link

{
+  "links": [
+    {
+      "link-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "linkConfig.openTSDBQuorum",
+                "value": "opentsdb-sp8afz7bgbps5ur.cloudtable.com:4242"
+              },
+              {
+                "name": "linkConfig.securityMode",
+                "value": "UNSAFE"
+              }
+            ],
+            "name": "linkConfig"
+          }
+        ]
+      },
+      "name": "opentsdb",
+      "connector-name": "opentsdb-connector"
+    }
+  ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.openTSDBQuorum

+

Yes

+

String

+

ZooKeeper Link of OpenTSDB

+

linkConfig.securityMode

+

Yes

+

String

+

Security or UNSAFE mode

+

If you select Security, enter the project ID, username, and AK/SK.

+

linkConfig.user

+

No

+

String

+

Username for accessing CloudTable

+

linkConfig.ak

+

No

+

String

+

AK for accessing CloudTable

+

linkConfig.sk

+

No

+

String

+

SK for accessing CloudTable

+

linkConfig.projectId

+

No

+

String

+

Project ID of CloudTable

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0280.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0280.html new file mode 100644 index 000000000..2f3c0990e --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0280.html @@ -0,0 +1,108 @@ + + +

Link to DMS Kafka

+

Description

By creating a DMS Kafka link, you can connect to Kafka Basic or Kafka Platinum on DMS. Currently, you can only export data from DMS Kafka to Cloud Search Service.

+
+

Sample Link

{
+  "links": [
+    {
+      "link-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "linkConfig.kafkaType",
+                "value": "Platinum"
+              },
+              {
+                "name": "linkConfig.brokerList",
+                "value": "100.85.121.112:9094,100.85.220.134:9094,100.85.127.232:9094"
+              },
+              {
+                "name": "linkConfig.isPlatinumInstance",
+                "value": "false"
+              }
+            ],
+            "name": "linkConfig"
+          }
+        ],
+        "extended-configs": {
+          "name": "linkConfig.extendedFields",
+          "value": "e30="
+        }
+      },
+      "name": "dms_kafka",
+      "connector-name": "dms-kafka-connector"
+    }
+  ]
+}
+
+

Link Parameters

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

linkConfig.kafkaType

+

Yes

+

Enumeration

+

DMS Kafka edition. Currently, only the Platinum edition is available.

+
  • Basic: DMS Kafka Basic
  • Platinum: DMS Kafka Platinum
+

linkConfig.brokerList

+

Yes

+

String

+

The DMS endpoint is in host1:port1,host2:port2 format.

+

linkConfig.isPlatinumInstance

+

Yes

+

Boolean

+

Whether to enable SSL authentication when a client connects to a Kafka premium instance.

+

If Kafka SASL_SSL is enabled, data will be encrypted before transmission for higher security, but performance will suffer.

+

linkConfig.user

+

No

+

String

+

Username for connecting to DMS Kafka. This parameter is displayed when Kafka SASL_SSL is enabled.

+

linkConfig.password

+

No

+

String

+

Password for connecting to DMS Kafka. This parameter is displayed when Kafka SASL_SSL is enabled.

+
+
+
+
+
+
+
Parent topic: Link Parameter Description
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0281.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0281.html new file mode 100644 index 000000000..ef44da834 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0281.html @@ -0,0 +1,39 @@ + + +

Source Job Parameters

+
+
+ + +
+
Parent topic: Public Data Structures
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0282.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0282.html new file mode 100644 index 000000000..c6e306eeb --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0282.html @@ -0,0 +1,152 @@ + + +

From a Relational Database

+

Sample JSON File

"from-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "fromJobConfig.useSql",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.schemaName",
+                "value": "rf_database"
+              },
+              {
+                "name": "fromJobConfig.tableName",
+                "value": "rf_from"
+              },
+              {
+                "name": "fromJobConfig.columnList",
+                "value": "AA&BB"
+              },
+              {
+                "name": "fromJobConfig.incrMigration",
+                "value": "false"
+              }
+            ],
+            "name": "fromJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

fromJobConfig.useSql

+

Yes

+

Boolean

+

Whether to use the customized SQL statement to export data when exporting relational database data

+

fromJobConfig.sql

+

No

+

String

+

Customized SQL statement. CDM executes the SQL statement to export data.

+

fromJobConfig.schemaName

+

Yes

+

String

+

Database mode or tablespace. For example, public.

+
NOTE:
The parameter value can contain wildcard characters (*), which is used to export all databases whose names start with a certain prefix or end with a certain suffix. The examples are as follows:
  • SCHEMA* indicates that all databases whose names starting with SCHEMA are exported.
  • *SCHEMA indicates that all databases whose names ending with SCHEMA are exported.
  • *SCHEMA* indicates that all databases whose names containing SCHEMA are exported.
+
+
+

fromJobConfig.tableName

+

Yes

+

String

+

Table name. For example, TBL_EXAMPLE.

+
NOTE:
The table name can contain wildcard characters (*), which is used to export all tables whose names start with a certain prefix or end with a certain suffix. The number and types of fields in the tables must be the same. The examples are as follows:
  • table* indicates that all tables whose names starting with table are exported.
  • *table indicates that all tables whose names ending with table are exported.
  • *table* indicates that all tables whose names containing table are exported.
+
+
+

fromJobConfig.whereClause

+

No

+

String

+

WHERE clause used to specify the data to be extracted. If no WHERE clause is configured, the entire table will be extracted. For example, age > 18 and age <= 60.

+

fromJobConfig.columnList

+

No

+

String

+

List of fields to be extracted. Use & to separate field names. For example, id&gid&name.

+

fromJobConfig.partitionColumn

+

No

+

String

+

Partition field to be extracted, by which a job is split in multiple sub-jobs executed concurrently. For example, id.

+

fromJobConfig.incrMigration

+

No

+

Boolean

+

Whether to enable incremental data migration using MySQL Binlog. This can be only used for table/file migration from MySQL databases to GaussDB(DWS).

+

If this function is enabled, data in the source table and destination table can be synchronized in real time. Each MySQL link or source table supports only one incremental migration job.

+

fromJobConfig.usePartition

+

No

+

Boolean

+

When data is exported from the Oracle database, data can be extracted from each partition in a partitioned table. When this function is enabled, you can use the fromJobConfig.partitionList parameter to specify the partitions in the Oracle table. This function does not support non-partitioned tables.

+

fromJobConfig.partitionList

+

No

+

String

+

Oracle table partitions to be migrated. Separate multiple partitions with ampersands (&). If you do not set this parameter, all partitions will be migrated.

+
+
+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0283.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0283.html new file mode 100644 index 000000000..dc5f29e21 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0283.html @@ -0,0 +1,331 @@ + + +

From Object Storage

+

Sample JSON File

"from-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "fromJobConfig.bucketName",
+                "value": "cdm-est"
+              },
+              {
+                "name": "fromJobConfig.inputDirectory",
+                "value": "/obsfrom/varchar.txt"
+              },
+              {
+                "name": "fromJobConfig.inputFormat",
+                "value": "CSV_FILE"
+              },
+              {
+                "name": "fromJobConfig.columnList",
+                "value": "1&2&3"
+              },
+              {
+                "name": "fromJobConfig.fieldSeparator",
+                "value": ","
+              },
+              {
+                "name": "fromJobConfig.quoteChar",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.regexSeparator",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.firstRowAsHeader",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.encodeType",
+                "value": "UTF-8"
+              },
+              {
+                "name": "fromJobConfig.fromCompression",
+                "value": "NONE"
+              },
+              {
+                "name": "fromJobConfig.splitType",
+                "value": "FILE"
+              }
+            ],
+            "name": "fromJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

fromJobConfig.bucketName

+

Yes

+

String

+

Bucket name

+

fromJobConfig.inputDirectory

+

Yes

+

String

+

Path for storing files to be extracted. You can enter a maximum of 50 file paths, which are separated by vertical bars (|). You can also customize the separators. For example, FROM/example.csv|FROM/b.txt.

+

fromJobConfig.inputFormat

+

Yes

+

Enumeration

+
File format required for data transmission. Currently, the following file formats are supported:
  • CSV_FILE: CSV format, used to migrate files to data tables
  • JSON_FILE: JSON format, used to migrate files to data tables
  • BINARY_FILE: Files (even not in binary format) will be directly transferred without resolution. It is applicable to file copy.
+
+

If you select BINARY_FILE, the migration destination must also be a file system.

+

fromJobConfig.lineSeparator

+

No

+

String

+

Line feed character in the file. For example, \n. By default, \n, \r, or \r\n is automatically identified.

+

fromJobConfig.columnList

+

No

+

String

+

Numbers of columns to be extracted. Use & to separate column numbers in ascending order. For example, 1&3&5.

+

fromJobConfig.regexSeparator

+

No

+

Boolean

+

Whether to use the regular expression to separate fields. This parameter is valid only when the file format is CSV_FILE.

+

fromJobConfig.regex

+

No

+

String

+

Regular expression. This parameter is valid only when the regular expression is used to separate fields.

+

fromJobConfig.fieldSeparator

+

No

+

String

+

Field delimiter. This parameter is valid only when the file format is CSV_FILE. The default value is ,.

+

fromJobConfig.quoteChar

+

No

+

Boolean

+

Whether to use the encircling symbol. If this parameter is set to true, the field delimiters in the encircling symbol are regarded as a part of the string value. Currently, the default encircling symbol of CDM is double quotation mark (").

+

fromJobConfig.firstRowAsHeader

+

No

+

Boolean

+

Whether to regard the first line as the heading line. This parameter is valid only when the file format is CSV_FILE. When you migrate a CSV file to a table, CDM writes all data to the table by default. If this parameter is set to true, CDM uses the first line of the CSV file as the heading line and does not write the line to the destination table.

+

fromJobConfig.fromCompression

+

No

+

Enumeration

+
Compression format. This parameter is valid only when the file format is CSV_FILE or JSON. The options are as follows:
  • NONE: Files in all formats are transferred.
  • GZIP: Files in gzip format are transferred.
  • ZIP: Files in Zip format are transferred.
+
+

fromJobConfig.jsonReferenceNode

+

No

+

String

+

Reference node. This parameter is valid when the file format is JSON_FILE. Resolve data on the JSON node. If the data corresponding to the node is a JSON array, the system extracts data from the array in the same mode. Nested JSON nodes are separated by periods (.). For example, data.list.

+

fromJobConfig.encodeType

+

No

+

String

+

Encoding type. For example, UTF_8 or GBK.

+

fromJobConfig.fromFileOpType

+

No

+

Enumeration

+

Source file processing mode. After a job is completed, operations on the source file can be performed. The source file can be renamed or deleted.

+

fromJobConfig.useMarkerFile

+

No

+

Boolean

+

Whether to start a job by a marker file. A job is started only when a marker file for starting the job exists in the source path. Otherwise, the job will be suspended for a period of time specified by fromJobConfig.waitTime.

+

fromJobConfig.markerFile

+

No

+

String

+

Name of the marker file for starting a job. After a marker file is specified, the task is executed only when the file exists in the source path. If the marker file is not specified, this function is disabled by default. For example, ok.txt.

+

fromJobConfig.waitTime

+

No

+

String

+

Period of waiting for a marker file. If you set Start Job by Marker File to Yes but no marker file exists in the source path, the job fails upon suspension timeout.

+

If you set this parameter to 0 and no marker file exists in the source path, the job will fail immediately.

+

Unit: second

+

fromJobConfig.filterType

+

No

+

Enumeration

+
Filter type. Possible values are as follows:
  • WILDCARD: Enter a wildcard character to filter paths or files. CDM will migrate the paths or files that meet the filter condition.
  • TIME: Specify a time filter. CDM will migrate the files modified after the specified time point.
+
+

fromJobConfig.pathFilter

+

No

+

String

+

Path filter, which is configured when the filter type is WILDCARD. It is used to filter the file directories. For example, *input.

+

fromJobConfig.fileFilter

+

No

+

String

+

File filter, which is configured when the filter type is WILDCARD. It is used to filter files in the specified directory. Use commas (,) to separate multiple files. For example, *.csv,*.txt.

+

fromJobConfig.startTime

+

No

+

String

+

If you set Filter Type to Time Filter, and specify a point in time for this parameter, only the files modified after the specified time are transferred. The time format must be yyyy-MM-dd HH:mm:ss.

+

This parameter can be set to a macro variable of date and time. For example, ${timestamp(dateformat(yyyy-MM-dd HH:mm:ss,-90,DAY))} indicates that only files generated within the latest 90 days are migrated.

+

fromJobConfig.endTime

+

No

+

String

+

If you set Filter Type to Time Filter, and specify a point in time for this parameter, only the files modified before the specified time are transferred. The time format must be yyyy-MM-dd HH:mm:ss.

+

This parameter can be set to a macro variable of date and time. For example, ${timestamp(dateformat(yyyy-MM-dd HH:mm:ss))} indicates that only the files whose modification time is earlier than the current time are migrated.

+

fromJobConfig.fileSeparator

+

No

+

String

+

File separator. If you enter multiple file paths in fromJobConfig.inputDirectory, CDM uses the file separator to separate files. The default value is |.

+

fromJobConfig.decryption

+

No

+

Enumeration

+
Whether to decrypt the encrypted file before export and the decryption method. The options are as follows:
  • NONE: Do not decrypt but directly export the file.
  • AES-256-GCM: Use the AES-256-GCM (NoPadding) algorithm to decrypt the file and then export the file.
+
+

fromJobConfig.dek

+

No

+

String

+

Data decryption key. The key is a string of 64-bit hexadecimal numbers and must be the same as the data encryption key toJobConfig.dek configured during encryption. If the encryption and decryption keys are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+

fromJobConfig.iv

+

No

+

String

+

Initialization vector required for decryption. The initialization vector is a string of 32-bit hexadecimal numbers and must be the same as the initialization vector toJobConfig.iv configured during encryption. If the initialization vectors are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+

fromJobConfig.md5FileSuffix

+

No

+

String

+

Check whether the files extracted by CDM are consistent with those in the migration source.

+
+
+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0284.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0284.html new file mode 100644 index 000000000..5826684a5 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0284.html @@ -0,0 +1,358 @@ + + +

From HDFS

+

Sample JSON File

"from-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "fromJobConfig.inputDirectory",
+                "value": "/hdfsfrom/from_hdfs_est.csv"
+              },
+              {
+                "name": "fromJobConfig.inputFormat",
+                "value": "CSV_FILE"
+              },
+              {
+                "name": "fromJobConfig.columnList",
+                "value": "1"
+              },
+              {
+                "name": "fromJobConfig.fieldSeparator",
+                "value": ","
+              },
+              {
+                "name": "fromJobConfig.quoteChar",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.regexSeparator",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.firstRowAsHeader",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.encodeType",
+                "value": "UTF-8"
+              },
+              {
+                "name": "fromJobConfig.fromCompression",
+                "value": "NONE"
+              },
+              {
+                "name": "fromJobConfig.compressedFileSuffix",
+                "value": "*"
+              },
+              {
+                "name": "fromJobConfig.splitType",
+                "value": "FILE"
+              },
+              {
+                "name": "fromJobConfig.useMarkerFile",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.fileSeparator",
+                "value": "|"
+              },
+              {
+                "name": "fromJobConfig.filterType",
+                "value": "NONE"
+              }
+            ],
+            "name": "fromJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ +
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0285.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0285.html new file mode 100644 index 000000000..cb638cc68 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0285.html @@ -0,0 +1,87 @@ + + +

From Hive

+

Sample JSON File

"from-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "fromJobConfig.hive",
+                "value": "hive"
+              },
+              {
+                "name": "fromJobConfig.database",
+                "value": "rf_database"
+              },
+              {
+                "name": "fromJobConfig.table",
+                "value": "rf_from"
+              },
+              {
+                "name": "fromJobConfig.columnList",
+                "value": "tiny&small&int&integer&bigint&float&double&timestamp&char&varchar&text"
+              }
+            ],
+            "name": "fromJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

fromJobConfig.hive

+

No

+

String

+

Data source to be extracted. If the data source is Hive, set this parameter to hive.

+

fromJobConfig.database

+

No

+

String

+

Database from which data is extracted. For example, default.

+

fromJobConfig.table

+

Yes

+

String

+

Name of the table from which data is extracted. For example, cdm.

+

fromJobConfig.columnList

+

No

+

String

+

Numbers of columns to be extracted. Use & to separate column numbers in ascending order. For example, 1&3&5.

+
+
+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0286.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0286.html new file mode 100644 index 000000000..0726eecf5 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0286.html @@ -0,0 +1,161 @@ + + +

From HBase/CloudTable

+

Sample JSON File

"from-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "fromJobConfig.table",
+                "value": "rf_from"
+              },
+              {
+                "name": "fromJobConfig.columnFamilies",
+                "value": "rowkey&f"
+              },
+              {
+                "name": "fromJobConfig.columns",
+                "value": "rowkey:rowkey&f:_small"
+              },
+              {
+                "name": "fromJobConfig.formats",
+                "value": {
+                  "f:_date": "yyyy-MM-dd",
+                  "f:_timestamp": "yyyy-MM-dd HH:mm:ss"
+                }
+              }
+            ],
+            "name": "fromJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0287.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0287.html new file mode 100644 index 000000000..06b8c4ed5 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0287.html @@ -0,0 +1,334 @@ + + +

From FTP/SFTP/NAS (to Be Brought Offline)/SFS (to Be Brought Offline)

+

Sample JSON File

"from-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "fromJobConfig.inputDirectory",
+                "value": "/sftpfrom/from_sftp.csv"
+              },
+              {
+                "name": "fromJobConfig.inputFormat",
+                "value": "CSV_FILE"
+              },
+              {
+                "name": "fromJobConfig.columnList",
+                "value": "1&2&3&4&5&6&7&8&9&10&11&12"
+              },
+              {
+                "name": "fromJobConfig.fieldSeparator",
+                "value": ","
+              },
+              {
+                "name": "fromJobConfig.regexSeparator",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.firstRowAsHeader",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.encodeType",
+                "value": "UTF-8"
+              },
+              {
+                "name": "fromJobConfig.fromCompression",
+                "value": "NONE"
+              },
+              {
+                "name": "fromJobConfig.splitType",
+                "value": "FILE"
+              }
+            ],
+            "name": "fromJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

Source link job parameters of FTP and SFTP are the same. Table 1 describes the parameters. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Source link job parameters of file systems

Parameter

+

Mandatory

+

Type

+

Description

+

fromJobConfig.inputDirectory

+

Yes

+

String

+

Path for storing files to be extracted. You can enter a maximum of 50 file paths, which are separated by vertical bars (|). You can also customize the separators. For example, FROM/example.csv|FROM/b.txt.

+

fromJobConfig.inputFormat

+

Yes

+

Enumeration

+
File format required for data transmission. Currently, the following file formats are supported:
  • CSV_FILE: CSV format, used to migrate files to data tables
  • JSON_FILE: JSON format, used to migrate files to data tables
  • BINARY_FILE: Files (even not in binary format) will be directly transferred without resolution. It is applicable to file copy.
+
+

If you select BINARY_FILE, the migration destination must also be a file system.

+

fromJobConfig.lineSeparator

+

No

+

String

+

Line feed character in the file. For example, \n. By default, \n, \r, or \r\n is automatically identified.

+

fromJobConfig.columnList

+

No

+

String

+

Numbers of columns to be extracted. Use & to separate column numbers in ascending order. For example, 1&3&5.

+

fromJobConfig.lineSeparator

+

No

+

String

+

Line feed character. This parameter is valid only when the file format is CSV_FILE. The default value is \r\n.

+

fromJobConfig.fieldSeparator

+

No

+

String

+

Field delimiter. This parameter is valid only when the file format is CSV_FILE. The default value is ,.

+

fromJobConfig.quoteChar

+

No

+

Boolean

+

Whether to use the encircling symbol. If this parameter is set to true, the field delimiters in the encircling symbol are regarded as a part of the string value. Currently, the default encircling symbol of CDM is double quotation mark (").

+

fromJobConfig.regexSeparator

+

No

+

Boolean

+

Whether to use the regular expression to separate fields. This parameter is valid only when the file format is CSV_FILE.

+

fromJobConfig.regex

+

No

+

String

+

Regular expression. This parameter is valid only when the regular expression is used to separate fields.

+

fromJobConfig.firstRowAsHeader

+

No

+

Boolean

+

Whether to regard the first line as the heading line. This parameter is valid only when the file format is CSV_FILE. When you migrate a CSV file to a table, CDM writes all data to the table by default. If this parameter is set to true, CDM uses the first line of the CSV file as the heading line and does not write the line to the destination table.

+

fromJobConfig.fromCompression

+

No

+

Enumeration

+
Compression format. This parameter is valid only when the file format is CSV_FILE or JSON. The options are as follows:
  • NONE: Files in all formats are transferred.
  • GZIP: Files in gzip format are transferred.
  • ZIP: Files in Zip format are transferred.
+
+

fromJobConfig.splitType

+

No

+

Enumeration

+

Whether to split files by file or size.

+
  • FILE: Split files by file quantity. If there are 10 files and throttlingConfig.numExtractors is set to 5, each shard consists of two files.
  • SIZE: Split files by file size. Files will not be split for balance. Suppose there are 10 files, among which nine are 10 MB and one is 200 MB in size. If throttlingConfig.numExtractors is set to 2, two shards will be created. One of them processes the nine 10 MB files, and the other processes the 200 MB file.
+

fromJobConfig.jsonReferenceNode

+

No

+

String

+

Reference node. This parameter is valid when the file format is JSON_FILE. Resolve data on the JSON node. If the data corresponding to the node is a JSON array, the system extracts data from the array in the same mode. Nested JSON nodes are separated by periods (.). For example, data.list.

+

fromJobConfig.encodeType

+

No

+

String

+

Encoding type. For example, UTF_8 or GBK.

+

fromJobConfig.fromFileOpType

+

No

+

Enumeration

+

Source file processing mode. After a job is completed, operations on the source file can be performed. The source file can be renamed or deleted.

+

fromJobConfig.useMarkerFile

+

No

+

Boolean

+

Whether to start a job by a marker file. A job is started only when a marker file for starting the job exists in the source path. Otherwise, the job will be suspended for a period of time specified by fromJobConfig.waitTime.

+

fromJobConfig.markerFile

+

No

+

String

+

Name of the marker file for starting a job. After a marker file is specified, the task is executed only when the file exists in the source path. If the marker file is not specified, this function is disabled by default. For example, ok.txt.

+

fromJobConfig.waitTime

+

No

+

String

+

Period of waiting for a marker file. If you set Start Job by Marker File to Yes but no marker file exists in the source path, the job fails upon suspension timeout.

+

If you set this parameter to 0 and no marker file exists in the source path, the job will fail immediately.

+

Unit: second

+

fromJobConfig.filterType

+

No

+

Enumeration

+
Filter type. Possible values are as follows:
  • WILDCARD: Enter a wildcard character to filter paths or files. CDM will migrate the paths or files that meet the filter condition.
  • TIME: Specify a time filter. CDM will migrate the files modified after the specified time point.
+
+

fromJobConfig.pathFilter

+

No

+

String

+

Path filter, which is configured when the filter type is WILDCARD. It is used to filter the file directories. For example, *input.

+

fromJobConfig.fileFilter

+

No

+

String

+

File filter, which is configured when the filter type is WILDCARD. It is used to filter files in the specified directory. Use commas (,) to separate multiple files. For example, *.csv,*.txt.

+

fromJobConfig.startTime

+

No

+

String

+

If you set Filter Type to Time Filter, and specify a point in time for this parameter, only the files modified after the specified time are transferred. The time format must be yyyy-MM-dd HH:mm:ss.

+

This parameter can be set to a macro variable of date and time. For example, ${timestamp(dateformat(yyyy-MM-dd HH:mm:ss,-90,DAY))} indicates that only files generated within the latest 90 days are migrated.

+

fromJobConfig.endTime

+

No

+

String

+

If you set Filter Type to Time Filter, and specify a point in time for this parameter, only the files modified before the specified time are transferred. The time format must be yyyy-MM-dd HH:mm:ss.

+

This parameter can be set to a macro variable of date and time. For example, ${timestamp(dateformat(yyyy-MM-dd HH:mm:ss))} indicates that only the files whose modification time is earlier than the current time are migrated.

+

fromJobConfig.fileSeparator

+

No

+

String

+

File separator. If you enter multiple file paths in fromJobConfig.inputDirectory, CDM uses the file separator to separate files. The default value is |.

+

fromJobConfig.decryption

+

No

+

Enumeration

+
Whether to decrypt the encrypted file before export and the decryption method. The options are as follows:
  • NONE: Do not decrypt but directly export the file.
  • AES-256-GCM: Use the AES-256-GCM (NoPadding) algorithm to decrypt the file and then export the file.
+
+

fromJobConfig.dek

+

No

+

String

+

Data decryption key. The key is a string of 64-bit hexadecimal numbers and must be the same as the data encryption key toJobConfig.dek configured during encryption. If the encryption and decryption keys are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+

fromJobConfig.iv

+

No

+

String

+

Initialization vector required for decryption. The initialization vector is a string of 32-bit hexadecimal numbers and must be the same as the initialization vector toJobConfig.iv configured during encryption. If the initialization vectors are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+

fromJobConfig.md5FileSuffix

+

No

+

String

+

Check whether the files extracted by CDM are consistent with those in the migration source.

+
+
+
+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0288.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0288.html new file mode 100644 index 000000000..9ef3d196e --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0288.html @@ -0,0 +1,148 @@ + + +

From HTTP/HTTPS

+

Sample JSON File

"from-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "fromJobConfig.inputDirectory",
+                "value": "http://10.114.196.186:8080/httpfrom/symbol.txt"
+              },
+              {
+                "name": "fromJobConfig.inputFormat",
+                "value": "BINARY_FILE"
+              },
+              {
+                "name": "fromJobConfig.fromCompression",
+                "value": "TARGZ"
+              },
+              {
+                "name": "fromJobConfig.compressedFileSuffix",
+                "value": "*"
+              },
+              {
+                "name": "fromJobConfig.fileSeparator",
+                "value": "|"
+              }
+            ],
+            "name": "fromJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

fromJobConfig.inputDirectory

+

Yes

+

String

+

URL of the file to be extracted

+

These connectors are used to read files with an HTTP/HTTPS URL, such as reading public files on the third-party object storage system and web disks.

+

fromJobConfig.inputFormat

+

Yes

+

Enumeration

+

File format required for data transmission. Currently, only the binary format is supported.

+

fromJobConfig.fromCompression

+

No

+

Enumeration

+
Compression format of the source files. The options are as follows:
  • NONE: Files in all formats are transferred.
  • GZIP: Files in gzip format are transferred.
  • ZIP: Files in Zip format are transferred.
  • TAR.GZ: Files in TAR.GZ format are transferred.
+
+

fromJobConfig.compressedFileSuffix

+

No

+

String

+

Extension of the files to be decompressed. The decompression operation is performed only when the file name extension is used in a batch of files. Otherwise, files are transferred in the original format. If you enter * or leave the parameter blank, all files are decompressed.

+

fromJobConfig.fileSeparator

+

No

+

String

+

File separator. When multiple files are transferred, CDM uses the file separator to separate files. The default value is |.

+

fromJobConfig.useQuery

+

No

+

Boolean

+
  • If you set this parameter to true, the name of the objects uploaded to OBS does not carry the query parameter.
  • If you set this parameter to false, the name of the objects uploaded to OBS carries the query parameter.
+

fromJobConfig.decryption

+

No

+

Enumeration

+
Whether to decrypt the encrypted file before export and the decryption method. The options are as follows:
  • NONE: Do not decrypt but directly export the file.
  • AES-256-GCM: Use the AES-256-GCM (NoPadding) algorithm to decrypt the file and then export the file.
+
+

fromJobConfig.dek

+

No

+

String

+

Data decryption key. The key is a string of 64-bit hexadecimal numbers and must be the same as the data encryption key toJobConfig.dek configured during encryption. If the encryption and decryption keys are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+

fromJobConfig.iv

+

No

+

String

+

Initialization vector required for decryption. The initialization vector is a string of 32-bit hexadecimal numbers and must be the same as the initialization vector toJobConfig.iv configured during encryption. If the initialization vectors are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+

fromJobConfig.md5FileSuffix

+

No

+

String

+

Check whether the files extracted by CDM are consistent with those in the migration source.

+
+
+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0289.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0289.html new file mode 100644 index 000000000..41bec74a5 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0289.html @@ -0,0 +1,101 @@ + + +

From MongoDB/DDS

+

Sample JSON File

"from-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "fromJobConfig.database",
+                "value": "cdm"
+              },
+              {
+                "name": "fromJobConfig.collectionName",
+                "value": "rf_from"
+              },
+              {
+                "name": "fromJobConfig.columnList",
+                "value": "TINYTEST&SMALLTEST&INTTEST&INTEGERTEST&BIGINTTEST&FLOATTEST"
+              },
+              {
+                "name": "fromJobConfig.isBatchMigration",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.filters",
+                "value": "{'last_name': 'Smith'}"
+              }
+            ],
+            "name": "fromJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

fromJobConfig.database

+

Yes

+

String

+

Name of the MongoDB/DDS database

+

fromJobConfig.collectionName

+

Yes

+

String

+

Name of the MongoDB/DDS collection

+

fromJobConfig.columnList

+

No

+

String

+

List of fields to be extracted. Use & to separate field names. For example, id&gid&name.

+

fromJobConfig.isBatchMigration

+

No

+

Boolean

+

Whether to migrate all data in the database

+

fromJobConfig.filters

+

No

+

String

+
Filter condition for files. CDM migrates only the data that meets the filter condition. Examples:
  1. Filter by expression: {'last_name': 'Smith'} indicates that all files whose last_name value is Smith are queried.
  2. Filter by parameter: { x : "john" }, { z : 1 } indicates that all z fields whose x is john are queried.
  3. Filter by condition: { "field" : { $gt: 5 } } indicates that the field values greater than 5 are queried.
+
+
+
+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0290.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0290.html new file mode 100644 index 000000000..2601fee76 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0290.html @@ -0,0 +1,157 @@ + + +

From Redis/DCS (to Be Brought Offline)

+

Sample JSON File

"from-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "fromJobConfig.isBatchMigration",
+                "value": "false"
+              },
+              {
+                "name": "fromJobConfig.keyPrefix",
+                "value": "rf_string_from"
+              },
+              {
+                "name": "fromJobConfig.keySeparator",
+                "value": ":"
+              },
+              {
+                "name": "fromJobConfig.valueStoreType",
+                "value": "STRING"
+              },
+              {
+                "name": "fromJobConfig.valueSeparator",
+                "value": ","
+              },
+              {
+                "name": "fromJobConfig.columnList",
+                "value": "1&2&3&4&5&6&7&8&9&10&11&12"
+              }
+            ],
+            "name": "fromJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0291.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0291.html new file mode 100644 index 000000000..be713cd2a --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0291.html @@ -0,0 +1,137 @@ + + +

From DIS

+

Sample JSON File

"from-config-values": {
+          "configs": [
+              {
+                  "inputs": [
+                      {
+                         "name": "fromJobConfig.streamName",
+                         "value": "cdm"
+                      },
+                      {
+                         "name": "fromJobConfig.disConsumerStrategy",
+                         "value": "FROM_LAST_STOP"
+                      },
+                      {
+                         "name": "fromJobConfig.isPermanency",
+                         "value": "true"
+                      },
+                      {
+                         "name": "fromJobConfig.maxPollRecords",
+                         "value": "100"
+                      },
+                      {
+                         "name": "fromJobConfig.shardId",
+                         "value": "0"
+                      },
+                      {
+                         "name": "fromJobConfig.dataFormat",
+                         "value": "BINARY"
+                      },
+                      {
+                         "name": "fromJobConfig.separator",
+                         "value": ","
+                      }
+                  ],
+                  "name": "fromJobConfig"
+              }
+          ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

fromJobConfig.streamName

+

Yes

+

String

+

DIS stream name

+

fromJobConfig.disConsumerStrategy

+

Yes

+

Enumeration

+

Used to set the initial offset when data is pulled from DIS. The options are as follows:

+
  • LATEST: maximum offset, that is, the latest data
  • FROM_LAST_STOP: data after the last stop
  • EARLIEST: minimum offset, that is, the earliest data
+

fromJobConfig.isPermanency

+

Yes

+

Boolean

+

Whether to run permanently

+

fromJobConfig.maxPollRecords

+

No

+

String

+

Maximum number of requests that can be sent to DIS each time

+

fromJobConfig.shardId

+

Yes

+

String

+

ID of the DIS partition. You can enter multiple partition IDs, which are separated by commas (,).

+

fromJobConfig.dataFormat

+

Yes

+

Enumeration

+

Format used for parsing data. The options are as follows:

+
  • BINARY: Data is transferred without being parsed, which is applicable to file migration.
  • CSV: Source data will be migrated after being parsed in CSV format.
+

fromJobConfig.separator

+

No

+

String

+

Field delimiter

+

fromJobConfig.appName

+

No

+

String

+

Unique identifier of the consumer application

+
+
+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0292.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0292.html new file mode 100644 index 000000000..cd2b4a3fc --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0292.html @@ -0,0 +1,122 @@ + + +

From Kafka

+

Sample JSON File

"from-config-values": {
+           "configs": [
+               {
+                   "inputs": [
+                       {
+                       "name": "fromJobConfig.topicsList",
+                       "value": "est1,est2"
+                       },
+                       {
+                       "name": "fromJobConfig.kafkaConsumerStrategy",
+                       "value": "EARLIEST"
+                       },
+                       {
+                       "name": "fromJobConfig.isPermanency",
+                       "value": "true"
+                       }
+                   ],
+                   "name": "fromJobConfig"
+               }
+           ]
+       }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

fromJobConfig.topicsList

+

Yes

+

String

+

List of Kafka topics. Separate multiple topics by commas (,).

+

fromJobConfig.kafkaConsumerStrategy

+

Yes

+

Enumeration

+

Used to set the initial offset when data is pulled from Kafka. The options are as follows:

+
  • LATEST: maximum offset, that is, the latest data
  • EARLIEST: minimum offset, that is, the earliest data
+

fromJobConfig.isPermanency

+

Yes

+

Boolean

+

Whether to run permanently

+

fromJobConfig.groupId

+

No

+

String

+

Consumer group ID

+

If you export data from DMS Kafka, enter any value for Kafka Platinum but a valid consumer group ID for Kafka Basic.

+

fromJobConfig.dataFormat

+

Yes

+

Enumeration

+

Format used for parsing data. The options are as follows:

+
  • BINARY: Data is transferred without being parsed, which is applicable to file migration.
  • CSV: Source data will be migrated after being parsed in CSV format.
+

fromJobConfig.maxPollRecords

+

No

+

String

+

Maximum number of requests that can be sent to Kafka each time

+

fromJobConfig.maxPollInterval

+

No

+

String

+

Maximum interval between each poll

+

fromJobConfig.separator

+

No

+

String

+

Field delimiter

+
+
+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0293.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0293.html new file mode 100644 index 000000000..3ddfebb07 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0293.html @@ -0,0 +1,100 @@ + + +

From Elasticsearch/Cloud Search Service

+

Sample JSON File

"from-config-values": {
+                "configs": [
+                    {
+                        "inputs": [
+                            {
+                                "name": "fromJobConfig.index",
+                                "value": "cdm"
+                            },
+                            {
+                                "name": "fromJobConfig.type",
+                                "value": "es"
+                            },
+                            {
+                                "name": "fromJobConfig.columnList",
+                                "value": "a1:numeric&s1:string"
+                            },
+                           {
+                               "name": "fromJobConfig.splitNestedField",
+                               "value": "true"
+                           },
+                           {
+                               "name": "fromJobConfig.queryString",
+                               "value": "last_name:Smith"
+                           }
+                        ],
+                        "name": "fromJobConfig"
+                    }
+                ]
+            }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

fromJobConfig.index

+

Yes

+

String

+

Index of the extracted data, which is similar to the database name in the relational database

+

fromJobConfig.type

+

Yes

+

String

+

Type of the extracted data, which is similar to the table name in the relational database

+

fromJobConfig.columnList

+

No

+

String

+

List of fields to be extracted. Use & to separate field names. For example, id&gid&name.

+

fromJobConfig.splitNestedField

+

No

+

Boolean

+

Whether to split the JSON content of the nested field. For example, a:{ b:{ c:1, d:{ e:2, f:3 } } } can be split into a.b.c, a.b.d.e, and a.b.d.f.

+

fromJobConfig.queryString

+

No

+

String

+

Whether to use the Elasticsearch query string to filter the source data. CDM migrates only the data that meets the filter criteria.

+
+
+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0294.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0294.html new file mode 100644 index 000000000..aed5cfd4b --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0294.html @@ -0,0 +1,105 @@ + + +

From OpenTSDB

+

Sample JSON File

      "from-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "fromJobConfig.start",
+                "value": "0"
+              },
+              {
+                "name": "fromJobConfig.metric",
+                "value": "city.temp"
+              },
+              {
+                "name": "fromJobConfig.aggregator",
+                "value": "sum"
+              },
+              {
+                "name": "fromJobConfig.columnList",
+                "value": "ps.timestample&metric&aggregator&dps.value"
+              }
+            ],
+            "name": "fromJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

fromJobConfig.start

+

Yes

+

String

+

Start time of the query. The value is a character string or timestamp in the format of yyyyMMddHHmmdd.

+

fromJobConfig.end

+

No

+

String

+

End time of the query. The value is a character string or timestamp in the format of yyyyMMddHHmmdd.

+

fromJobConfig.metric

+

Yes

+

String

+

Metric of the data to be migrated

+

fromJobConfig.aggregator

+

Yes

+

String

+

Aggregate function

+

fromJobConfig.tags

+

No

+

String

+

Data tag. If you specify this parameter, only the tagged data will be migrated.

+

fromJobConfig.columnList

+

No

+

String

+

Field list

+
+
+
+
+
+
+
Parent topic: Source Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0295.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0295.html new file mode 100644 index 000000000..8a4682755 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0295.html @@ -0,0 +1,37 @@ + + +

Destination Job Parameters

+
+
+ + +
+
Parent topic: Public Data Structures
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0296.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0296.html new file mode 100644 index 000000000..4daeb7cf9 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0296.html @@ -0,0 +1,173 @@ + + +

To a Relational Database

+

Sample JSON File

"to-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "toJobConfig.schemaName",
+                "value": "cdm"
+              },
+              {
+                "name": "toJobConfig.tablePreparation",
+                "value": "DROP_AND_CREATE"
+              },
+              {
+                "name": "toJobConfig.tableName",
+                "value": "rf_to"
+              },
+              {
+                "name": "toJobConfig.columnList",
+                "value": "id&gid&name"
+              },
+              {
+                "name": "toJobConfig.isCompress",
+                "value": "false"
+              },
+              {
+                "name": "toJobConfig.orientation",
+                "value": "ROW"
+              },
+              {
+                "name": "toJobConfig.useStageTable",
+                "value": "false"
+              },
+              {
+                "name": "toJobConfig.shouldClearTable",
+                "value": "false"
+              },
+              {
+                "name": "toJobConfig.extendCharLength",
+                "value": "false"
+              }
+            ],
+            "name": "toJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

toJobConfig.schemaName

+

Yes

+

String

+

Database mode or tablespace

+

toJobConfig.tablePreparation

+

Yes

+

Enumeration

+
This parameter is available only when both the source and destination databases are relational databases. The options for data write to tables are as follows:
  • DO_NOTHING: Do not create the table automatically.
  • CREATE_WHEN_NOT_EXIST: If the destination database does not contain the table specified by tableName, CDM automatically creates the table.
  • DROP_AND_CREATE: Delete the table specified by tableName, and then create the table again.
+
+

toJobConfig.tableName

+

Yes

+

String

+

Name of the table to which data is written

+

toJobConfig.columnList

+

No

+

String

+

List of fields to be loaded. Use & to separate field names. For example, id&gid&name.

+

toJobConfig.beforeImportType

+

No

+

Enumeration

+
Whether to clear the data in the target table before data import. The options are as follows:
  • none: Do not clear the data in the target table before data import but append data to the table.
  • shouldClearTable: Clear the data in the target table before data import.
  • whereClause: To clear data in the target table based on the WHERE clause, set the toJobConfig.whereClause parameter. CDM deletes the data from the target table as specified.
+
+

toJobConfig.whereClause

+

No

+

String

+

WHERE clause used to delete data from the target table before data import

+

toJobConfig.orientation

+

No

+

Enumeration

+

Storage mode. This parameter is enabled only for the DWS database. When the DWS database table needs to be automatically created, the optional data storage modes of the table are as follows:

+
  • ROW: Data in the table is stored in rows.
  • COLUMN: Data in the table is stored in columns.
+

toJobConfig.isCompress

+

No

+

Boolean

+

Whether to perform compression. This parameter is enabled only for the DWS database. When the DWS database table needs to be automatically created, you can specify whether to store the data in the table after compression.

+

oJobConfig.useStageTable

+

No

+

Boolean

+

Whether to import data to the phase table first. If this parameter is set to true, the data is imported to the phase table before it is imported to the destination table. After the data is successfully imported to the phase table, it is then imported from the phase table to the destination table. In this way, the data that is successfully imported to the destination table remains in case the data import fails.

+

toJobConfig.extendCharLength

+

No

+

Boolean

+

Whether to extend the length of the character string field. If this parameter is set to true, the length of the character string field in the destination table is three times the length of the corresponding field in the source table when the destination table needs to be automatically created.

+

toJobConfig.useNullable

+

No

+

Boolean

+

If you choose to create a target table automatically and specify the NOT NULL constraint, keep the NOT NULL constraints of the source and target tables consistent.

+
+
+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0297.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0297.html new file mode 100644 index 000000000..ef9122408 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0297.html @@ -0,0 +1,317 @@ + + +

To OBS

+

Sample JSON File

"to-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "toJobConfig.bucketName",
+                "value": "cdm"
+              },
+              {
+                "name": "toJobConfig.outputDirectory",
+                "value": "/obsfrom/advance/"
+              },
+              {
+                "name": "toJobConfig.outputFormat",
+                "value": "CSV_FILE"
+              },
+              {
+                "name": "toJobConfig.fieldSeparator",
+                "value": ","
+              },
+              {
+                "name": "toJobConfig.writeToTempFile",
+                "value": "false"
+              },
+              {
+                "name": "toJobConfig.validateMD5",
+                "value": "false"
+              },
+              {
+                "name": "toJobConfig.recordMD5Result",
+                "value": "false"
+              },
+              {
+                "name": "toJobConfig.encodeType",
+                "value": "UTF-8"
+              },
+              {
+                "name": "toJobConfig.markerFile",
+                "value": "finish.txt"
+              },
+              {
+                "name": "toJobConfig.duplicateFileOpType",
+                "value": "REPLACE"
+              },
+              {
+                "name": "toJobConfig.columnList",
+                "value": "1&2"
+              },
+              {
+                "name": "toJobConfig.quoteChar",
+                "value": "false"
+              },
+              {
+                "name": "toJobConfig.encryption",
+                "value": "NONE"
+              },
+              {
+                "name": "toJobConfig.copyContentType",
+                "value": "false"
+              }
+            ],
+            "name": "toJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

toJobConfig.bucketName

+

Yes

+

String

+

OBS bucket name. For example, cdm.

+

toJobConfig.outputDirectory

+

Yes

+

String

+

Path to which data is written. For example, data_dir.

+

toJobConfig.outputFormat

+

Yes

+

Enumeration

+
File format required for data writes (except the binary format). Currently, the following file formats are supported:
  • CSV_FILE: Write data in CSV format.
  • BINARY_FILE: Files are directly transferred without resolving the content. CDM writes the file without changing the file format.
+
+

If you select BINARY_FILE, the migration source must also be a file system.

+

toJobConfig.fieldSeparator

+

No

+

String

+

Column delimiter. This parameter is valid only when toJobConfig.outputFormat is CSV_FILE. The default value is ,.

+

toJobConfig.lineSeparator

+

No

+

String

+

Line feed character. This parameter is valid only when toJobConfig.outputFormat is CSV_FILE. The default value is \r\n.

+

toJobConfig.writeFileSize

+

No

+

String

+

Whether to fragment multiple files by size so that the files are exported in proper size. The unit is MB. This parameter is valid when the migration source is a database.

+

toJobConfig.duplicateFileOpType

+

No

+

Enumeration

+
Method for processing duplicate files. If the name and size of a file are the same as those of another file, the file is regarded as a duplicate file. Duplicate files can be processed in the following ways:
  • REPLACE: Replace duplicate files.
  • SKIP: Skip duplicate files.
  • ABANDON: Stop the job when any duplicate file is found.
+
+

toJobConfig.columnList

+

No

+

String

+

List of the fields to be extracted. Use & to separate field names, for example, id&gid&name.

+

toJobConfig.encryption

+

No

+

Enumeration

+
Whether to encrypt the uploaded data and the encryption method. The options are as follows:
  • NONE: Directly write data without encryption.
  • KMS: Use KMS in Data Encryption Workshop (DEW) for encryption. If KMS encryption is enabled, MD5 verification for data cannot be performed.
  • AES-256-GCM: Use the AES 256-bit encryption algorithm to encrypt data. Currently, only the AES-256-GCM (NoPadding) encryption algorithm is supported.
+
+

toJobConfig.dek

+

No

+

String

+

Data encryption key. This parameter is available when toJobConfig.encryption is set to AES-256-GCM. The key is a string of 64-bit hexadecimal numbers.

+

Remember the key configured here because the decryption key must be the same as that configured here. If the encryption and decryption keys are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+

toJobConfig.iv

+

No

+

String

+

Initialization vector. This parameter is available when toJobConfig.encryption is set to AES-256-GCM. The initialization vector is a string of 32-bit hexadecimal numbers.

+

Remember the initialization vector configured here because the initialization vector used for decryption must be the same as that configured here. If the initialization vectors are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+

toJobConfig.kmsID

+

No

+

String

+

Key used for encryption during data upload. You must create a key in KMS before data upload.

+

toJobConfig.projectID

+

No

+

String

+

ID of the project to which the KMS key belongs.

+

toJobConfig.writeToTempFile

+

No

+

Boolean

+

The binary file is written to a .tmp file first. After the migration is successful, run the rename or move command at the destination to restore the file.

+

toJobConfig.validateMD5

+

No

+

Boolean

+

Whether to verify the MD5 value. MD5 verification cannot be used together with KMS encryption. MD5 values can be verified only when files are transferred in binary format.

+

Calculate the MD5 value of the source file and verify it with the MD5 value returned by OBS. If an MD5 file exists on the source end, directly read the MD5 file and verify the MD5 file with the MD5 value returned by OBS.

+

toJobConfig.recordMD5Result

+

No

+

Boolean

+

Whether to record the verification result when the MD5 value is verified

+

toJobConfig.recordMD5Link

+

No

+

String

+

OBS link where the bucket to which the MD5 verification result is written resides

+

toJobConfig.recordMD5Bucket

+

No

+

String

+

OBS bucket to which the MD5 verification result is written

+

toJobConfig.recordMD5Directory

+

No

+

String

+

Directory to which the MD5 verification result is written

+

toJobConfig.encodeType

+

No

+

String

+

Encoding type. For example, UTF_8 or GBK.

+

toJobConfig.markerFile

+

No

+

String

+

Whether to generate a marker file with a custom name in the destination directory after a job is executed successfully. If you do not specify a file name, this function is disabled by default.

+

toJobConfig.copyContentType

+

No

+

Boolean

+

This parameter is displayed only when toJobConfig.outputFormat is Binary and both the migration source and destination are object storage.

+

If you set this parameter to Yes, the Content-Type attribute of the source file is copied during object file migration. This function is mainly used for static website migration

+

The Content-Type attribute cannot be written to Archive buckets. Therefore, if you set this parameter to Yes, the migration destination must be a non-Archive bucket.

+

toJobConfig.quoteChar

+

No

+

Boolean

+

This parameter is available only when toJobConfig.outputFormat is CSV. It is used when database tables are migrated to file systems.

+

If you set this parameter to Yes and a field in the source data table contains a field delimiter or line separator, CDM uses double quotation marks (") as the quote character to quote the field content as a whole to prevent a field delimiter from dividing a field into two fields, or a line separator from dividing a field into different lines. For example, if the hello,world field in the database is quoted, it will be exported to the CSV file as a whole.

+

toJobConfig.firstRowAsHeader

+

No

+

Boolean

+

This parameter is available only when toJobConfig.outputFormat is CSV. When a table is migrated to a CSV file, CDM does not migrate the heading line of the table by default. If this parameter is set to Yes, CDM writes the heading line of the table to the file.

+
+
+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0298.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0298.html new file mode 100644 index 000000000..e6f6e93af --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0298.html @@ -0,0 +1,165 @@ + + +

To HDFS

+

Sample JSON File

"to-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "toJobConfig.outputDirectory",
+                "value": "/hdfsto"
+              },
+              {
+                "name": "toJobConfig.outputFormat",
+                "value": "BINARY_FILE"
+              },
+              {
+                "name": "toJobConfig.writeToTempFile",
+                "value": "false"
+              },
+              {
+                "name": "toJobConfig.duplicateFileOpType",
+                "value": "REPLACE"
+              },
+              {
+                "name": "toJobConfig.compression",
+                "value": "NONE"
+              },
+              {
+                "name": "toJobConfig.appendMode",
+                "value": "true"
+              }
+            ],
+            "name": "toJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

toJobConfig.outputDirectory

+

Yes

+

String

+

Path to which data is written. For example, /data_dir.

+

toJobConfig.outputFormat

+

Yes

+

Enumeration

+
File format required for data writes (except the binary format). Currently, the following file formats are supported:
  • CSV_FILE: Write data in CSV format.
  • BINARY_FILE: Files are directly transferred without resolving the content. CDM writes the file without changing the file format.
+
+

If you select BINARY_FILE, the migration source must also be a file system.

+

toJobConfig.lineSeparator

+

No

+

String

+

Line feed character. This parameter is valid only when toJobConfig.outputFormat is CSV_FILE. The default value is \r\n.

+

toJobConfig.fieldSeparator

+

No

+

String

+

Column delimiter. This parameter is valid only when toJobConfig.outputFormat is CSV_FILE. The default value is ,.

+

toJobConfig.writeToTempFile

+

No

+

Boolean

+

The binary file is written to a .tmp file first. After the migration is successful, run the rename or move command at the migration destination to restore the file.

+

toJobConfig.duplicateFileOpType

+

No

+

Enumeration

+
Method for processing duplicate files. If the name and size of a file are the same as those of another file, the file is regarded as a duplicate file. Duplicate files can be processed in the following ways:
  • REPLACE: Replace duplicate files.
  • SKIP: Skip duplicate files.
  • ABANDON: Stop the job when any duplicate file is found.
+
+

toJobConfig.compression

+

No

+

Enumeration

+
After the file is written, select the compression format of the file. The following compression formats are supported:
  • NONE: Do not compress the file.
  • DEFLATE: Compress the file in DEFLATE format.
  • GZIP: Compress the file in gzip format.
  • BZIP2: Compress the file in bzip2 format.
  • LZ4: Compress the file in LZ4 format.
  • SNAPPY: Compress the file in Snappy format.
+
+

toJobConfig.appendMode

+

Yes

+

Boolean

+

Whether to write data when one or more files exist in the loading path. The default value is false.

+

toJobConfig.encryption

+

No

+

Enumeration

+
This parameter is available only when toJobConfig.outputFormat is set to BINARY_FILE. It specifies whether to encrypt the uploaded data, and the encryption method. The options are as follows:
  • NONE: Directly write data without encryption.
  • AES-256-GCM: Use the AES 256-bit encryption algorithm to encrypt data. Currently, only the AES-256-GCM (NoPadding) encryption algorithm is supported.
+
+

toJobConfig.dek

+

No

+

String

+

Data encryption key. This parameter is available when toJobConfig.encryption is set to AES-256-GCM. The key is a string of 64-bit hexadecimal numbers.

+

Remember the key configured here because the decryption key must be the same as that configured here. If the encryption and decryption keys are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+

toJobConfig.iv

+

No

+

String

+

Initialization vector. This parameter is available when toJobConfig.encryption is set to AES-256-GCM. The initialization vector is a string of 32-bit hexadecimal numbers.

+

Remember the initialization vector configured here because the initialization vector used for decryption must be the same as that configured here. If the initialization vectors are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+
+
+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0299.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0299.html new file mode 100644 index 000000000..879ff6e4a --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0299.html @@ -0,0 +1,114 @@ + + +

To Hive

+

Sample JSON File

"to-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "toJobConfig.hive",
+                "value": "hive"
+              },
+              {
+                "name": "toJobConfig.database",
+                "value": "rf_database"
+              },
+              {
+                "name": "toJobConfig.table",
+                "value": "rf_to"
+              },
+              {
+                "name": "toJobConfig.tablePreparation",
+                "value": "DO_NOTHING"
+              },
+              {
+                "name": "toJobConfig.columnList",
+                "value": "aa&bb&cc&dd"
+              },
+              {
+                "name": "toJobConfig.shouldClearTable",
+                "value": "true"
+              }
+            ],
+            "name": "toJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

toJobConfig.hive

+

No

+

String

+

Data source to which data is written

+

toJobConfig.database

+

No

+

String

+

Name of the database to which data is written. For example, default.

+

toJobConfig.table

+

Yes

+

String

+

Name of the table to which data is written

+

toJobConfig.tablePreparation

+

Yes

+

Enumeration

+
The options for data write to tables are as follows:
  • DO_NOTHING: Do not create the table automatically.
  • CREATE_WHEN_NOT_EXIST: If the destination database does not contain the table specified by tableName, CDM automatically creates the table.
  • DROP_AND_CREATE: Delete the table specified by tableName, and then create the table again.
+
+

toJobConfig.columnList

+

No

+

String

+

List of fields to be loaded. Use & to separate field names. For example, id&gid&name.

+

toJobConfig.shouldClearTable

+

No

+

Boolean

+

Whether to clear the data in the target table before data import. If this parameter is set to true, the data in the target table is cleared before the job is started.

+
+
+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0300.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0300.html new file mode 100644 index 000000000..357090893 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0300.html @@ -0,0 +1,164 @@ + + +

To HBase/CloudTable

+

Sample JSON File

"to-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "toJobConfig.table",
+                "value": "rf_to"
+              },
+              {
+                "name": "toJobConfig.storageType",
+                "value": "PUTLIST"
+              },
+              {
+                "name": "toJobConfig.columns",
+                "value": "AA:AA&BB:BB&CC:CC&DD:DD"
+              },
+              {
+                "name": "toJobConfig.rowKeyColumn",
+                "value": "AA:AA"
+              },
+              {
+                "name": "toJobConfig.isOverride",
+                "value": "false"
+              },
+              {
+                "name": "toJobConfig.isRowkeyRedundancy",
+                "value": "false"
+              },
+              {
+                "name": "toJobConfig.algorithm",
+                "value": "NONE"
+              },
+              {
+                "name": "toJobConfig.writeToWAL",
+                "value": "true"
+              },
+              {
+                "name": "toJobConfig.transType",
+                "value": "false"
+              }
+            ],
+            "name": "toJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

toJobConfig.table

+

Yes

+

String

+

Name of the table to which data is written. For example, TBL_EXAMPLE.

+

toJobConfig.storageType

+

Yes

+

Enumeration

+
Mode for writing data to an HBase table. The options are as follows:
  • BULKLOAD: The BULKLOAD mode is recommended to improve the loading performance.
  • PUTLIST: The PUTLIST mode is recommended only when the data volume is small.
+
+

toJobConfig.columns

+

No

+

String

+

Columns to be extracted. Use & to separate column numbers and : to separate column families and columns. For example, cf1:c1&cf2:c2.

+

toJobConfig.rowKeyColumn

+

Yes

+

String

+

Columns serve as rowkeys. Use & to separate column numbers and : to separate column families and columns. For example, cf1:c1&cf2:c2.

+

toJobConfig.isOverride

+

No

+

Boolean

+

Whether to clear data when data is imported in BULKLOAD mode. For example, true.

+

toJobConfig.delimiter

+

No

+

String

+

Delimiter used for separating columns when multiple columns are used as rowkeys. For example, vertical bars (|).

+

toJobConfig.isRowkeyRedundancy

+

No

+

Boolean

+

Whether to write rowkey data to the HBase column at the same time

+

toJobConfig.algorithm

+

No

+

Enumeration

+

Compression algorithm used when a new HBase table is created. The Snappy and GZ algorithms are supported. The default value is None.

+

toJobConfig.writeToWAL

+

No

+

Boolean

+

Whether to enable Write Ahead Log (WAL) of HBase. The options are as follows:

+
  • Yes: If the HBase server breaks down after the function is enabled, you can replay the operations that have not been performed in WAL.
  • No: If you set this parameter to No, the write performance is improved. However, if the HBase server breaks down, data may be lost.
+

toJobConfig.transType

+

No

+

Boolean

+
  • true: Data of the Short, Int, Long, Float, Double, and Decimal columns in the source database is converted into Byte[] arrays (binary) and written into HBase. Other types of data are written as character strings. If several types of data mentioned above are combined as rowkeys, they will be written as character strings.

    This function saves storage space. In specific scenarios, the rowkey distribution is evener.

    +
  • false: All types of data in the source database are written into HBase as character strings.
+
+
+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0301.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0301.html new file mode 100644 index 000000000..673f764f3 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0301.html @@ -0,0 +1,183 @@ + + +

To FTP/SFTP/NAS (to Be Brought Offline)/SFS (to Be Brought Offline)

+

Sample JSON File

"to-config-values": {
+                "configs": [
+                    {
+                        "inputs": [
+                          {
+                           "name": "toJobConfig.outputDirectory",
+                           "value": "/opt/data"
+                          },
+                          {
+                           "name": "toJobConfig.outputFormat",
+                           "value": "CSV_FILE"
+                          },
+                          {
+                           "name": "toJobConfig.fieldSeparator",
+                           "value": ","
+                          },
+                          {
+                           "name": "toJobConfig.duplicateFileOpType",
+                           "value": "REPLACE"
+                            }
+                        ],
+                        "name": "toJobConfig"
+                    }
+                ]
+            }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

toJobConfig.outputDirectory

+

Yes

+

String

+

Path to which data is written. For example, /data_dir.

+

toJobConfig.outputFormat

+

Yes

+

Enumeration

+
File format required for data writes (except the binary format). Currently, the following file formats are supported:
  • CSV_FILE: Write data in CSV format.
  • BINARY_FILE: Files are directly transferred without resolving the content. CDM writes the file without changing the file format.
+
+

If you select BINARY_FILE, the migration source must also be a file system.

+

toJobConfig.duplicateFileOpType

+

No

+

Enumeration

+
Method for processing duplicate files. If the name and size of a file are the same as those of another file, the file is regarded as a duplicate file. Duplicate files can be processed in the following ways:
  • REPLACE: Replace duplicate files.
  • SKIP: Skip duplicate files.
  • ABANDON: Stop the job when any duplicate file is found.
+
+

toJobConfig.lineSeparator

+

No

+

String

+

Line feed character. This parameter is valid only when toJobConfig.outputFormat is CSV_FILE. The default value is \r\n.

+

toJobConfig.fieldSeparator

+

No

+

String

+

Column delimiter. This parameter is valid only when toJobConfig.outputFormat is CSV_FILE. The default value is ,.

+

toJobConfig.encodeType

+

No

+

String

+

Encoding type. For example, UTF_8 or GBK.

+

toJobConfig.writeToTempFile

+

No

+

Boolean

+

The binary file is written to a .tmp file first. After the migration is successful, run the rename or move command at the migration destination to restore the file.

+

toJobConfig.recordMD5Result

+

No

+

Boolean

+

This parameter is invalid when File Format is set to Binary. An MD5 hash value is generated for each transferred file, and the value is recorded in a new .md5 file. You can specify the directory where the MD5 value is generated.

+

toJobConfig.recordMD5Directory

+

No

+

String

+

Directory for storing MD5 values

+

toJobConfig.markerFile

+

No

+

String

+

Whether to generate a marker file with a custom name in the destination directory after a job is executed successfully. If you do not specify a file name, this function is disabled by default.

+

toJobConfig.firstRowAsHeader

+

No

+

Boolean

+

This parameter is available only when toJobConfig.outputFormat is CSV. When a table is migrated to a CSV file, CDM does not migrate the heading line of the table by default. If this parameter is set to Yes, CDM writes the heading line of the table to the file.

+

toJobConfig.encryption

+

No

+

Enumeration

+
Whether to encrypt the uploaded data and the encryption method. The options are as follows:
  • NONE: Directly write data without encryption.
  • AES-256-GCM: Use the AES 256-bit encryption algorithm to encrypt data. Currently, only the AES-256-GCM (NoPadding) encryption algorithm is supported.
+
+

toJobConfig.dek

+

No

+

String

+

Data encryption key. This parameter is available when toJobConfig.encryption is set to AES-256-GCM. The key is a string of 64-bit hexadecimal numbers.

+

Remember the key configured here because the decryption key must be the same as that configured here. If the encryption and decryption keys are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+

toJobConfig.iv

+

No

+

String

+

Initialization vector. This parameter is available when toJobConfig.encryption is set to AES-256-GCM. The initialization vector is a string of 32-bit hexadecimal numbers.

+

Remember the initialization vector configured here because the initialization vector used for decryption must be the same as that configured here. If the initialization vectors are inconsistent, the system does not report an exception, but the decrypted data is incorrect.

+
+
+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0302.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0302.html new file mode 100644 index 000000000..9422c4a00 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0302.html @@ -0,0 +1,87 @@ + + +

To DDS

+

Sample JSON File

"to-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "toJobConfig.database",
+                "value": "demo"
+              },
+              {
+                "name": "toJobConfig.collectionName",
+                "value": "cdmbase"
+              },
+              {
+                "name": "toJobConfig.columnList",
+                "value": "_char&_varchar"
+              },
+              {
+                "name": "toJobConfig.isBatchMigration",
+                "value": "false"
+              }
+            ],
+            "name": "toJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

toJobConfig.database

+

Yes

+

String

+

Name of the MongoDB/DDS database

+

toJobConfig.collectionName

+

Yes

+

String

+

Name of the MongoDB/DDS collection

+

toJobConfig.columnList

+

No

+

String

+

List of fields to be extracted. Use & to separate field names. For example, id&gid&name.

+

toJobConfig.isBatchMigration

+

No

+

Boolean

+

Whether to migrate all data in the database

+
+
+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0303.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0303.html new file mode 100644 index 000000000..ae73fd75c --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0303.html @@ -0,0 +1,183 @@ + + +

To DCS

+

Sample JSON File

 "to-config-values": {
+           "configs": [
+               {
+                   "inputs": [
+                       {
+                          "name": "toJobConfig.isBatchMigration",
+                          "value": "false"
+                       },
+                       {
+                          "name": "toJobConfig.shouldClearDatabase",
+                          "value": "false"
+                       },
+                       {
+                           "name": "toJobConfig.keyPrefix",
+                           "value": "cdm_string"
+                       },
+                       {
+                           "name": "toJobConfig.keySeparator",
+                           "value": ":"
+                       },
+                       {
+                           "name": "toJobConfig.primaryKeyList",
+                           "value": "1"
+                       },
+                       {
+                           "name": "toJobConfig.valueStoreType",
+                           "value": "STRING"
+                       },
+                       {
+                           "name": "toJobConfig.valueSeparator",
+                           "value": ","
+                       },
+                       {
+                           "name": "toJobConfig.columnList",
+                           "value": "1&2&3&4&5&6&7&8&9&10&11&12"
+                       }
+                   ],
+                   "name": "toJobConfig"
+               }
+           ]
+       }
+
+

Parameter Description

+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0304.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0304.html new file mode 100644 index 000000000..4bfe879c9 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0304.html @@ -0,0 +1,116 @@ + + +

To Elasticsearch/Cloud Search Service

+

Sample JSON File

"to-config-values": {
+                "configs": [
+                    {
+                        "inputs": [
+                            {
+                              "name": "toJobConfig.index",
+                              "value": "cdm"
+                            },
+                            {
+                              "name": "toJobConfig.type",
+                              "value": "type1"
+                            },
+                            {
+                              "name": "toJobConfig.shouldClearType",
+                              "value": "false"
+                            },
+                           {
+                              "name": "toJobConfig.pipeLine",
+                              "value": "es_03"
+                           }
+                        ],
+                        "name": "toJobConfig"
+                    }
+                ]
+            }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

toJobConfig.index

+

Yes

+

String

+

Index of the written data, which is similar to the database name in the relational database

+

toJobConfig.type

+

Yes

+

String

+

Type of the written data, which is similar to the table name in the relational database

+

toJobConfig.shouldClearType

+

No

+

Boolean

+

Whether to clear data before import

+

toJobConfig.primaryKey

+

No

+

String

+

Primary key or unique index

+

toJobConfig.columnList

+

No

+

String

+

List of fields to be written. Use & to separate field names. For example, id&gid&name.

+

toJobConfig.pipeLine

+

No

+

String

+

This parameter is available only after a pipeline ID is created in Kibana. It is used to convert the data format using the data transformation pipeline of Cloud Search Service or Elasticsearch after data is transferred to Cloud Search Service or Elasticsearch.

+

toJobConfig.createIndexStrategy

+

No

+

Enumeration

+
For streaming jobs that continuously write data to Elasticsearch, CDM periodically creates indexes and writes data to the indexes, which helps you delete expired data. The indexes can be created based on the following periods:
  • EveryHour: CDM creates indexes on the hour. The new indexes are named in the format of Index name+Year+Month+Day+Hour, for example, index2018121709.
  • EveryDay: CDM creates indexes at 00:00 every day. The new indexes are named in the format of Index name+Year+Month+Day, for example, index20181217.
  • EveryWeek: CDM creates indexes at 00:00 every Monday. The new indexes are named in the format of Index name+Year+Week, for example, index201842.
  • EveryMonth: CDM creates indexes at 00:00 on the first day of each month. The new indexes are named in the format of Index name+Year+Month, for example, index201812.
+
+

When extracting data from a file, you must configure a single extractor, which means setting Concurrent Extractors to 1. Otherwise, this parameter is invalid.

+
+
+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0305.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0305.html new file mode 100644 index 000000000..253e53351 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0305.html @@ -0,0 +1,100 @@ + + +

To DLI

+

Sample JSON File

"to-config-values": {
+                "configs": [
+                    {
+                        "inputs": [
+                            {
+                                "name": "toJobConfig.queue",
+                                "value": "cdm"
+                            },
+                            {
+                                "name": "toJobConfig.database",
+                                "value": "sqoop"
+                            },
+                            {
+                                "name": "toJobConfig.table",
+                                "value": "est1"
+                            },
+                            {
+                                "name": "toJobConfig.columnList",
+                                "value": "string_&int_&date_&double_&boolean_&short_&timestamp_&long_&smallint_&bigint_"
+                            },
+                            {
+                                "name": "toJobConfig.shouldClearTable",
+                                "value": "false"
+                            }
+                        ],
+                        "name": "toJobConfig"
+                    }
+                ]
+            }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

toJobConfig.queue

+

Yes

+

String

+

Resource queue to which data is written

+

toJobConfig.database

+

Yes

+

String

+

DLI database to which data is written

+

toJobConfig.table

+

Yes

+

String

+

Name of the table to which data is written

+

toJobConfig.columnList

+

No

+

String

+

List of fields to be loaded. Use & to separate field names. For example, id&gid&name.

+

toJobConfig.shouldClearTable

+

No

+

Boolean

+

Whether to clear data in the resource queue before data import

+
+
+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0306.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0306.html new file mode 100644 index 000000000..0b3cb7307 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0306.html @@ -0,0 +1,74 @@ + + +

To DIS

+

Sample JSON File

"to-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "toJobConfig.streamName",
+                "value": "cdm"
+              },
+              {
+                "name": "toJobConfig.separator",
+                "value": ","
+              },
+              {
+                "name": "toJobConfig.identifierEnclose",
+                "value": "'"
+              }
+            ],
+            "name": "toJobConfig"
+          }
+        ]
+      }
+
+

Parameter description

+
+ + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

toJobConfig.streamName

+

Yes

+

String

+

DIS stream name

+

toJobConfig.separator

+

No

+

String

+

Field separator. The default value is a space.

+

toJobConfig.identifierEnclose

+

No

+

String

+

Delimiter used to separate referenced table names or column names. By default, this parameter is left blank.

+
+
+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0307.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0307.html new file mode 100644 index 000000000..cbf1acf37 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0307.html @@ -0,0 +1,87 @@ + + +

To OpenTSDB

+

Sample JSON File

"to-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "toJobConfig.metric",
+                "value": "city.temp"
+              },
+              {
+                "name": "toJobConfig.timeStamp",
+                "value": "0"
+              },
+              {
+                "name": "toJobConfig.tags",
+                "value": "tagk1:tagv1"
+              },
+              {
+                "name": "toJobConfig.columnList",
+                "value": "metric:1&tag:2&timestamp&value"
+              }
+            ],
+            "name": "toJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

toJobConfig.metric

+

No

+

String

+

Data metric

+

toJobConfig.timeStamp

+

No

+

String

+

Data point. The value is a character string or timestamp in the format of yyyyMMddHHmmdd.

+

toJobConfig.tags

+

No

+

String

+

Data tag

+

toJobConfig.columnList

+

No

+

String

+

Field list

+
+
+
+
+
+
+
Parent topic: Destination Job Parameters
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0308.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0308.html new file mode 100644 index 000000000..1ebdd29dd --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0308.html @@ -0,0 +1,231 @@ + + +

Job Parameter Description

+

When Creating a Job in a Specified Cluster or Creating and Executing a Job in a Random Cluster, the driver-config-values parameter specifies the job configuration, which includes the following functions:

+ +

Sample JSON File

"driver-config-values": {
+        "configs": [
+          {
+            "inputs": [
+              {
+                "name": "throttlingConfig.numExtractors",
+                "value": "1"
+              },
+              {
+                "name": "throttlingConfig.numLoaders",
+                "value": "1"
+              },
+              {
+                "name": "throttlingConfig.recordDirtyData",
+                "value": "false"
+              }
+            ],
+            "name": "throttlingConfig"
+          },
+          {
+            "inputs": [],
+            "name": "jarConfig"
+          },
+          {
+            "inputs": [
+              {
+                "name": "schedulerConfig.isSchedulerJob",
+                "value": "false"
+              },
+              {
+                "name": "schedulerConfig.disposableType",
+                "value": "NONE"
+              }
+            ],
+            "name": "schedulerConfig"
+          },
+          {
+            "inputs": [],
+            "name": "transformConfig"
+          },
+          {
+            "inputs": [
+              {
+                "name": "retryJobConfig.retryJobType",
+                "value": "NONE"
+              }
+            ],
+            "name": "retryJobConfig"
+          }
+        ]
+      }
+
+

Parameter Description

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Parameter

+

Mandatory

+

Type

+

Description

+

throttlingConfig.numExtractors

+

No

+

Integer

+

Maximum number of concurrent extraction jobs. For example, 20.

+

groupJobConfig.groupName

+

No

+

Enumeration

+

Group to which a job belongs. The default group is DEFAULT.

+

throttlingConfig.numLoaders

+

No

+

Integer

+

This parameter is available only when HBase or Hive serves as the destination data source.

+

Maximum number of loading jobs. For example, 5.

+

throttlingConfig.recordDirtyData

+

No

+

Boolean

+

Whether to write dirty data. For example, true.

+

throttlingConfig.writeToLink

+

No

+

String

+

Link to which dirty data is written. Currently, dirty data can be written only to OBS or HDFS. For example, obslink.

+

throttlingConfig.obsBucket

+

No

+

String

+

Name of the OBS bucket to which dirty data is written. This parameter is valid only when dirty data is written to OBS. For example, dirtyData.

+

throttlingConfig.dirtyDataDirectory

+

No

+

String

+

Directory to which dirty data is written

+
  • To write dirty data to HDFS, set this parameter to the specified HDFS directory.
  • To write dirty data to OBS, set this parameter to the directory in the OBS bucket. For example, /data/dirtydata/.
+

throttlingConfig.maxErrorRecords

+

No

+

String

+

Maximum number of error records in a single shard. When the number of error records of a map exceeds the upper limit, the task automatically ends. The imported data will not be rolled back.

+

schedulerConfig.isSchedulerJob

+

No

+

Boolean

+

Whether to enable a scheduled task. For example, true.

+

schedulerConfig.cycleType

+

No

+

String

+

Cycle type of a scheduled task. The options are as follows:

+
  • minute: minute
  • hour: hour
  • day: day
  • week: week
  • month: month
+

schedulerConfig.cycle

+

No

+

Integer

+

Cycle of a scheduled task. If cycleType is set to minute and cycle is set to 10, the scheduled task is executed every 10 minutes.

+

schedulerConfig.runAt

+

No

+

String

+

Time when a scheduled task is triggered in a cycle. This parameter is valid only when cycleType is set to hour, week, or month.

+
  • If cycleType is set to month, cycle is set to 1, and runAt is set to 15, the scheduled task is executed on the 15th day of each month. You can set runAt to multiple values and separate the values with commas (,).

    For example, if runAt is set to 1,2,3,4,5, the scheduled task is executed on the first day, second day, third day, fourth day, and fifth day of each month.

    +
  • If cycleType is set to week and runAt is set to mon,tue,wed,thu,fri, the scheduled task is executed on Monday to Friday.
  • If cycleType is set to hour and runAt is set to 27,57, the scheduled task is executed at the 27th and 57th minute in the cycle.
+

schedulerConfig.startDate

+

No

+

String

+

Start time of a scheduled task. For example, 2018-01-24 19:56:19.

+

schedulerConfig.stopDate

+

No

+

String

+

End time of a scheduled task. For example, 2018-01-27 23:59:00.

+

If you do not set the end time, the scheduled task is always executed and will never stop.

+

schedulerConfig.disposableType

+

No

+

Enumeration

+
Whether to delete a job after the job is executed. The options are as follows:
  • NONE: A job will not be deleted after it is executed.
  • DELETE_AFTER_SUCCEED: A job will be deleted only after it is successfully executed. It is applicable to massive one-time jobs.
  • DELETE: A job will be deleted after it is executed, regardless of the execution result.
+
+

retryJobConfig.retryJobType

+

No

+

Enumeration

+

Whether to automatically retry if a job fails to be executed. The options are as follows:

+
  • NONE: Do not retry.
  • RETRY_TRIPLE: Retry three times.
+
+
+
+
+
+
+
Parent topic: Public Data Structures
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0309.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0309.html new file mode 100644 index 000000000..a79ef04b0 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0309.html @@ -0,0 +1,21 @@ + + + +

Appendix

+ +

+
+ +
+ +
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0310.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0310.html new file mode 100644 index 000000000..39b6bab1a --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0310.html @@ -0,0 +1,329 @@ + + +

Status Codes

+
A status code consists of three digits. The first digit defines the class of a response. There are five values for the first digit: +
+

Table 1 describes status codes.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Status codes

Status Code

+

Message

+

Description

+

100

+

Continue

+

The client should continue with its request.

+

This interim response is used to inform the client that part of the request has been received and has not yet been rejected by the server.

+

101

+

Switching Protocols

+

The protocol should be switched. The protocol can only be switched to a newer protocol.

+

For example, the current HTTP protocol is switched to a later version of HTTP.

+

200

+

OK

+

The request has been fulfilled.

+

201

+

Created

+

The request has been fulfilled and a new resource has been created.

+

202

+

Accepted

+

The request has been accepted, but the processing has not been completed.

+

203

+

Non-Authoritative Information

+

The server has successfully processed the request, but is returning information that may be from another source.

+

204

+

NoContent

+

The request has been fulfilled, but the HTTP response does not contain a response body.

+

The status code is returned in response to an HTTPS OPTIONS request.

+

205

+

Reset Content

+

The server has fulfilled the request, but the requester is required to reset the content.

+

206

+

Partial Content

+

The server has successfully processed the partial GET request.

+

300

+

Multiple Choices

+

There are multiple options for the location of the requested resource. The response contains a list of resource characteristics and addresses from which the user or user agent (such as a browser) can choose the most appropriate one.

+

301

+

Moved Permanently

+

The requested resource has been assigned a new permanent URI, and the new URI is contained in the response.

+

302

+

Found

+

The requested resource resides temporarily under a different URI.

+

303

+

See Other

+

The response to the request can be found under a different URI, and should be retrieved using a GET or POST method.

+

304

+

Not Modified

+

The requested resource has not been modified. When the server returns this status code, it does not return any resources.

+

305

+

Use Proxy

+

The requested resource is available only through a proxy.

+

306

+

Unused

+

The HTTP status code is no longer used.

+

400

+

BadRequest

+

The request is invalid.

+

The client should not repeat the request without modifications.

+

401

+

Unauthorized

+

This status code is returned after the client provides the authentication information, indicating that the authentication information is incorrect or invalid.

+

402

+

Payment Required

+

This status code is reserved for future use.

+

403

+

Forbidden

+

The server understood the request, but is refusing to fulfill it.

+

The client should not repeat the request without modifications.

+

404

+

NotFound

+

The requested resource cannot be found.

+

The client should not repeat the request without modifications.

+

405

+

MethodNotAllowed

+

The method specified in the request is not supported for the requested resource.

+

The client should not repeat the request without modifications.

+

406

+

Not Acceptable

+

The server cannot fulfill the request according to the content characteristics of the request.

+

407

+

Proxy Authentication Required

+

This status code is similar to 401, but indicates that the client must first authenticate itself with the proxy.

+

408

+

Request Time-out

+

The server has timed out waiting for the request.

+

The client may repeat the request without modifications at a later time.

+

409

+

Conflict

+

The request could not be processed due to a conflict with the current state of the resource.

+

This status code indicates that the resource that the client is attempting to create already exists, or that the request has failed to be processed because of the update of the conflict request.

+

410

+

Gone

+

The requested resource has been deleted permanently and is no longer available.

+

411

+

Length Required

+

The server is refusing to process the request without a defined Content-Length.

+

412

+

Precondition Failed

+

The server did not meet one of the preconditions that the requester put on the request.

+

413

+

Request Entity Too Large

+

The server is refusing to process a request because the request entity is too large for the server to process. The server may close the connection to prevent the client from continuing the request. If the server is only temporarily unable to process the request, the response will contain a Retry-After header field.

+

414

+

Request-URI Too Large

+

The Request-URI is too long for the server to process.

+

415

+

Unsupported Media Type

+

The server is unable to process the media format in the request.

+

416

+

Requested range not satisfiable

+

The requested range is invalid.

+

417

+

Expectation Failed

+

The server has failed to meet the requirements of the Expect request-header field.

+

422

+

UnprocessableEntity

+

The request is well-formed but cannot be processed due to semantic errors.

+

429

+

TooManyRequests

+

The client has sent excessive number of requests to the server within a given time (exceeding the limit on the access frequency of the client), or the server has received an excessive number of requests within a given time (beyond its processing capability). In this case, the client should resend the request after the time specified in the Retry-After header of the response has elapsed.

+

500

+

InternalServerError

+

The server is able to receive the request but unable to understand it.

+

501

+

Not Implemented

+

The server does not support the function required to fulfill the request.

+

502

+

Bad Gateway

+

The server was acting as a gateway or proxy and received an invalid request from the remote server.

+

503

+

ServiceUnavailable

+

The requested service is invalid.

+

The client should not repeat the request without modifications.

+

504

+

ServerTimeout

+

The request cannot be fulfilled within a given time. This status code is returned to the client only if the Timeout parameter is specified in the request.

+

505

+

HTTP Version not supported

+

The server does not support the HTTP protocol version used in the request.

+
+
+
+
+
+
Parent topic: Appendix
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0311.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0311.html new file mode 100644 index 000000000..862894eb0 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0311.html @@ -0,0 +1,1837 @@ + + +

CDM Error Codes

+

If an error occurs in API calling, no result is returned. Identify the error cause based on the error codes of each API. If an error occurs in API calling, HTTP status code 4xx or 5xx is returned. The response body contains the specific error code and information. If you are unable to identify the cause of an error, contact customer service and provide the error code so that we can help you solve the problem as soon as possible.

+ +
+
+
+
Parent topic: Error Codes
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0313.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0313.html new file mode 100644 index 000000000..7620bed64 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0313.html @@ -0,0 +1,346 @@ + + +

DLF Error Codes

+

The error response is in the following format:

+

{

+

"error_code":"DLF.0100",

+

"error_msg":"The job not found"

+

}

+

Table 1 describes the error codes.

+

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Error codes

Status Code

+

Error Code

+

Error Message

+

Solution

+

400

+

DLF.0100

+

The job does not exist.

+

Ensure that the job exists.

+

400

+

DLF.0101

+

The job quota has reached the upper limit.

+

Apply for a higher quota.

+

400

+

DLF.0102

+

The job name has been used by another job.

+

Try another name.

+

400

+

DLF.0136

+

This API can be called only for batch jobs.

+

Check the job.

+

400

+

DLF.0137

+

The job instance does not exist.

+

Ensure that the job instance exists.

+

400

+

DLF.0201

+

The script fails to be created.

+

Check the script.

+

400

+

DLF.0202

+

The script name has been used by another script.

+

Try another name.

+

400

+

DLF.0203

+

The script fails to be modified.

+

Check the script.

+

400

+

DLF.0802

+

The specified parameter is not configured.

+

Check the parameter.

+

400

+

DLF.0803

+

The OBS path is invalid.

+

Check the OBS path.

+

400

+

DLF.0810

+

The queried task does not exist.

+

Check that the task exists.

+

400

+

DLF.0815

+

The OBS file fails to be accessed.

+

Check the OBS file.

+

400

+

DLF.1006

+

The job node is empty.

+

Check the node.

+

400

+

DLF.1242

+

The OBS bucket does not exist.

+

Ensure that the OBS bucket exists.

+

400

+

DLF.3004

+

The job name cannot be left blank.

+

Check the job name.

+

400

+

DLF.3018

+

The job name is invalid.

+

Check the job name.

+

400

+

DLF.3025

+

The job scheduling parameter is invalid.

+

Check the job scheduling parameter.

+

400

+

DLF.3050

+

The job description is invalid.

+

Check the job description.

+

400

+

DLF.6201

+

The script does not exist.

+

Ensure that the script exists.

+

400

+

DLF.6205

+

The script execution instance does not exist.

+

Check that the script execution instance exists.

+

400

+

DLF.6241

+

The resource to be queried does not exist.

+

Ensure that the resource exists.

+

400

+

DLF.6247

+

The script type is not specified.

+

Check the script.

+

400

+

DLF.6253

+

The script quota has reached the upper limit.

+

Apply for a higher quota.

+

400

+

DLF.6258

+

The directory contains data that cannot be deleted.

+

Delete the data in the directory first.

+

400

+

DLF.6259

+

A resource with the same name already exists in the directory.

+

Try another name.

+

400

+

DLF.6263

+

The resource type is invalid.

+

Check the resource type.

+

400

+

DLF.6264

+

The OBS path of the resource file is invalid.

+

Check the OBS path.

+

400

+

DLF.6265

+

The resource description is invalid.

+

Check the resource description.

+

400

+

DLF.6271

+

Each user can create only one DLI connection.

+

Check the DLI data connection.

+

400

+

DLF.6309

+

The connection name has been used by another connection.

+

Try another connection name.

+

400

+

DLF.6322

+

The data connection does not exist.

+

Check that the data connection exists.

+

400

+

DLF.6323

+

The connection type cannot be modified.

+

Check the parameter type.

+

400

+

DLF.6416

+

The cluster has been occupied by another connection.

+

Check the cluster.

+

400

+

DLF.6418

+

Either the DWS cluster name or the access address and port number of the cluster must be configured.

+

Check the DWS cluster name or the access address and port of the cluster.

+

400

+

DLF.6888

+

Invalid account permissions.

+

Check the account permissions.

+
+
+
+
+
+
Parent topic: Error Codes
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0314.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0314.html new file mode 100644 index 000000000..3bb56edf5 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0314.html @@ -0,0 +1,42 @@ + + +

Project ID and Account ID

+

Obtaining a Project ID and Account ID

You can obtain the project ID and account ID by performing the following steps:
  1. Register with and log in to the management console.
  2. Hover the cursor on the username in the upper right corner and select My Credentials from the drop-down list.
  3. On the My Credentials page, obtain the account name and account ID, and obtain the project ID from the project list.
+
+
+

Obtaining a Project ID by Calling an API

You can obtain a project ID by calling the API used to query projects based on specified criteria.

+

The API for obtaining a project ID is GET https://{Endpoint}/v3/projects/, where {Endpoint} indicates the endpoint of IAM.

+

An endpoint is the request address for calling an API. Endpoints vary depending on services and regions. You can obtain endpoints from Regions and Endpoints.

+

For details about API authentication, see Authentication.

+

The following is a response example. The value of id under projects is the project ID. If multiple IDs are returned, obtain the desired one based on your actual region (name).

+
{
+    "projects": [
+        {
+            "domain_id": "65382450e8f64ac0870cd180d14e684b",
+            "is_domain": false,
+            "parent_id": "65382450e8f64ac0870cd180d14e684b",
+            "name": "region-name",
+            "description": "",
+            "links": {
+                "next": null,
+                "previous": null,
+                "self": "https://www.example.com/v3/projects/a4a5d4098fb4474fa22cd05f897d6b99"
+            },
+            "id": "a4a5d4098fb4474fa22cd05f897d6b99",
+            "enabled": true
+        }
+    ],
+    "links": {
+        "next": null,
+        "previous": null,
+        "self": "https://www.example.com/v3/projects"
+    }
+}
+
+
+
+
+
Parent topic: Before You Start
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0316.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0316.html new file mode 100644 index 000000000..c89ce3c09 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0316.html @@ -0,0 +1,15 @@ + + +

Data Development Job ID

+

A job ID (jobid) is required for some URLs when an API is called. To obtain a job ID, perform the following operations:

+
  1. Log in to the DataArts Studio console. Locate an instance and click Access. On the displayed page, locate a workspace and click DataArts Factory.
    Figure 1 DataArts Factory
    +
  1. In the navigation pane, choose Monitoring > Monitor Job, and click Batch Job Monitoring or Real-Time Job Monitoring based on the job type.
  2. Click a job name to go to the job monitoring page.
    In the URL of the page, the job ID is after the job type (batchJobDetail or realTimeJobDetail) and job name, as shown in the following figure.
    Figure 2 Obtaining a job ID
    +
    +
+
+
+
+
Parent topic: Before You Start
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0317.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0317.html new file mode 100644 index 000000000..289c52571 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0317.html @@ -0,0 +1,30 @@ + + +

Parsing a Stream in a Response Message

+

The response messages of the job export API and connection export API are streams that need to be converted to files. For details, see the following sample code:

+
String EXPORT_JOB_URL = "https://{endpoint}/v1/{project_id}/jobs/{job_name}/export";
+
+try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
+    HttpPost httpPost = new HttpPost(EXPORT_JOB_URL);
+    httpPost.setHeader("Content-Type", "application/json; charset=UTF-8");
+    httpPost.setHeader("Accept", "application/octet-stream");
+    httpPost.setHeader("X-Auth-Token", token);
+
+    HttpResponse response = httpClient.execute(httpPost);
+    int statusCode = response.getStatusLine().getStatusCode();
+    if (statusCode == 200) {
+        String filePath = "d:";
+        String fileName = "job.zip";
+        FileOutputStream fileOutputStream = new FileOutputStream(filePath + fileName);
+        response.getEntity().writeTo(fileOutputStream);
+    } else {
+        System.out.println(statusCode);
+    }
+}
+
+
+
+
Parent topic: Appendix
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0318.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0318.html new file mode 100644 index 000000000..793d452e4 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0318.html @@ -0,0 +1,150 @@ + + +

Common Message Headers

+

Common request header fields

Table 1 describes common request headers.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Common request header fields

Field

+

Description

+

Mandatory

+

Example

+

Content-Type

+

Specifies the request body MIME type. This field is mandatory and its default value is application/json. Other values of this field will be provided for specific APIs if any.

+

Yes

+

The default value is application/json.

+

Content-Length

+

Specifies the length of the request body. The unit is byte.

+

No

+

3495

+

X-Language

+

Specifies the request language.

+

No

+

en_us

+

X-Project-Id

+

Specifies a project ID. Obtain the project ID by following the instructions in Project ID and Account ID.

+

No

+

e9993fc787d94b6c886cbaa340f9c0f4

+

X-Auth-Token

+

Specifies a user token.

+

The user token is a response to the API used to obtain a user token. This API is the only one that does not require authentication.

+

The token is the value of X-Subject-Token in the response header.

+

This field is mandatory for token authentication.

+

The following is part of an example token:

+

MIIPAgYJKoZIhvcNAQcCo...ggg1BBIINPXsidG9rZ

+

X-Sdk-Date

+

Time when the request is sent. The time is in YYYYMMDD'T'HHMMSS'Z' format.

+

The value is the current GMT time of the system.

+

This field is mandatory for AK/SK-based authentication.

+

20150907T101459Z

+

Host

+

Specifies the server domain name and port number of the resources being requested. The value can be obtained from the URL of the service API. The value is in the format of hostname[:port]. If the port number is not specified, the default port is used. The default port number for HTTPS is 443.

+

This field is mandatory for AK/SK-based authentication.

+

code.test.com

+

or

+

code.test.com:443

+

Authorization

+

Authentication information.

+

The value can be obtained from the request signing result.

+

This field is mandatory for AK/SK-based authentication.

+

SDK-HMAC-SHA256 Credential=QRUP2R3QFNAOVAWMYHZW/20160202/northchina/test/sdk_request, SignedHeaders=host;x-sdk-date, Signature=9d8b56b055c0e1f7a9498d881a7cb726be91b4f0cde1773b0b1557e987a480ce

+

workspace

+

Workspace ID. For details about how to obtain it, see DataArts Studio Instance ID and Workspace ID.

+
  • You do not need to set this field when calling a CDM API because in that case, only the default workspace can be used.
  • When a DataArts Factory API is called, data in the default workspace is queried by default. To query data in other workspaces, include this header in the query request.
+

d1cd7861478748a6925bc02f47c69279

+

X-Dlm-Type

+

DataArts DataService edition, which is Exclusive

+

This parameter is mandatory for calling DataArts DataService APIs.

+

EXCLUSIVE: exclusive edition

+
+
+
+

Common response headers

Table 2 describes common response headers.

+ +
+ + + + + + + + + + + + + +
Table 2 Common response headers

Parameter

+

Description

+

Content-Length

+

Length of the response body. The unit is byte.

+

Date

+

Time when a response is returned

+

Content-type

+

Request body MIME type

+
+
+
+
+
+
+
Parent topic: Appendix
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0330.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0330.html new file mode 100644 index 000000000..99771158b --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0330.html @@ -0,0 +1,32 @@ + + +

Change History

+
+
+ + + + + + + + + + + + + +

Release Date

+

What's New

+

2023-05-30

+

Update the following sections:

+ +

2023-04-30

+

Update the following sections:

+ +

2023-01-30

+

This is the first official release.

+
+
+
+ diff --git a/docs/dataartsstudio/api-ref/dataartsstudio_02_0350.html b/docs/dataartsstudio/api-ref/dataartsstudio_02_0350.html new file mode 100644 index 000000000..331921096 --- /dev/null +++ b/docs/dataartsstudio/api-ref/dataartsstudio_02_0350.html @@ -0,0 +1,16 @@ + + +

DataArts Studio Instance ID and Workspace ID

+

Obtaining a DataArts Studio Instance ID and Workspace ID

To obtain a DataArts Studio instance ID and workspace ID, perform the following steps:
  1. On the DataArts Studio console, locate a workspace and click any module, such as Management Center.
    Figure 1 Management Center
    +
  2. On the Management Center page, obtain the values of instanceId and workspace in the browser address bar, which are the instance ID and workspace ID, respectively.
    As shown in Figure 2, the instance ID is 6b88…2688, and the workspace ID is 1dd3bc…d93f0.
    Figure 2 Obtaining the instance ID and workspace ID
    +
    +
+
+
+
+
+
+
Parent topic: Before You Start
+
+
+ diff --git a/docs/dataartsstudio/api-ref/en-us_image_0000001321929240.gif b/docs/dataartsstudio/api-ref/en-us_image_0000001321929240.gif new file mode 100644 index 000000000..cfbc91a43 Binary files /dev/null and b/docs/dataartsstudio/api-ref/en-us_image_0000001321929240.gif differ diff --git a/docs/dataartsstudio/api-ref/en-us_image_0000001322088924.png b/docs/dataartsstudio/api-ref/en-us_image_0000001322088924.png new file mode 100644 index 000000000..1a1115328 Binary files /dev/null and b/docs/dataartsstudio/api-ref/en-us_image_0000001322088924.png differ diff --git a/docs/dataartsstudio/api-ref/en-us_image_0000001322248848.png b/docs/dataartsstudio/api-ref/en-us_image_0000001322248848.png new file mode 100644 index 000000000..d4a37a976 Binary files /dev/null and b/docs/dataartsstudio/api-ref/en-us_image_0000001322248848.png differ diff --git a/docs/dataartsstudio/api-ref/en-us_image_0000001373088745.png b/docs/dataartsstudio/api-ref/en-us_image_0000001373088745.png new file mode 100644 index 000000000..01ad57325 Binary files /dev/null and b/docs/dataartsstudio/api-ref/en-us_image_0000001373088745.png differ diff --git a/docs/dataartsstudio/api-ref/en-us_image_0000001373088765.png b/docs/dataartsstudio/api-ref/en-us_image_0000001373088765.png new file mode 100644 index 000000000..80a2a93c8 Binary files /dev/null and b/docs/dataartsstudio/api-ref/en-us_image_0000001373088765.png differ diff --git a/docs/dataartsstudio/api-ref/en-us_image_0000001373289273.png b/docs/dataartsstudio/api-ref/en-us_image_0000001373289273.png new file mode 100644 index 000000000..d17b6fda2 Binary files /dev/null and b/docs/dataartsstudio/api-ref/en-us_image_0000001373289273.png differ diff --git a/docs/dataartsstudio/api-ref/en-us_image_0000001373289293.png b/docs/dataartsstudio/api-ref/en-us_image_0000001373289293.png new file mode 100644 index 000000000..d17b6fda2 Binary files /dev/null and b/docs/dataartsstudio/api-ref/en-us_image_0000001373289293.png differ diff --git a/docs/dataartsstudio/api-ref/en-us_image_0000001373408961.png b/docs/dataartsstudio/api-ref/en-us_image_0000001373408961.png new file mode 100644 index 000000000..50edae770 Binary files /dev/null and b/docs/dataartsstudio/api-ref/en-us_image_0000001373408961.png differ diff --git a/docs/dataartsstudio/api-ref/en-us_image_0000001373408969.png b/docs/dataartsstudio/api-ref/en-us_image_0000001373408969.png new file mode 100644 index 000000000..323d460df Binary files /dev/null and b/docs/dataartsstudio/api-ref/en-us_image_0000001373408969.png differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/caution_3.0-en-us.png b/docs/dataartsstudio/api-ref/public_sys-resources/caution_3.0-en-us.png new file mode 100644 index 000000000..60f607621 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/caution_3.0-en-us.png differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/danger_3.0-en-us.png b/docs/dataartsstudio/api-ref/public_sys-resources/danger_3.0-en-us.png new file mode 100644 index 000000000..47a9c7235 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/danger_3.0-en-us.png differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/delta.gif b/docs/dataartsstudio/api-ref/public_sys-resources/delta.gif new file mode 100644 index 000000000..0d1b1f674 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/delta.gif differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/deltaend.gif b/docs/dataartsstudio/api-ref/public_sys-resources/deltaend.gif new file mode 100644 index 000000000..cc7da0fc8 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/deltaend.gif differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/icon-arrowdn.gif b/docs/dataartsstudio/api-ref/public_sys-resources/icon-arrowdn.gif new file mode 100644 index 000000000..379428032 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/icon-arrowdn.gif differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/icon-arrowrt.gif b/docs/dataartsstudio/api-ref/public_sys-resources/icon-arrowrt.gif new file mode 100644 index 000000000..6aaaa11c2 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/icon-arrowrt.gif differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/icon-caution.gif b/docs/dataartsstudio/api-ref/public_sys-resources/icon-caution.gif new file mode 100644 index 000000000..079c79b26 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/icon-caution.gif differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/icon-danger.gif b/docs/dataartsstudio/api-ref/public_sys-resources/icon-danger.gif new file mode 100644 index 000000000..079c79b26 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/icon-danger.gif differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/icon-huawei.gif b/docs/dataartsstudio/api-ref/public_sys-resources/icon-huawei.gif new file mode 100644 index 000000000..a31d60f89 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/icon-huawei.gif differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/icon-note.gif b/docs/dataartsstudio/api-ref/public_sys-resources/icon-note.gif new file mode 100644 index 000000000..31be2b039 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/icon-note.gif differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/icon-notice.gif b/docs/dataartsstudio/api-ref/public_sys-resources/icon-notice.gif new file mode 100644 index 000000000..409070650 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/icon-notice.gif differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/icon-tip.gif b/docs/dataartsstudio/api-ref/public_sys-resources/icon-tip.gif new file mode 100644 index 000000000..c47bae05c Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/icon-tip.gif differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/icon-warning.gif b/docs/dataartsstudio/api-ref/public_sys-resources/icon-warning.gif new file mode 100644 index 000000000..079c79b26 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/icon-warning.gif differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/note_3.0-en-us.png b/docs/dataartsstudio/api-ref/public_sys-resources/note_3.0-en-us.png new file mode 100644 index 000000000..57a0e1f53 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/note_3.0-en-us.png differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/notice_3.0-en-us.png b/docs/dataartsstudio/api-ref/public_sys-resources/notice_3.0-en-us.png new file mode 100644 index 000000000..fa4b64990 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/notice_3.0-en-us.png differ diff --git a/docs/dataartsstudio/api-ref/public_sys-resources/warning_3.0-en-us.png b/docs/dataartsstudio/api-ref/public_sys-resources/warning_3.0-en-us.png new file mode 100644 index 000000000..def5c3565 Binary files /dev/null and b/docs/dataartsstudio/api-ref/public_sys-resources/warning_3.0-en-us.png differ diff --git a/docs/dataartsstudio/api-ref/topic_300000000_3.html b/docs/dataartsstudio/api-ref/topic_300000000_3.html new file mode 100644 index 000000000..8c4f8aa81 --- /dev/null +++ b/docs/dataartsstudio/api-ref/topic_300000000_3.html @@ -0,0 +1,28 @@ + + +

Cluster Management

+

+
+
+ + +
+
Parent topic: DataArts Migration APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/topic_300000001_2.html b/docs/dataartsstudio/api-ref/topic_300000001_2.html new file mode 100644 index 000000000..955e9ec1d --- /dev/null +++ b/docs/dataartsstudio/api-ref/topic_300000001_2.html @@ -0,0 +1,32 @@ + + +

Job Management

+

+
+
+ + +
+
Parent topic: DataArts Migration APIs
+
+
+ diff --git a/docs/dataartsstudio/api-ref/topic_300000002_2.html b/docs/dataartsstudio/api-ref/topic_300000002_2.html new file mode 100644 index 000000000..4a3679339 --- /dev/null +++ b/docs/dataartsstudio/api-ref/topic_300000002_2.html @@ -0,0 +1,22 @@ + + +

Link Management

+

+
+
+ + +
+
Parent topic: DataArts Migration APIs
+
+
+