From 6d6608a45debccc04d375197645841881348d290 Mon Sep 17 00:00:00 2001 From: "Su, Xiaomeng" Date: Fri, 6 Mar 2026 08:27:45 +0000 Subject: [PATCH] dli_umn_20260211 Reviewed-by: Pruthi, Vineet Co-authored-by: Su, Xiaomeng Co-committed-by: Su, Xiaomeng --- docs/dli/api-ref/ALL_META.TXT.json | 4 +- docs/dli/api-ref/CLASS.TXT.json | 2 +- docs/dli/api-ref/dli_02_00003.html | 8 +- docs/dli/api-ref/dli_02_0016.html | 18 - docs/dli/api-ref/dli_02_0020.html | 2 +- docs/dli/api-ref/dli_02_0021.html | 4 +- docs/dli/api-ref/dli_02_0022.html | 2 +- docs/dli/api-ref/dli_02_0024.html | 2 +- docs/dli/api-ref/dli_02_0025.html | 8 +- docs/dli/api-ref/dli_02_0102.html | 4 +- docs/dli/api-ref/dli_02_0105.html | 4 +- docs/dli/api-ref/dli_02_0124.html | 6 +- docs/dli/api-ref/dli_02_0126.html | 4 +- docs/dli/api-ref/dli_02_0127.html | 2 +- docs/dli/api-ref/dli_02_0130.html | 4 +- docs/dli/api-ref/dli_02_0187.html | 3 +- docs/dli/api-ref/dli_02_0189.html | 10 + docs/dli/api-ref/dli_02_0190.html | 12 +- docs/dli/api-ref/dli_02_0194.html | 7 +- docs/dli/api-ref/dli_02_0196.html | 38 +- docs/dli/api-ref/dli_02_0201.html | 951 ------------- docs/dli/api-ref/dli_02_0228.html | 30 +- docs/dli/api-ref/dli_02_0229.html | 28 +- docs/dli/api-ref/dli_02_0230.html | 60 +- docs/dli/api-ref/dli_02_0231.html | 50 +- docs/dli/api-ref/dli_02_0233.html | 8 +- docs/dli/api-ref/dli_02_0234.html | 8 +- docs/dli/api-ref/dli_02_0235.html | 115 +- docs/dli/api-ref/dli_02_0236.html | 2 +- docs/dli/api-ref/dli_02_0241.html | 2 +- docs/dli/api-ref/dli_02_0242.html | 2 +- docs/dli/api-ref/dli_02_0243.html | 2 +- docs/dli/api-ref/dli_02_0245.html | 4 +- docs/dli/api-ref/dli_02_0246.html | 2 +- docs/dli/api-ref/dli_02_0247.html | 2 +- docs/dli/api-ref/dli_02_0248.html | 4 +- docs/dli/api-ref/dli_02_0250.html | 2 +- docs/dli/api-ref/dli_02_0252.html | 2 +- docs/dli/api-ref/dli_02_0254.html | 2 +- docs/dli/api-ref/dli_02_0255.html | 2 +- docs/dli/api-ref/dli_02_0312.html | 2 +- docs/dli/api-ref/dli_02_0316.html | 2 +- docs/dli/api-ref/dli_02_0326.html | 13 +- docs/dli/api-ref/dli_02_0327.html | 21 +- docs/dli/api-ref/dli_02_0336.html | 4 +- docs/dli/api-ref/dli_02_0371.html | 2 +- docs/dli/api-ref/en-us_topic_0206808210.html | 949 +++++++++++++ docs/dli/umn/ALL_META.TXT.json | 1282 ++++++++++-------- docs/dli/umn/CLASS.TXT.json | 1228 +++++++++-------- docs/dli/umn/dli_01_00006.html | 16 +- docs/dli/umn/dli_01_0001.html | 8 +- docs/dli/umn/dli_01_0002.html | 2 +- docs/dli/umn/dli_01_0005.html | 12 +- docs/dli/umn/dli_01_0006.html | 25 +- docs/dli/umn/dli_01_0010.html | 4 +- docs/dli/umn/dli_01_0014.html | 9 + docs/dli/umn/dli_01_0016.html | 2 +- docs/dli/umn/dli_01_0017.html | 6 +- docs/dli/umn/dli_01_0019.html | 2 +- docs/dli/umn/dli_01_0020.html | 2 +- docs/dli/umn/dli_01_0320.html | 12 +- docs/dli/umn/dli_01_0363.html | 8 +- docs/dli/umn/dli_01_0375.html | 6 +- docs/dli/umn/dli_01_0378.html | 47 +- docs/dli/umn/dli_01_0384.html | 10 +- docs/dli/umn/dli_01_0385.html | 2 +- docs/dli/umn/dli_01_0403.html | 2 +- docs/dli/umn/dli_01_0410.html | 4 +- docs/dli/umn/dli_01_0417.html | 37 +- docs/dli/umn/dli_01_0419.html | 14 +- docs/dli/umn/dli_01_0445.html | 509 ++++--- docs/dli/umn/dli_01_0448.html | 10 +- docs/dli/umn/dli_01_0451.html | 260 ++-- docs/dli/umn/dli_01_0457.html | 186 ++- docs/dli/umn/dli_01_0461.html | 18 +- docs/dli/umn/dli_01_0463.html | 4 +- docs/dli/umn/dli_01_0464.html | 20 +- docs/dli/umn/dli_01_0489.html | 2 +- docs/dli/umn/dli_01_0494.html | 4 +- docs/dli/umn/dli_01_0498.html | 82 +- docs/dli/umn/dli_01_0504.html | 7 +- docs/dli/umn/dli_01_0505.html | 21 +- docs/dli/umn/dli_01_0506.html | 8 +- docs/dli/umn/dli_01_0512.html | 12 +- docs/dli/umn/dli_01_0515.html | 4 +- docs/dli/umn/dli_01_0525.html | 2 +- docs/dli/umn/dli_01_0531.html | 27 +- docs/dli/umn/dli_01_0538.html | 6 +- docs/dli/umn/dli_01_0550.html | 14 +- docs/dli/umn/dli_01_0552.html | 2 +- docs/dli/umn/dli_01_0559.html | 2 +- docs/dli/umn/dli_01_0561.html | 10 +- docs/dli/umn/dli_01_0566.html | 10 +- docs/dli/umn/dli_01_0612.html | 1 - docs/dli/umn/dli_01_0616.html | 6 +- docs/dli/umn/dli_01_0617.html | 4 +- docs/dli/umn/dli_01_0618.html | 2 +- docs/dli/umn/dli_01_0622.html | 12 +- docs/dli/umn/dli_01_0623.html | 4 +- docs/dli/umn/dli_01_0632.html | 2 +- docs/dli/umn/dli_01_0636.html | 8 +- docs/dli/umn/dli_01_0638.html | 2 +- docs/dli/umn/dli_01_0639.html | 2 +- docs/dli/umn/dli_01_0687.html | 23 +- docs/dli/umn/dli_01_0698.html | 51 + docs/dli/umn/dli_03_0031.html | 4 +- docs/dli/umn/dli_03_0032.html | 6 +- docs/dli/umn/dli_03_0040.html | 2 +- docs/dli/umn/dli_03_0054.html | 2 +- docs/dli/umn/dli_03_0056.html | 2 +- docs/dli/umn/dli_03_0060.html | 2 +- docs/dli/umn/dli_03_0067.html | 19 +- docs/dli/umn/dli_03_0076.html | 4 +- docs/dli/umn/dli_03_0086.html | 253 +++- docs/dli/umn/dli_03_0098.html | 2 +- docs/dli/umn/dli_03_0156.html | 2 +- docs/dli/umn/dli_03_0173.html | 4 +- docs/dli/umn/dli_03_0175.html | 4 +- docs/dli/umn/dli_03_0184.html | 2 +- docs/dli/umn/dli_03_0211.html | 2 +- docs/dli/umn/dli_07_0003.html | 61 +- docs/dli/umn/dli_07_0004.html | 14 +- docs/dli/umn/dli_07_0005.html | 11 +- docs/dli/umn/dli_07_0006.html | 14 +- docs/dli/umn/dli_07_0007.html | 10 +- docs/dli/umn/dli_07_0009.html | 16 +- docs/dli/umn/dli_07_0027.html | 9 +- docs/dli/umn/dli_09_0210.html | 180 +++ docs/dli/umn/dli_09_0211.html | 421 ++++++ docs/dli/umn/dli_09_0215.html | 186 +++ 130 files changed, 4651 insertions(+), 3073 deletions(-) delete mode 100644 docs/dli/api-ref/dli_02_0201.html create mode 100644 docs/dli/api-ref/en-us_topic_0206808210.html create mode 100644 docs/dli/umn/dli_01_0698.html create mode 100644 docs/dli/umn/dli_09_0210.html create mode 100644 docs/dli/umn/dli_09_0211.html create mode 100644 docs/dli/umn/dli_09_0215.html diff --git a/docs/dli/api-ref/ALL_META.TXT.json b/docs/dli/api-ref/ALL_META.TXT.json index 7b786e695..a19e973db 100644 --- a/docs/dli/api-ref/ALL_META.TXT.json +++ b/docs/dli/api-ref/ALL_META.TXT.json @@ -1719,8 +1719,8 @@ "githuburl":"" }, { - "uri":"dli_02_0201.html", - "node_id":"dli_02_0201.xml", + "uri":"en-us_topic_0206808210.html", + "node_id":"en-us_topic_0206808210.xml", "product_code":"dli", "code":"79", "des":"This section describes fine-grained permissions management for your DLI. If your account does not need individual IAM users, then you may skip this section.By default, ne", diff --git a/docs/dli/api-ref/CLASS.TXT.json b/docs/dli/api-ref/CLASS.TXT.json index 51a4ddfd7..c2b7b277c 100644 --- a/docs/dli/api-ref/CLASS.TXT.json +++ b/docs/dli/api-ref/CLASS.TXT.json @@ -705,7 +705,7 @@ "desc":"This section describes fine-grained permissions management for your DLI. If your account does not need individual IAM users, then you may skip this section.By default, ne", "product_code":"dli", "title":"Permissions Policies and Supported Actions", - "uri":"dli_02_0201.html", + "uri":"en-us_topic_0206808210.html", "doc_type":"api", "p_code":"", "code":"79" diff --git a/docs/dli/api-ref/dli_02_00003.html b/docs/dli/api-ref/dli_02_00003.html index 8df774254..816003356 100644 --- a/docs/dli/api-ref/dli_02_00003.html +++ b/docs/dli/api-ref/dli_02_00003.html @@ -8,7 +8,13 @@ -

2025-09-25

+

2026-02-10

+ +

Modified the following sections:

+ + + +

2025-09-25

Modified the following sections:

diff --git a/docs/dli/api-ref/dli_02_0016.html b/docs/dli/api-ref/dli_02_0016.html index 6355094c7..cb339852a 100644 --- a/docs/dli/api-ref/dli_02_0016.html +++ b/docs/dli/api-ref/dli_02_0016.html @@ -214,24 +214,6 @@

Default Spark version of the queue

-

support_hetu_engine_versions

- -

No

- -

Array[String]

- -

HetuEngine version supported by the queue

- - -

default_hetu_engine_version

- -

No

- -

String

- -

Default HetuEngine version of the queue

- -

support_flink_sql_versions

No

diff --git a/docs/dli/api-ref/dli_02_0020.html b/docs/dli/api-ref/dli_02_0020.html index 716eb6542..4719cf9b1 100644 --- a/docs/dli/api-ref/dli_02_0020.html +++ b/docs/dli/api-ref/dli_02_0020.html @@ -114,7 +114,7 @@

Boolean

Whether to export column names when exporting CSV and JSON data.

-
  • If this parameter is set to true, the column names are exported.
  • If this parameter is set to false, the column names are not exported.
  • If this parameter is left blank, the default value false is used.
+
  • If this parameter is set to true, the column names are exported.
  • If this parameter is set to false, the column names are not exported.
  • If this parameter is left blank, the default value false is used.
diff --git a/docs/dli/api-ref/dli_02_0021.html b/docs/dli/api-ref/dli_02_0021.html index 092789e29..219db53e8 100644 --- a/docs/dli/api-ref/dli_02_0021.html +++ b/docs/dli/api-ref/dli_02_0021.html @@ -28,7 +28,7 @@

Constraints

None

Range

-

The value can contain 1 to 64 characters. Only letters and digits are allowed.

+

The value can contain up to 64 characters. Only letters and digits are allowed.

Default Value

None

@@ -45,7 +45,7 @@

Constraints

None

Range

-

The value can contain 1 to 64 characters. Only letters and digits are allowed.

+

The value can contain up to 64 characters. Only letters and digits are allowed.

Default Value

None

diff --git a/docs/dli/api-ref/dli_02_0022.html b/docs/dli/api-ref/dli_02_0022.html index 65579e394..e52efcb3c 100644 --- a/docs/dli/api-ref/dli_02_0022.html +++ b/docs/dli/api-ref/dli_02_0022.html @@ -123,7 +123,7 @@

String

-

Path to imported or exported files.

+

Path for imported or exported files.

data_type

diff --git a/docs/dli/api-ref/dli_02_0024.html b/docs/dli/api-ref/dli_02_0024.html index e2abd1b1d..66cb29efa 100644 --- a/docs/dli/api-ref/dli_02_0024.html +++ b/docs/dli/api-ref/dli_02_0024.html @@ -103,7 +103,7 @@

Boolean

Whether to export column names when exporting CSV and JSON data.

-
  • If this parameter is set to true, the column names are exported.
  • If this parameter is set to false, the column names are not exported.
  • If this parameter is left blank, the default value false is used.
+
  • If this parameter is set to true, the column names are exported.
  • If this parameter is set to false, the column names are not exported.
  • If this parameter is left blank, the default value false is used.

limit_num

diff --git a/docs/dli/api-ref/dli_02_0025.html b/docs/dli/api-ref/dli_02_0025.html index 21ea61f8d..a5c9e5245 100644 --- a/docs/dli/api-ref/dli_02_0025.html +++ b/docs/dli/api-ref/dli_02_0025.html @@ -232,11 +232,11 @@

String

Definition

-

Engine type

+

Engine type Only the Spark engine is currently supported.

Constraints

None

Range

-
  • spark: Spark engine
+
  • spark: indicates the Spark engine.

Default Value

None

@@ -535,9 +535,9 @@

String

Definition

-

Engine type

+

Engine type configured for the queue.

Range

-
  • spark: Spark engine
+
  • spark: the Spark engine.

statement

diff --git a/docs/dli/api-ref/dli_02_0102.html b/docs/dli/api-ref/dli_02_0102.html index 1a96feeed..5997153c7 100644 --- a/docs/dli/api-ref/dli_02_0102.html +++ b/docs/dli/api-ref/dli_02_0102.html @@ -32,7 +32,7 @@

Constraints

None

Range

-

The value can contain 1 to 64 characters. Only letters and digits are allowed.

+

The value can contain up to 64 characters. Only letters and digits are allowed.

Default Value

None

@@ -144,7 +144,7 @@

Constraints

None

Range

-
  • spark: Spark engine
+

Only the Spark engine is currently supported. The default value is spark.

Default Value

spark

diff --git a/docs/dli/api-ref/dli_02_0105.html b/docs/dli/api-ref/dli_02_0105.html index 8fef57d4f..9f3e55f2c 100644 --- a/docs/dli/api-ref/dli_02_0105.html +++ b/docs/dli/api-ref/dli_02_0105.html @@ -267,7 +267,7 @@

Long

-

Size of a DLI table. Set parameter to 0 for non-DLI tables. The unit is byte.

+

Size of a DLI table. This parameter is consistently 0 for non-DLI tables. The unit is byte.

table_type

@@ -295,7 +295,7 @@

Integer

-

Paging size. The minimum value is 1 and the maximum value is 100.

+

Paging size. The minimum value is 1 and the maximum value is 100.

current-page

diff --git a/docs/dli/api-ref/dli_02_0124.html b/docs/dli/api-ref/dli_02_0124.html index 7251a71ab..e7362279f 100644 --- a/docs/dli/api-ref/dli_02_0124.html +++ b/docs/dli/api-ref/dli_02_0124.html @@ -29,7 +29,7 @@

Constraints

None

Range

-

The value can contain 1 to 64 characters. Only letters and digits are allowed.

+

The value can contain up to 64 characters. Only letters and digits are allowed.

Default Value

None

@@ -347,7 +347,7 @@

Integer

Definition

-

Number of Executors in a Spark application. This configuration item replaces the default parameter in sc_type.

+

Number of Executors in a Spark application. This configuration item replaces the default parameter in sc_type.

Constraints

None

Range

@@ -739,7 +739,7 @@

Definition

Queue name. Set this parameter to the name of the created DLI queue.

-
  • This parameter is compatible with the cluster_name parameter. That is, if cluster_name is used to specify a queue, the queue is still valid.
  • You are advised to use the queue parameter. The queue and cluster_name parameters cannot coexist.
+
  • This parameter is compatible with the cluster_name parameter. That is, if cluster_name is used to specify a queue, the queue is still valid.
  • You are advised to use the queue parameter. The queue and cluster_name parameters cannot coexist.

Range

None

diff --git a/docs/dli/api-ref/dli_02_0126.html b/docs/dli/api-ref/dli_02_0126.html index 4711ae798..cfa3a87a6 100644 --- a/docs/dli/api-ref/dli_02_0126.html +++ b/docs/dli/api-ref/dli_02_0126.html @@ -27,7 +27,7 @@

Constraints

None

Range

-

The value can contain 1 to 64 characters. Only letters and digits are allowed.

+

The value can contain up to 64 characters. Only letters and digits are allowed.

Default Value

None

@@ -155,7 +155,7 @@

No

-

Array of Strings

+

Array of strings

Definition

Last 10 records of the current batch processing job.

diff --git a/docs/dli/api-ref/dli_02_0127.html b/docs/dli/api-ref/dli_02_0127.html index 6ccdcea3a..f97cbebe7 100644 --- a/docs/dli/api-ref/dli_02_0127.html +++ b/docs/dli/api-ref/dli_02_0127.html @@ -23,7 +23,7 @@

Constraints

None

Range

-

The value can contain 1 to 64 characters. Only letters and digits are allowed.

+

The value can contain up to 64 characters. Only letters and digits are allowed.

Default Value

None

diff --git a/docs/dli/api-ref/dli_02_0130.html b/docs/dli/api-ref/dli_02_0130.html index 6cd8ed215..5933ec353 100644 --- a/docs/dli/api-ref/dli_02_0130.html +++ b/docs/dli/api-ref/dli_02_0130.html @@ -211,7 +211,7 @@

Boolean

-

Whether to upload resource packages in asynchronous mode. The default value is false, indicating that the asynchronous mode is not used. You are advised to upload resource packages in asynchronous mode.

+

Whether to upload resource packages in asynchronous mode. The default value is false, indicating that the asynchronous mode is not used. You are advised to upload resource packages in asynchronous mode.

owner

@@ -297,7 +297,7 @@

Boolean

-

Whether to upload resource packages in asynchronous mode. The default value is false, indicating that the asynchronous mode is not used. You are advised to upload resource packages in asynchronous mode.

+

Whether to upload resource packages in asynchronous mode. The default value is false, indicating that the asynchronous mode is not used. You are advised to upload resource packages in asynchronous mode.

diff --git a/docs/dli/api-ref/dli_02_0187.html b/docs/dli/api-ref/dli_02_0187.html index d5918860e..d50988c46 100644 --- a/docs/dli/api-ref/dli_02_0187.html +++ b/docs/dli/api-ref/dli_02_0187.html @@ -233,8 +233,7 @@

Example Response

{
   "is_success": true,
-  "message": "Create peer connection for queues:{Queue list in the request parameter}",
-  "connection_id": "2a620c33-5609-40c9-affd-2b6453071b0f"
+  "message": "Create peer connection for queues:{Queue list in the request parameter}",   "connection_id": "2a620c33-5609-40c9-affd-2b6453071b0f"
 }

Status Codes

Table 6 describes the status code.

diff --git a/docs/dli/api-ref/dli_02_0189.html b/docs/dli/api-ref/dli_02_0189.html index d5441270d..c279f927e 100644 --- a/docs/dli/api-ref/dli_02_0189.html +++ b/docs/dli/api-ref/dli_02_0189.html @@ -142,6 +142,16 @@

User-defined host information. For details, see hosts parameter description.

+

ipv6_enable

+ +

No

+ +

Boolean

+ +

Whether to enable IPv6

+
  • true: Enable IPv6.
  • false: Do not enable IPv6.
+ +
diff --git a/docs/dli/api-ref/dli_02_0190.html b/docs/dli/api-ref/dli_02_0190.html index 7451dc479..2b8bf48b7 100644 --- a/docs/dli/api-ref/dli_02_0190.html +++ b/docs/dli/api-ref/dli_02_0190.html @@ -239,6 +239,16 @@

User-defined host information. For details, see Table 7.

+

ipv6_enable

+ +

No

+ +

Boolean

+ +

Whether to enable IPv6

+
  • true: Enable IPv6.
  • false: Do not enable IPv6.
+ + @@ -345,7 +355,7 @@

String

-

Detailed error message when the status is FAILED.

+

Detailed error message when the status is FAILED.

update_time

diff --git a/docs/dli/api-ref/dli_02_0194.html b/docs/dli/api-ref/dli_02_0194.html index b5db982cd..6e7698ca9 100644 --- a/docs/dli/api-ref/dli_02_0194.html +++ b/docs/dli/api-ref/dli_02_0194.html @@ -169,7 +169,7 @@

No

-

Array of Strings

+

Array of strings

Definition

Tag information of the queue to be created. Currently, the tag information includes whether the queue is cross-AZ (JSON string). The value can only be 2, that is, a dual-AZ queue whose compute resources are distributed in two AZs is created.

@@ -208,8 +208,7 @@

Constraints

This parameter is available only when queue_type is set to sql.

Range

-

The options are spark and hetuEngine.

-
  • spark: Spark engine
+

You can choose the Spark engine, configured as spark.

Default Value

The default value is spark.

@@ -362,7 +361,7 @@

500

-

Internal service error.

+

Internal server error.

diff --git a/docs/dli/api-ref/dli_02_0196.html b/docs/dli/api-ref/dli_02_0196.html index e79e6e447..896850722 100644 --- a/docs/dli/api-ref/dli_02_0196.html +++ b/docs/dli/api-ref/dli_02_0196.html @@ -95,24 +95,6 @@

Query results are filtered by tag.

-

limit

- -

No

- -

Integer

- -

Number of records to be displayed of the page-based query.

- - -

offset

- -

No

- -

Integer

- -

Specifies the offset of the page-based query.

- - @@ -153,7 +135,7 @@

No

-

Array of Object

+

Array of objects

Queue information For details, see Table 4.

@@ -394,24 +376,6 @@

Default Spark version of the queue

-

support_hetu_engine_versions

- -

No

- -

Array[String]

- -

HetuEngine version supported by the queue

- - -

default_hetu_engine_version

- -

No

- -

String

- -

Default HetuEngine version of the queue

- -

support_flink_sql_versions

No

diff --git a/docs/dli/api-ref/dli_02_0201.html b/docs/dli/api-ref/dli_02_0201.html deleted file mode 100644 index ff91492f5..000000000 --- a/docs/dli/api-ref/dli_02_0201.html +++ /dev/null @@ -1,951 +0,0 @@ - - -

Permissions Policies and Supported Actions

-

This section describes fine-grained permissions management for your DLI. If your account does not need individual IAM users, then you may skip this section.

-

By default, new IAM users do not have permissions assigned. You need to add them to one or more groups, and attach permissions policies or roles to these groups. Users inherit permissions from the groups to which they are added After authorization, the user can perform specified operations on MRS based on the permissions.

-

-
  • Roles are a type of coarse-grained authorization mechanism that defines permissions related to user responsibilities.
  • Policies define API-based permissions for operations on specific resources under certain conditions, allowing for more fine-grained, secure access control of cloud resources.
-

Policy-based authorization is useful if you want to allow or deny the access to an API.

-
-

An account has all the permissions required to call all APIs, but IAM users must be assigned the required permissions. The permissions required for calling an API are determined by the actions supported by the API. Only users who have been granted permissions allowing the actions can call the API successfully. For example, if an IAM user needs to create buckets using an API, the user must have been granted permissions that allow the dli:queue:create_queue action.

-

Supported Actions

VBS provides system-defined policies that can be directly used in IAM. You can also create custom policies and use them to supplement system-defined policies, implementing more refined access control. Operations supported by policies are specific to APIs. The following are common concepts related to policies:

-
  • Permissions: Allow or deny operations on specified resources under specific conditions.
  • APIs: RESTful APIs that can be called in a custom policy.
  • Actions: added to a custom policy to control permissions for specific operations.
  • Dependencies: actions which a specific action depends on. When allowing an action for a user, you also need to allow any existing action dependencies for that user.
  • IAM or enterprise projects: Type of projects for which an action will take effect. Policies that contain actions supporting both IAM and enterprise projects can be assigned to user groups and take effect in both IAM and Enterprise Management. Policies that only contain actions supporting IAM projects can be assigned to user groups and only take effect for IAM. Such policies will not take effect if they are assigned to user groups in Enterprise Project.

    -

    Only the APIs listed in Table 1 support custom authorization.

    -
-

The check mark (√) indicates that an action takes effect. The cross mark (x) indicates that an action does not take effect.

-
-

DLI supports the following actions that can be defined in custom policies:

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table 1 Actions

Permission

-

API

-

Actions

-

IAM Project

-

(Project)

-

Enterprise Project

-

(Enterprise Project)

-

Creating a queue

-

POST /v1.0/{project_id}/queues

-

dli:queue:createQueue

-

√

-

√

-

Deleting a queue

-

DELETE /v1.0/{project_id}/queues/{queue_name}

-

dli:queue:dropQueue

-

√

-

√

-

Submitting a job

-

POST /v1.0/{project_id}/jobs/submit-job

-

dli:queue:submitJob

-

√

-

×

-

Canceling a job

-

DELETE /v1.0/{project_id}/jobs/{job_id}

-

dli:queue:cancelJob

-

√

-

×

-

Viewing queue permissions of other users

-

GET /v1.0/{project_id}/queues/{queue_name}/users

-

dli:queue:showPrivileges

-

√

-

×

-

Restarting a queue

-

PUT /v1.0/{project_id}/queues/{queue_name}/action

-

dli:queue:restart

-

√

-

×

-

Scaling out/in a queue

-

PUT /v1.0/{project_id}/queues/{queue_name}/action

-

dli:queue:scaleQueue

-

√

-

×

-

Granting elastic resource pool permissions

-

PUT /v1.0/{project_id}/user-authorization

-

dli:elasticresourcepool:grantPrivilege

-

√

-

×

-

Revoking elastic resource pool permissions

-

PUT /v1.0/{project_id}/user-authorization

-

dli:elasticresourcepool:revokePrivilege

-

√

-

×

-

Obtaining information about all catalogs in a project

-

GET /v3/{project_id}/catalogs

-

dli:catalog:list

-

√

-

×

-

Querying details about a specified catalog

-

GET /v3/{project_id}/catalogs/{catalog_name}

-

dli:catalog:get

-

√

-

×

-

Binding a catalog

-

POST /v3/{project_id}/catalogs/action

-

dli:catalog:bind

-

√

-

×

-

Unbinding a catalog

-

POST /v3/{project_id}/catalogs/action

-

dli:catalog:unbind

-

√

-

×

-

Creating a database

-

POST /v1.0/{project_id}/databases

-

dli:database:createDatabase

-

√

-

×

-

Deleting a database

-

DELETE /v1.0/{project_id}/databases/{database_name}

-

dli:database:dropDatabase

-

√

-

×

-

Modifying database configuration

-

POST /v1.0/{project_id}/jobs/submit-job

-

dli:database:alterDatabaseProperties

-

√

-

×

-

Explaining the SQL statement as an execution plan

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:explain

-

√

-

×

-

Creating a table

-

POST /v1.0/{project_id}/databases/{database_name}/tables

-

dli:database:createTable

-

√

-

×

-

Creating a view

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:createView

-

√

-

×

-

Creating a function

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:createFunction

-

√

-

×

-

Describing a function

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:describeFunction

-

√

-

×

-

Deleting a function

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:dropFunction

-

√

-

×

-

Displaying a function

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:showFunctions

-

√

-

×

-

Creating a role

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:createRole

-

√

-

×

-

Deleting a role

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:dropRole

-

√

-

×

-

Displaying a role

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:showRoles

-

√

-

×

-

Displaying all roles

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:showAllRoles

-

√

-

×

-

Binding a role

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:grantRole

-

√

-

×

-

Unbinding a role

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:revokeRole

-

√

-

×

-

Displaying the binding relationships between all roles and users

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:database:showUsers

-

√

-

×

-

Viewing database permissions of other users

-

GET /v1.0/{project_id}/databases/{database_name}/users

-

dli:database:showPrivileges

-

√

-

×

-

Displaying a database

-

GET /v1.0/{project_id}/databases

-

dli:database:displayDatabase

-

√

-

×

-

Displaying all databases

-

GET /v1.0/{project_id}/databases

-

dli:database:displayAllDatabases

-

√

-

×

-

Displaying all tables

-

GET /v1.0/{project_id}/databases

-

dli:database:displayAllTables

-

√

-

×

-

Granting database permissions to a specified user

-

PUT /v1.0/{project_id}/user-authorization

-

dli:database:grantPrivilege

-

√

-

×

-

Removing database permissions of a specified user

-

PUT /v1.0/{project_id}/user-authorization

-

dli:database:revokePrivilege

-

√

-

×

-

Deleting a table

-

DELETE /v1.0/{project_id}/databases/{database_name}/tables/{table_name}

-

dli:table:dropTable

-

√

-

×

-

Displaying the table structure

-

GET /v1.0/{project_id}/databases/{database_name}/tables/{table_name}

-

dli:table:describeTable

-

√

-

×

-

Querying a table

-

GET /v1.0/{project_id}/databases/{database_name}/tables/{table_name}/preview

-

dli:table:select

-

√

-

×

-

Displaying table configuration

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:showTableProperties

-

√

-

×

-

Displaying the table creation statement

-

GET /v1.0/{project_id}/databases/{database_name}/tables/{table_name}/preview

-

dli:table:showCreateTable

-

√

-

×

-

Displaying all partitions

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:showPartitions

-

√

-

×

-

Setting table configuration

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:alterTableSetProperties

-

√

-

×

-

Adding a column

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:alterTableAddColumns

-

√

-

×

-

Adding partitions to a partitioned table

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:alterTableAddPartition

-

√

-

×

-

Renaming a table partition

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:alterTableRenamePartition

-

√

-

×

-

Deleting partitions from a partitioned table

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:alterTableDropPartition

-

√

-

×

-

Restoring a table partition

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:alterTableRecoverPartition

-

√

-

×

-

Renaming a table

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:alterTableRename

-

√

-

×

-

Setting the partition path

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:alterTableSetLocation

-

√

-

×

-

Inserting table data

-

POST /v1.0/{project_id}/jobs/submit-job, statement invoking

-

dli:table:insertIntoTable

-

√

-

×

-

Rewriting table data

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:insertOverwriteTable

-

√

-

×

-

Viewing table permissions of other users

-

GET /v1.0/{project_id}/databases/{database_name}/tables/{table_name}/users

-

dli:table:showPrivileges

-

√

-

×

-

Clearing a table

-

POST /v1.0/{project_id}/jobs/submit-job

-

dli:table:truncateTable

-

√

-

×

-

Updating a table

-

POST /v1.0/{project_id}/jobs/submit-job

-

dli:table:update

-

√

-

×

-

Deleting table data

-

POST /v1.0/{project_id}/jobs/submit-job

-

dli:table:delete

-

√

-

×

-

Modifying column information

-

POST /v1.0/{project_id}/jobs/submit-job

-

dli:table:alterTableChangeColumn

-

√

-

×

-

Deleting a column

-

POST /v1.0/{project_id}/jobs/submit-job

-

dli:table:alterTableDropColumns

-

√

-

×

-

Displaying data segments

-

POST /v1.0/{project_id}/jobs/submit-job

-

dli:table:showSegments

-

√

-

×

-

Merging data segments

-

POST /v1.0/{project_id}/jobs/submit-job

-

dli:table:compaction

-

√

-

×

-

Modifying a view

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:table:alterView

-

√

-

×

-

Displaying the table structure

-

POST /v1.0/{project_id}/jobs/submit-job, executing corresponding SQL statements

-

dli:table:describeTable

-

√

-

×

-

Granting data table permissions to a specified user

-

PUT /v1.0/{project_id}/user-authorization

-

dli:table:grantPrivilege

-

√

-

×

-

Removing data table permissions of a specified user

-

PUT /v1.0/{project_id}/user-authorization

-

dli:table:revokePrivilege

-

√

-

×

-

Viewing the security authentication information permission list of other users

-

GET /v1.0/{project_id}/datasource/auth-infos/{auth_name}/users

-

dli:datasourceauth:showPrivileges

-

√

-

×

-

Using security authentication information

-

POST /v1.0/{project_id}/jobs/submit-job

-

dli:datasourceauth:useAuth

-

√

-

×

-

Deleting security authentication information

-

DELETE /v2.0/{project_id}/datasource/auth-infos/{auth_info_name}

-

dli:datasourceauth:dropAuth

-

√

-

×

-

Updating security authentication information

-

PUT /v2.0/{project_id}/datasource/auth-infos

-

dli:datasourceauth:updateAuth

-

√

-

×

-

Granting security authentication permissions to a specified user

-

PUT /v1.0/{project_id}/user-authorization

-

dli:datasourceauth:grantPrivilege

-

√

-

×

-

Removing security authentication permissions of a specified user

-

PUT /v1.0/{project_id}/user-authorization

-

dli:datasourceauth:revokePrivilege

-

√

-

×

-

Removing job permissions of a specified user

-

PUT /v1.0/{{project_id}}/authorization

-

dli:jobs:revokePrivilege

-

√

-

×

-

Querying a Column

-

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

-

dli:column:select

-

√

-

×

-

Granting permissions to a specified user queue

-

PUT /v1.0/{project_id}/user-authorization

-

dli:column:grantPrivilege

-

√

-

×

-

Removing permissions of a specified user queue

-

PUT /v1.0/{project_id}/user-authorization

-

dli:column:revokePrivilege

-

√

-

×

-

Listing Flink jobs

-

GET /v1.0/{project_id}/streaming/jobs

-

dli:jobs:listAll

-

√

-

×

-

Querying Flink job details

-

GET /v1.0/{project_id}/streaming/jobs/{job_id}

-

dli:jobs:get

-

√

-

×

-

Creating a Flink SQL job

-

POST /v1.0/{project_id}/streaming/sql-jobs

-

dli:jobs:create

-

√

-

×

-

Creating a Flink Jar job

-

POST /v1.0/{project_id}/streaming/flink-jobs

-

dli:jobs:create

-

√

-

×

-

Updating a Flink SQL job

-

PUT /v1.0/{project_id}/streaming/sql-jobs/{job_id}

-

dli:jobs:update

-

√

-

×

-

Updating a Flink Jar job

-

PUT /v1.0/{project_id}/streaming/flink-jobs/{job_id}

-

dli:jobs:update

-

√

-

×

-

Deleting a Flink job

-

DELETE /v1.0/{project_id}/streaming/jobs/{job_id}

-

dli:jobs:delete

-

√

-

×

-

Running Flink jobs in batches

-

POST /v1.0/{project_id}/streaming/jobs/run

-

dli:jobs:start

-

√

-

×

-

Stopping a Flink job

-

POST /v1.0/{project_id}/streaming/jobs/stopob_id/stop

-

dli:jobs:stop

-

√

-

×

-
-
-
-
- diff --git a/docs/dli/api-ref/dli_02_0228.html b/docs/dli/api-ref/dli_02_0228.html index 6b6ec2f40..d41701bd7 100644 --- a/docs/dli/api-ref/dli_02_0228.html +++ b/docs/dli/api-ref/dli_02_0228.html @@ -46,7 +46,7 @@

String

-

Name of the job. The value can contain 1 to 57 characters.

+

Job name. The value can contain up to 57 characters.

desc

@@ -74,7 +74,7 @@

String

-

Name of a queue. The value can contain 0 to 128 characters.

+

Queue name. The value can contain 0 to 128 characters.

sql_body

@@ -124,8 +124,8 @@

Boolean

-

Whether to enable the automatic job snapshot function.

-
  • true: indicates to enable the automatic job snapshot function.
  • false: indicates to disable the automatic job snapshot function.
  • Default value: false
+

Whether to enable the automatic job snapshot function. Options:

+
  • true: Enable this function.
  • false: Do not enable this function.
  • Default value: false.

checkpoint_mode

@@ -135,8 +135,8 @@

Integer

Snapshot mode. Options:

-
  • 1: ExactlyOnce, indicates that data is processed only once.
  • 2: AtLeastOnce, indicates that data is processed at least once.
-

The default value is 1.

+
  • 1: ExactlyOnce, meaning data is consumed only once.
  • 2: AtLeastOnce, meaning data is consumed at least once.
+

Default value: 1.

checkpoint_interval

@@ -173,7 +173,7 @@

String

-

SMN topic. If a job fails, the system will send a message to users subscribed to the SMN topic.

+

SMN topic. If a job fails, the system will send a message to users subscribed to this SMN topic.

restart_when_exception

@@ -315,8 +315,8 @@

Resource configuration version. The value can be v1 or v2. The default value is v1.

Compared with the v1 template, the v2 template does not support the setting of the number of CUs. The v2 template supports the setting of Job Manager Memory and Task Manager Memory.

-

v1: applicable to Flink 1.12, 1.13, and 1.15

-

v2: applicable to Flink 1.13, 1.15, and 1.17.

+

v1: applicable to Flink 1.12 and 1.15.

+

v2: applicable to Flink 1.15 and 1.17.

You are advised to use the parameter settings of v2.

@@ -350,7 +350,7 @@

String

-

Tag key

+

Tag key.

NOTE:

A tag key can contain up to 128 characters, cannot start or end with a space, and cannot start with _sys_. Only letters, digits, spaces, and the following special characters are allowed: _.:+-@

NOTE:

A tag key can contain a maximum of 36 characters. Only letters, digits, hyphens (-), underscores (_), and spaces are allowed. The key cannot start or end with a space.

@@ -442,7 +442,8 @@

double

-

Number of CPU cores available for JobManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

Number of CPU cores available for JobManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the JobManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

memory

@@ -474,7 +475,8 @@

double

-

Number of CPU cores available for TaskManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

Number of CPU cores available for TaskManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the TaskManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

memory

@@ -507,7 +509,7 @@

String

-

Indicates whether the request is successfully executed. Value true indicates that the request is successfully executed.

+

Whether the request is successfully executed. Value true indicates that the request is successfully executed.

message

@@ -628,7 +630,7 @@
-

Error Codes

If an error occurs when this API is invoked, the system does not return the result similar to the preceding example, but returns the error code and error information. For details, see Error Codes.

+

Error Codes

If an error occurs when this API is called, the system does not return the result similar to the preceding example, but returns an error code and error message. For details, see Error Codes.

diff --git a/docs/dli/api-ref/dli_02_0229.html b/docs/dli/api-ref/dli_02_0229.html index 4beace899..42269dbc5 100644 --- a/docs/dli/api-ref/dli_02_0229.html +++ b/docs/dli/api-ref/dli_02_0229.html @@ -55,7 +55,7 @@

String

-

Name of a job. Length range: 0 to 57 characters.

+

Job name. Length range: 0 to 57 characters.

desc

@@ -73,7 +73,7 @@

String

-

Name of a queue. The value can contain 0 to 128 characters.

+

Queue name. The value can contain 0 to 128 characters.

sql_body

@@ -120,8 +120,8 @@

Boolean

-

Whether to enable the automatic job snapshot function.

-
  • true: indicates to enable the automatic job snapshot function.
  • false: indicates to disable the automatic job snapshot function.
  • Default value: false
+

Whether to enable the automatic job snapshot function. Options:

+
  • true: Enable this function.
  • false: Do not enable this function.
  • Default value: false.

checkpoint_mode

@@ -131,8 +131,8 @@

Integer

Snapshot mode. Options:

-
  • 1: ExactlyOnce, indicates that data is processed only once.
  • 2: at_least_once, indicates that data is processed at least once.
-

The default value is 1.

+
  • 1: ExactlyOnce, meaning data is consumed only once.
  • 2: AtLeastOnce, meaning data is consumed at least once.
+

Default value: 1.

checkpoint_interval

@@ -169,7 +169,7 @@

String

-

SMN topic. If a job fails, the system will send a message to users subscribed to the SMN topic.

+

SMN topic. If a job fails, the system will send a message to users subscribed to this SMN topic.

restart_when_exception

@@ -319,8 +319,8 @@

Resource configuration version. The value can be v1 or v2. The default value is v1.

Compared with the v1 template, the v2 template does not support the setting of the number of CUs. The v2 template supports the setting of Job Manager Memory and Task Manager Memory.

-

v1: applicable to Flink 1.12, 1.13, and 1.15

-

v2: applicable to Flink 1.13, 1.15, and 1.17.

+

v1: applicable to Flink 1.12 and 1.15.

+

v2: applicable to Flink 1.15 and 1.17.

You are advised to use the parameter settings of v2.

@@ -406,7 +406,8 @@

double

-

Number of CPU cores available for JobManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

Number of CPU cores available for JobManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the JobManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

memory

@@ -438,7 +439,8 @@

double

-

Number of CPU cores available for TaskManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

Number of CPU cores available for TaskManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the TaskManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

memory

@@ -471,7 +473,7 @@

String

-

Indicates whether the request is successfully executed. Value true indicates that the request is successfully executed.

+

Whether the request is successfully executed. Value true indicates that the request is successfully executed.

message

@@ -575,7 +577,7 @@
-

Error Codes

If an error occurs when this API is invoked, the system does not return the result similar to the preceding example, but returns the error code and error information. For details, see Error Codes.

+

Error Codes

If an error occurs when this API is called, the system does not return the result similar to the preceding example, but returns an error code and error message. For details, see Error Codes.

diff --git a/docs/dli/api-ref/dli_02_0230.html b/docs/dli/api-ref/dli_02_0230.html index 5f0b57447..a37fe4488 100644 --- a/docs/dli/api-ref/dli_02_0230.html +++ b/docs/dli/api-ref/dli_02_0230.html @@ -30,7 +30,7 @@

Request Parameters

-
- + + + + + + + + + + @@ -75,7 +75,7 @@ diff --git a/docs/dli/api-ref/dli_02_0371.html b/docs/dli/api-ref/dli_02_0371.html index 44386005b..6a7a65257 100644 --- a/docs/dli/api-ref/dli_02_0371.html +++ b/docs/dli/api-ref/dli_02_0371.html @@ -55,7 +55,7 @@ -
Table 2 Parameter description

Parameter

+
@@ -46,7 +46,7 @@ - - + + + + + + + + + + + + + + + - - @@ -283,7 +313,7 @@
Table 2 Parameter descriptions

Parameter

Mandatory

String

Name of the job. The value can contain 1 to 57 characters.

+

Job name. The value can contain up to 57 characters.

desc

@@ -64,7 +64,7 @@

String

Name of a queue. The value can contain 0 to 128 characters.

+

Queue name. The value can contain 0 to 128 characters.

cu_number

@@ -94,14 +94,44 @@

Number of parallel operations selected for a job.

checkpoint_enabled

+

No

+

Boolean

+

Whether to enable the automatic job snapshot function. Options:

+
  • true: Enable this function.
  • false: Do not enable this function.
  • Default value: false.
+

checkpoint_mode

+

No

+

Integer

+

Snapshot mode. Options:

+
  • 1: ExactlyOnce, meaning data is consumed only once.
  • 2: AtLeastOnce, meaning data is consumed at least once.
+

Default value: 1.

+

checkpoint_interval

+

No

+

Integer

+

Snapshot interval. The unit is second. The default value is 10.

+

log_enabled

No

Boolean

Whether to enable the job log function.

-
  • true: indicates to enable the job log function.
  • false: indicates to disable the job log function.
  • Default value: false
+

Whether to enable the job log function. Options:

+
  • true: Enable this function.
  • false: Do not enable this function.
  • Default value: false.

obs_bucket

@@ -119,7 +149,7 @@

String

SMN topic. If a job fails, the system will send a message to users subscribed to the SMN topic.

+

SMN topic. If a job fails, the system will send a message to users subscribed to this SMN topic.

main_class

@@ -264,8 +294,8 @@

Resource configuration version. The value can be v1 or v2. The default value is v1.

Compared with the v1 template, the v2 template does not support the setting of the number of CUs. The v2 template supports the setting of Job Manager Memory and Task Manager Memory.

-

v1: applicable to Flink 1.12, 1.13, and 1.15

-

v2: applicable to Flink 1.13, 1.15, and 1.17.

+

v1: applicable to Flink 1.12 and 1.15.

+

v2: applicable to Flink 1.15 and 1.17.

You are advised to use the parameter settings of v2.
-
- - + + + + + + + + + + + + + + + - - @@ -359,7 +389,8 @@ - - @@ -931,7 +931,7 @@ @@ -1089,7 +1089,7 @@

Example Response

{
     "is_success": "true",
-    "message": "Querying of the job list succeeds.",
+    "message": "Jobs are successfully listed.",
     "job_list": {
         "total_count": 26,
         "jobs": [
diff --git a/docs/dli/api-ref/dli_02_0235.html b/docs/dli/api-ref/dli_02_0235.html
index f343dc1df..bb2e32efd 100644
--- a/docs/dli/api-ref/dli_02_0235.html
+++ b/docs/dli/api-ref/dli_02_0235.html
@@ -57,7 +57,7 @@
- - - - - @@ -350,9 +350,9 @@ - - @@ -426,7 +426,7 @@ - - - @@ -484,7 +484,7 @@ - - @@ -582,7 +582,7 @@ - @@ -650,6 +650,81 @@ + + + + + + + + + + + +
Table 3 tags parameter

Parameter

+
@@ -312,7 +342,7 @@ - @@ -389,7 +419,8 @@ - - -
Table 3 tags parameters

Parameter

Mandatory

String

Tag key.

+

Tag value.

NOTE:

A tag value can contain a maximum of 43 characters. Special characters (=*<>\|) are not allowed, and the value cannot start with a space.

double

Number of CPU cores available for JobManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

Number of CPU cores available for JobManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the JobManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

memory

@@ -421,7 +452,8 @@

double

Number of CPU cores available for TaskManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

Number of CPU cores available for TaskManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the TaskManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

memory

@@ -454,7 +486,7 @@

String

Indicates whether the request is successfully executed. Value true indicates that the request is successfully executed.

+

Whether the request is successfully executed. Value true indicates that the request is successfully executed.

message

@@ -580,7 +612,7 @@
-

Error Codes

If an error occurs when this API is invoked, the system does not return the result similar to the preceding example, but returns the error code and error information. For details, see Error Codes.

+

Error Codes

If an error occurs when this API is called, the system does not return the result similar to the preceding example, but returns an error code and error message. For details, see Error Codes.

diff --git a/docs/dli/api-ref/dli_02_0231.html b/docs/dli/api-ref/dli_02_0231.html index 034fadf0d..c0370e508 100644 --- a/docs/dli/api-ref/dli_02_0231.html +++ b/docs/dli/api-ref/dli_02_0231.html @@ -55,7 +55,7 @@

String

Name of the job. Length range: 0 to 57 characters.

+

Job name. Length range: 0 to 57 characters.

desc

@@ -100,7 +100,37 @@

Integer

Number of parallel operations selected for a job. The default value is 1.

+

Number of parallel operations selected for a job. The default value is 1.

+

checkpoint_enabled

+

No

+

Boolean

+

Whether to enable the automatic job snapshot function. Options:

+
  • true: Enable this function.
  • false: Do not enable this function.
  • Default value: false.
+

checkpoint_mode

+

No

+

Integer

+

Snapshot mode. Options:

+
  • 1: ExactlyOnce, meaning data is consumed only once.
  • 2: AtLeastOnce, meaning data is consumed at least once.
+

Default value: 1.

+

checkpoint_interval

+

No

+

Integer

+

Snapshot interval. The unit is second. The default value is 10.

log_enabled

@@ -109,8 +139,8 @@

Boolean

Whether to enable the job log function.

-
  • true: indicates to enable the job log function.
  • false: indicates to disable the job log function.
  • Default value: false
+

Whether to enable the job log function. Options:

+
  • true: Enable this function.
  • false: Do not enable this function.
  • Default value: false.

obs_bucket

@@ -128,7 +158,7 @@

String

SMN topic. If a job fails, the system will send a message to users subscribed to the SMN topic.

+

SMN topic. If a job fails, the system will send a message to users subscribed to this SMN topic.

main_class

@@ -272,8 +302,8 @@

Resource configuration version. The value can be v1 or v2. The default value is v1.

Compared with the v1 template, the v2 template does not support the setting of the number of CUs. The v2 template supports the setting of Job Manager Memory and Task Manager Memory.

-

v1: applicable to Flink 1.12, 1.13, and 1.15.

-

v2: applicable to Flink 1.13, 1.15, and 1.17.

+

v1: applicable to Flink 1.12 and 1.15.

+

v2: applicable to Flink 1.15 and 1.17.

You are advised to use the parameter settings of v2.

double

Number of CPU cores available for JobManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

Number of CPU cores available for JobManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the JobManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

memory

@@ -391,7 +422,8 @@

double

Number of CPU cores available for TaskManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

Number of CPU cores available for TaskManager. The default value is 1.0 CPU core, with a minimum of no less than 0.5 CPU cores.

+

If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the TaskManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

memory

diff --git a/docs/dli/api-ref/dli_02_0233.html b/docs/dli/api-ref/dli_02_0233.html index 75f771162..f0936474b 100644 --- a/docs/dli/api-ref/dli_02_0233.html +++ b/docs/dli/api-ref/dli_02_0233.html @@ -129,19 +129,19 @@

Example Response

[
     {
         "is_success": "true",
-        "message": "The request for submitting DLI jobs is delivered successfully."
+        "message": "The request for submitting DLI jobs is successfully delivered."
     },
     {
         "is_success": "true",
-        "message": "The request for submitting DLI jobs is delivered successfully."
+        "message": "The request for submitting DLI jobs is successfully delivered."
     },
     {
         "is_success": "true",
-        "message": "The request for submitting DLI jobs is delivered successfully."
+        "message": "The request for submitting DLI jobs is successfully delivered."
     },
     {
         "is_success": "true",
-        "message": "The request for submitting DLI jobs is delivered successfully."
+        "message": "The request for submitting DLI jobs is successfully delivered."
     }
 ]
diff --git a/docs/dli/api-ref/dli_02_0234.html b/docs/dli/api-ref/dli_02_0234.html index 955f43cfc..c3d79fa2e 100644 --- a/docs/dli/api-ref/dli_02_0234.html +++ b/docs/dli/api-ref/dli_02_0234.html @@ -382,7 +382,7 @@

Definition

Job status.

Range

-
  • job_init: The job is in the draft state.
  • job_submitting: The job is being submitted.
  • job_submit_fail: The job fails to be submitted.
  • job_running: The job is running.
  • job_running_exception: The job is running abnormally.
  • job_downloading: The job is being downloaded.
  • job_idle: The job is idle.
  • job_canceling: The job is being stopped.
  • job_cancel_success: The job has been stopped.
  • job_cancel_fail: The job fails to be stopped.
  • job_savepointing: A savepoint is being created.
  • job_arrearage_stopped: The job is stopped due to arrears.
  • job_arrearage_recovering: The job in arrears is being restored.
  • job_finish: The job is completed.
+
  • job_init: The job is in the draft state.
  • job_submitting: The job is being submitted.
  • job_submit_fail: The job fails to be submitted.
  • job_running: The job is running.
  • job_running_exception: The job is running abnormally.
  • job_downloading: The job is being downloaded.
  • job_idle: The job is idle.
  • job_canceling: The job is being stopped.
  • job_cancel_success: The job has been stopped.
  • job_cancel_fail: The job fails to be stopped.
  • job_savepointing: A savepoint is being created.
  • job_arrearage_stopped: The job is stopped due to arrears.
  • job_arrearage_recovering: The job in arrears is being restored.
  • job_finish: The job is completed.

status_desc

@@ -453,7 +453,7 @@

Boolean

Definition

-

Whether the flow diagram can be edited. Value true indicates that the flow diagram can be edited, and false indicates that the flow diagram cannot be edited.

+

Whether the stream graph of a job can be edited. Value true indicates that the stream graph can be edited, and false indicates that the stream graph cannot be edited.

Range

None

Boolean

Definition

-

Whether to enable flow diagram editing. The default value is false.

+

Whether to enable stream graph editing. The default value is false.

Range

None

String

Indicates whether the request is successfully executed. Value true indicates that the request is successfully executed.

+

Whether the request is successfully executed. Value true indicates that the request is successfully executed.

message

@@ -107,7 +107,7 @@

String

Name of the job. Length range: 0 to 57 characters.

+

Job name. Length range: 0 to 57 characters.

desc

@@ -182,7 +182,7 @@

String

Name of a queue. Length range: 1 to 128 characters.

+

Queue name. Length range: 1 to 128 characters.

project_id

@@ -300,7 +300,7 @@

Boolean

Whether the stream graph of a job can be edited. Value true indicates that the flow diagram can be edited, and false indicates that the flow diagram cannot be edited.

+

Whether the stream graph of a job can be edited. Value true indicates that the stream graph can be edited, and false indicates that the stream graph cannot be edited.

has_savepoint

@@ -316,7 +316,7 @@

No

Array of Strings

+

Array of strings

List of edge computing group IDs. Use commas (,) to separate multiple IDs.

Boolean

Whether to enable the automatic job snapshot function.

-
  • true: The automatic job snapshot function is enabled.
  • false: The automatic job snapshot function is disabled.
-

The default value is false.

+

Whether to enable the automatic job snapshot function. Options:

+
  • true: Enable this function.
  • false: Do not enable this function.
+

Default value: false.

checkpoint_interval

@@ -371,8 +371,8 @@

String

Snapshot mode. Options:

-
  • exactly_once: indicates that data is processed only once.
  • at_least_once: indicates that data is processed at least once.
-

The default value is exactly_once.

+
  • exactly_once: Data is consumed only once.
  • at_least_once: Data is consumed at least once.
+

Default value: exactly_once.

log_enabled

@@ -406,7 +406,7 @@

No

Array of Strings

+

Array of strings

List of edge computing group IDs. Use commas (,) to separate multiple IDs.

Boolean

Whether to enable flow diagram editing. The default value is false.

+

Whether stream graph editing is enabled. The default value is false.

graph_editor_data

@@ -435,7 +435,7 @@

String

Data of flow diagram editing. The default value is null.

+

Edited stream graph data. The default value is null.

executor_number

@@ -473,7 +473,7 @@

Integer

Number of concurrent jobs set by a user. This parameter is valid only when show_detail is set to true.

+

Number of parallel jobs set by a user. This parameter is valid only when show_detail is set to true.

  • Minimum value: 1
  • Maximum value: 2000

The default value is 1.

String

SMN topic name. If a job fails, the system will send a message to users subscribed to this SMN topic.

+

SMN topic. If a job fails, the system will send a message to users subscribed to this SMN topic.

restart_when_exception

@@ -573,7 +573,7 @@

No

Array of Strings

+

Array of strings

Name of the package that has been uploaded to the DLI resource management system. This parameter is used to customize other dependency packages.

No

Array of Strings

+

Array of strings

Name of the resource package that has been uploaded to the DLI resource management system. This parameter is used to customize dependency files.

Number of actually used CUs. The default value is 0, indicating that the value of cu_number is used.

resource_config_version

+

No

+

String

+

Resource configuration version. The value can be v1 or v2. The default value is v1.

+

Compared with the v1 template, the v2 template does not support the setting of the number of CUs. The v2 template supports the setting of Job Manager Memory and Task Manager Memory.

+

v1: applicable to Flink 1.12 and 1.15.

+

v2: applicable to Flink 1.15 and 1.17.

+

You are advised to use the parameter settings of v2.

+

resource_config

+

No

+

Object

+

Resource configuration of a Flink job. For detailed parameter descriptions, refer to Table 5.

+

When the resource configuration version is v2, the configuration takes effect; when the resource configuration version is v1, the configuration is invalid.

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + +
Table 5 resource_config parameters

Parameter

+

Mandatory

+

Type

+

Description

+

max_slot

+

No

+

integer

+

Number of parallel tasks that a single TaskManager can support. Each task slot can execute one task in parallel. Increasing task slots enhances the parallel processing capacity of the TaskManager but also increases resource consumption.

+

The number of task slots is linked to the CPU count of the TaskManager since each CPU can offer one task slot.

+

By default, a single TM slot is set to 1. The minimum parallelism must not be less than 1.

+

parallel_number

+

No

+

integer

+

Number of tasks concurrently executed by each operator in a job. The default value is 1.

+

jobmanager_resource_spec

+

No

+

Object

+

JobManager resource specifications. For details about the parameters, see Table 5.

+

taskmanager_resource_spec

+

No

+

Object

+

TaskManager resource specifications. For details about the parameters, see Table 6.

+
@@ -713,9 +788,9 @@ } -

Status Codes

Table 5 describes the status codes.

+

Status Codes

Table 6 describes status codes.

-
- - diff --git a/docs/dli/api-ref/dli_02_0254.html b/docs/dli/api-ref/dli_02_0254.html index d3f32ab27..6f5c5d5e0 100644 --- a/docs/dli/api-ref/dli_02_0254.html +++ b/docs/dli/api-ref/dli_02_0254.html @@ -124,7 +124,7 @@

Example Response

{
     "is_success": true,
-    "message": "The job is exported successfully.",
+    "message": "The job is successfully exported.",
     "zip_file": ["obs-test/aggregate_1582677879475.zip"]
 }
diff --git a/docs/dli/api-ref/dli_02_0255.html b/docs/dli/api-ref/dli_02_0255.html index 48fa83e00..92b411573 100644 --- a/docs/dli/api-ref/dli_02_0255.html +++ b/docs/dli/api-ref/dli_02_0255.html @@ -155,7 +155,7 @@

Example Response

{
     "is_success": true,
-    "message": "The job is imported successfully.",
+    "message": "The job is successfully imported.",
     "job_mapping": [
         {
             "old_job_id": "100",
diff --git a/docs/dli/api-ref/dli_02_0312.html b/docs/dli/api-ref/dli_02_0312.html
index a70ac899a..c9470553a 100644
--- a/docs/dli/api-ref/dli_02_0312.html
+++ b/docs/dli/api-ref/dli_02_0312.html
@@ -122,7 +122,7 @@
diff --git a/docs/dli/api-ref/dli_02_0316.html b/docs/dli/api-ref/dli_02_0316.html index d084aa29d..055de8e65 100644 --- a/docs/dli/api-ref/dli_02_0316.html +++ b/docs/dli/api-ref/dli_02_0316.html @@ -216,7 +216,7 @@ 
{
   "job_type": "flink_opensource_sql_job",
   "graph_type": "job_graph",
-  "sql_body": "create table orders(\r\n  name string,\r\n  num int\r\n) with (\r\n  'connector' = 'datagen',\r\n  'rows-per-second' = '1', --Generates a data record per second.\r\n  'fields.name.kind' = 'random', --Specify a random generator for the user_id field.\r\n  'fields.name.length' = '5' --Limit the length of user_id to 3.\r\n);\r\n \r\nCREATE TABLE sink_table (\r\n  name string,\r\n  num int\r\n) WITH (\r\n   'connector' = 'print'\r\n);\r\nINSERT into sink_table SELECT * FROM orders;",
+  "sql_body": "create table orders(\r\n  name string,\r\n  num int\r\n) with (\r\n  'connector' = 'datagen',\r\n  'rows-per-second' = '1', --Generate a data record per second.\r\n  'fields.name.kind' = 'random', --Specify a random generator for the user_id field.\r\n  'fields.name.length' = '5' --Limit the length of user_id to 3.\r\n);\r\n \r\nCREATE TABLE sink_table (\r\n  name string,\r\n  num int\r\n) WITH (\r\n   'connector' = 'print'\r\n);\r\nINSERT into sink_table SELECT * FROM orders;",
   "cu_number": 2,
   "manager_cu_number": 1,
   "parallel_number": 2,
diff --git a/docs/dli/api-ref/dli_02_0326.html b/docs/dli/api-ref/dli_02_0326.html
index 16a10f2e5..38c0add74 100644
--- a/docs/dli/api-ref/dli_02_0326.html
+++ b/docs/dli/api-ref/dli_02_0326.html
@@ -46,7 +46,7 @@
- @@ -127,6 +127,17 @@

If not set, the default is the standard version.

+ + + + +
Table 5 Status codes

Status Code

+
@@ -735,7 +810,7 @@
Table 6 Status codes

Status Code

Description

-

Error Codes

If an error occurs when this API is invoked, the system does not return the result similar to the preceding example, but returns the error code and error information. For details, see Error Codes.

+

Error Codes

If an error occurs when this API is called, the system does not return the result similar to the preceding example, but returns an error code and error message. For details, see Error Codes.

diff --git a/docs/dli/api-ref/dli_02_0236.html b/docs/dli/api-ref/dli_02_0236.html index 80c130220..4c5afccde 100644 --- a/docs/dli/api-ref/dli_02_0236.html +++ b/docs/dli/api-ref/dli_02_0236.html @@ -164,7 +164,7 @@

Example Response

{
     "is_success": "true",
-  "message": "Querying the job execution graph succeeds.",
+    "message": "The job execution stream graph is successfully queried.",
     "execute_graph": {
         "jid": "4e966f43f2c90b0e1bf3188ecf55504b",
         "name": "",
diff --git a/docs/dli/api-ref/dli_02_0241.html b/docs/dli/api-ref/dli_02_0241.html
index 1d8606507..cdf82feb5 100644
--- a/docs/dli/api-ref/dli_02_0241.html
+++ b/docs/dli/api-ref/dli_02_0241.html
@@ -126,7 +126,7 @@
 }

Example Response

[{"is_success":"true",
-"message": "The request for stopping DLI jobs is delivered successfully."}]
+"message": "The request for stopping DLI jobs is successfully delivered."}]

Status Codes

Table 5 describes status codes.

diff --git a/docs/dli/api-ref/dli_02_0242.html b/docs/dli/api-ref/dli_02_0242.html index d1af7584a..739a324b0 100644 --- a/docs/dli/api-ref/dli_02_0242.html +++ b/docs/dli/api-ref/dli_02_0242.html @@ -79,7 +79,7 @@

Example Response

{
     "is_success": "true",
-    "message": "The job is deleted successfully.",
+    "message": "The job is successfully deleted.",
 }

Status Codes

Table 3 describes status codes.

diff --git a/docs/dli/api-ref/dli_02_0243.html b/docs/dli/api-ref/dli_02_0243.html index b530234d2..5227ee56d 100644 --- a/docs/dli/api-ref/dli_02_0243.html +++ b/docs/dli/api-ref/dli_02_0243.html @@ -116,7 +116,7 @@

Example Response

[{
     "is_success": "true",
-    "message": "The job is deleted successfully.",
+    "message": "The job is successfully deleted.",
 }]

Status Codes

Table 5 describes status codes.

diff --git a/docs/dli/api-ref/dli_02_0245.html b/docs/dli/api-ref/dli_02_0245.html index 9a8333cf1..44527df25 100644 --- a/docs/dli/api-ref/dli_02_0245.html +++ b/docs/dli/api-ref/dli_02_0245.html @@ -46,7 +46,7 @@

String

Template name. The value can contain 1 to 64 characters.

+

Template name. The value can contain up to 64 characters.

desc

@@ -240,7 +240,7 @@

Example Response

{
     "is_success": true,
-    "message": "A template is created successfully.",
+    "message": "A template is successfully created.",
     "template": {
         "template_id": 0,
         "name": "IoT_example",
diff --git a/docs/dli/api-ref/dli_02_0246.html b/docs/dli/api-ref/dli_02_0246.html
index 39d1b4312..f3e2deeab 100644
--- a/docs/dli/api-ref/dli_02_0246.html
+++ b/docs/dli/api-ref/dli_02_0246.html
@@ -122,7 +122,7 @@

Example Response

{
     "is_success": "true",
-    "message": "The template is updated successfully.",
+    "message": "The template is successfully updated.",
 }

Status Codes

Table 4 describes status codes.

diff --git a/docs/dli/api-ref/dli_02_0247.html b/docs/dli/api-ref/dli_02_0247.html index fd98826e0..4a6ee9556 100644 --- a/docs/dli/api-ref/dli_02_0247.html +++ b/docs/dli/api-ref/dli_02_0247.html @@ -110,7 +110,7 @@

Example Response

{
     "is_success": "true",
-    "message": "The template is deleted successfully.",
+    "message": "The template is successfully deleted.",
     "template": {
         "template_id": 2
     }
diff --git a/docs/dli/api-ref/dli_02_0248.html b/docs/dli/api-ref/dli_02_0248.html
index 0b83edc05..50510165a 100644
--- a/docs/dli/api-ref/dli_02_0248.html
+++ b/docs/dli/api-ref/dli_02_0248.html
@@ -248,14 +248,14 @@

Example Response

{
     "is_success": "true",
-    "message": "The template list is obtained successfully.",
+    "message": "The template list is successfully obtained.",
     "template_list": {
         "total_count": 2,
         "templates": [
             {
                 "template_id": 2,
                 "name": "updatetest",
-                 "desc": "Example of quick start",
+                "desc": "Example of quick start",
                 "create_time": 1578748092000,
                 "update_time": 1578748092000,
                 "sql_body": "select * from source_table",
diff --git a/docs/dli/api-ref/dli_02_0250.html b/docs/dli/api-ref/dli_02_0250.html
index f172fe796..5cbc35b91 100644
--- a/docs/dli/api-ref/dli_02_0250.html
+++ b/docs/dli/api-ref/dli_02_0250.html
@@ -210,7 +210,7 @@

Array of Strings

Path. This parameter is displayed only for non-DLI tables.

+

Path. This parameter is returned only for non-DLI tables.

last_ddl_time

diff --git a/docs/dli/api-ref/dli_02_0252.html b/docs/dli/api-ref/dli_02_0252.html index 1acf27231..15dc0410f 100644 --- a/docs/dli/api-ref/dli_02_0252.html +++ b/docs/dli/api-ref/dli_02_0252.html @@ -29,7 +29,7 @@

Constraints

None

Range

-

The value can contain 1 to 64 characters. Only letters and digits are allowed.

+

The value can contain up to 64 characters. Only letters and digits are allowed.

Default Value

None

String

Job type. Options: DDL, DCL, IMPORT, EXPORT, QUERY, INSERT, DATA_MIGRATION, UPDATE, DELETE, RESTART_QUEUE and SCALE_QUEUE.

-

Currently, you can only query execution results of jobs of the QUERY type.

+

Currently, you can only query execution results of jobs of the QUERY type.

  • DDL: jobs that create, modify, and delete metadata files
  • DCL: jobs that grant and revoke permissions

    When job_type is set to DCL, the operation is synchronous.

  • IMPORT: jobs that import external data into the database
  • EXPORT: jobs that export data to an external database
  • QUERY: jobs that run query statements
  • INSERT: jobs that add new data to tables
  • DATA_MIGRATION: data migration jobs
  • UPDATE: jobs that update table data
  • DELETE: jobs that delete data from tables
  • RESTART_QUEUE: jobs that restart specified queues
  • SCALE_QUEUE: jobs that scale in or out specified queues

String

Name of a new elastic resource pool. Only digits, letters, and underscores (_) are allowed, but the value cannot contain only digits or start with an underscore (_). The value can contain 1 to 128 characters.

+

Name of a new elastic resource pool. Only digits, letters, and underscores (_) are allowed, but the value cannot contain only digits or start with an underscore (_). The value can contain up to 128 characters.

NOTE:

If the name contains uppercase letters, the system automatically converts them to lowercase letters.

ipv6_enable

+

No

+

Boolean

+

Whether to enable IPv6

+

Once enabled, the resource pool will be automatically assigned an IPv6 CIDR block. Custom IPv6 CIDR blocks are not currently supported. IPv6 cannot be disabled once enabled. The default value is false.

+
  • true: Enable IPv6.
  • false: Do not enable IPv6.
+
diff --git a/docs/dli/api-ref/dli_02_0327.html b/docs/dli/api-ref/dli_02_0327.html index 03dab791f..03a9f1e19 100644 --- a/docs/dli/api-ref/dli_02_0327.html +++ b/docs/dli/api-ref/dli_02_0327.html @@ -321,7 +321,26 @@

Map<string, string>

Label used to identify the elastic resource pool. Currently, only the developer label is supported. The value is "label": {"billing_spec_code":"developer"}.

+

Identifies the type of the elastic resource pool.

+

ipv6_enable

+

No

+

Boolean

+

Whether to enable IPv6

+
  • true: Enable IPv6.
  • false: Do not enable IPv6.
+

ipv6_cidr_in_subnet

+

No

+

String

+

IPv6 subnet CIDR block

id

diff --git a/docs/dli/api-ref/dli_02_0336.html b/docs/dli/api-ref/dli_02_0336.html index 2222c2824..f2e353c3b 100644 --- a/docs/dli/api-ref/dli_02_0336.html +++ b/docs/dli/api-ref/dli_02_0336.html @@ -31,7 +31,7 @@

String

Elastic resource pool name

-

The value can contain 1 to 128 characters.

+

The value can contain up to 128 characters.

String

Scaling status

-

Enumerated values:

+

Options:

  • success
  • fail

String

Route name. The value can contain 1 to 64 characters.

+

Route name. The value can contain up to 64 characters.

cidr

diff --git a/docs/dli/api-ref/en-us_topic_0206808210.html b/docs/dli/api-ref/en-us_topic_0206808210.html new file mode 100644 index 000000000..e875f09e0 --- /dev/null +++ b/docs/dli/api-ref/en-us_topic_0206808210.html @@ -0,0 +1,949 @@ + + +

Permissions Policies and Supported Actions

+

This section describes fine-grained permissions management for your DLI. If your account does not need individual IAM users, then you may skip this section.

+

By default, new IAM users do not have permissions assigned. You need to add them to one or more groups, and attach permissions policies or roles to these groups. Users inherit permissions from the groups to which they are added After authorization, the users can perform specified operations.

+

+
  • Roles are a type of coarse-grained authorization mechanism that defines permissions related to user responsibilities.
  • Policies define API-based permissions for operations on specific resources under certain conditions, allowing for more fine-grained, secure access control of cloud resources.
+

Policy-based authorization is useful if you want to allow or deny the access to an API.

+
+

An account has all the permissions required to call all APIs, but IAM users must be assigned the required permissions. The permissions required for calling an API are determined by the actions supported by the API. Only users who have been granted permissions allowing the actions can call the API successfully. For example, if an IAM user needs to create buckets using an API, the user must have been granted permissions that allow the dli:queue:create_queue action.

+

Supported Actions

DLI provides system-defined policies that can be directly used in IAM. You can also create custom policies to supplement system-defined policies for more refined access control. Operations supported by policies are specific to APIs. The following are common concepts related to policies:

+
  • Permissions: Allow or deny operations on specified resources under specific conditions.
  • APIs: RESTful APIs that can be called in a custom policy.
  • Actions: added to a custom policy to control permissions for specific operations.
  • Dependencies: actions which a specific action depends on. When allowing an action for a user, you also need to allow any existing action dependencies for that user.
  • IAM or enterprise projects: Type of projects for which an action will take effect. Policies that contain actions supporting both IAM and enterprise projects can be assigned to user groups and take effect in both IAM and Enterprise Management. Policies that only contain actions supporting IAM projects can be assigned to user groups and only take effect for IAM. Such policies will not take effect if they are assigned to user groups in Enterprise Project.

    +

    Only the APIs listed in Table 1 support custom authorization.

    +
+

The check mark (√) indicates that an action takes effect. The cross mark (x) indicates that an action does not take effect.

+
+

DLI supports the following actions that can be defined in custom policies:

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 1 Actions

Permission

+

API

+

Action

+

IAM Project

+

Enterprise Project

+

Creating a queue

+

POST /v1.0/{project_id}/queues

+

dli:queue:createQueue

+

√

+

√

+

Deleting a queue

+

DELETE /v1.0/{project_id}/queues/{queue_name}

+

dli:queue:dropQueue

+

√

+

√

+

Submitting a job

+

POST /v1.0/{project_id}/jobs/submit-job

+

dli:queue:submitJob

+

√

+

×

+

Canceling a job

+

DELETE /v1.0/{project_id}/jobs/{job_id}

+

dli:queue:cancelJob

+

√

+

×

+

Viewing queue permissions of other users

+

GET /v1.0/{project_id}/queues/{queue_name}/users

+

dli:queue:showPrivileges

+

√

+

×

+

Restarting a queue

+

PUT /v1.0/{project_id}/queues/{queue_name}/action

+

dli:queue:restart

+

√

+

×

+

Scaling out/in a queue

+

PUT /v1.0/{project_id}/queues/{queue_name}/action

+

dli:queue:scaleQueue

+

√

+

×

+

Granting elastic resource pool permissions

+

PUT /v1.0/{project_id}/user-authorization

+

dli:elasticresourcepool:grantPrivilege

+

√

+

×

+

Revoking elastic resource pool permissions

+

PUT /v1.0/{project_id}/user-authorization

+

dli:elasticresourcepool:revokePrivilege

+

√

+

×

+

Obtaining information about all catalogs in a project

+

GET /v3/{project_id}/catalogs

+

dli:catalog:list

+

√

+

×

+

Querying details about a specified catalog

+

GET /v3/{project_id}/catalogs/{catalog_name}

+

dli:catalog:get

+

√

+

×

+

Binding a catalog

+

POST /v3/{project_id}/catalogs/action

+

dli:catalog:bind

+

√

+

×

+

Unbinding a catalog

+

POST /v3/{project_id}/catalogs/action

+

dli:catalog:unbind

+

√

+

×

+

Creating a database

+

POST /v1.0/{project_id}/databases

+

dli:database:createDatabase

+

√

+

×

+

Deleting a database

+

DELETE /v1.0/{project_id}/databases/{database_name}

+

dli:database:dropDatabase

+

√

+

×

+

Modifying database configuration

+

POST /v1.0/{project_id}/jobs/submit-job

+

dli:database:alterDatabaseProperties

+

√

+

×

+

Explaining the SQL statement as an execution plan

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:explain

+

√

+

×

+

Creating a table

+

POST /v1.0/{project_id}/databases/{database_name}/tables

+

dli:database:createTable

+

√

+

×

+

Creating a view

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:createView

+

√

+

×

+

Creating a function

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:createFunction

+

√

+

×

+

Describing a function

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:describeFunction

+

√

+

×

+

Deleting a function

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:dropFunction

+

√

+

×

+

Displaying a function

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:showFunctions

+

√

+

×

+

Creating a role

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:createRole

+

√

+

×

+

Deleting a role

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:dropRole

+

√

+

×

+

Displaying a role

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:showRoles

+

√

+

×

+

Displaying all roles

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:showAllRoles

+

√

+

×

+

Binding a role

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:grantRole

+

√

+

×

+

Unbinding a role

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:revokeRole

+

√

+

×

+

Displaying the binding relationships between all roles and users

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:database:showUsers

+

√

+

×

+

Viewing database permissions of other users

+

GET /v1.0/{project_id}/databases/{database_name}/users

+

dli:database:showPrivileges

+

√

+

×

+

Displaying a database

+

GET /v1.0/{project_id}/databases

+

dli:database:displayDatabase

+

√

+

×

+

Displaying all databases

+

GET /v1.0/{project_id}/databases

+

dli:database:displayAllDatabases

+

√

+

×

+

Displaying all tables

+

GET /v1.0/{project_id}/databases

+

dli:database:displayAllTables

+

√

+

×

+

Granting database permissions to a specified user

+

PUT /v1.0/{project_id}/user-authorization

+

dli:database:grantPrivilege

+

√

+

×

+

Removing database permissions of a specified user

+

PUT /v1.0/{project_id}/user-authorization

+

dli:database:revokePrivilege

+

√

+

×

+

Deleting a table

+

DELETE /v1.0/{project_id}/databases/{database_name}/tables/{table_name}

+

dli:table:dropTable

+

√

+

×

+

Displaying the table structure

+

GET /v1.0/{project_id}/databases/{database_name}/tables/{table_name}

+

dli:table:describeTable

+

√

+

×

+

Querying a table

+

GET /v1.0/{project_id}/databases/{database_name}/tables/{table_name}/preview

+

dli:table:select

+

√

+

×

+

Displaying table configuration

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:showTableProperties

+

√

+

×

+

Displaying the table creation statement

+

GET /v1.0/{project_id}/databases/{database_name}/tables/{table_name}/preview

+

dli:table:showCreateTable

+

√

+

×

+

Displaying all partitions

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:showPartitions

+

√

+

×

+

Setting table configuration

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:alterTableSetProperties

+

√

+

×

+

Adding a column

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:alterTableAddColumns

+

√

+

×

+

Adding partitions to a partitioned table

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:alterTableAddPartition

+

√

+

×

+

Renaming a table partition

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:alterTableRenamePartition

+

√

+

×

+

Deleting partitions from a partitioned table

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:alterTableDropPartition

+

√

+

×

+

Restoring a table partition

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:alterTableRecoverPartition

+

√

+

×

+

Renaming a table

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:alterTableRename

+

√

+

×

+

Setting the partition path

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:alterTableSetLocation

+

√

+

×

+

Inserting table data

+

POST /v1.0/{project_id}/jobs/submit-job, statement invoking

+

dli:table:insertIntoTable

+

√

+

×

+

Rewriting table data

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:insertOverwriteTable

+

√

+

×

+

Viewing table permissions of other users

+

GET /v1.0/{project_id}/databases/{database_name}/tables/{table_name}/users

+

dli:table:showPrivileges

+

√

+

×

+

Clearing a table

+

POST /v1.0/{project_id}/jobs/submit-job

+

dli:table:truncateTable

+

√

+

×

+

Updating a table

+

POST /v1.0/{project_id}/jobs/submit-job

+

dli:table:update

+

√

+

×

+

Deleting table data

+

POST /v1.0/{project_id}/jobs/submit-job

+

dli:table:delete

+

√

+

×

+

Modifying column information

+

POST /v1.0/{project_id}/jobs/submit-job

+

dli:table:alterTableChangeColumn

+

√

+

×

+

Deleting a column

+

POST /v1.0/{project_id}/jobs/submit-job

+

dli:table:alterTableDropColumns

+

√

+

×

+

Displaying data segments

+

POST /v1.0/{project_id}/jobs/submit-job

+

dli:table:showSegments

+

√

+

×

+

Merging data segments

+

POST /v1.0/{project_id}/jobs/submit-job

+

dli:table:compaction

+

√

+

×

+

Modifying a view

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:table:alterView

+

√

+

×

+

Displaying the table structure

+

POST /v1.0/{project_id}/jobs/submit-job, executing corresponding SQL statements

+

dli:table:describeTable

+

√

+

×

+

Granting data table permissions to a specified user

+

PUT /v1.0/{project_id}/user-authorization

+

dli:table:grantPrivilege

+

√

+

×

+

Removing data table permissions of a specified user

+

PUT /v1.0/{project_id}/user-authorization

+

dli:table:revokePrivilege

+

√

+

×

+

Viewing the security authentication information permission list of other users

+

GET /v1.0/{project_id}/datasource/auth-infos/{auth_name}/users

+

dli:datasourceauth:showPrivileges

+

√

+

×

+

Using security authentication information

+

POST /v1.0/{project_id}/jobs/submit-job

+

dli:datasourceauth:useAuth

+

√

+

×

+

Deleting security authentication information

+

DELETE /v2.0/{project_id}/datasource/auth-infos/{auth_info_name}

+

dli:datasourceauth:dropAuth

+

√

+

×

+

Updating security authentication information

+

PUT /v2.0/{project_id}/datasource/auth-infos

+

dli:datasourceauth:updateAuth

+

√

+

×

+

Granting security authentication permissions to a specified user

+

PUT /v1.0/{project_id}/user-authorization

+

dli:datasourceauth:grantPrivilege

+

√

+

×

+

Removing security authentication permissions of a specified user

+

PUT /v1.0/{project_id}/user-authorization

+

dli:datasourceauth:revokePrivilege

+

√

+

×

+

Removing job permissions of a specified user

+

PUT /v1.0/{{project_id}}/authorization

+

dli:jobs:revokePrivilege

+

√

+

×

+

Querying a Column

+

POST /v1.0/{project_id}/jobs/submit-job, SQL statement invoking

+

dli:column:select

+

√

+

×

+

Granting permissions to a specified user queue

+

PUT /v1.0/{project_id}/user-authorization

+

dli:column:grantPrivilege

+

√

+

×

+

Removing permissions of a specified user queue

+

PUT /v1.0/{project_id}/user-authorization

+

dli:column:revokePrivilege

+

√

+

×

+

Listing Flink jobs

+

GET /v1.0/{project_id}/streaming/jobs

+

dli:jobs:listAll

+

√

+

×

+

Querying Flink job details

+

GET /v1.0/{project_id}/streaming/jobs/{job_id}

+

dli:jobs:get

+

√

+

×

+

Creating a Flink SQL job

+

POST /v1.0/{project_id}/streaming/sql-jobs

+

dli:jobs:create

+

√

+

×

+

Creating a Flink Jar job

+

POST /v1.0/{project_id}/streaming/flink-jobs

+

dli:jobs:create

+

√

+

×

+

Updating a Flink SQL job

+

PUT /v1.0/{project_id}/streaming/sql-jobs/{job_id}

+

dli:jobs:update

+

√

+

×

+

Updating a Flink Jar job

+

PUT /v1.0/{project_id}/streaming/flink-jobs/{job_id}

+

dli:jobs:update

+

√

+

×

+

Deleting a Flink job

+

DELETE /v1.0/{project_id}/streaming/jobs/{job_id}

+

dli:jobs:delete

+

√

+

×

+

Running Flink jobs in batches

+

POST /v1.0/{project_id}/streaming/jobs/run

+

dli:jobs:start

+

√

+

×

+

Stopping a Flink job

+

POST /v1.0/{project_id}/streaming/jobs/{job_id}/stop

+

dli:jobs:stop

+

√

+

×

+
+
+
+
+ diff --git a/docs/dli/umn/ALL_META.TXT.json b/docs/dli/umn/ALL_META.TXT.json index b6c0ac1a8..9c698ba4d 100644 --- a/docs/dli/umn/ALL_META.TXT.json +++ b/docs/dli/umn/ALL_META.TXT.json @@ -28,17 +28,17 @@ "node_id":"dli_01_0378.xml", "product_code":"dli", "code":"2", - "des":"Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark and Apache Flink ecosystems. It frees you from managing an", + "des":"Data Lake Insight (DLI) is a fully-managed, serverless service for data processing and analytics. It seamlessly combines stream processing, batch processing, and interact", "doc_type":"usermanual", "kw":"What Is Data Lake Insight,Service Overview,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"What Is Data Lake Insight", @@ -49,17 +49,17 @@ "node_id":"dli_07_0007.xml", "product_code":"dli", "code":"3", - "des":"You do not need a background in big data to use DLI for data analysis. You only need to know SQL, and you are good to go. The SQL syntax is fully compatible with the stan", + "des":"DLI offers standard SQL APIs, enabling you to perform massive data query and analysis using only SQL. Its syntax is fully compatible with ANSI SQL 2003 standards.This sig", "doc_type":"usermanual", "kw":"Advantages,Service Overview,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Advantages", @@ -87,24 +87,24 @@ "githuburl":"" }, { - "uri":"dli_07_0005.html", - "node_id":"dli_07_0005.xml", + "uri":"dli_01_0698.html", + "node_id":"dli_01_0698.xml", "product_code":"dli", "code":"5", - "des":"For more notes and constraints on elastic resource pools, see Notes and Constraints on Using an Elastic Resource Pool.For more notes and constraints on using a queue, see", + "des":"This section outlines the key features supported by DLI. For detailed information on regional availability of each feature, you can refer to the console.Before submitting", "doc_type":"usermanual", - "kw":"Notes and Constraints,Service Overview,User Guide", + "kw":"Features,Service Overview,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], - "title":"Notes and Constraints", + "title":"Features", "githuburl":"" }, { @@ -112,17 +112,17 @@ "node_id":"dli_07_0027.xml", "product_code":"dli", "code":"6", - "des":"DLI compute resources are the foundation for job execution. Both DLI's elastic resource pools and queues fall under compute resources. This section introduces the types a", + "des":"When performing big data analysis with DLI, you need to select the appropriate compute resources based on specific business needs. DLI offers a variety of resource types ", "doc_type":"usermanual", "kw":"Compute Resource Types and Specifications,Service Overview,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Compute Resource Types and Specifications", @@ -133,48 +133,69 @@ "node_id":"dli_07_0006.xml", "product_code":"dli", "code":"7", - "des":"If you need to assign different permissions to employees in your enterprise to access your DLI resources, IAM is a good choice for fine-grained permissions management. IA", + "des":"After purchasing DLI resources, you can use IAM to assign different access rights to employees, ensuring proper isolation between roles. IAM provides identity authenticat", "doc_type":"usermanual", "kw":"Permission Management,Service Overview,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Permission Management", "githuburl":"" }, { - "uri":"dli_07_0009.html", - "node_id":"dli_07_0009.xml", + "uri":"dli_07_0005.html", + "node_id":"dli_07_0005.xml", "product_code":"dli", "code":"8", - "des":"A quota limits the quantity of a resource available to users, thereby preventing spikes in the usage of the resource.You can also request for an increased quota if your e", + "des":"For more notes and constraints on elastic resource pools, see Notes and Constraints on Using an Elastic Resource Pool.For more notes and constraints on using a queue, see", "doc_type":"usermanual", - "kw":"Quotas,Service Overview,User Guide", + "kw":"Notes and Constraints,Service Overview,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], - "title":"Quotas", + "title":"Notes and Constraints", + "githuburl":"" + }, + { + "uri":"dli_07_0009.html", + "node_id":"dli_07_0009.xml", + "product_code":"dli", + "code":"9", + "des":"Quotas are enforced for service resources on the platform to prevent unforeseen spikes in resource usage. Quotas can limit the quantity and capacity of resources availabl", + "doc_type":"usermanual", + "kw":"Quota Management,Service Overview,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No;Yes;NO", + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" + } + ], + "title":"Quota Management", "githuburl":"" }, { "uri":"dli_07_0004.html", "node_id":"dli_07_0004.xml", "product_code":"dli", - "code":"9", + "code":"10", "des":"OBS works as the data source and data storage system for DLI, and delivers the following capabilities:Data source: DLI provides an API for you to import data from corresp", "doc_type":"usermanual", "kw":"Related Services,Service Overview,User Guide", @@ -182,10 +203,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Related Services", @@ -195,18 +216,18 @@ "uri":"dli_07_0003.html", "node_id":"dli_07_0003.xml", "product_code":"dli", - "code":"10", - "des":"An elastic resource pool consists of dedicated compute resources where the resources in different pools are completely isolated from each other. Within a single elastic r", + "code":"11", + "des":"Actual CUs: The current allocated resource size of the elastic resource pool, measured in CUs.When no queues exist in the resource pool: The actual CUs equal the minimum ", "doc_type":"usermanual", "kw":"Basic Concepts,Service Overview,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Basic Concepts", @@ -216,7 +237,7 @@ "uri":"dli_01_0220.html", "node_id":"dli_01_0220.xml", "product_code":"dli", - "code":"11", + "code":"12", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Getting Started", @@ -237,7 +258,7 @@ "uri":"dli_01_0002.html", "node_id":"dli_01_0002.xml", "product_code":"dli", - "code":"12", + "code":"13", "des":"DLI can query data stored in OBS. This section describes how to use DLI to submit a SQL job to query OBS data.To illustrate, create a file called sampledata.csv and uploa", "doc_type":"usermanual", "kw":"Submitting a SQL Job to Query OBS Data Using DLI,Getting Started,User Guide", @@ -245,10 +266,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Submitting a SQL Job to Query OBS Data Using DLI", @@ -258,7 +279,7 @@ "uri":"dli_01_0020.html", "node_id":"dli_01_0020.xml", "product_code":"dli", - "code":"13", + "code":"14", "des":"DLI can query data stored in RDS. This section describes how to use DLI to submit a SQL job to query RDS for MySQL data.In this example, we will create an RDS for MySQL D", "doc_type":"usermanual", "kw":"Submitting a SQL Job to Query RDS for MySQL Data Using DLI,Getting Started,User Guide", @@ -266,10 +287,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Submitting a SQL Job to Query RDS for MySQL Data Using DLI", @@ -279,7 +300,7 @@ "uri":"dli_01_0531.html", "node_id":"dli_01_0531.xml", "product_code":"dli", - "code":"14", + "code":"15", "des":"DLI Flink jobs can use other cloud services as data sources and sink streams for real-time compute.This example describes how to create and submit a Flink OpenSource SQL ", "doc_type":"usermanual", "kw":"Submitting a Flink OpenSource SQL Job to Query RDS for MySQL Data Using DLI,Getting Started,User Gui", @@ -287,10 +308,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Submitting a Flink OpenSource SQL Job to Query RDS for MySQL Data Using DLI", @@ -300,7 +321,7 @@ "uri":"dli_01_0512.html", "node_id":"dli_01_0512.xml", "product_code":"dli", - "code":"15", + "code":"16", "des":"Flink Jar jobs are suitable for data analysis scenarios that require custom stream processing logic, complex state management, or integration with specific libraries. You", "doc_type":"usermanual", "kw":"Submitting a Flink Jar Job Using DLI,Getting Started,User Guide", @@ -308,10 +329,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Submitting a Flink Jar Job Using DLI", @@ -321,7 +342,7 @@ "uri":"dli_01_0375.html", "node_id":"dli_01_0375.xml", "product_code":"dli", - "code":"16", + "code":"17", "des":"DLI allows you to submit Spark jobs compiled as JAR files, which contain the necessary code and dependency information for executing the job. These files are used for spe", "doc_type":"usermanual", "kw":"Submitting a Spark Jar Job Using DLI,Getting Started,User Guide", @@ -329,10 +350,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Submitting a Spark Jar Job Using DLI", @@ -342,7 +363,7 @@ "uri":"dli_01_0001.html", "node_id":"dli_01_0001.xml", "product_code":"dli", - "code":"17", + "code":"18", "des":"This chapter walks you through on how to develop a DLI job.To manage fine-grained permissions for your DLI resources using IAM, create an IAM user and grant them permissi", "doc_type":"usermanual", "kw":"DLI Job Development Process,User Guide", @@ -350,10 +371,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"DLI Job Development Process", @@ -363,7 +384,7 @@ "uri":"dli_01_0611.html", "node_id":"dli_01_0611.xml", "product_code":"dli", - "code":"18", + "code":"19", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Preparations", @@ -384,7 +405,7 @@ "uri":"dli_01_0618.html", "node_id":"dli_01_0618.xml", "product_code":"dli", - "code":"19", + "code":"20", "des":"Before using DLI, configure DLI agency permissions. This section describes the scenarios and steps for configuring DLI agency permissions (dli_management_agency).If you u", "doc_type":"usermanual", "kw":"Configuring DLI Agency Permissions,Preparations,User Guide", @@ -392,10 +413,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Configuring DLI Agency Permissions", @@ -405,7 +426,7 @@ "uri":"dli_01_0418.html", "node_id":"dli_01_0418.xml", "product_code":"dli", - "code":"20", + "code":"21", "des":"You can use Identity and Access Management (IAM) to implement fine-grained permissions management for your DLI resources. For details, see Overview.If your cloud account ", "doc_type":"usermanual", "kw":"Creating an IAM User and Granting Permissions,Preparations,User Guide", @@ -413,10 +434,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating an IAM User and Granting Permissions", @@ -426,7 +447,7 @@ "uri":"dli_01_0536.html", "node_id":"dli_01_0536.xml", "product_code":"dli", - "code":"21", + "code":"22", "des":"Before using DLI, you need to configure a DLI job bucket. The bucket is used to store temporary data generated during DLI job running, such as job logs and results.Config", "doc_type":"usermanual", "kw":"Configuring a DLI Job Bucket,Preparations,User Guide", @@ -434,10 +455,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Configuring a DLI Job Bucket", @@ -447,7 +468,7 @@ "uri":"dli_01_0508.html", "node_id":"dli_01_0508.xml", "product_code":"dli", - "code":"22", + "code":"23", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Creating an Elastic Resource Pool and Creating Queues Within It", @@ -468,7 +489,7 @@ "uri":"dli_01_0504.html", "node_id":"dli_01_0504.xml", "product_code":"dli", - "code":"23", + "code":"24", "des":"DLI compute resources are the foundation to run jobs. This section describes the modes of DLI compute resources and queue types.Before we dive into the compute resource m", "doc_type":"usermanual", "kw":"Overview of DLI Elastic Resource Pools and Queues,Creating an Elastic Resource Pool and Creating Que", @@ -476,10 +497,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Overview of DLI Elastic Resource Pools and Queues", @@ -489,7 +510,7 @@ "uri":"dli_01_0505.html", "node_id":"dli_01_0505.xml", "product_code":"dli", - "code":"24", + "code":"25", "des":"An elastic resource pool offers compute resources (CPU and memory) required for running DLI jobs, which can adapt to the changing demands of services.You can create multi", "doc_type":"usermanual", "kw":"Creating an Elastic Resource Pool and Creating Queues Within It,Creating an Elastic Resource Pool an", @@ -497,10 +518,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating an Elastic Resource Pool and Creating Queues Within It", @@ -510,7 +531,7 @@ "uri":"dli_01_0529.html", "node_id":"dli_01_0529.xml", "product_code":"dli", - "code":"25", + "code":"26", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Managing Elastic Resource Pools", @@ -531,7 +552,7 @@ "uri":"dli_01_0622.html", "node_id":"dli_01_0622.xml", "product_code":"dli", - "code":"26", + "code":"27", "des":"After creating an elastic resource pool, you can check and manage it on the management console.This section describes how to check basic information about an elastic reso", "doc_type":"usermanual", "kw":"Checking Basic Information,Managing Elastic Resource Pools,User Guide", @@ -539,10 +560,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Checking Basic Information", @@ -552,7 +573,7 @@ "uri":"dli_01_0526.html", "node_id":"dli_01_0526.xml", "product_code":"dli", - "code":"27", + "code":"28", "des":"Administrators can assign permissions of different operation scopes to users for each elastic resource pool.The administrator and elastic resource pool owner have all per", "doc_type":"usermanual", "kw":"Managing Permissions,Managing Elastic Resource Pools,User Guide", @@ -573,7 +594,7 @@ "uri":"dli_01_0530.html", "node_id":"dli_01_0530.xml", "product_code":"dli", - "code":"28", + "code":"29", "des":"If you want a queue to use resources in an elastic resource pool, bind the queue to the pool.You can click Associate Queue on the Resource Pool page to bind a queue to an", "doc_type":"usermanual", "kw":"Binding a Queue,Managing Elastic Resource Pools,User Guide", @@ -594,7 +615,7 @@ "uri":"dli_01_0686.html", "node_id":"dli_01_0686.xml", "product_code":"dli", - "code":"29", + "code":"30", "des":"Scaling out or in an elastic resource pool essentially means adjusting the actual CUs of the resource pool.In an elastic resource pool:Actual CUs: actual size of resource", "doc_type":"usermanual", "kw":"Scaling Out or In an Elastic Resource Pool,Managing Elastic Resource Pools,User Guide", @@ -602,10 +623,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Scaling Out or In an Elastic Resource Pool", @@ -615,7 +636,7 @@ "uri":"dli_01_0507.html", "node_id":"dli_01_0507.xml", "product_code":"dli", - "code":"30", + "code":"31", "des":"CU settings are used to control the maximum and minimum CU ranges for elastic resource pool scaling to prevent unlimited resource expansion risks.For example, if the curr", "doc_type":"usermanual", "kw":"Setting CUs,Managing Elastic Resource Pools,User Guide", @@ -623,10 +644,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Setting CUs", @@ -636,7 +657,7 @@ "uri":"dli_01_0525.html", "node_id":"dli_01_0525.xml", "product_code":"dli", - "code":"31", + "code":"32", "des":"A tag is a user-defined key-value pair used to identify cloud resources, helping you classify and search for them. A tag consists of a tag key and a tag value.If you use ", "doc_type":"usermanual", "kw":"Managing Tags,Managing Elastic Resource Pools,User Guide", @@ -644,10 +665,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Managing Tags", @@ -657,18 +678,18 @@ "uri":"dli_01_0506.html", "node_id":"dli_01_0506.xml", "product_code":"dli", - "code":"32", - "des":"Multiple queues can be added to an elastic resource pool. For how to add a queue, see Creating an Elastic Resource Pool and Creating Queues Within It. You can configure t", + "code":"33", + "des":"Elastic resource pools support multiple queues to manage job execution efficiently. For details about how to create queues in a pool, see Creating an Elastic Resource Poo", "doc_type":"usermanual", "kw":"Adjusting Scaling Policies for Queues in an Elastic Resource Pool,Managing Elastic Resource Pools,Us", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Adjusting Scaling Policies for Queues in an Elastic Resource Pool", @@ -678,7 +699,7 @@ "uri":"dli_01_0532.html", "node_id":"dli_01_0532.xml", "product_code":"dli", - "code":"33", + "code":"34", "des":"If you added a queue to or deleted one from an elastic resource pool, or you scaled an added queue, the CU quantity of the elastic resource pool may be changed. You can v", "doc_type":"usermanual", "kw":"Viewing Scaling History,Managing Elastic Resource Pools,User Guide", @@ -699,18 +720,18 @@ "uri":"dli_01_0566.html", "node_id":"dli_01_0566.xml", "product_code":"dli", - "code":"34", - "des":"You can create enterprise projects matching the organizational structure of your enterprises to centrally manage cloud resources across regions by project. Then you can c", + "code":"35", + "des":"An enterprise project is a cloud resource management approach that allows organizations to plan resources based on their organizational structure. It enables unified mana", "doc_type":"usermanual", "kw":"Allocating to an Enterprise Project,Managing Elastic Resource Pools,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Allocating to an Enterprise Project", @@ -720,7 +741,7 @@ "uri":"dli_01_0528.html", "node_id":"dli_01_0528.xml", "product_code":"dli", - "code":"35", + "code":"36", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Managing Queues", @@ -741,7 +762,7 @@ "uri":"dli_01_0663.html", "node_id":"dli_01_0663.xml", "product_code":"dli", - "code":"36", + "code":"37", "des":"This section walks you through how to view basic information about a queue on the management console, including the engine type and version.Log in to the DLI management c", "doc_type":"usermanual", "kw":"Viewing Basic Information About a Queue,Managing Queues,User Guide", @@ -762,7 +783,7 @@ "uri":"dli_01_0015.html", "node_id":"dli_01_0015.xml", "product_code":"dli", - "code":"37", + "code":"38", "des":"Administrators and queue owners have full operation permissions on queues. They can grant operation permissions to other users based on service needs. This ensures that u", "doc_type":"usermanual", "kw":"Queue Permission Management,Managing Queues,User Guide", @@ -783,7 +804,7 @@ "uri":"dli_01_0565.html", "node_id":"dli_01_0565.xml", "product_code":"dli", - "code":"38", + "code":"39", "des":"You can create enterprise projects matching the organizational structure of your enterprises to centrally manage cloud resources across regions by project. Then you can c", "doc_type":"usermanual", "kw":"Allocating a Queue to an Enterprise Project,Managing Queues,User Guide", @@ -804,7 +825,7 @@ "uri":"dli_01_0421.html", "node_id":"dli_01_0421.xml", "product_code":"dli", - "code":"39", + "code":"40", "des":"Once you have created an SMN topic, you can easily subscribe to it by going to the Topic Management > Topics page of the SMN console. You can choose to receive notificati", "doc_type":"usermanual", "kw":"Creating an SMN Topic,Managing Queues,User Guide", @@ -825,7 +846,7 @@ "uri":"dli_01_0022.html", "node_id":"dli_01_0022.xml", "product_code":"dli", - "code":"40", + "code":"41", "des":"A tag is a user-defined key-value pair used to identify cloud resources, helping you classify and search for them. A tag consists of a tag key and a tag value.If you use ", "doc_type":"usermanual", "kw":"Managing Queue Tags,Managing Queues,User Guide", @@ -833,10 +854,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Managing Queue Tags", @@ -846,7 +867,7 @@ "uri":"dli_01_0563.html", "node_id":"dli_01_0563.xml", "product_code":"dli", - "code":"41", + "code":"42", "des":"DLI allows you to set properties for queues.You can currently set the following property parameters:Spark driver parameters: Set them to improve the scheduling efficiency", "doc_type":"usermanual", "kw":"Setting Queue Properties,Managing Queues,User Guide", @@ -854,10 +875,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Setting Queue Properties", @@ -867,7 +888,7 @@ "uri":"dli_01_0685.html", "node_id":"dli_01_0685.xml", "product_code":"dli", - "code":"42", + "code":"43", "des":"Spark Native is a core component of Apache Spark designed to enhance the performance of Spark SQL computations. By utilizing vectorized C++ acceleration libraries, it acc", "doc_type":"usermanual", "kw":"Enabling Spark Native Operator Optimization,Managing Queues,User Guide", @@ -888,7 +909,7 @@ "uri":"dli_01_0489.html", "node_id":"dli_01_0489.xml", "product_code":"dli", - "code":"43", + "code":"44", "des":"DLI's address connectivity testing feature can be used to verify network connectivity between DLI queues and destination addresses.This feature is typically utilized for ", "doc_type":"usermanual", "kw":"Testing the Network Connectivity Between a Queue and a Data Source,Managing Queues,User Guide", @@ -896,10 +917,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Testing the Network Connectivity Between a Queue and a Data Source", @@ -909,7 +930,7 @@ "uri":"dli_01_0016.html", "node_id":"dli_01_0016.xml", "product_code":"dli", - "code":"44", + "code":"45", "des":"You can delete a queue based on actual conditions.This operation will fail if there are jobs in the Submitting or Running state on this queue.Deleting a queue does not ca", "doc_type":"usermanual", "kw":"Deleting a Queue,Managing Queues,User Guide", @@ -917,10 +938,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Deleting a Queue", @@ -930,7 +951,7 @@ "uri":"dli_01_0487.html", "node_id":"dli_01_0487.xml", "product_code":"dli", - "code":"45", + "code":"46", "des":"Newly created queues need to run jobs before they can be scaled in or out.The operations described in this section only apply to queues in non-elastic resource pools.Queu", "doc_type":"usermanual", "kw":"Enabling Elastic Scaling for a Queue in a Non-Elastic Resource Pool,Managing Queues,User Guide", @@ -951,7 +972,7 @@ "uri":"dli_01_0488.html", "node_id":"dli_01_0488.xml", "product_code":"dli", - "code":"46", + "code":"47", "des":"When services are busy, you might need to use more compute resources to process services in a period. After this period, you do not require the same amount of resources. ", "doc_type":"usermanual", "kw":"Creating a Scheduled Elastic Scaling Task for a Queue in a Non-Elastic Resource Pool,Managing Queues", @@ -972,7 +993,7 @@ "uri":"dli_01_0443.html", "node_id":"dli_01_0443.xml", "product_code":"dli", - "code":"47", + "code":"48", "des":"If the CIDR block of the DLI queue conflicts with that of the user data source, you can change the CIDR block of the queue.If the queue whose CIDR block is to be modified", "doc_type":"usermanual", "kw":"Changing the CIDR Block of a Queue in a Non-Elastic Resource Pool,Managing Queues,User Guide", @@ -993,7 +1014,7 @@ "uri":"dli_01_0515.html", "node_id":"dli_01_0515.xml", "product_code":"dli", - "code":"48", + "code":"49", "des":"This section walks you through the procedure of adding a queue to an elastic resource pool and binding an enhanced datasource connection to the elastic resource pool.Proc", "doc_type":"usermanual", "kw":"Example Use Case: Creating an Elastic Resource Pool and Running Jobs,Creating an Elastic Resource Po", @@ -1001,10 +1022,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Example Use Case: Creating an Elastic Resource Pool and Running Jobs", @@ -1014,7 +1035,7 @@ "uri":"dli_01_0516.html", "node_id":"dli_01_0516.xml", "product_code":"dli", - "code":"49", + "code":"50", "des":"A company has multiple departments that perform data analysis in different periods during a day.Department A requires a large number of compute resources from 00:00 a.m. ", "doc_type":"usermanual", "kw":"Example Use Case: Configuring Scaling Policies for Queues in an Elastic Resource Pool,Creating an El", @@ -1035,7 +1056,7 @@ "uri":"dli_01_0363.html", "node_id":"dli_01_0363.xml", "product_code":"dli", - "code":"50", + "code":"51", "des":"Queues in the non-elastic resource pool mode are the previous-gen of resource management for DLI. It involved purchasing and releasing resources based on usage demands, r", "doc_type":"usermanual", "kw":"Creating a Non-Elastic Resource Pool Queue (Deprecated, Not Recommended),Creating an Elastic Resourc", @@ -1043,10 +1064,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating a Non-Elastic Resource Pool Queue (Deprecated, Not Recommended)", @@ -1056,7 +1077,7 @@ "uri":"dli_01_0390.html", "node_id":"dli_01_0390.xml", "product_code":"dli", - "code":"51", + "code":"52", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Creating a Data Directory, Database, and Table", @@ -1077,7 +1098,7 @@ "uri":"dli_01_0228.html", "node_id":"dli_01_0228.xml", "product_code":"dli", - "code":"52", + "code":"53", "des":"Databases and tables are the basis for developing SQL and Spark jobs. Before running a job, you need to define databases and tables based on your service scenarios.Flink ", "doc_type":"usermanual", "kw":"Understanding Data Catalogs, Databases, and Tables,Creating a Data Directory, Database, and Table,Us", @@ -1085,10 +1106,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Understanding Data Catalogs, Databases, and Tables", @@ -1098,7 +1119,7 @@ "uri":"dli_01_0005.html", "node_id":"dli_01_0005.xml", "product_code":"dli", - "code":"53", + "code":"54", "des":"A database is a repository of data organized, stored, and managed on computer storage devices according to data structures.A table is one of the most essential components", "doc_type":"usermanual", "kw":"Creating a Data Catalog, Database, and Table on the DLI Console,Creating a Data Directory, Database,", @@ -1106,10 +1127,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating a Data Catalog, Database, and Table on the DLI Console", @@ -1119,7 +1140,7 @@ "uri":"dli_01_0008.html", "node_id":"dli_01_0008.xml", "product_code":"dli", - "code":"54", + "code":"55", "des":"Metadata is data used to define the type of data. It primarily describes information about the data itself, including its source, size, format, or other characteristics. ", "doc_type":"usermanual", "kw":"Viewing Table Metadata,Creating a Data Directory, Database, and Table,User Guide", @@ -1140,7 +1161,7 @@ "uri":"dli_01_0671.html", "node_id":"dli_01_0671.xml", "product_code":"dli", - "code":"55", + "code":"56", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Managing Data Catalogs on the DLI Console", @@ -1161,7 +1182,7 @@ "uri":"dli_01_0672.html", "node_id":"dli_01_0672.xml", "product_code":"dli", - "code":"56", + "code":"57", "des":"DLI data catalogs support authorization on the DLI console or authentication through IAM. By setting permissions, you can grant varying data catalog permissions to differ", "doc_type":"usermanual", "kw":"Configuring Data Catalog Permissions on the DLI Console,Managing Data Catalogs on the DLI Console,Us", @@ -1182,7 +1203,7 @@ "uri":"dli_01_0004.html", "node_id":"dli_01_0004.xml", "product_code":"dli", - "code":"57", + "code":"58", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Managing Database Resources on the DLI Console", @@ -1203,7 +1224,7 @@ "uri":"dli_01_0447.html", "node_id":"dli_01_0447.xml", "product_code":"dli", - "code":"58", + "code":"59", "des":"By setting permissions, you can assign varying database permissions to different users.The administrator and database owner have all permissions, which cannot be set or m", "doc_type":"usermanual", "kw":"Configuring Database Permissions on the DLI Console,Managing Database Resources on the DLI Console,U", @@ -1211,10 +1232,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Configuring Database Permissions on the DLI Console", @@ -1224,7 +1245,7 @@ "uri":"dli_01_0011.html", "node_id":"dli_01_0011.xml", "product_code":"dli", - "code":"59", + "code":"60", "des":"You can delete an unused database from the DLI console: when a database is no longer needed, such as after a test database has completed testing; if a database has errors", "doc_type":"usermanual", "kw":"Deleting a Database on the DLI Console,Managing Database Resources on the DLI Console,User Guide", @@ -1245,7 +1266,7 @@ "uri":"dli_01_0376.html", "node_id":"dli_01_0376.xml", "product_code":"dli", - "code":"60", + "code":"61", "des":"In practical use, developers create databases and tables, which are then handed over to testers for testing. Once testing is complete, the databases and tables are handed", "doc_type":"usermanual", "kw":"Changing the Database Owner on the DLI Console,Managing Database Resources on the DLI Console,User G", @@ -1266,7 +1287,7 @@ "uri":"dli_01_0552.html", "node_id":"dli_01_0552.xml", "product_code":"dli", - "code":"61", + "code":"62", "des":"A tag is a user-defined key-value pair used to identify cloud resources, helping you classify and search for them. A tag consists of a tag key and a tag value. If you use", "doc_type":"usermanual", "kw":"Managing Tags for Database Tables,Managing Database Resources on the DLI Console,User Guide", @@ -1274,10 +1295,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Managing Tags for Database Tables", @@ -1287,7 +1308,7 @@ "uri":"dli_01_0625.html", "node_id":"dli_01_0625.xml", "product_code":"dli", - "code":"62", + "code":"63", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Managing Table Resources on the DLI Console", @@ -1308,7 +1329,7 @@ "uri":"dli_01_0448.html", "node_id":"dli_01_0448.xml", "product_code":"dli", - "code":"63", + "code":"64", "des":"By setting permissions, you can assign varying table permissions to different users.The administrator and table owner have all permissions, which cannot be set or modifie", "doc_type":"usermanual", "kw":"Configuring Table Permissions on the DLI Console,Managing Table Resources on the DLI Console,User Gu", @@ -1316,10 +1337,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Configuring Table Permissions on the DLI Console", @@ -1329,7 +1350,7 @@ "uri":"dli_01_0626.html", "node_id":"dli_01_0626.xml", "product_code":"dli", - "code":"64", + "code":"65", "des":"You can delete an unused data table from the DLI console: when a data table is no longer needed, such as after a test data table has completed testing; if a data table ha", "doc_type":"usermanual", "kw":"Deleting a Table on the DLI Console,Managing Table Resources on the DLI Console,User Guide", @@ -1350,7 +1371,7 @@ "uri":"dli_01_0627.html", "node_id":"dli_01_0627.xml", "product_code":"dli", - "code":"65", + "code":"66", "des":"In practical use, developers create databases and tables, which are then handed over to testers for testing. Once testing is complete, the databases and tables are handed", "doc_type":"usermanual", "kw":"Changing the Table Owner on the DLI Console,Managing Table Resources on the DLI Console,User Guide", @@ -1371,7 +1392,7 @@ "uri":"dli_01_0253.html", "node_id":"dli_01_0253.xml", "product_code":"dli", - "code":"66", + "code":"67", "des":"This section describes how to import data stored in OBS to a table on the DLI console. Data can be imported to both DLI tables (table type: MANAGED) and OBS tables (table", "doc_type":"usermanual", "kw":"Importing OBS Data to a DLI Table,Managing Table Resources on the DLI Console,User Guide", @@ -1392,7 +1413,7 @@ "uri":"dli_01_0010.html", "node_id":"dli_01_0010.xml", "product_code":"dli", - "code":"67", + "code":"68", "des":"You can export data from a DLI table to OBS. During the export, a folder is created in OBS or the content in the existing folder is overwritten.The exported file can be i", "doc_type":"usermanual", "kw":"Exporting DLI Table Data to OBS,Managing Table Resources on the DLI Console,User Guide", @@ -1400,10 +1421,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Exporting DLI Table Data to OBS", @@ -1413,7 +1434,7 @@ "uri":"dli_01_0007.html", "node_id":"dli_01_0007.xml", "product_code":"dli", - "code":"68", + "code":"69", "des":"The Preview page displays the first 10 records in the table.You can preview data on either the Data Management page or the SQL Editor page.To preview data on the Data Man", "doc_type":"usermanual", "kw":"Previewing Table Data on the DLI Console,Managing Table Resources on the DLI Console,User Guide", @@ -1434,7 +1455,7 @@ "uri":"dli_01_0661.html", "node_id":"dli_01_0661.xml", "product_code":"dli", - "code":"69", + "code":"70", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Data Import and Migration", @@ -1455,7 +1476,7 @@ "uri":"dli_01_0662.html", "node_id":"dli_01_0662.xml", "product_code":"dli", - "code":"70", + "code":"71", "des":"DLI enables direct access to data stored in OBS for query and analysis, eliminating the need for data migration.To begin using DLI for data analysis, just import your loc", "doc_type":"usermanual", "kw":"Overview,Data Import and Migration,User Guide", @@ -1463,10 +1484,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Overview", @@ -1476,7 +1497,7 @@ "uri":"dli_01_0631.html", "node_id":"dli_01_0631.xml", "product_code":"dli", - "code":"71", + "code":"72", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Migrating Data from External Data Sources to DLI", @@ -1497,7 +1518,7 @@ "uri":"dli_01_0632.html", "node_id":"dli_01_0632.xml", "product_code":"dli", - "code":"72", + "code":"73", "des":"To centrally analyze and manage scattered data from various systems, you can utilize migration tools like Cloud Data Migration (CDM) to migrate the data to DLI. Once the ", "doc_type":"usermanual", "kw":"Overview of Data Migration Scenarios,Migrating Data from External Data Sources to DLI,User Guide", @@ -1505,10 +1526,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Overview of Data Migration Scenarios", @@ -1518,7 +1539,7 @@ "uri":"dli_01_0633.html", "node_id":"dli_01_0633.xml", "product_code":"dli", - "code":"73", + "code":"74", "des":"On its GUI, CDM enables you to create data migration tasks from multiple data sources to a data lake.This section describes how to use CDM to migrate data from data sourc", "doc_type":"usermanual", "kw":"Using CDM to Migrate Data to DLI,Migrating Data from External Data Sources to DLI,User Guide", @@ -1526,10 +1547,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Using CDM to Migrate Data to DLI", @@ -1539,7 +1560,7 @@ "uri":"dli_01_0634.html", "node_id":"dli_01_0634.xml", "product_code":"dli", - "code":"74", + "code":"75", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Configuring DLI to Read and Write Data from and to External Data Sources", @@ -1560,7 +1581,7 @@ "uri":"dli_01_0635.html", "node_id":"dli_01_0635.xml", "product_code":"dli", - "code":"75", + "code":"76", "des":"To read and write external data sources when running DLI jobs, two conditions must be met:Establish network connectivity between DLI and the external data source to ensur", "doc_type":"usermanual", "kw":"Configuring DLI to Read and Write External Data Sources,Configuring DLI to Read and Write Data from ", @@ -1568,10 +1589,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Configuring DLI to Read and Write External Data Sources", @@ -1581,7 +1602,7 @@ "uri":"dli_01_0426.html", "node_id":"dli_01_0426.xml", "product_code":"dli", - "code":"76", + "code":"77", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Configuring the Network Connection Between DLI and Data Sources (Enhanced Datasource Connection)", @@ -1602,7 +1623,7 @@ "uri":"dli_01_0003.html", "node_id":"dli_01_0003.xml", "product_code":"dli", - "code":"77", + "code":"78", "des":"Imagine DLI elastic resource pools and data sources as two isolated islands surrounded by water.To enable traffic between these two islands, you need to first construct a", "doc_type":"usermanual", "kw":"Overview of Enhanced Datasource Connections,Configuring the Network Connection Between DLI and Data ", @@ -1610,10 +1631,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Overview of Enhanced Datasource Connections", @@ -1623,18 +1644,18 @@ "uri":"dli_01_0006.html", "node_id":"dli_01_0006.xml", "product_code":"dli", - "code":"78", - "des":"Create an enhanced datasource connection for DLI to access, import, query, and analyze data of other data sources.For example, to connect DLI to the MRS, RDS, CSS, Kafka,", + "code":"79", + "des":"Before using DLI to access data from other sources, you need to create an enhanced datasource connection to enable network communication between DLI and the destination d", "doc_type":"usermanual", "kw":"Creating an Enhanced Datasource Connection,Configuring the Network Connection Between DLI and Data S", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating an Enhanced Datasource Connection", @@ -1644,7 +1665,7 @@ "uri":"dli_01_0410.html", "node_id":"dli_01_0410.xml", "product_code":"dli", - "code":"79", + "code":"80", "des":"If DLI needs to access external data sources, you need to establish enhanced datasource connections to enable the network between DLI and the data sources, and then devel", "doc_type":"usermanual", "kw":"Common Development Methods for DLI Cross-Source Analysis,Configuring the Network Connection Between ", @@ -1652,10 +1673,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Common Development Methods for DLI Cross-Source Analysis", @@ -1665,7 +1686,7 @@ "uri":"dli_01_0636.html", "node_id":"dli_01_0636.xml", "product_code":"dli", - "code":"80", + "code":"81", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Managing Data Source Access Credentials Using DEW", @@ -1686,10 +1707,31 @@ "uri":"dli_01_0687.html", "node_id":"dli_01_0687.xml", "product_code":"dli", - "code":"81", - "des":"When submitting Flink or Spark jobs through DLI to access external data sources (such as OBS and Kafka), there is a risk of plaintext exposure if AK/SK, usernames/passwor", + "code":"82", + "des":"When you use DLI to perform big data processing or cross-service data queries, authentication is a critical prerequisite to ensure legitimate access and data security. Em", "doc_type":"usermanual", - "kw":"Overview,Managing Data Source Access Credentials Using DEW,User Guide", + "kw":"Using an Agency to Obtain Access Credentials in DLI,Managing Data Source Access Credentials Using DE", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No;Yes;NO", + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" + } + ], + "title":"Using an Agency to Obtain Access Credentials in DLI", + "githuburl":"" + }, + { + "uri":"dli_09_0210.html", + "node_id":"dli_09_0210.xml", + "product_code":"dli", + "code":"83", + "des":"When DLI writes the output data of Flink jobs to MySQL or DWS, you need to set sensitive parameters such as the username and password in the connector. Storing this infor", + "doc_type":"usermanual", + "kw":"Flink OpenSource SQL Jobs Using DEW to Manage Access Credentials,Managing Data Source Access Credent", "search_title":"", "metedata":[ { @@ -1700,14 +1742,56 @@ "IsBot":"No;Yes" } ], - "title":"Overview", + "title":"Flink OpenSource SQL Jobs Using DEW to Manage Access Credentials", + "githuburl":"" + }, + { + "uri":"dli_09_0211.html", + "node_id":"dli_09_0211.xml", + "product_code":"dli", + "code":"84", + "des":"When writing output data from Flink Jar jobs to OBS, you need to configure an AK/SK for accessing OBS. To ensure the security of AK/SK data, you can use DEW and CSMS for ", + "doc_type":"usermanual", + "kw":"Flink Jar Jobs Using DEW to Acquire Access Credentials for Reading and Writing Data from and to OBS,", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;Yes;NO", + "IsBot":"No;Yes" + } + ], + "title":"Flink Jar Jobs Using DEW to Acquire Access Credentials for Reading and Writing Data from and to OBS", + "githuburl":"" + }, + { + "uri":"dli_09_0215.html", + "node_id":"dli_09_0215.xml", + "product_code":"dli", + "code":"85", + "des":"A temporary security credential grants temporary access rights. It includes a temporary AK/SK and a security token, both of which must be used together.When writing outpu", + "doc_type":"usermanual", + "kw":"Spark Jar Jobs Using DEW to Acquire Access Credentials for Reading and Writing Data from and to OBS,", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;Yes;NO", + "IsBot":"No;Yes" + } + ], + "title":"Spark Jar Jobs Using DEW to Acquire Access Credentials for Reading and Writing Data from and to OBS", "githuburl":"" }, { "uri":"dli_01_0422.html", "node_id":"dli_01_0422.xml", "product_code":"dli", - "code":"82", + "code":"86", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Using DLI Datasource Authentication to Manage Access Credentials for Data Sources", @@ -1728,7 +1812,7 @@ "uri":"dli_01_0561.html", "node_id":"dli_01_0561.xml", "product_code":"dli", - "code":"83", + "code":"87", "des":"When analyzing across multiple sources, you are advised not to configure authentication information directly in a job as it can lead to password leakage. Instead, you are", "doc_type":"usermanual", "kw":"Overview,Using DLI Datasource Authentication to Manage Access Credentials for Data Sources,User Guid", @@ -1736,10 +1820,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Overview", @@ -1749,7 +1833,7 @@ "uri":"dli_01_0427.html", "node_id":"dli_01_0427.xml", "product_code":"dli", - "code":"84", + "code":"88", "des":"Create a CSS datasource authentication on the DLI console to store the authentication information of the CSS security cluster to DLI. This will allow you to access to the", "doc_type":"usermanual", "kw":"Creating a CSS Datasource Authentication,Using DLI Datasource Authentication to Manage Access Creden", @@ -1770,7 +1854,7 @@ "uri":"dli_01_0558.html", "node_id":"dli_01_0558.xml", "product_code":"dli", - "code":"85", + "code":"89", "des":"Create a Kerberos datasource authentication on the DLI console to store the authentication information of the data source to DLI. This will allow you to access to the dat", "doc_type":"usermanual", "kw":"Creating a Kerberos Datasource Authentication,Using DLI Datasource Authentication to Manage Access C", @@ -1778,10 +1862,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating a Kerberos Datasource Authentication", @@ -1791,7 +1875,7 @@ "uri":"dli_01_0560.html", "node_id":"dli_01_0560.xml", "product_code":"dli", - "code":"86", + "code":"90", "des":"Create a Kafka_SSL datasource authentication on the DLI console to store the Kafka authentication information to DLI. This will allow you to access to Kafka instances wit", "doc_type":"usermanual", "kw":"Creating a Kafka_SSL Datasource Authentication,Using DLI Datasource Authentication to Manage Access ", @@ -1799,10 +1883,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating a Kafka_SSL Datasource Authentication", @@ -1812,18 +1896,18 @@ "uri":"dli_01_0559.html", "node_id":"dli_01_0559.xml", "product_code":"dli", - "code":"87", - "des":"Create a password datasource authentication on the DLI console to store passwords of the GaussDB(DWS), RDS, DCS, and DDS data sources to DLI. This will allow you to acces", + "code":"91", + "des":"Create a password datasource authentication on the DLI console to store passwords of the DWS, RDS, DCS, and DDS data sources to DLI. This will allow you to access to the ", "doc_type":"usermanual", "kw":"Creating a Password Datasource Authentication,Using DLI Datasource Authentication to Manage Access C", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating a Password Datasource Authentication", @@ -1833,7 +1917,7 @@ "uri":"dli_01_0480.html", "node_id":"dli_01_0480.xml", "product_code":"dli", - "code":"88", + "code":"92", "des":"Grant permissions on a datasource authentication to users so multiple user jobs can use the datasource authentication without affecting each other.The administrator and t", "doc_type":"usermanual", "kw":"Datasource Authentication Permission Management,Using DLI Datasource Authentication to Manage Access", @@ -1841,10 +1925,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Datasource Authentication Permission Management", @@ -1854,7 +1938,7 @@ "uri":"dli_01_0509.html", "node_id":"dli_01_0509.xml", "product_code":"dli", - "code":"89", + "code":"93", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Managing Enhanced Datasource Connections", @@ -1875,7 +1959,7 @@ "uri":"dli_01_0623.html", "node_id":"dli_01_0623.xml", "product_code":"dli", - "code":"90", + "code":"94", "des":"After creating an enhanced datasource connection, you can view and manage it on the management console.This section describes how to view basic information about an enhan", "doc_type":"usermanual", "kw":"Viewing Basic Information About an Enhanced Datasource Connection,Managing Enhanced Datasource Conne", @@ -1883,10 +1967,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Viewing Basic Information About an Enhanced Datasource Connection", @@ -1896,7 +1980,7 @@ "uri":"dli_01_0018.html", "node_id":"dli_01_0018.xml", "product_code":"dli", - "code":"91", + "code":"95", "des":"Enhanced connections support user authorization by project. After authorization, users in the project have the permission to perform operations on the enhanced connection", "doc_type":"usermanual", "kw":"Enhanced Connection Permission Management,Managing Enhanced Datasource Connections,User Guide", @@ -1917,7 +2001,7 @@ "uri":"dli_01_0009.html", "node_id":"dli_01_0009.xml", "product_code":"dli", - "code":"92", + "code":"96", "des":"To connect other resource pools to data sources through enhanced datasource connections, bind enhanced datasource connections to resource pools on the Enhanced tab page.T", "doc_type":"usermanual", "kw":"Binding an Enhanced Datasource Connection to an Elastic Resource Pool,Managing Enhanced Datasource C", @@ -1938,7 +2022,7 @@ "uri":"dli_01_0555.html", "node_id":"dli_01_0555.xml", "product_code":"dli", - "code":"93", + "code":"97", "des":"Unbind an enhanced datasource connection from an elastic resource pool that does not need to access a data source through an enhanced datasoruce connection.If the status ", "doc_type":"usermanual", "kw":"Unbinding an Enhanced Datasource Connection from an Elastic Resource Pool,Managing Enhanced Datasour", @@ -1959,7 +2043,7 @@ "uri":"dli_01_0014.html", "node_id":"dli_01_0014.xml", "product_code":"dli", - "code":"94", + "code":"98", "des":"Enhanced datasource connections establish peering connections between DLI elastic resource pools and data sources to interconnect two VPC networks. If we liken these peer", "doc_type":"usermanual", "kw":"Adding a Route for an Enhanced Datasource Connection,Managing Enhanced Datasource Connections,User G", @@ -1967,10 +2051,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Adding a Route for an Enhanced Datasource Connection", @@ -1980,7 +2064,7 @@ "uri":"dli_01_0556.html", "node_id":"dli_01_0556.xml", "product_code":"dli", - "code":"95", + "code":"99", "des":"Delete a route that is no longer used.A custom route table cannot be deleted if it is associated with a subnet.Log in to the DLI management console.In the navigation pane", "doc_type":"usermanual", "kw":"Deleting the Route for an Enhanced Datasource Connection,Managing Enhanced Datasource Connections,Us", @@ -2001,7 +2085,7 @@ "uri":"dli_01_0013.html", "node_id":"dli_01_0013.xml", "product_code":"dli", - "code":"96", + "code":"100", "des":"Host information is the mapping between host IP addresses and domain names. After you configure host information, jobs can only use the configured domain names to access ", "doc_type":"usermanual", "kw":"Modifying Host Information in an Elastic Resource Pool,Managing Enhanced Datasource Connections,User", @@ -2022,7 +2106,7 @@ "uri":"dli_01_0019.html", "node_id":"dli_01_0019.xml", "product_code":"dli", - "code":"97", + "code":"101", "des":"A tag is a user-defined key-value pair used to identify cloud resources, helping you classify and search for them. A tag consists of a tag key and a tag value.If you use ", "doc_type":"usermanual", "kw":"Enhanced Datasource Connection Tag Management,Managing Enhanced Datasource Connections,User Guide", @@ -2030,10 +2114,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Enhanced Datasource Connection Tag Management", @@ -2043,7 +2127,7 @@ "uri":"dli_01_0553.html", "node_id":"dli_01_0553.xml", "product_code":"dli", - "code":"98", + "code":"102", "des":"Delete an enhanced datasource connection that is no longer used on the console.Log in to the DLI management console.In the navigation pane on the left, choose Datasource ", "doc_type":"usermanual", "kw":"Deleting an Enhanced Datasource Connection,Managing Enhanced Datasource Connections,User Guide", @@ -2064,7 +2148,7 @@ "uri":"dli_01_0486.html", "node_id":"dli_01_0486.xml", "product_code":"dli", - "code":"99", + "code":"103", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Configuring an Agency to Allow DLI to Access Other Cloud Services", @@ -2085,18 +2169,18 @@ "uri":"dli_01_0419.html", "node_id":"dli_01_0419.xml", "product_code":"dli", - "code":"100", - "des":"Cloud services often interact with each other, with some of which dependent on other services. You can create an agency to delegate DLI to use other cloud services and pe", + "code":"104", + "des":"Cloud services often need to interact with each other for business operations. In some cases, one cloud service may require collaboration with another. To facilitate this", "doc_type":"usermanual", "kw":"DLI Agency Overview,Configuring an Agency to Allow DLI to Access Other Cloud Services,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"DLI Agency Overview", @@ -2106,7 +2190,7 @@ "uri":"dli_01_0616.html", "node_id":"dli_01_0616.xml", "product_code":"dli", - "code":"101", + "code":"105", "des":"When Flink 1.15, Spark 3.3, or a later version is used to execute jobs and the required agency is not included in the DLI system agency dli_management_agency, you need to", "doc_type":"usermanual", "kw":"Creating a Custom DLI Agency,Configuring an Agency to Allow DLI to Access Other Cloud Services,User ", @@ -2114,10 +2198,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating a Custom DLI Agency", @@ -2127,7 +2211,7 @@ "uri":"dli_01_0617.html", "node_id":"dli_01_0617.xml", "product_code":"dli", - "code":"102", + "code":"106", "des":"This section provides agency permission policies for common scenarios, which can be used to configure agency permission policies when you customize your permissions. The ", "doc_type":"usermanual", "kw":"Agency Permission Policies in Common Scenarios,Configuring an Agency to Allow DLI to Access Other Cl", @@ -2135,10 +2219,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Agency Permission Policies in Common Scenarios", @@ -2148,7 +2232,7 @@ "uri":"dli_01_0402.html", "node_id":"dli_01_0402.xml", "product_code":"dli", - "code":"103", + "code":"107", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Submitting a SQL Job on the DLI Management Console", @@ -2169,7 +2253,7 @@ "uri":"dli_01_0320.html", "node_id":"dli_01_0320.xml", "product_code":"dli", - "code":"104", + "code":"108", "des":"DLI offers a SQL editor for executing data query operations using SQL statements.This section describes how to create and submit a SQL job using the DLI SQL editor.Before", "doc_type":"usermanual", "kw":"Creating and Submitting a SQL Job,Submitting a SQL Job on the DLI Management Console,User Guide", @@ -2177,10 +2261,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating and Submitting a SQL Job", @@ -2190,7 +2274,7 @@ "uri":"dli_01_0621.html", "node_id":"dli_01_0621.xml", "product_code":"dli", - "code":"105", + "code":"109", "des":"Store the data results of analyzed SQL jobs in a specified location in the desired format.By default, DLI stores SQL job results in its job bucket. You can also download ", "doc_type":"usermanual", "kw":"Exporting SQL Job Results,Submitting a SQL Job on the DLI Management Console,User Guide", @@ -2198,10 +2282,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Exporting SQL Job Results", @@ -2211,7 +2295,7 @@ "uri":"dli_01_0612.html", "node_id":"dli_01_0612.xml", "product_code":"dli", - "code":"106", + "code":"110", "des":"There are numerous SQL engines in the big data field, which bring diversity to the solutions but also expose some issues such as varying quality of SQL input statements, ", "doc_type":"usermanual", "kw":"Creating a SQL Inspection Rule,Submitting a SQL Job on the DLI Management Console,User Guide", @@ -2219,10 +2303,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating a SQL Inspection Rule", @@ -2232,7 +2316,7 @@ "uri":"dli_01_0535.html", "node_id":"dli_01_0535.xml", "product_code":"dli", - "code":"107", + "code":"111", "des":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", "doc_type":"usermanual", "kw":"Setting the Priority for a SQL Job,Submitting a SQL Job on the DLI Management Console,User Guide", @@ -2253,7 +2337,7 @@ "uri":"dli_01_0637.html", "node_id":"dli_01_0637.xml", "product_code":"dli", - "code":"108", + "code":"112", "des":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", "doc_type":"usermanual", "kw":"Querying Logs for SQL Jobs,Submitting a SQL Job on the DLI Management Console,User Guide", @@ -2274,7 +2358,7 @@ "uri":"dli_01_0017.html", "node_id":"dli_01_0017.xml", "product_code":"dli", - "code":"109", + "code":"113", "des":"The SQL Jobs page displays all SQL jobs, which may span multiple pages if there are many jobs. You can navigate to a specific page as needed. DLI allows you to view jobs ", "doc_type":"usermanual", "kw":"Managing SQL Jobs,Submitting a SQL Job on the DLI Management Console,User Guide", @@ -2282,10 +2366,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Managing SQL Jobs", @@ -2295,7 +2379,7 @@ "uri":"dli_01_0638.html", "node_id":"dli_01_0638.xml", "product_code":"dli", - "code":"110", + "code":"114", "des":"A SQL execution plan is a logical flowchart of a database query that shows how a database management system executes a specific SQL query. The execution plan details the ", "doc_type":"usermanual", "kw":"Viewing a SQL Execution Plan,Submitting a SQL Job on the DLI Management Console,User Guide", @@ -2303,10 +2387,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Viewing a SQL Execution Plan", @@ -2316,7 +2400,7 @@ "uri":"dli_01_05110.html", "node_id":"dli_01_05110.xml", "product_code":"dli", - "code":"111", + "code":"115", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Creating and Managing SQL Job Templates", @@ -2337,7 +2421,7 @@ "uri":"dli_01_0021.html", "node_id":"dli_01_0021.xml", "product_code":"dli", - "code":"112", + "code":"116", "des":"DLI allows you to create custom templates or save currently used SQL statements as templates for quick and convenient SQL operations. Once a template is saved, you can ex", "doc_type":"usermanual", "kw":"Creating a SQL Job Template,Creating and Managing SQL Job Templates,User Guide", @@ -2358,7 +2442,7 @@ "uri":"dli_01_0639.html", "node_id":"dli_01_0639.xml", "product_code":"dli", - "code":"113", + "code":"117", "des":"DLI allows you to create custom templates or save currently used SQL statements as templates for quick and convenient SQL operations. Once a template is saved, you can ex", "doc_type":"usermanual", "kw":"Developing and Submitting a SQL Job Using a SQL Job Template,Creating and Managing SQL Job Templates", @@ -2366,10 +2450,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Developing and Submitting a SQL Job Using a SQL Job Template", @@ -2379,7 +2463,7 @@ "uri":"dli_01_05111.html", "node_id":"dli_01_05111.xml", "product_code":"dli", - "code":"114", + "code":"118", "des":"TPC-H is a test set developed by the Transaction Processing Performance Council (TPC) to simulate decision-making support applications. It is widely used in academia and ", "doc_type":"usermanual", "kw":"TPC-H Sample Data in the SQL Templates Preset on DLI,Creating and Managing SQL Job Templates,User Gu", @@ -2400,7 +2484,7 @@ "uri":"dli_01_0389.html", "node_id":"dli_01_0389.xml", "product_code":"dli", - "code":"115", + "code":"119", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Submitting a Flink Job on the DLI Management Console", @@ -2421,7 +2505,7 @@ "uri":"dli_01_0403.html", "node_id":"dli_01_0403.xml", "product_code":"dli", - "code":"116", + "code":"120", "des":"DLI supports two types of Flink jobs:Flink OpenSource SQL job:It is fully compatible with Flink of the community edition, ensuring that jobs can run smoothly on these Fli", "doc_type":"usermanual", "kw":"Flink Job Overview,Submitting a Flink Job on the DLI Management Console,User Guide", @@ -2429,10 +2513,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Flink Job Overview", @@ -2442,7 +2526,7 @@ "uri":"dli_01_0498.html", "node_id":"dli_01_0498.xml", "product_code":"dli", - "code":"117", + "code":"121", "des":"This section describes how to create a Flink OpenSource SQL job.DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink provided by the community. In ", "doc_type":"usermanual", "kw":"Creating a Flink OpenSource SQL Job,Submitting a Flink Job on the DLI Management Console,User Guide", @@ -2450,10 +2534,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating a Flink OpenSource SQL Job", @@ -2463,7 +2547,7 @@ "uri":"dli_01_0457.html", "node_id":"dli_01_0457.xml", "product_code":"dli", - "code":"118", + "code":"122", "des":"A Flink Jar job involves developing a custom application Jar package based on Flink's capabilities and submitting it to a DLI queue for execution.To create a Flink Jar jo", "doc_type":"usermanual", "kw":"Creating a Flink Jar Job,Submitting a Flink Job on the DLI Management Console,User Guide", @@ -2471,10 +2555,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating a Flink Jar Job", @@ -2484,7 +2568,7 @@ "uri":"dli_01_0479.html", "node_id":"dli_01_0479.xml", "product_code":"dli", - "code":"119", + "code":"123", "des":"You can isolate Flink jobs allocated to different users by setting permissions to ensure data query performance.The administrator and job creator have all permissions, wh", "doc_type":"usermanual", "kw":"Configuring Flink Job Permissions,Submitting a Flink Job on the DLI Management Console,User Guide", @@ -2505,7 +2589,7 @@ "uri":"dli_01_0377.html", "node_id":"dli_01_0377.xml", "product_code":"dli", - "code":"120", + "code":"124", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Managing Flink Jobs", @@ -2526,7 +2610,7 @@ "uri":"dli_01_0462.html", "node_id":"dli_01_0462.xml", "product_code":"dli", - "code":"121", + "code":"125", "des":"After creating a Flink job, you can check the basic information, job details, task list, and execution plan of the job on the DLI console.This section describes how to ch", "doc_type":"usermanual", "kw":"Viewing Flink Job Details,Managing Flink Jobs,User Guide", @@ -2534,10 +2618,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Viewing Flink Job Details", @@ -2547,7 +2631,7 @@ "uri":"dli_01_0650.html", "node_id":"dli_01_0650.xml", "product_code":"dli", - "code":"122", + "code":"126", "des":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", "doc_type":"usermanual", "kw":"Setting the Priority for a Flink Job,Managing Flink Jobs,User Guide", @@ -2568,7 +2652,7 @@ "uri":"dli_01_0534.html", "node_id":"dli_01_0534.xml", "product_code":"dli", - "code":"123", + "code":"127", "des":"In actual job operations, the compute resources required by a job vary depending on the data volume. As a result, compute resources are wasted when the volume is small an", "doc_type":"usermanual", "kw":"Enabling Dynamic Scaling for Flink Jobs,Managing Flink Jobs,User Guide", @@ -2589,7 +2673,7 @@ "uri":"dli_01_0651.html", "node_id":"dli_01_0651.xml", "product_code":"dli", - "code":"124", + "code":"128", "des":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", "doc_type":"usermanual", "kw":"Querying Logs for Flink Jobs,Managing Flink Jobs,User Guide", @@ -2610,7 +2694,7 @@ "uri":"dli_01_0461.html", "node_id":"dli_01_0461.xml", "product_code":"dli", - "code":"125", + "code":"129", "des":"After creating a job, you can manage it by performing various operations such as editing its basic information, starting or stopping it, and importing or exporting it.You", "doc_type":"usermanual", "kw":"Common Operations of Flink Jobs,Managing Flink Jobs,User Guide", @@ -2618,10 +2702,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Common Operations of Flink Jobs", @@ -2631,7 +2715,7 @@ "uri":"dli_01_0464.html", "node_id":"dli_01_0464.xml", "product_code":"dli", - "code":"126", + "code":"130", "des":"Flink templates include sample templates and custom templates. You can modify an existing sample template to meet the actual job logic requirements and save time for edit", "doc_type":"usermanual", "kw":"Managing Flink Job Templates,Submitting a Flink Job on the DLI Management Console,User Guide", @@ -2639,10 +2723,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Managing Flink Job Templates", @@ -2652,7 +2736,7 @@ "uri":"dli_01_0463.html", "node_id":"dli_01_0463.xml", "product_code":"dli", - "code":"127", + "code":"131", "des":"A tag is a user-defined key-value pair used to identify cloud resources, helping you classify and search for them. A tag consists of a tag key and a tag value.DLI allows ", "doc_type":"usermanual", "kw":"Adding Tags to a Flink Job,Submitting a Flink Job on the DLI Management Console,User Guide", @@ -2660,10 +2744,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Adding Tags to a Flink Job", @@ -2673,7 +2757,7 @@ "uri":"dli_01_0465.html", "node_id":"dli_01_0465.xml", "product_code":"dli", - "code":"128", + "code":"132", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Submitting a Spark Job on the DLI Management Console", @@ -2694,7 +2778,7 @@ "uri":"dli_01_0384.html", "node_id":"dli_01_0384.xml", "product_code":"dli", - "code":"129", + "code":"133", "des":"DLI Spark jobs provide fully managed Spark computing services.On the Overview page, click Create Job in the upper right corner of the Spark Jobs tab or click Create Job i", "doc_type":"usermanual", "kw":"Creating a Spark Job,Submitting a Spark Job on the DLI Management Console,User Guide", @@ -2702,10 +2786,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating a Spark Job", @@ -2715,7 +2799,7 @@ "uri":"dli_01_0652.html", "node_id":"dli_01_0652.xml", "product_code":"dli", - "code":"130", + "code":"134", "des":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", "doc_type":"usermanual", "kw":"Setting the Priority for a Spark Job,Submitting a Spark Job on the DLI Management Console,User Guide", @@ -2736,7 +2820,7 @@ "uri":"dli_01_0379.html", "node_id":"dli_01_0379.xml", "product_code":"dli", - "code":"131", + "code":"135", "des":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", "doc_type":"usermanual", "kw":"Querying Logs for Spark Jobs,Submitting a Spark Job on the DLI Management Console,User Guide", @@ -2757,18 +2841,18 @@ "uri":"dli_01_0385.html", "node_id":"dli_01_0385.xml", "product_code":"dli", - "code":"132", - "des":"On the Overview page, click Spark Jobs to go to the SQL job management page. Alternatively, you can click Job Management > Spark Jobs. The page displays all Spark jobs. I", + "code":"136", + "des":"To access the Spark job management page, click Spark Jobs on the Overview page or choose Job Management > Spark Jobs in the navigation pane on the left. The page displays", "doc_type":"usermanual", "kw":"Managing Spark Jobs,Submitting a Spark Job on the DLI Management Console,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Managing Spark Jobs", @@ -2778,7 +2862,7 @@ "uri":"dli_01_0551.html", "node_id":"dli_01_0551.xml", "product_code":"dli", - "code":"133", + "code":"137", "des":"You can modify a sample template to meet the Spark job requirements, saving time for editing SQL statements.Currently, the cloud platform does not provide preset Spark te", "doc_type":"usermanual", "kw":"Managing Spark Job Templates,Submitting a Spark Job on the DLI Management Console,User Guide", @@ -2799,7 +2883,7 @@ "uri":"dli_01_0445.html", "node_id":"dli_01_0445.xml", "product_code":"dli", - "code":"134", + "code":"138", "des":"This section describes metrics reported by DLI to Cloud Eye as well as their namespaces and dimensions. You can use the management console or APIs provided by Cloud Eye t", "doc_type":"usermanual", "kw":"Monitoring DLI Using Cloud Eye,User Guide", @@ -2807,10 +2891,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Monitoring DLI Using Cloud Eye", @@ -2820,7 +2904,7 @@ "uri":"dli_01_0318.html", "node_id":"dli_01_0318.xml", "product_code":"dli", - "code":"135", + "code":"139", "des":"With CTS, you can log operations related to DLI, making it easier to search, audit, and trace in the future.", "doc_type":"usermanual", "kw":"Using CTS to Audit DLI,User Guide", @@ -2828,10 +2912,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Using CTS to Audit DLI", @@ -2841,7 +2925,7 @@ "uri":"dli_01_0408.html", "node_id":"dli_01_0408.xml", "product_code":"dli", - "code":"136", + "code":"140", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"DLI Permission Management", @@ -2849,10 +2933,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"DLI Permission Management", @@ -2862,7 +2946,7 @@ "uri":"dli_01_0440.html", "node_id":"dli_01_0440.xml", "product_code":"dli", - "code":"137", + "code":"141", "des":"DLI features two sets of permission systems. These two permission control mechanisms are used in conjunction, with their permissions overlapping and complementing each ot", "doc_type":"usermanual", "kw":"Overview of the DLI Permission System,DLI Permission Management,User Guide", @@ -2870,10 +2954,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Overview of the DLI Permission System", @@ -2883,7 +2967,7 @@ "uri":"dli_01_0689.html", "node_id":"dli_01_0689.xml", "product_code":"dli", - "code":"138", + "code":"142", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"DLI Permission Management", @@ -2891,10 +2975,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"DLI Permission Management", @@ -2904,7 +2988,7 @@ "uri":"dli_01_0690.html", "node_id":"dli_01_0690.xml", "product_code":"dli", - "code":"139", + "code":"143", "des":"The permission management function of DLI itself primarily applies to the permission control of internal DLI resources, rather than relying on the unified identity authen", "doc_type":"usermanual", "kw":"Overview,DLI Permission Management,User Guide", @@ -2912,10 +2996,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Overview", @@ -2925,7 +3009,7 @@ "uri":"dli_01_0691.html", "node_id":"dli_01_0691.xml", "product_code":"dli", - "code":"140", + "code":"144", "des":"For details about authorization through the DLI console, see Managing Permissions.For details about authorization through the DLI console, see Queue Permission Management", "doc_type":"usermanual", "kw":"DLI Permissions,DLI Permission Management,User Guide", @@ -2933,10 +3017,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"DLI Permissions", @@ -2946,7 +3030,7 @@ "uri":"dli_01_0692.html", "node_id":"dli_01_0692.xml", "product_code":"dli", - "code":"141", + "code":"145", "des":"This section describes how to authorize resources on the DLI console.", "doc_type":"usermanual", "kw":"DLI Authorization Operation Guide,DLI Permission Management,User Guide", @@ -2954,10 +3038,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"DLI Authorization Operation Guide", @@ -2967,7 +3051,7 @@ "uri":"dli_01_0475.html", "node_id":"dli_01_0475.xml", "product_code":"dli", - "code":"142", + "code":"146", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"IAM Permission Management", @@ -2975,10 +3059,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"IAM Permission Management", @@ -2988,7 +3072,7 @@ "uri":"dli_01_0417.html", "node_id":"dli_01_0417.xml", "product_code":"dli", - "code":"143", + "code":"147", "des":"IAM is a foundational service for permission management, offering functions like user identity authentication, permission allocation, and access control, enabling you to ", "doc_type":"usermanual", "kw":"Overview,IAM Permission Management,User Guide", @@ -2996,10 +3080,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Overview", @@ -3009,7 +3093,7 @@ "uri":"dli_01_0693.html", "node_id":"dli_01_0693.xml", "product_code":"dli", - "code":"144", + "code":"148", "des":"In IAM, authorization entities are primarily categorized into users and user groups. Integration with enterprise projects enables resource isolation by group and facilita", "doc_type":"usermanual", "kw":"IAM Authorization Entity,IAM Permission Management,User Guide", @@ -3017,10 +3101,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"IAM Authorization Entity", @@ -3030,7 +3114,7 @@ "uri":"dli_01_0441.html", "node_id":"dli_01_0441.xml", "product_code":"dli", - "code":"145", + "code":"149", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"IAM Permissions,IAM Permission Management,User Guide", @@ -3038,10 +3122,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"IAM Permissions", @@ -3051,7 +3135,7 @@ "uri":"dli_01_0451.html", "node_id":"dli_01_0451.xml", "product_code":"dli", - "code":"146", + "code":"150", "des":"The role/policy-based authorization model provided by Identity and Access Management (IAM) lets you control access to DLI resources. With IAM, you can:Based on the organi", "doc_type":"usermanual", "kw":"Using IAM Roles or Policies to Grant Access to DLI,IAM Permission Management,User Guide", @@ -3059,10 +3143,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Using IAM Roles or Policies to Grant Access to DLI", @@ -3072,7 +3156,7 @@ "uri":"dli_01_0695.html", "node_id":"dli_01_0695.xml", "product_code":"dli", - "code":"147", + "code":"151", "des":"An e-commerce company conducts data analysis operations in region A. The marketing department's employees, user 1 and user 2, along with the technical department's employ", "doc_type":"usermanual", "kw":"Example Use Case: Creating a User Group and Granting Permissions for Using DLI Elastic Resource Pool", @@ -3080,10 +3164,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Example Use Case: Creating a User Group and Granting Permissions for Using DLI Elastic Resource Pools and Databases", @@ -3093,7 +3177,7 @@ "uri":"dli_01_0567.html", "node_id":"dli_01_0567.xml", "product_code":"dli", - "code":"148", + "code":"152", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Common DLI Management Operations", @@ -3114,7 +3198,7 @@ "uri":"dli_01_0494.html", "node_id":"dli_01_0494.xml", "product_code":"dli", - "code":"149", + "code":"153", "des":"To enhance the functions and performance of Spark and Flink jobs, you can create custom images by downloading the base images provided by DLI and adding dependencies (fil", "doc_type":"usermanual", "kw":"Enhancing the Job Runtime Environment Using a Custom Image,Common DLI Management Operations,User Gui", @@ -3122,10 +3206,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Enhancing the Job Runtime Environment Using a Custom Image", @@ -3135,7 +3219,7 @@ "uri":"dli_01_0476.html", "node_id":"dli_01_0476.xml", "product_code":"dli", - "code":"150", + "code":"154", "des":"DLI allows you to set variables that are frequently used during job development as global variables on the DLI management console. This avoids repeated definitions during", "doc_type":"usermanual", "kw":"Managing DLI Global Variables,Common DLI Management Operations,User Guide", @@ -3156,7 +3240,7 @@ "uri":"dli_01_0366.html", "node_id":"dli_01_0366.xml", "product_code":"dli", - "code":"151", + "code":"155", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Managing Program Packages of Jar Jobs", @@ -3177,7 +3261,7 @@ "uri":"dli_01_0407.html", "node_id":"dli_01_0407.xml", "product_code":"dli", - "code":"152", + "code":"156", "des":"Before running DLI jobs, UDF JAR files or Jar job packages need to be uploaded to the cloud platform for unified management and maintenance.There are two ways to manage p", "doc_type":"usermanual", "kw":"Package Management Overview,Managing Program Packages of Jar Jobs,User Guide", @@ -3198,7 +3282,7 @@ "uri":"dli_01_0367.html", "node_id":"dli_01_0367.xml", "product_code":"dli", - "code":"153", + "code":"157", "des":"DLI allows you to submit program packages in batches to the general-use queue for running.If you need to update a package, you can use the same package or file to upload ", "doc_type":"usermanual", "kw":"Creating a DLI Package,Managing Program Packages of Jar Jobs,User Guide", @@ -3206,10 +3290,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Creating a DLI Package", @@ -3219,7 +3303,7 @@ "uri":"dli_01_0477.html", "node_id":"dli_01_0477.xml", "product_code":"dli", - "code":"154", + "code":"158", "des":"By configuring permissions, you can grant different package groups or packages to various users, ensuring that job efficiency remains unaffected and job performance is ma", "doc_type":"usermanual", "kw":"Configuring DLI Package Permissions,Managing Program Packages of Jar Jobs,User Guide", @@ -3240,7 +3324,7 @@ "uri":"dli_01_0478.html", "node_id":"dli_01_0478.xml", "product_code":"dli", - "code":"155", + "code":"159", "des":"DLI allows you to change the owner of a package group or package.Log in to the DLI management console and choose Data Management > Package Management.On the Package Manag", "doc_type":"usermanual", "kw":"Changing the DLI Package Owner,Managing Program Packages of Jar Jobs,User Guide", @@ -3261,7 +3345,7 @@ "uri":"dli_01_0369.html", "node_id":"dli_01_0369.xml", "product_code":"dli", - "code":"156", + "code":"160", "des":"Tags are key-value pairs that you can define to identify cloud resources. They assist you in categorizing and searching for cloud resources. A tag consists of a key and a", "doc_type":"usermanual", "kw":"Managing DLI Package Tags,Managing Program Packages of Jar Jobs,User Guide", @@ -3269,10 +3353,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Managing DLI Package Tags", @@ -3282,7 +3366,7 @@ "uri":"dli_01_0397.html", "node_id":"dli_01_0397.xml", "product_code":"dli", - "code":"157", + "code":"161", "des":"DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar ", "doc_type":"usermanual", "kw":"DLI Built-in Dependencies,Managing Program Packages of Jar Jobs,User Guide", @@ -3290,10 +3374,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"DLI Built-in Dependencies", @@ -3303,18 +3387,18 @@ "uri":"dli_01_0550.html", "node_id":"dli_01_0550.xml", "product_code":"dli", - "code":"158", - "des":"A quota limits the quantity of a resource available to users, thereby preventing spikes in the usage of the resource.You can also request for an increased quota if your e", + "code":"162", + "des":"Quotas are enforced for service resources on the platform to prevent unforeseen spikes in resource usage. Quotas can limit the quantity and capacity of resources availabl", "doc_type":"usermanual", "kw":"Managing DLI Resource Quotas,Common DLI Management Operations,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Managing DLI Resource Quotas", @@ -3324,7 +3408,7 @@ "uri":"dli_01_0539.html", "node_id":"dli_01_0539.xml", "product_code":"dli", - "code":"159", + "code":"163", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"FAQ", @@ -3345,7 +3429,7 @@ "uri":"dli_03_0001.html", "node_id":"dli_03_0001.xml", "product_code":"dli", - "code":"160", + "code":"164", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"DLI Basics", @@ -3365,7 +3449,7 @@ "uri":"dli_03_0115.html", "node_id":"dli_03_0115.xml", "product_code":"dli", - "code":"161", + "code":"165", "des":"Both DLI and MRS support Spark, but there are some differences in service mode, interface, application scenarios, and performance characteristics.The Spark component of D", "doc_type":"usermanual", "kw":"What Are the Differences Between MRS Spark and DLI Spark?,DLI Basics,User Guide", @@ -3385,7 +3469,7 @@ "uri":"dli_03_0029.html", "node_id":"dli_03_0029.xml", "product_code":"dli", - "code":"162", + "code":"166", "des":"Supported data formats:ParquetCSVORCJSONAvroOBS: Data used by SQL jobs, Spark jobs, and Flink jobs can be stored in OBS, reducing storage costs.DLI: The column-based Parq", "doc_type":"usermanual", "kw":"Where Can Data Be Stored in DLI?,DLI Basics,User Guide", @@ -3405,7 +3489,7 @@ "uri":"dli_03_0129.html", "node_id":"dli_03_0129.xml", "product_code":"dli", - "code":"163", + "code":"167", "des":"DLI supports importing data from OBS buckets shared by IAM users under the same tenant, but not from OBS buckets shared by other tenants.This ensures data security and is", "doc_type":"usermanual", "kw":"Can I Import OBS Bucket Data Shared by Other Tenants into DLI?,DLI Basics,User Guide", @@ -3425,7 +3509,7 @@ "uri":"dli_03_0263.html", "node_id":"dli_03_0263.xml", "product_code":"dli", - "code":"164", + "code":"168", "des":"No, a global variable can only be used by the user who created it. Global variables can be used to simplify complex parameters. For example, long and difficult variables ", "doc_type":"usermanual", "kw":"Can a Member Account Use Global Variables Created by Other Member Accounts?,DLI Basics,User Guide", @@ -3445,7 +3529,7 @@ "uri":"dli_03_0260.html", "node_id":"dli_03_0260.xml", "product_code":"dli", - "code":"165", + "code":"169", "des":"No.The spark.acls.enable configuration item is not used in DLI. The Apache Spark command injection vulnerability (CVE-2022-33891) does not exist in DLI.This vulnerability", "doc_type":"usermanual", "kw":"Is DLI Affected by the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?,DLI Basics,Use", @@ -3465,7 +3549,7 @@ "uri":"dli_03_0126.html", "node_id":"dli_03_0126.xml", "product_code":"dli", - "code":"166", + "code":"170", "des":"To manage a large number of DLI jobs, you can use the following methods:Manage jobs by group.Group tens of thousands of jobs by type and run each group on a queue.Group t", "doc_type":"usermanual", "kw":"How Do I Manage Jobs Running on DLI?,DLI Basics,User Guide", @@ -3485,7 +3569,7 @@ "uri":"dli_03_0162.html", "node_id":"dli_03_0162.xml", "product_code":"dli", - "code":"167", + "code":"171", "des":"DLI does not support directly changing the field names of a table. However, you can solve this issue by migrating the table data using the following steps:Create a table:", "doc_type":"usermanual", "kw":"How Do I Change the Field Names of an Existing Table on DLI?,DLI Basics,User Guide", @@ -3505,7 +3589,7 @@ "uri":"dli_03_0049.html", "node_id":"dli_03_0049.xml", "product_code":"dli", - "code":"168", + "code":"172", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"DLI Elastic Resource Pools and Queues", @@ -3525,7 +3609,7 @@ "uri":"dli_03_0276.html", "node_id":"dli_03_0276.xml", "product_code":"dli", - "code":"169", + "code":"173", "des":"In daily big data analysis work, it is important to allocate and manage compute resources properly to provide a good job execution environment.You can allocate resources ", "doc_type":"usermanual", "kw":"How Can I Check the Actual and Used CUs for an Elastic Resource Pool as Well as the Required CUs for", @@ -3545,7 +3629,7 @@ "uri":"dli_03_0183.html", "node_id":"dli_03_0183.xml", "product_code":"dli", - "code":"170", + "code":"174", "des":"You need to check the large number of jobs in the Submitting and Running states on the queue.Use Cloud Eye to view jobs in different states on the queue. The procedure is", "doc_type":"usermanual", "kw":"How Do I Check for a Backlog of Jobs in the Current DLI Queue?,DLI Elastic Resource Pools and Queues", @@ -3565,7 +3649,7 @@ "uri":"dli_03_0095.html", "node_id":"dli_03_0095.xml", "product_code":"dli", - "code":"171", + "code":"175", "des":"To check the running status of the DLI queue and determine whether to run more jobs on that queue, you need to check the queue load.Search for Cloud Eye on the console.In", "doc_type":"usermanual", "kw":"How Do I View the Load of a DLI Queue?,DLI Elastic Resource Pools and Queues,User Guide", @@ -3585,7 +3669,7 @@ "uri":"dli_03_0098.html", "node_id":"dli_03_0098.xml", "product_code":"dli", - "code":"172", + "code":"176", "des":"DLI allows you to subscribe to an SMN topic for failed jobs.Log in to the DLI console.In the navigation pane on the left, choose Queue Management.On the Queue Management ", "doc_type":"usermanual", "kw":"How Do I Monitor Job Exceptions on a DLI Queue?,DLI Elastic Resource Pools and Queues,User Guide", @@ -3605,7 +3689,7 @@ "uri":"dli_03_0065.html", "node_id":"dli_03_0065.xml", "product_code":"dli", - "code":"173", + "code":"177", "des":"Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earl", "doc_type":"usermanual", "kw":"How Do I Migrate an Old Version Spark Queue to a General-Purpose Queue?,DLI Elastic Resource Pools a", @@ -3625,7 +3709,7 @@ "uri":"dli_03_0171.html", "node_id":"dli_03_0171.xml", "product_code":"dli", - "code":"174", + "code":"178", "des":"After a SQL job was submitted to the default queue, the job runs abnormally. The job log reported that the execution timed out. The exception logs are as follows:[ERROR] ", "doc_type":"usermanual", "kw":"How Do I Do If I Encounter a Timeout Exception When Executing DLI SQL Statements on the default Queu", @@ -3645,7 +3729,7 @@ "uri":"dli_03_0211.html", "node_id":"dli_03_0211.xml", "product_code":"dli", - "code":"175", + "code":"179", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"DLI Databases and Tables", @@ -3665,7 +3749,7 @@ "uri":"dli_03_0184.html", "node_id":"dli_03_0184.xml", "product_code":"dli", - "code":"176", + "code":"180", "des":"A DLI table exists but cannot be queried on the DLI console.If a table exists but cannot be queried, there is a high probability that the current user does not have the p", "doc_type":"usermanual", "kw":"Why Am I Unable to Query a Table on the DLI Console?,DLI Databases and Tables,User Guide", @@ -3685,7 +3769,7 @@ "uri":"dli_03_0013.html", "node_id":"dli_03_0013.xml", "product_code":"dli", - "code":"177", + "code":"181", "des":"When submitting a job to import data into a DLI table, if the compression rate of the Parquet/ORC file corresponding to the OBS table is high, exceeding 5 times the compr", "doc_type":"usermanual", "kw":"How Do I Do If the Compression Rate of an OBS Table Is High?,DLI Databases and Tables,User Guide", @@ -3705,7 +3789,7 @@ "uri":"dli_03_0009.html", "node_id":"dli_03_0009.xml", "product_code":"dli", - "code":"178", + "code":"182", "des":"To avoid garbled characters caused by inconsistent character encoding, you are advised to unify the encoding format of your data source when executing jobs in DLI.DLI onl", "doc_type":"usermanual", "kw":"How Do I Do If Inconsistent Character Encoding Leads to Garbled Characters?,DLI Databases and Tables", @@ -3725,10 +3809,10 @@ "uri":"dli_03_0175.html", "node_id":"dli_03_0175.xml", "product_code":"dli", - "code":"179", + "code":"183", "des":"User A created the testTable table in a database through a SQL job and granted user B the permission to insert and delete table data. User A deleted the testTable table a", "doc_type":"usermanual", - "kw":"Do I Need to to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With", + "kw":"Do I Need to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With th", "search_title":"", "metedata":[ { @@ -3738,14 +3822,14 @@ "IsBot":"Yes" } ], - "title":"Do I Need to to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With the Same Name?", + "title":"Do I Need to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With the Same Name?", "githuburl":"" }, { "uri":"dli_03_0177.html", "node_id":"dli_03_0177.xml", "product_code":"dli", - "code":"180", + "code":"184", "des":"A CSV file is imported to a DLI partitioned table, but the imported file data does not contain the data in the partitioning column. The partitioning column needs to be sp", "doc_type":"usermanual", "kw":"How Do I Do If Files Imported Into a DLI Partitioned Table Lack Data for the Partition Columns, Caus", @@ -3765,7 +3849,7 @@ "uri":"dli_03_0181.html", "node_id":"dli_03_0181.xml", "product_code":"dli", - "code":"181", + "code":"185", "des":"When an OBS foreign table is created, a field in the specified OBS file contains a carriage return line feed (CRLF) character. As a result, the data is incorrect.The stat", "doc_type":"usermanual", "kw":"How Do I Fix Incorrect Data in an OBS Foreign Table Caused by Newline Characters in OBS File Fields?", @@ -3785,7 +3869,7 @@ "uri":"dli_03_0187.html", "node_id":"dli_03_0187.xml", "product_code":"dli", - "code":"182", + "code":"186", "des":"The on clause was not added to the SQL statement for joining tables. As a result, the Cartesian product query occurs due to multi-table association, and the queue resourc", "doc_type":"usermanual", "kw":"How Do I Prevent a Cartesian Product Query and Resource Overload Due to Missing \"ON\" Conditions in T", @@ -3805,7 +3889,7 @@ "uri":"dli_03_0190.html", "node_id":"dli_03_0190.xml", "product_code":"dli", - "code":"183", + "code":"187", "des":"Partition data is manually uploaded to a partition of an OBS table. However, the data cannot be queried using DLI SQL editor.After manually adding partition data, you nee", "doc_type":"usermanual", "kw":"How Do I Do If I Can't Query Data After Manually Adding It to the Partition Directory of an OBS Tabl", @@ -3825,7 +3909,7 @@ "uri":"dli_03_0212.html", "node_id":"dli_03_0212.xml", "product_code":"dli", - "code":"184", + "code":"188", "des":"When using the INSERT OVERWRITE statement to overwrite data in a partitioned table, if you find that it overwrites all data instead of the expected partitioned data, it m", "doc_type":"usermanual", "kw":"Why Does the \"insert overwrite\" Operation Affect All Data in a Partitioned Table Instead of Just the", @@ -3845,7 +3929,7 @@ "uri":"dli_03_0214.html", "node_id":"dli_03_0214.xml", "product_code":"dli", - "code":"185", + "code":"189", "des":"Spark does not have the datetime type and uses the TIMESTAMP type instead.You can use a function to convert data types.The following is an example.select cast(create_date", "doc_type":"usermanual", "kw":"Why Does the \"create_date\" Field in an RDS Table (Datetime Data Type) Appear as a Timestamp in DLI Q", @@ -3865,7 +3949,7 @@ "uri":"dli_03_0215.html", "node_id":"dli_03_0215.xml", "product_code":"dli", - "code":"186", + "code":"190", "des":"Changing a table name immediately after executing a SQL job may result in an incorrect data size for the table.This is because DLI updates the metadata of the table when ", "doc_type":"usermanual", "kw":"How Do I Do If Renaming a Table After a SQL Job Causes Incorrect Data Size?,DLI Databases and Tables", @@ -3885,7 +3969,7 @@ "uri":"dli_03_0231.html", "node_id":"dli_03_0231.xml", "product_code":"dli", - "code":"187", + "code":"191", "des":"When DLI is used to insert data into an OBS temporary table, only part of data is imported.Possible causes are as follows:The amount of data read during job execution is ", "doc_type":"usermanual", "kw":"How Can I Resolve Data Inconsistencies When Importing Data from DLI to OBS?,DLI Databases and Tables", @@ -3905,7 +3989,7 @@ "uri":"dli_03_0277.html", "node_id":"dli_03_0277.xml", "product_code":"dli", - "code":"188", + "code":"192", "des":"The Hudi table created using Flink SQL statements is not displayed on the DLI console, making it impossible to manage or perform query operations on the table through the", "doc_type":"usermanual", "kw":"Why Is a Hudi Table Not Displayed on the DLI Console?,DLI Databases and Tables,User Guide", @@ -3925,7 +4009,7 @@ "uri":"dli_03_0022.html", "node_id":"dli_03_0022.xml", "product_code":"dli", - "code":"189", + "code":"193", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Enhanced Datasource Connections", @@ -3945,7 +4029,7 @@ "uri":"dli_03_0238.html", "node_id":"dli_03_0238.xml", "product_code":"dli", - "code":"190", + "code":"194", "des":"The outbound rule had been configured for the security group of the queue associated with the enhanced datasource connection. The datasource authentication used a passwor", "doc_type":"usermanual", "kw":"How Do I Resolve a Failure in Connecting DLI to GaussDB(DWS) Through an Enhanced Datasource Connecti", @@ -3965,7 +4049,7 @@ "uri":"dli_03_0179.html", "node_id":"dli_03_0179.xml", "product_code":"dli", - "code":"191", + "code":"195", "des":"A datasource connection is created and bound to a queue. The connectivity test fails and the following error information is displayed:failed to connect to specified addre", "doc_type":"usermanual", "kw":"How Do I Do If the Datasource Connection Is Successfully Created but the Network Connectivity Test F", @@ -3985,7 +4069,7 @@ "uri":"dli_03_0186.html", "node_id":"dli_03_0186.xml", "product_code":"dli", - "code":"192", + "code":"196", "des":"Configuring the Connection Between a DLI Queue and a Data Source in a Private NetworkIf your DLI job needs to connect to a data source, for example, MRS, RDS, CSS, Kafka,", "doc_type":"usermanual", "kw":"How Do I Configure Network Connectivity Between a DLI Queue and a Data Source?,Enhanced Datasource C", @@ -4005,7 +4089,7 @@ "uri":"dli_03_0128.html", "node_id":"dli_03_0128.xml", "product_code":"dli", - "code":"193", + "code":"197", "des":"The main reason for creating a VPC peering connection for DLI's enhanced datasource connection is to establish network connectivity between DLI and data sources in differ", "doc_type":"usermanual", "kw":"Why Is Creating a VPC Peering Connection Necessary for Enhanced Datasource Connections in DLI?,Enhan", @@ -4025,7 +4109,7 @@ "uri":"dli_03_0257.html", "node_id":"dli_03_0257.xml", "product_code":"dli", - "code":"194", + "code":"198", "des":"The possible causes and solutions are as follows:If you have created a queue, do not bind it to a datasource connection immediately. Wait for 5 to 10 minutes. After the c", "doc_type":"usermanual", "kw":"How Do I Do If Creating a Datasource Connection in DLI Gets Stuck in the \"Creating\" State When Bindi", @@ -4045,7 +4129,7 @@ "uri":"dli_03_0047.html", "node_id":"dli_03_0047.xml", "product_code":"dli", - "code":"195", + "code":"199", "des":"The network connectivity is abnormal. Check whether the security group is correctly selected and whether the VPC is correctly configured.Example: When you create an RDS d", "doc_type":"usermanual", "kw":"How Do I Resolve the \"communication link failure\" Error When Using a Newly Created Datasource Connec", @@ -4065,7 +4149,7 @@ "uri":"dli_03_0080.html", "node_id":"dli_03_0080.xml", "product_code":"dli", - "code":"196", + "code":"200", "des":"If you have not added the cluster host information to the datasource connection, it can lead to KRB authentication failure, resulting in a connection timeout. In this cas", "doc_type":"usermanual", "kw":"How Do I Troubleshoot a Connection Timeout Issue That Isn't Recorded in Logs When Accessing MRS HBas", @@ -4085,7 +4169,7 @@ "uri":"dli_03_0239.html", "node_id":"dli_03_0239.xml", "product_code":"dli", - "code":"197", + "code":"201", "des":"A datasource RDS table was created in the DataArts Studio, and the insert overwrite statement was executed to write data into RDS. DLI.0999: BatchUpdateException: Incorre", "doc_type":"usermanual", "kw":"How Do I Do If I Encounter the \"Incorrect string value\" Error When Executing insert overwrite on a D", @@ -4105,7 +4189,7 @@ "uri":"dli_03_0250.html", "node_id":"dli_03_0250.xml", "product_code":"dli", - "code":"198", + "code":"202", "des":"The system failed to create a datasource RDS table, and null pointer error was reported.The following table creation statement was used:The RDS database is in a PostGre c", "doc_type":"usermanual", "kw":"How Do I Resolve the Null Pointer Error When Creating an RDS Datasource Table?,Enhanced Datasource C", @@ -4125,7 +4209,7 @@ "uri":"dli_03_0251.html", "node_id":"dli_03_0251.xml", "product_code":"dli", - "code":"199", + "code":"203", "des":"The system failed to execute insert overwrite on the datasource GaussDB(DWS) table, and org.postgresql.util.PSQLException: ERROR: tuple concurrently updated was displayed", "doc_type":"usermanual", "kw":"Error Message \"org.postgresql.util.PSQLException: ERROR: tuple concurrently updated\" Is Displayed Wh", @@ -4145,7 +4229,7 @@ "uri":"dli_03_0252.html", "node_id":"dli_03_0252.xml", "product_code":"dli", - "code":"200", + "code":"204", "des":"A datasource table was used to import data to a CloudTable HBase table. This HBase table contains a column family and a rowkey for 100 million simulating data records. Th", "doc_type":"usermanual", "kw":"RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Datas", @@ -4165,7 +4249,7 @@ "uri":"dli_03_0253.html", "node_id":"dli_03_0253.xml", "product_code":"dli", - "code":"201", + "code":"205", "des":"A table was created on GaussDB(DWS) and then a datasource connection was created on DLI to read and write data. An error message was displayed during data writing, indica", "doc_type":"usermanual", "kw":"How Do I Do If A Null Value Is Written Into a Non-Null Field When Using a DLI Datasource Connection ", @@ -4185,7 +4269,7 @@ "uri":"dli_03_0254.html", "node_id":"dli_03_0254.xml", "product_code":"dli", - "code":"202", + "code":"206", "des":"A datasource GaussDB(DWS) table and the datasource connection were created in DLI, and the schema of the source table in GaussDB(DWS) were updated. During the job executi", "doc_type":"usermanual", "kw":"How Do I Do If an Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updat", @@ -4205,7 +4289,7 @@ "uri":"dli_03_0028.html", "node_id":"dli_03_0028.xml", "product_code":"dli", - "code":"203", + "code":"207", "des":"When creating an associated RDS table in DLI, if the RDS table contains an auto-increment primary key or other auto-populated fields, you can take the following measures ", "doc_type":"usermanual", "kw":"How Do I Insert Data into an RDS Table with an Auto-Increment Primary Key Using DLI?,Enhanced Dataso", @@ -4225,7 +4309,7 @@ "uri":"dli_03_0020.html", "node_id":"dli_03_0020.xml", "product_code":"dli", - "code":"204", + "code":"208", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"SQL Jobs", @@ -4245,7 +4329,7 @@ "uri":"dli_03_0204.html", "node_id":"dli_03_0204.xml", "product_code":"dli", - "code":"205", + "code":"209", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"SQL Job Development", @@ -4265,7 +4349,7 @@ "uri":"dli_03_0200.html", "node_id":"dli_03_0200.xml", "product_code":"dli", - "code":"206", + "code":"210", "des":"A temporary table is used to store intermediate results. When a transaction or session ends, the data in the temporary table can be automatically deleted. For example, in", "doc_type":"usermanual", "kw":"SQL Jobs,SQL Job Development,User Guide", @@ -4285,8 +4369,8 @@ "uri":"dli_03_0086.html", "node_id":"dli_03_0086.xml", "product_code":"dli", - "code":"207", - "des":"If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.You a", + "code":"211", + "des":"In distributed file systems, data is stored in blocks. A small file refers to a file whose size is significantly smaller than the block size of the storage system. Large ", "doc_type":"usermanual", "kw":"How Do I Merge Small Files?,SQL Job Development,User Guide", "search_title":"", @@ -4305,7 +4389,7 @@ "uri":"dli_03_0092.html", "node_id":"dli_03_0092.xml", "product_code":"dli", - "code":"208", + "code":"212", "des":"When creating an OBS table, you must specify a table path in the database.The path format is as follows: obs://xxx/database name/table name.If the specified path is akdc.", "doc_type":"usermanual", "kw":"How Do I Specify an OBS Path When Creating an OBS Table?,SQL Job Development,User Guide", @@ -4325,7 +4409,7 @@ "uri":"dli_03_0108.html", "node_id":"dli_03_0108.xml", "product_code":"dli", - "code":"209", + "code":"213", "des":"To associate JSON data nested in an OBS bucket, you can create a table in asynchronous mode.The following is an example of a table creation statement that shows how to us", "doc_type":"usermanual", "kw":"How Do I Create a Table Using JSON Data in an OBS Bucket?,SQL Job Development,User Guide", @@ -4345,7 +4429,7 @@ "uri":"dli_03_0069.html", "node_id":"dli_03_0069.xml", "product_code":"dli", - "code":"210", + "code":"214", "des":"The correct method for using the count function to perform aggregation is as follows:OrIf an incorrect method is used, an error will be reported.", "doc_type":"usermanual", "kw":"How Can I Use the count Function to Perform Aggregation?,SQL Job Development,User Guide", @@ -4365,7 +4449,7 @@ "uri":"dli_03_0072.html", "node_id":"dli_03_0072.xml", "product_code":"dli", - "code":"211", + "code":"215", "des":"You can use the cross-region replication function of OBS. The procedure is as follows:Export the DLI table data in region 1 to the user-defined OBS bucket.Use the OBS cro", "doc_type":"usermanual", "kw":"How Do I Synchronize DLI Table Data Across Regions?,SQL Job Development,User Guide", @@ -4385,7 +4469,7 @@ "uri":"dli_03_0191.html", "node_id":"dli_03_0191.xml", "product_code":"dli", - "code":"212", + "code":"216", "des":"If you need to insert data into a table but only want to specify certain fields, you can use the INSERT INTO statement combined with the SELECT clause.However, DLI curren", "doc_type":"usermanual", "kw":"How Do I Insert Table Data into Specific Fields of a Table Using a SQL Job?,SQL Job Development,User", @@ -4405,7 +4489,7 @@ "uri":"dli_03_0196.html", "node_id":"dli_03_0196.xml", "product_code":"dli", - "code":"213", + "code":"217", "des":"If the job runs slowly, perform the following steps to find the causes and rectify the fault:Check whether the problem is caused by FullGC.Log in to the DLI console. In t", "doc_type":"usermanual", "kw":"How Do I Troubleshoot Slow SQL Jobs?,SQL Job Development,User Guide", @@ -4425,7 +4509,7 @@ "uri":"dli_03_0091.html", "node_id":"dli_03_0091.xml", "product_code":"dli", - "code":"214", + "code":"218", "des":"You can view SQL job logs for routine O&M.Obtain the ID of the DLI job executed on the DataArts Studio console.Job IDOn the DLI console, choose Job Management > SQL Jobs.", "doc_type":"usermanual", "kw":"How Do I View DLI SQL Logs?,SQL Job Development,User Guide", @@ -4445,7 +4529,7 @@ "uri":"dli_03_0116.html", "node_id":"dli_03_0116.xml", "product_code":"dli", - "code":"215", + "code":"219", "des":"You can view the job execution records when a job is running.Log in to the DLI management console.In the navigation pane on the left, choose Job Management > SQL Jobs.Ent", "doc_type":"usermanual", "kw":"How Do I View SQL Execution Records in DLI?,SQL Job Development,User Guide", @@ -4465,7 +4549,7 @@ "uri":"dli_03_0093.html", "node_id":"dli_03_0093.xml", "product_code":"dli", - "code":"216", + "code":"220", "des":"Data skew is a common issue during the execution of SQL jobs. When data is unevenly distributed, some compute nodes process significantly more data than others, which can", "doc_type":"usermanual", "kw":"How Do I Do When Data Skew Occurs During the Execution of a SQL Job?,SQL Job Development,User Guide", @@ -4485,7 +4569,7 @@ "uri":"dli_03_0182.html", "node_id":"dli_03_0182.xml", "product_code":"dli", - "code":"217", + "code":"221", "des":"A SQL job contains join operations. After the job is submitted, it is stuck in the Running state and no result is returned.When a Spark SQL job has join operations on sma", "doc_type":"usermanual", "kw":"Why Does a SQL Job That Has Join Operations Stay in the Running State?,SQL Job Development,User Guid", @@ -4505,7 +4589,7 @@ "uri":"dli_03_0213.html", "node_id":"dli_03_0213.xml", "product_code":"dli", - "code":"218", + "code":"222", "des":"The possible causes and solutions are as follows:After you purchase a DLI queue and submit a SQL job for the first time, wait for 5 to 10 minutes. After the cluster is st", "doc_type":"usermanual", "kw":"Why Is a SQL Job Stuck in the Submitting State?,SQL Job Development,User Guide", @@ -4525,7 +4609,7 @@ "uri":"dli_03_0206.html", "node_id":"dli_03_0206.xml", "product_code":"dli", - "code":"219", + "code":"223", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"SQL Job O&M", @@ -4545,7 +4629,7 @@ "uri":"dli_03_0014.html", "node_id":"dli_03_0014.xml", "product_code":"dli", - "code":"220", + "code":"224", "des":"This message indicates that you are exporting data to an existing OBS path.Solution:Create an OBS directory.You can create an OBS directory that does not exist to store t", "doc_type":"usermanual", "kw":"Why Is Error \"path obs://xxx already exists\" Reported When Data Is Exported to OBS?,SQL Job O&M,User", @@ -4565,7 +4649,7 @@ "uri":"dli_03_0066.html", "node_id":"dli_03_0066.xml", "product_code":"dli", - "code":"221", + "code":"225", "des":"This message indicates that the two tables to be joined contain the same column, but the owner of the column is not specified when the command is executed.For example, ta", "doc_type":"usermanual", "kw":"Why Is Error \"SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;\" Displayed W", @@ -4585,7 +4669,7 @@ "uri":"dli_03_0071.html", "node_id":"dli_03_0071.xml", "product_code":"dli", - "code":"222", + "code":"226", "des":"This message indicates that your operation is restricted because your account is in arrears or there is insufficient balance in your account.Solution:Check the account st", "doc_type":"usermanual", "kw":"Why Is Error \"The current account does not have permission to perform this operation,the current acc", @@ -4605,7 +4689,7 @@ "uri":"dli_03_0145.html", "node_id":"dli_03_0145.xml", "product_code":"dli", - "code":"223", + "code":"227", "des":"When you query the partitioned table XX.YYY, the partition column is not specified in the search criteria.A partitioned table can be queried only when the query condition", "doc_type":"usermanual", "kw":"Why Is Error \"There should be at least one partition pruning predicate on partitioned table XX.YYY\" ", @@ -4625,7 +4709,7 @@ "uri":"dli_03_0169.html", "node_id":"dli_03_0169.xml", "product_code":"dli", - "code":"224", + "code":"228", "des":"The following error message is displayed when the LOAD DATA command is executed by a Spark SQL job to import data to a DLI table:In some cases ,the following error messag", "doc_type":"usermanual", "kw":"Why Is Error \"IllegalArgumentException: Buffer size too small. size\" Reported When Data Is Loaded to", @@ -4645,7 +4729,7 @@ "uri":"dli_03_0189.html", "node_id":"dli_03_0189.xml", "product_code":"dli", - "code":"225", + "code":"229", "des":"An error is reported during SQL job execution:Please contact DLI service. DLI.0002: FileNotFoundException: getFileStatus on obs://xxx: status [404]Check whether there is ", "doc_type":"usermanual", "kw":"Why Is Error \"DLI.0002 FileNotFoundException\" Reported During SQL Job Running?,SQL Job O&M,User Guid", @@ -4665,7 +4749,7 @@ "uri":"dli_03_0046.html", "node_id":"dli_03_0046.xml", "product_code":"dli", - "code":"226", + "code":"230", "des":"Currently, DLI supports the creation of TEXTFILE, SEQUENCEFILE, RCFILE, ORC, AVRO, and PARQUET tables using the Hive syntax.If you create a table using CTAS statements, s", "doc_type":"usermanual", "kw":"Why Is a Schema Parsing Error Reported When I Create a Hive Table Using CTAS?,SQL Job O&M,User Guide", @@ -4685,7 +4769,7 @@ "uri":"dli_03_0173.html", "node_id":"dli_03_0173.xml", "product_code":"dli", - "code":"227", + "code":"231", "des":"When you run a DLI SQL script on DataArts Studio, the log shows that the statements fail to be executed. The error information is as follows:DLI.0999: RuntimeException: o", "doc_type":"usermanual", "kw":"Why Is Error \"org.apache.hadoop.fs.obs.OBSIOException\" Reported When I Run DLI SQL Scripts on DataAr", @@ -4705,7 +4789,7 @@ "uri":"dli_03_0207.html", "node_id":"dli_03_0207.xml", "product_code":"dli", - "code":"228", + "code":"232", "des":"Error message \"File not Found\" is displayed when a SQL job is accessed.The system may not be able to locate the specified file path or file due to an incorrect file path ", "doc_type":"usermanual", "kw":"Why Is Error \"File not Found\" Reported When I Access a SQL Job?,SQL Job O&M,User Guide", @@ -4725,7 +4809,7 @@ "uri":"dli_03_0208.html", "node_id":"dli_03_0208.xml", "product_code":"dli", - "code":"229", + "code":"233", "des":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.Check the permissions of the OBS bucket to ensure that the account has access ", "doc_type":"usermanual", "kw":"Why Is Error \"DLI.0003: AccessControlException XXX\" Reported When I Access a SQL Job?,SQL Job O&M,Us", @@ -4745,7 +4829,7 @@ "uri":"dli_03_0209.html", "node_id":"dli_03_0209.xml", "product_code":"dli", - "code":"230", + "code":"234", "des":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Ch", "doc_type":"usermanual", "kw":"Why Is Error \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{b", @@ -4765,7 +4849,7 @@ "uri":"dli_03_0210.html", "node_id":"dli_03_0210.xml", "product_code":"dli", - "code":"231", + "code":"235", "des":"Error message \"The current account does not have permission to perform this operation,the current account was restricted.\" is reported during SQL statement execution.Chec", "doc_type":"usermanual", "kw":"Why Am I Seeing the Error Message \"The current account does not have permission to perform this oper", @@ -4785,7 +4869,7 @@ "uri":"dli_03_0037.html", "node_id":"dli_03_0037.xml", "product_code":"dli", - "code":"232", + "code":"236", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Flink Jobs", @@ -4805,7 +4889,7 @@ "uri":"dli_03_0137.html", "node_id":"dli_03_0137.xml", "product_code":"dli", - "code":"233", + "code":"237", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Flink Job Consulting", @@ -4825,7 +4909,7 @@ "uri":"dli_03_0139.html", "node_id":"dli_03_0139.xml", "product_code":"dli", - "code":"234", + "code":"238", "des":"A sub-user can view queues but cannot view Flink jobs. You can authorize the sub-user using DLI or IAM.Authorization on DLILog in to the DLI console using a tenant accoun", "doc_type":"usermanual", "kw":"How Do I Authorize a Subuser to View Flink Jobs?,Flink Job Consulting,User Guide", @@ -4845,7 +4929,7 @@ "uri":"dli_03_0090.html", "node_id":"dli_03_0090.xml", "product_code":"dli", - "code":"235", + "code":"239", "des":"DLI Flink jobs are highly available. You can enable the automatic restart function to automatically restart your jobs after short-time faults of peripheral services are r", "doc_type":"usermanual", "kw":"How Do I Configure Auto Restart upon Exception for a Flink Job?,Flink Job Consulting,User Guide", @@ -4865,7 +4949,7 @@ "uri":"dli_03_0099.html", "node_id":"dli_03_0099.xml", "product_code":"dli", - "code":"236", + "code":"240", "des":"When you create a Flink SQL job or Flink Jar job, you can select Save Job Log on the job editing page to save job running logs to OBS.To set the OBS bucket for storing th", "doc_type":"usermanual", "kw":"How Do I Save Logs for Flink Jobs?,Flink Job Consulting,User Guide", @@ -4885,7 +4969,7 @@ "uri":"dli_03_0160.html", "node_id":"dli_03_0160.xml", "product_code":"dli", - "code":"237", + "code":"241", "des":"Choose Job Management > Flink Jobs. In the Operation column of the target job, choose More > Permissions. When a new user is authorized, No such user. userName:xxxx. is d", "doc_type":"usermanual", "kw":"Why Is Error \"No such user. userName:xxxx.\" Reported on the Flink Job Management Page When I Grant P", @@ -4905,7 +4989,7 @@ "uri":"dli_03_0180.html", "node_id":"dli_03_0180.xml", "product_code":"dli", - "code":"238", + "code":"242", "des":"Checkpoint was enabled when a Flink job is created, and the OBS bucket for storing checkpoints was specified. I am not sure how to restore a Flink job from a specific che", "doc_type":"usermanual", "kw":"How Do I Restore a Flink Job from a Specific Checkpoint After Manually Stopping the Job?,Flink Job C", @@ -4925,7 +5009,7 @@ "uri":"dli_03_0036.html", "node_id":"dli_03_0036.xml", "product_code":"dli", - "code":"239", + "code":"243", "des":"When you set running parameters of a DLI Flink job, you can enable Alarm Generation upon Job Exception to receive alarms when the job runs abnormally or is in arrears.If ", "doc_type":"usermanual", "kw":"Why Is a Message Displayed Indicating That the SMN Topic Does Not Exist When I Use the SMN Topic in ", @@ -4945,7 +5029,7 @@ "uri":"dli_03_0131.html", "node_id":"dli_03_0131.xml", "product_code":"dli", - "code":"240", + "code":"244", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Flink SQL Jobs", @@ -4965,7 +5049,7 @@ "uri":"dli_03_0089.html", "node_id":"dli_03_0089.xml", "product_code":"dli", - "code":"241", + "code":"245", "des":"When using a Flink SQL job, you need to create an OBS partition table for subsequent batch processing.In the following example, the day field is used as the partition fie", "doc_type":"usermanual", "kw":"How Do I Map an OBS Table to a DLI Partitioned Table?,Flink SQL Jobs,User Guide", @@ -4985,7 +5069,7 @@ "uri":"dli_03_0120.html", "node_id":"dli_03_0120.xml", "product_code":"dli", - "code":"242", + "code":"246", "des":"You used Flink 1.10 to run a Flink Opensource SQL job. You set the number of Kafka partitions for the job a small value at the beginning and need to increase the number n", "doc_type":"usermanual", "kw":"How Do I Change the Number of Kafka Partitions in a Flink SQL Job Without Stopping It?,Flink SQL Job", @@ -5005,7 +5089,7 @@ "uri":"dli_03_0167.html", "node_id":"dli_03_0167.xml", "product_code":"dli", - "code":"243", + "code":"247", "des":"When I run the creation statement with an EL expression in the table name in a Flink SQL job, the following error message is displayed:DLI.0005: AnalysisException: t_user", "doc_type":"usermanual", "kw":"How Do I Fix the DLI.0005 Error When Using EL Expressions to Create a Table in a Flink SQL Job?,Flin", @@ -5025,7 +5109,7 @@ "uri":"dli_03_0168.html", "node_id":"dli_03_0168.xml", "product_code":"dli", - "code":"244", + "code":"248", "des":"After data is written to OBS through the Flink job output stream, data cannot be queried from the DLI table created in the OBS file path.For example, use the following Fl", "doc_type":"usermanual", "kw":"Why Is No Data Queried in the DLI Table Created Using the OBS File Path When Data Is Written to OBS ", @@ -5045,7 +5129,7 @@ "uri":"dli_03_0174.html", "node_id":"dli_03_0174.xml", "product_code":"dli", - "code":"245", + "code":"249", "des":"After a Flink SQL job is submitted on DLI, the job fails to be executed. The following error information is displayed in the job log:connect to DIS failed java.lang.Illeg", "doc_type":"usermanual", "kw":"Why Does a Flink SQL Job Fails to Be Executed, and Is \"connect to DIS failed java.lang.IllegalArgume", @@ -5065,7 +5149,7 @@ "uri":"dli_03_0232.html", "node_id":"dli_03_0232.xml", "product_code":"dli", - "code":"246", + "code":"250", "des":"After a Flink SQL job consumed Kafka and sent data to the Elasticsearch cluster, the job was successfully executed, but no data is available.Possible causes are as follow", "doc_type":"usermanual", "kw":"Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster,F", @@ -5085,7 +5169,7 @@ "uri":"dli_03_0121.html", "node_id":"dli_03_0121.xml", "product_code":"dli", - "code":"247", + "code":"251", "des":"You used Flink 1.10 to run a Flink Opensource SQL job. The job failed after the following error was reported when Flink Sink wrote data to Kafka.The CPU usage is too high", "doc_type":"usermanual", "kw":"How Do I Configure Connection Retries for Kafka Sink If it is Disconnected?,Flink SQL Jobs,User Guid", @@ -5105,7 +5189,7 @@ "uri":"dli_03_0048.html", "node_id":"dli_03_0048.xml", "product_code":"dli", - "code":"248", + "code":"252", "des":"In a Flink job, you can use CREATE statements to define source and sink tables, and specify their connector types and related attributes.If you need to write data to diff", "doc_type":"usermanual", "kw":"How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?,Flink SQL Jobs,User Guide", @@ -5125,7 +5209,7 @@ "uri":"dli_03_0040.html", "node_id":"dli_03_0040.xml", "product_code":"dli", - "code":"249", + "code":"253", "des":"To rectify this fault, perform the following steps:Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statemen", "doc_type":"usermanual", "kw":"Why Does DIS Stream Not Exist During Job Semantic Check?,Flink SQL Jobs,User Guide", @@ -5145,7 +5229,7 @@ "uri":"dli_03_0265.html", "node_id":"dli_03_0265.xml", "product_code":"dli", - "code":"250", + "code":"254", "des":"If the Flink JobManager prompts \"Timeout expired while fetching topic metadata\", it means that the Flink job timed out while trying to fetch metadata for the Kafka topic.", "doc_type":"usermanual", "kw":"Why Is Error \"Timeout expired while fetching topic metadata\" Repeatedly Reported in Flink JobManager", @@ -5165,7 +5249,7 @@ "uri":"dli_03_0132.html", "node_id":"dli_03_0132.xml", "product_code":"dli", - "code":"251", + "code":"255", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Flink Jar Jobs", @@ -5185,7 +5269,7 @@ "uri":"dli_03_0044.html", "node_id":"dli_03_0044.xml", "product_code":"dli", - "code":"252", + "code":"256", "des":"You can upload configuration files for custom jobs (Jar).Upload the configuration file to DLI through Package Management.In the Other Dependencies area of the Flink Jar j", "doc_type":"usermanual", "kw":"Can I Upload Configuration Files for Flink Jar Jobs?,Flink Jar Jobs,User Guide", @@ -5205,7 +5289,7 @@ "uri":"dli_03_0119.html", "node_id":"dli_03_0119.xml", "product_code":"dli", - "code":"253", + "code":"257", "des":"The dependency of your Flink job conflicts with a built-in dependency of the DLI Flink platform. As a result, the job submission fails.Check whether there are conflicting", "doc_type":"usermanual", "kw":"Why Does a Flink Jar Package Conflict Result in Job Submission Failure?,Flink Jar Jobs,User Guide", @@ -5225,7 +5309,7 @@ "uri":"dli_03_0161.html", "node_id":"dli_03_0161.xml", "product_code":"dli", - "code":"254", + "code":"258", "des":"When a Flink Jar job is submitted to access GaussDB(DWS), an error message is displayed indicating that the job fails to be started. The job log contains the following er", "doc_type":"usermanual", "kw":"Why Does a Flink Jar Job Fail to Access GaussDB(DWS) and a Message Is Displayed Indicating Too Many ", @@ -5245,7 +5329,7 @@ "uri":"dli_03_0233.html", "node_id":"dli_03_0233.xml", "product_code":"dli", - "code":"255", + "code":"259", "des":"The storage path of the Flink Jar job checkpoints was set to an OBS bucket. The job failed to be submitted, and an error message indicating an invalid OBS bucket name was", "doc_type":"usermanual", "kw":"Why Is Error Invalid OBS Bucket Name Reported After a Flink Job Submission Failed?,Flink Jar Jobs,Us", @@ -5265,7 +5349,7 @@ "uri":"dli_03_0234.html", "node_id":"dli_03_0234.xml", "product_code":"dli", - "code":"256", + "code":"260", "des":"Flink Job submission failed. The exception information is as follows:Flink JAR files conflicted. The submitted Flink JAR file conflicted with the HDFS JAR file of the DLI", "doc_type":"usermanual", "kw":"Why Does the Flink Submission Fail Due to Hadoop JAR File Conflict?,Flink Jar Jobs,User Guide", @@ -5285,7 +5369,7 @@ "uri":"dli_03_0103.html", "node_id":"dli_03_0103.xml", "product_code":"dli", - "code":"257", + "code":"261", "des":"On the Flink job management page, hover the cursor on the status of the job that fails to be submitted to view the brief information about the failure.The possible causes", "doc_type":"usermanual", "kw":"How Do I Locate a Flink Job Submission Error?,Flink Jar Jobs,User Guide", @@ -5305,7 +5389,7 @@ "uri":"dli_03_0133.html", "node_id":"dli_03_0133.xml", "product_code":"dli", - "code":"258", + "code":"262", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Flink Job Performance Tuning", @@ -5325,7 +5409,7 @@ "uri":"dli_03_0106.html", "node_id":"dli_03_0106.xml", "product_code":"dli", - "code":"259", + "code":"263", "des":"Data Stacking in a Consumer GroupThe accumulated data of a consumer group can be calculated by the following formula: Total amount of data to be consumed by the consumer ", "doc_type":"usermanual", "kw":"Flink Job Performance Tuning,Flink Job Performance Tuning,User Guide", @@ -5345,7 +5429,7 @@ "uri":"dli_03_0096.html", "node_id":"dli_03_0096.xml", "product_code":"dli", - "code":"260", + "code":"264", "des":"The DLI Flink checkpoint/savepoint mechanism is complete and reliable. You can use this mechanism to prevent data loss when a job is manually restarted or restarted due t", "doc_type":"usermanual", "kw":"How Do I Prevent Data Loss After Flink Job Restart?,Flink Job Performance Tuning,User Guide", @@ -5365,7 +5449,7 @@ "uri":"dli_03_0105.html", "node_id":"dli_03_0105.xml", "product_code":"dli", - "code":"261", + "code":"265", "des":"On the Flink job management, click Edit in the Operation column of the target job. On the displayed page, check whether Save Job Log in the Running Parameters tab is enab", "doc_type":"usermanual", "kw":"How Do I Locate a Flink Job Running Error?,Flink Job Performance Tuning,User Guide", @@ -5385,7 +5469,7 @@ "uri":"dli_03_0136.html", "node_id":"dli_03_0136.xml", "product_code":"dli", - "code":"262", + "code":"266", "des":"Flink's checkpointing is a fault tolerance and recovery mechanism. This mechanism ensures that real-time programs can self-recover in case of exceptions or machine issues", "doc_type":"usermanual", "kw":"How Can I Check if a Flink Job Can Be Restored From a Checkpoint After Restarting It?,Flink Job Perf", @@ -5405,7 +5489,7 @@ "uri":"dli_03_0064.html", "node_id":"dli_03_0064.xml", "product_code":"dli", - "code":"263", + "code":"267", "des":"Mode for storing generated job logs when a DLI Flink job fails to be submitted or executed. The options are as follows:If the submission fails, a submission log is genera", "doc_type":"usermanual", "kw":"Why Are Logs Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?,", @@ -5425,7 +5509,7 @@ "uri":"dli_03_0236.html", "node_id":"dli_03_0236.xml", "product_code":"dli", - "code":"264", + "code":"268", "des":"JobManager and TaskManager heartbeats timed out. As a result, the Flink job is abnormal.Check whether the network is intermittently disconnected and whether the cluster l", "doc_type":"usermanual", "kw":"Why Is the Flink Job Abnormal Due to Heartbeat Timeout Between JobManager and TaskManager?,Flink Job", @@ -5445,7 +5529,7 @@ "uri":"dli_03_0021.html", "node_id":"dli_03_0021.xml", "product_code":"dli", - "code":"265", + "code":"269", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Spark Jobs", @@ -5465,7 +5549,7 @@ "uri":"dli_03_0217.html", "node_id":"dli_03_0217.xml", "product_code":"dli", - "code":"266", + "code":"270", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Spark Job Development", @@ -5485,7 +5569,7 @@ "uri":"dli_03_0201.html", "node_id":"dli_03_0201.xml", "product_code":"dli", - "code":"267", + "code":"271", "des":"DLI Spark does not support job scheduling. You can use other services, such as DataArts Studio, or use APIs or SDKs to customize job schedule.The Spark SQL syntax does no", "doc_type":"usermanual", "kw":"Spark Jobs,Spark Job Development,User Guide", @@ -5505,7 +5589,7 @@ "uri":"dli_03_0107.html", "node_id":"dli_03_0107.xml", "product_code":"dli", - "code":"268", + "code":"272", "des":"To use Spark to write data into a DLI table, configure the following parameters:fs.obs.access.keyfs.obs.secret.keyfs.obs.implfs.obs.endpointThe following is an example:", "doc_type":"usermanual", "kw":"How Do I Use Spark to Write Data into a DLI Table?,Spark Job Development,User Guide", @@ -5525,7 +5609,7 @@ "uri":"dli_03_0017.html", "node_id":"dli_03_0017.xml", "product_code":"dli", - "code":"269", + "code":"273", "des":"To obtain the AK/SK, set the parameters as follows:Create a SparkContext using code.val sc: SparkContext = new SparkContext()\nsc.hadoopConfiguration.set(\"fs.obs.access.ke", "doc_type":"usermanual", "kw":"How Do I Set Up AK/SK So That a General Queue Can Access Tables Stored in OBS?,Spark Job Development", @@ -5545,7 +5629,7 @@ "uri":"dli_03_0102.html", "node_id":"dli_03_0102.xml", "product_code":"dli", - "code":"270", + "code":"274", "des":"Log in to the DLI console. In the navigation pane, choose Job Management > Spark Jobs. In the job list, locate the target job and click next to Job ID to view the parame", "doc_type":"usermanual", "kw":"How Do I View the Resource Usage of DLI Spark Jobs?,Spark Job Development,User Guide", @@ -5565,7 +5649,7 @@ "uri":"dli_03_0076.html", "node_id":"dli_03_0076.xml", "product_code":"dli", - "code":"271", + "code":"275", "des":"If the pymysql module is missing, check whether the corresponding EGG package exists. If the package does not exist, upload the pyFile package on the Package Management p", "doc_type":"usermanual", "kw":"How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the S", @@ -5585,7 +5669,7 @@ "uri":"dli_03_0082.html", "node_id":"dli_03_0082.xml", "product_code":"dli", - "code":"272", + "code":"276", "des":"DLI natively supports PySpark.For most cases, Python is preferred for data analysis, and PySpark is the best choice for big data analysis. Generally, JVM programs are pac", "doc_type":"usermanual", "kw":"How Do I Run a Complex PySpark Program in DLI?,Spark Job Development,User Guide", @@ -5605,7 +5689,7 @@ "uri":"dli_03_0068.html", "node_id":"dli_03_0068.xml", "product_code":"dli", - "code":"273", + "code":"277", "des":"When shuffle statements, such as GROUP BY and JOIN, are executed in Spark jobs, data skew occurs, which slows down the job execution.To solve this problem, you can config", "doc_type":"usermanual", "kw":"How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?", @@ -5625,7 +5709,7 @@ "uri":"dli_03_0118.html", "node_id":"dli_03_0118.xml", "product_code":"dli", - "code":"274", + "code":"278", "des":"You can use SparkFiles to read the file submitted using –-file form a local path: SparkFiles.get(\"Name of the uploaded file\").The file path in the Driver is different fro", "doc_type":"usermanual", "kw":"How Do I Read Uploaded Files for a Spark Jar Job?,Spark Job Development,User Guide", @@ -5645,7 +5729,7 @@ "uri":"dli_03_0279.html", "node_id":"dli_03_0279.xml", "product_code":"dli", - "code":"275", + "code":"279", "des":"When running a job with Spark 3.3.1, certain fields have empty default attribute values when queried in the view attribute on the client. In contrast, when using Spark 3.", "doc_type":"usermanual", "kw":"Why Are View Attributes Empty in Spark 3.3.1 Client?,Spark Job Development,User Guide", @@ -5665,7 +5749,7 @@ "uri":"dli_03_0077.html", "node_id":"dli_03_0077.xml", "product_code":"dli", - "code":"276", + "code":"280", "des":"I cannot find the specified Python environment after adding the Python 3 package.Set spark.yarn.appMasterEnv.PYSPARK_PYTHON to python3 in the conf file to specify the Pyt", "doc_type":"usermanual", "kw":"Why Can't I Find the Specified Python Environment After Adding the Python Package?,Spark Job Develop", @@ -5685,7 +5769,7 @@ "uri":"dli_03_0220.html", "node_id":"dli_03_0220.xml", "product_code":"dli", - "code":"277", + "code":"281", "des":"The remaining CUs in the queue may be insufficient. As a result, the job cannot be submitted.To view the remaining CUs of a queue, perform the following steps:Check the C", "doc_type":"usermanual", "kw":"Why Is a Spark Jar Job Stuck in the Submitting State?,Spark Job Development,User Guide", @@ -5705,7 +5789,7 @@ "uri":"dli_03_0218.html", "node_id":"dli_03_0218.xml", "product_code":"dli", - "code":"278", + "code":"282", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Spark Job O&M", @@ -5725,7 +5809,7 @@ "uri":"dli_03_0156.html", "node_id":"dli_03_0156.xml", "product_code":"dli", - "code":"279", + "code":"283", "des":"The following error is reported when a Spark job accesses OBS data:Set the AK/SK to enable Spark jobs to access OBS data.For details, see How Do I Set Up AK/SK So That a ", "doc_type":"usermanual", "kw":"Why Do I Get \"ResponseCode: 403\" and \"ResponseStatus: Forbidden\" Errors When a Spark Job Accesses OB", @@ -5745,7 +5829,7 @@ "uri":"dli_03_0164.html", "node_id":"dli_03_0164.xml", "product_code":"dli", - "code":"280", + "code":"284", "des":"This error message may be due to the OBS bucket being set as the DLI log bucket, which cannot be used for other purposes.You can follow these steps to check:Check if the ", "doc_type":"usermanual", "kw":"Why Do I Encounter the Error \"verifyBucketExists on XXXX: status [403]\" When Using a Spark Job to Ac", @@ -5765,7 +5849,7 @@ "uri":"dli_03_0157.html", "node_id":"dli_03_0157.xml", "product_code":"dli", - "code":"281", + "code":"285", "des":"When running large amounts of data in a Spark job, if a timeout exception error occurs, it is usually due to insufficient resource configuration, data skew, network issue", "doc_type":"usermanual", "kw":"Why Does a Job Running Timeout Occur When Processing a Large Amount of Data with a Spark Job?,Spark ", @@ -5785,7 +5869,7 @@ "uri":"dli_03_0188.html", "node_id":"dli_03_0188.xml", "product_code":"dli", - "code":"282", + "code":"286", "des":"Spark jobs cannot access SFTP. Upload the files you want to access to OBS and then you can analyze the data using Spark jobs.Upload data to an OBS bucket: Upload data sto", "doc_type":"usermanual", "kw":"Why Does a Spark Job Fail to Execute with an Abnormal Access Directory Error When Accessing Files in", @@ -5805,7 +5889,7 @@ "uri":"dli_03_0192.html", "node_id":"dli_03_0192.xml", "product_code":"dli", - "code":"283", + "code":"287", "des":"When a Spark job is running, an error message is displayed, indicating that the user does not have the database permission. The error information is as follows:org.apache", "doc_type":"usermanual", "kw":"Why Does the Job Fail to Be Executed Due to Insufficient Database and Table Permissions?,Spark Job O", @@ -5825,7 +5909,7 @@ "uri":"dli_03_0272.html", "node_id":"dli_03_0272.xml", "product_code":"dli", - "code":"284", + "code":"288", "des":"I cannot find the global_temp database in the Spark 3.x job log.The global_temp database is the default built-in database of Spark 3.x and is Spark's global temporary vie", "doc_type":"usermanual", "kw":"Why Is the global_temp Database Missing in the Job Log of Spark 3.x?,Spark Job O&M,User Guide", @@ -5845,7 +5929,7 @@ "uri":"dli_03_0275.html", "node_id":"dli_03_0275.xml", "product_code":"dli", - "code":"285", + "code":"289", "des":"I failed to use the DataSource syntax to create an OBS table in Avro format when selecting Spark to access metadata.Spark 2.3.x does not support creating OBS tables in Av", "doc_type":"usermanual", "kw":"Why Does Using DataSource Syntax to Create an OBS Table of Avro Type Fail When Accessing Metadata Wi", @@ -5865,7 +5949,7 @@ "uri":"dli_03_0278.html", "node_id":"dli_03_0278.xml", "product_code":"dli", - "code":"286", + "code":"290", "des":"When the rand function is used in SQL statements of Spark 3.3.1, the error message \"Input argument to rand must be a constant\" is displayed.In Spark 3.3.1, the rand funct", "doc_type":"usermanual", "kw":"How Do I Resolve the \"Input argument to rand must be a constant\" Error in Spark 3.3.1 SQL Statements", @@ -5885,7 +5969,7 @@ "uri":"dli_03_0273.html", "node_id":"dli_03_0273.xml", "product_code":"dli", - "code":"287", + "code":"291", "des":"When using Spark 3.3.1 client to create a subquery view with aggregate functions, an error occurs when attempting to write the data, stating \"Not allowed to create a perm", "doc_type":"usermanual", "kw":"When Using Spark 3.3.1 Client to Create a View and Perform a Join Query, an Error Occurs Stating \"No", @@ -5905,7 +5989,7 @@ "uri":"dli_03_0053.html", "node_id":"dli_03_0053.xml", "product_code":"dli", - "code":"288", + "code":"292", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"DLI Resource Quotas", @@ -5925,8 +6009,8 @@ "uri":"dli_03_0031.html", "node_id":"dli_03_0031.xml", "product_code":"dli", - "code":"289", - "des":"Log in to the management console.Click in the upper left corner and select a region and a project.Click the My Quota icon in the upper right corner of the page.The Serv", + "code":"293", + "des":"Log in to the management console.Click in the upper left corner and select a region and a project.Click the My Quota icon in the upper right corner of the page.This wil", "doc_type":"usermanual", "kw":"How Do I View My Quotas?,DLI Resource Quotas,User Guide", "search_title":"", @@ -5945,8 +6029,8 @@ "uri":"dli_03_0032.html", "node_id":"dli_03_0032.xml", "product_code":"dli", - "code":"290", - "des":"The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your applica", + "code":"294", + "des":"The system currently does not support online quota adjustments. If you need to modify your quota, please contact our customer service team via phone or email. They will p", "doc_type":"usermanual", "kw":"How Do I Apply for a Higher Quota?,DLI Resource Quotas,User Guide", "search_title":"", @@ -5965,7 +6049,7 @@ "uri":"dli_03_0054.html", "node_id":"dli_03_0054.xml", "product_code":"dli", - "code":"291", + "code":"295", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"DLI Permissions Management", @@ -5985,7 +6069,7 @@ "uri":"dli_03_0008.html", "node_id":"dli_03_0008.xml", "product_code":"dli", - "code":"292", + "code":"296", "des":"You are unable to perform permission operations on the partition columns of partitioned tables.However, when you grant the permission of any non-partition column in a par", "doc_type":"usermanual", "kw":"What Is Column-Level Authorization for DLI Partitioned Tables?,DLI Permissions Management,User Guide", @@ -6005,7 +6089,7 @@ "uri":"dli_03_0195.html", "node_id":"dli_03_0195.xml", "product_code":"dli", - "code":"293", + "code":"297", "des":"When the user update an existing program package, the following error information is displayed:\"error_code\"*DLI.0003\",\"error_msg\":\"Permission denied for resource 'resourc", "doc_type":"usermanual", "kw":"How Do I Do If I Encounter Insufficient Permissions While Updating Packages?,DLI Permissions Managem", @@ -6025,7 +6109,7 @@ "uri":"dli_03_0227.html", "node_id":"dli_03_0227.xml", "product_code":"dli", - "code":"294", + "code":"298", "des":"When the SQL query statement is executed, the system displays a message indicating that the user does not have the permission to query resources.Error information: DLI.00", "doc_type":"usermanual", "kw":"Why Is Error \"DLI.0003: Permission denied for resource...\" Reported When I Run a SQL Statement?,DLI ", @@ -6045,7 +6129,7 @@ "uri":"dli_03_0228.html", "node_id":"dli_03_0228.xml", "product_code":"dli", - "code":"295", + "code":"299", "des":"If you have already granted authorization to a table and a test query was successful, but you encounter an error when trying to query it again after some time, you should", "doc_type":"usermanual", "kw":"How Do I Do If I Can't Query Table Data After Being Granted Table Permissions?,DLI Permissions Manag", @@ -6065,7 +6149,7 @@ "uri":"dli_03_0057.html", "node_id":"dli_03_0057.xml", "product_code":"dli", - "code":"296", + "code":"300", "des":"If a table inherits database permissions, you do not need to regrant the inherited permissions to the table.Re-authorizing may cause confusion in table permission managem", "doc_type":"usermanual", "kw":"Will Granting Duplicate Permissions to a Table After Inheriting Database Permissions Cause an Error?", @@ -6085,10 +6169,10 @@ "uri":"dli_03_0067.html", "node_id":"dli_03_0067.xml", "product_code":"dli", - "code":"297", - "des":"User A created Table1.User B created View1 based on Table1.After the Select Table permission on Table1 is granted to user C, user C fails to query View1.User B does not h", + "code":"301", + "des":"User A created a table named Table1.User B created a view named View1 based on Table1.After granting user C the select permission on Table1, user C failed to query the vi", "doc_type":"usermanual", - "kw":"Why Can't I Query a View After I'm Granted the Select Table Permission on the View?,DLI Permissions ", + "kw":"Why Can't I Query a View Despite Having the Select Permission?,DLI Permissions Management,User Guide", "search_title":"", "metedata":[ { @@ -6098,14 +6182,14 @@ "IsBot":"Yes" } ], - "title":"Why Can't I Query a View After I'm Granted the Select Table Permission on the View?", + "title":"Why Can't I Query a View Despite Having the Select Permission?", "githuburl":"" }, { "uri":"dli_03_0225.html", "node_id":"dli_03_0225.xml", "product_code":"dli", - "code":"298", + "code":"302", "des":"Despite configuring a job bucket and authorizing DLI to access it, I still receive an error message stating that DLI is not authorized to access the bucket when attemptin", "doc_type":"usermanual", "kw":"How Do I Do If I Receive a Message Saying I Don't Have Sufficient Permissions to Submit My Jobs to t", @@ -6125,7 +6209,7 @@ "uri":"dli_03_0056.html", "node_id":"dli_03_0056.xml", "product_code":"dli", - "code":"299", + "code":"303", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"DLI APIs", @@ -6145,10 +6229,10 @@ "uri":"dli_03_0060.html", "node_id":"dli_03_0060.xml", "product_code":"dli", - "code":"300", + "code":"304", "des":"In the REST API provided by DLI, the request header can be added to the request URI, for example, Content-Type.Content-Type indicates the request body type or format. The", "doc_type":"usermanual", - "kw":"Why Is Error \"unsupported media Type\" Reported When I Subimt a SQL Job?,DLI APIs,User Guide", + "kw":"Why Am I Getting an \"unsupported media Type\" Error When Submitting a SQL Job?,DLI APIs,User Guide", "search_title":"", "metedata":[ { @@ -6158,14 +6242,14 @@ "IsBot":"Yes" } ], - "title":"Why Is Error \"unsupported media Type\" Reported When I Subimt a SQL Job?", + "title":"Why Am I Getting an \"unsupported media Type\" Error When Submitting a SQL Job?", "githuburl":"" }, { "uri":"dli_03_0178.html", "node_id":"dli_03_0178.xml", "product_code":"dli", - "code":"301", + "code":"305", "des":"When the API call for submitting a SQL job times out, and the following error information is displayed:There are currently no resources tracked in the state, so there is ", "doc_type":"usermanual", "kw":"What Can I Do If an Error Is Reported When the Execution of the API for Creating a SQL Job Times Out", @@ -6185,7 +6269,7 @@ "uri":"dli_01_00006.html", "node_id":"dli_01_00006.xml", "product_code":"dli", - "code":"302", + "code":"306", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Change History,User Guide", @@ -6193,10 +6277,10 @@ "metedata":[ { "prodname":"dli", - "opensource":"true", - "documenttype":"usermanual", "IsMulti":"No;Yes;NO", - "IsBot":"No;Yes" + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], "title":"Change History", diff --git a/docs/dli/umn/CLASS.TXT.json b/docs/dli/umn/CLASS.TXT.json index 6ccfd45c9..c580e8517 100644 --- a/docs/dli/umn/CLASS.TXT.json +++ b/docs/dli/umn/CLASS.TXT.json @@ -9,7 +9,7 @@ "code":"1" }, { - "desc":"Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark and Apache Flink ecosystems. It frees you from managing an", + "desc":"Data Lake Insight (DLI) is a fully-managed, serverless service for data processing and analytics. It seamlessly combines stream processing, batch processing, and interact", "product_code":"dli", "title":"What Is Data Lake Insight", "uri":"dli_01_0378.html", @@ -18,7 +18,7 @@ "code":"2" }, { - "desc":"You do not need a background in big data to use DLI for data analysis. You only need to know SQL, and you are good to go. The SQL syntax is fully compatible with the stan", + "desc":"DLI offers standard SQL APIs, enabling you to perform massive data query and analysis using only SQL. Its syntax is fully compatible with ANSI SQL 2003 standards.This sig", "product_code":"dli", "title":"Advantages", "uri":"dli_07_0007.html", @@ -36,16 +36,16 @@ "code":"4" }, { - "desc":"For more notes and constraints on elastic resource pools, see Notes and Constraints on Using an Elastic Resource Pool.For more notes and constraints on using a queue, see", + "desc":"This section outlines the key features supported by DLI. For detailed information on regional availability of each feature, you can refer to the console.Before submitting", "product_code":"dli", - "title":"Notes and Constraints", - "uri":"dli_07_0005.html", + "title":"Features", + "uri":"dli_01_0698.html", "doc_type":"usermanual", "p_code":"1", "code":"5" }, { - "desc":"DLI compute resources are the foundation for job execution. Both DLI's elastic resource pools and queues fall under compute resources. This section introduces the types a", + "desc":"When performing big data analysis with DLI, you need to select the appropriate compute resources based on specific business needs. DLI offers a variety of resource types ", "product_code":"dli", "title":"Compute Resource Types and Specifications", "uri":"dli_07_0027.html", @@ -54,7 +54,7 @@ "code":"6" }, { - "desc":"If you need to assign different permissions to employees in your enterprise to access your DLI resources, IAM is a good choice for fine-grained permissions management. IA", + "desc":"After purchasing DLI resources, you can use IAM to assign different access rights to employees, ensuring proper isolation between roles. IAM provides identity authenticat", "product_code":"dli", "title":"Permission Management", "uri":"dli_07_0006.html", @@ -63,14 +63,23 @@ "code":"7" }, { - "desc":"A quota limits the quantity of a resource available to users, thereby preventing spikes in the usage of the resource.You can also request for an increased quota if your e", + "desc":"For more notes and constraints on elastic resource pools, see Notes and Constraints on Using an Elastic Resource Pool.For more notes and constraints on using a queue, see", "product_code":"dli", - "title":"Quotas", - "uri":"dli_07_0009.html", + "title":"Notes and Constraints", + "uri":"dli_07_0005.html", "doc_type":"usermanual", "p_code":"1", "code":"8" }, + { + "desc":"Quotas are enforced for service resources on the platform to prevent unforeseen spikes in resource usage. Quotas can limit the quantity and capacity of resources availabl", + "product_code":"dli", + "title":"Quota Management", + "uri":"dli_07_0009.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"9" + }, { "desc":"OBS works as the data source and data storage system for DLI, and delivers the following capabilities:Data source: DLI provides an API for you to import data from corresp", "product_code":"dli", @@ -78,16 +87,16 @@ "uri":"dli_07_0004.html", "doc_type":"usermanual", "p_code":"1", - "code":"9" + "code":"10" }, { - "desc":"An elastic resource pool consists of dedicated compute resources where the resources in different pools are completely isolated from each other. Within a single elastic r", + "desc":"Actual CUs: The current allocated resource size of the elastic resource pool, measured in CUs.When no queues exist in the resource pool: The actual CUs equal the minimum ", "product_code":"dli", "title":"Basic Concepts", "uri":"dli_07_0003.html", "doc_type":"usermanual", "p_code":"1", - "code":"10" + "code":"11" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -96,7 +105,7 @@ "uri":"dli_01_0220.html", "doc_type":"usermanual", "p_code":"", - "code":"11" + "code":"12" }, { "desc":"DLI can query data stored in OBS. This section describes how to use DLI to submit a SQL job to query OBS data.To illustrate, create a file called sampledata.csv and uploa", @@ -104,8 +113,8 @@ "title":"Submitting a SQL Job to Query OBS Data Using DLI", "uri":"dli_01_0002.html", "doc_type":"usermanual", - "p_code":"11", - "code":"12" + "p_code":"12", + "code":"13" }, { "desc":"DLI can query data stored in RDS. This section describes how to use DLI to submit a SQL job to query RDS for MySQL data.In this example, we will create an RDS for MySQL D", @@ -113,8 +122,8 @@ "title":"Submitting a SQL Job to Query RDS for MySQL Data Using DLI", "uri":"dli_01_0020.html", "doc_type":"usermanual", - "p_code":"11", - "code":"13" + "p_code":"12", + "code":"14" }, { "desc":"DLI Flink jobs can use other cloud services as data sources and sink streams for real-time compute.This example describes how to create and submit a Flink OpenSource SQL ", @@ -122,8 +131,8 @@ "title":"Submitting a Flink OpenSource SQL Job to Query RDS for MySQL Data Using DLI", "uri":"dli_01_0531.html", "doc_type":"usermanual", - "p_code":"11", - "code":"14" + "p_code":"12", + "code":"15" }, { "desc":"Flink Jar jobs are suitable for data analysis scenarios that require custom stream processing logic, complex state management, or integration with specific libraries. You", @@ -131,8 +140,8 @@ "title":"Submitting a Flink Jar Job Using DLI", "uri":"dli_01_0512.html", "doc_type":"usermanual", - "p_code":"11", - "code":"15" + "p_code":"12", + "code":"16" }, { "desc":"DLI allows you to submit Spark jobs compiled as JAR files, which contain the necessary code and dependency information for executing the job. These files are used for spe", @@ -140,8 +149,8 @@ "title":"Submitting a Spark Jar Job Using DLI", "uri":"dli_01_0375.html", "doc_type":"usermanual", - "p_code":"11", - "code":"16" + "p_code":"12", + "code":"17" }, { "desc":"This chapter walks you through on how to develop a DLI job.To manage fine-grained permissions for your DLI resources using IAM, create an IAM user and grant them permissi", @@ -150,7 +159,7 @@ "uri":"dli_01_0001.html", "doc_type":"usermanual", "p_code":"", - "code":"17" + "code":"18" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -159,7 +168,7 @@ "uri":"dli_01_0611.html", "doc_type":"usermanual", "p_code":"", - "code":"18" + "code":"19" }, { "desc":"Before using DLI, configure DLI agency permissions. This section describes the scenarios and steps for configuring DLI agency permissions (dli_management_agency).If you u", @@ -167,8 +176,8 @@ "title":"Configuring DLI Agency Permissions", "uri":"dli_01_0618.html", "doc_type":"usermanual", - "p_code":"18", - "code":"19" + "p_code":"19", + "code":"20" }, { "desc":"You can use Identity and Access Management (IAM) to implement fine-grained permissions management for your DLI resources. For details, see Overview.If your cloud account ", @@ -176,8 +185,8 @@ "title":"Creating an IAM User and Granting Permissions", "uri":"dli_01_0418.html", "doc_type":"usermanual", - "p_code":"18", - "code":"20" + "p_code":"19", + "code":"21" }, { "desc":"Before using DLI, you need to configure a DLI job bucket. The bucket is used to store temporary data generated during DLI job running, such as job logs and results.Config", @@ -185,8 +194,8 @@ "title":"Configuring a DLI Job Bucket", "uri":"dli_01_0536.html", "doc_type":"usermanual", - "p_code":"18", - "code":"21" + "p_code":"19", + "code":"22" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -195,7 +204,7 @@ "uri":"dli_01_0508.html", "doc_type":"usermanual", "p_code":"", - "code":"22" + "code":"23" }, { "desc":"DLI compute resources are the foundation to run jobs. This section describes the modes of DLI compute resources and queue types.Before we dive into the compute resource m", @@ -203,8 +212,8 @@ "title":"Overview of DLI Elastic Resource Pools and Queues", "uri":"dli_01_0504.html", "doc_type":"usermanual", - "p_code":"22", - "code":"23" + "p_code":"23", + "code":"24" }, { "desc":"An elastic resource pool offers compute resources (CPU and memory) required for running DLI jobs, which can adapt to the changing demands of services.You can create multi", @@ -212,8 +221,8 @@ "title":"Creating an Elastic Resource Pool and Creating Queues Within It", "uri":"dli_01_0505.html", "doc_type":"usermanual", - "p_code":"22", - "code":"24" + "p_code":"23", + "code":"25" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -221,8 +230,8 @@ "title":"Managing Elastic Resource Pools", "uri":"dli_01_0529.html", "doc_type":"usermanual", - "p_code":"22", - "code":"25" + "p_code":"23", + "code":"26" }, { "desc":"After creating an elastic resource pool, you can check and manage it on the management console.This section describes how to check basic information about an elastic reso", @@ -230,8 +239,8 @@ "title":"Checking Basic Information", "uri":"dli_01_0622.html", "doc_type":"usermanual", - "p_code":"25", - "code":"26" + "p_code":"26", + "code":"27" }, { "desc":"Administrators can assign permissions of different operation scopes to users for each elastic resource pool.The administrator and elastic resource pool owner have all per", @@ -239,8 +248,8 @@ "title":"Managing Permissions", "uri":"dli_01_0526.html", "doc_type":"usermanual", - "p_code":"25", - "code":"27" + "p_code":"26", + "code":"28" }, { "desc":"If you want a queue to use resources in an elastic resource pool, bind the queue to the pool.You can click Associate Queue on the Resource Pool page to bind a queue to an", @@ -248,8 +257,8 @@ "title":"Binding a Queue", "uri":"dli_01_0530.html", "doc_type":"usermanual", - "p_code":"25", - "code":"28" + "p_code":"26", + "code":"29" }, { "desc":"Scaling out or in an elastic resource pool essentially means adjusting the actual CUs of the resource pool.In an elastic resource pool:Actual CUs: actual size of resource", @@ -257,8 +266,8 @@ "title":"Scaling Out or In an Elastic Resource Pool", "uri":"dli_01_0686.html", "doc_type":"usermanual", - "p_code":"25", - "code":"29" + "p_code":"26", + "code":"30" }, { "desc":"CU settings are used to control the maximum and minimum CU ranges for elastic resource pool scaling to prevent unlimited resource expansion risks.For example, if the curr", @@ -266,8 +275,8 @@ "title":"Setting CUs", "uri":"dli_01_0507.html", "doc_type":"usermanual", - "p_code":"25", - "code":"30" + "p_code":"26", + "code":"31" }, { "desc":"A tag is a user-defined key-value pair used to identify cloud resources, helping you classify and search for them. A tag consists of a tag key and a tag value.If you use ", @@ -275,17 +284,17 @@ "title":"Managing Tags", "uri":"dli_01_0525.html", "doc_type":"usermanual", - "p_code":"25", - "code":"31" + "p_code":"26", + "code":"32" }, { - "desc":"Multiple queues can be added to an elastic resource pool. For how to add a queue, see Creating an Elastic Resource Pool and Creating Queues Within It. You can configure t", + "desc":"Elastic resource pools support multiple queues to manage job execution efficiently. For details about how to create queues in a pool, see Creating an Elastic Resource Poo", "product_code":"dli", "title":"Adjusting Scaling Policies for Queues in an Elastic Resource Pool", "uri":"dli_01_0506.html", "doc_type":"usermanual", - "p_code":"25", - "code":"32" + "p_code":"26", + "code":"33" }, { "desc":"If you added a queue to or deleted one from an elastic resource pool, or you scaled an added queue, the CU quantity of the elastic resource pool may be changed. You can v", @@ -293,17 +302,17 @@ "title":"Viewing Scaling History", "uri":"dli_01_0532.html", "doc_type":"usermanual", - "p_code":"25", - "code":"33" + "p_code":"26", + "code":"34" }, { - "desc":"You can create enterprise projects matching the organizational structure of your enterprises to centrally manage cloud resources across regions by project. Then you can c", + "desc":"An enterprise project is a cloud resource management approach that allows organizations to plan resources based on their organizational structure. It enables unified mana", "product_code":"dli", "title":"Allocating to an Enterprise Project", "uri":"dli_01_0566.html", "doc_type":"usermanual", - "p_code":"25", - "code":"34" + "p_code":"26", + "code":"35" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -311,8 +320,8 @@ "title":"Managing Queues", "uri":"dli_01_0528.html", "doc_type":"usermanual", - "p_code":"22", - "code":"35" + "p_code":"23", + "code":"36" }, { "desc":"This section walks you through how to view basic information about a queue on the management console, including the engine type and version.Log in to the DLI management c", @@ -320,8 +329,8 @@ "title":"Viewing Basic Information About a Queue", "uri":"dli_01_0663.html", "doc_type":"usermanual", - "p_code":"35", - "code":"36" + "p_code":"36", + "code":"37" }, { "desc":"Administrators and queue owners have full operation permissions on queues. They can grant operation permissions to other users based on service needs. This ensures that u", @@ -329,8 +338,8 @@ "title":"Queue Permission Management", "uri":"dli_01_0015.html", "doc_type":"usermanual", - "p_code":"35", - "code":"37" + "p_code":"36", + "code":"38" }, { "desc":"You can create enterprise projects matching the organizational structure of your enterprises to centrally manage cloud resources across regions by project. Then you can c", @@ -338,8 +347,8 @@ "title":"Allocating a Queue to an Enterprise Project", "uri":"dli_01_0565.html", "doc_type":"usermanual", - "p_code":"35", - "code":"38" + "p_code":"36", + "code":"39" }, { "desc":"Once you have created an SMN topic, you can easily subscribe to it by going to the Topic Management > Topics page of the SMN console. You can choose to receive notificati", @@ -347,8 +356,8 @@ "title":"Creating an SMN Topic", "uri":"dli_01_0421.html", "doc_type":"usermanual", - "p_code":"35", - "code":"39" + "p_code":"36", + "code":"40" }, { "desc":"A tag is a user-defined key-value pair used to identify cloud resources, helping you classify and search for them. A tag consists of a tag key and a tag value.If you use ", @@ -356,8 +365,8 @@ "title":"Managing Queue Tags", "uri":"dli_01_0022.html", "doc_type":"usermanual", - "p_code":"35", - "code":"40" + "p_code":"36", + "code":"41" }, { "desc":"DLI allows you to set properties for queues.You can currently set the following property parameters:Spark driver parameters: Set them to improve the scheduling efficiency", @@ -365,8 +374,8 @@ "title":"Setting Queue Properties", "uri":"dli_01_0563.html", "doc_type":"usermanual", - "p_code":"35", - "code":"41" + "p_code":"36", + "code":"42" }, { "desc":"Spark Native is a core component of Apache Spark designed to enhance the performance of Spark SQL computations. By utilizing vectorized C++ acceleration libraries, it acc", @@ -374,8 +383,8 @@ "title":"Enabling Spark Native Operator Optimization", "uri":"dli_01_0685.html", "doc_type":"usermanual", - "p_code":"35", - "code":"42" + "p_code":"36", + "code":"43" }, { "desc":"DLI's address connectivity testing feature can be used to verify network connectivity between DLI queues and destination addresses.This feature is typically utilized for ", @@ -383,8 +392,8 @@ "title":"Testing the Network Connectivity Between a Queue and a Data Source", "uri":"dli_01_0489.html", "doc_type":"usermanual", - "p_code":"35", - "code":"43" + "p_code":"36", + "code":"44" }, { "desc":"You can delete a queue based on actual conditions.This operation will fail if there are jobs in the Submitting or Running state on this queue.Deleting a queue does not ca", @@ -392,8 +401,8 @@ "title":"Deleting a Queue", "uri":"dli_01_0016.html", "doc_type":"usermanual", - "p_code":"35", - "code":"44" + "p_code":"36", + "code":"45" }, { "desc":"Newly created queues need to run jobs before they can be scaled in or out.The operations described in this section only apply to queues in non-elastic resource pools.Queu", @@ -401,8 +410,8 @@ "title":"Enabling Elastic Scaling for a Queue in a Non-Elastic Resource Pool", "uri":"dli_01_0487.html", "doc_type":"usermanual", - "p_code":"35", - "code":"45" + "p_code":"36", + "code":"46" }, { "desc":"When services are busy, you might need to use more compute resources to process services in a period. After this period, you do not require the same amount of resources. ", @@ -410,8 +419,8 @@ "title":"Creating a Scheduled Elastic Scaling Task for a Queue in a Non-Elastic Resource Pool", "uri":"dli_01_0488.html", "doc_type":"usermanual", - "p_code":"35", - "code":"46" + "p_code":"36", + "code":"47" }, { "desc":"If the CIDR block of the DLI queue conflicts with that of the user data source, you can change the CIDR block of the queue.If the queue whose CIDR block is to be modified", @@ -419,8 +428,8 @@ "title":"Changing the CIDR Block of a Queue in a Non-Elastic Resource Pool", "uri":"dli_01_0443.html", "doc_type":"usermanual", - "p_code":"35", - "code":"47" + "p_code":"36", + "code":"48" }, { "desc":"This section walks you through the procedure of adding a queue to an elastic resource pool and binding an enhanced datasource connection to the elastic resource pool.Proc", @@ -428,8 +437,8 @@ "title":"Example Use Case: Creating an Elastic Resource Pool and Running Jobs", "uri":"dli_01_0515.html", "doc_type":"usermanual", - "p_code":"22", - "code":"48" + "p_code":"23", + "code":"49" }, { "desc":"A company has multiple departments that perform data analysis in different periods during a day.Department A requires a large number of compute resources from 00:00 a.m. ", @@ -437,8 +446,8 @@ "title":"Example Use Case: Configuring Scaling Policies for Queues in an Elastic Resource Pool", "uri":"dli_01_0516.html", "doc_type":"usermanual", - "p_code":"22", - "code":"49" + "p_code":"23", + "code":"50" }, { "desc":"Queues in the non-elastic resource pool mode are the previous-gen of resource management for DLI. It involved purchasing and releasing resources based on usage demands, r", @@ -446,8 +455,8 @@ "title":"Creating a Non-Elastic Resource Pool Queue (Deprecated, Not Recommended)", "uri":"dli_01_0363.html", "doc_type":"usermanual", - "p_code":"22", - "code":"50" + "p_code":"23", + "code":"51" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -456,7 +465,7 @@ "uri":"dli_01_0390.html", "doc_type":"usermanual", "p_code":"", - "code":"51" + "code":"52" }, { "desc":"Databases and tables are the basis for developing SQL and Spark jobs. Before running a job, you need to define databases and tables based on your service scenarios.Flink ", @@ -464,8 +473,8 @@ "title":"Understanding Data Catalogs, Databases, and Tables", "uri":"dli_01_0228.html", "doc_type":"usermanual", - "p_code":"51", - "code":"52" + "p_code":"52", + "code":"53" }, { "desc":"A database is a repository of data organized, stored, and managed on computer storage devices according to data structures.A table is one of the most essential components", @@ -473,8 +482,8 @@ "title":"Creating a Data Catalog, Database, and Table on the DLI Console", "uri":"dli_01_0005.html", "doc_type":"usermanual", - "p_code":"51", - "code":"53" + "p_code":"52", + "code":"54" }, { "desc":"Metadata is data used to define the type of data. It primarily describes information about the data itself, including its source, size, format, or other characteristics. ", @@ -482,8 +491,8 @@ "title":"Viewing Table Metadata", "uri":"dli_01_0008.html", "doc_type":"usermanual", - "p_code":"51", - "code":"54" + "p_code":"52", + "code":"55" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -491,8 +500,8 @@ "title":"Managing Data Catalogs on the DLI Console", "uri":"dli_01_0671.html", "doc_type":"usermanual", - "p_code":"51", - "code":"55" + "p_code":"52", + "code":"56" }, { "desc":"DLI data catalogs support authorization on the DLI console or authentication through IAM. By setting permissions, you can grant varying data catalog permissions to differ", @@ -500,8 +509,8 @@ "title":"Configuring Data Catalog Permissions on the DLI Console", "uri":"dli_01_0672.html", "doc_type":"usermanual", - "p_code":"55", - "code":"56" + "p_code":"56", + "code":"57" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -509,8 +518,8 @@ "title":"Managing Database Resources on the DLI Console", "uri":"dli_01_0004.html", "doc_type":"usermanual", - "p_code":"51", - "code":"57" + "p_code":"52", + "code":"58" }, { "desc":"By setting permissions, you can assign varying database permissions to different users.The administrator and database owner have all permissions, which cannot be set or m", @@ -518,8 +527,8 @@ "title":"Configuring Database Permissions on the DLI Console", "uri":"dli_01_0447.html", "doc_type":"usermanual", - "p_code":"57", - "code":"58" + "p_code":"58", + "code":"59" }, { "desc":"You can delete an unused database from the DLI console: when a database is no longer needed, such as after a test database has completed testing; if a database has errors", @@ -527,8 +536,8 @@ "title":"Deleting a Database on the DLI Console", "uri":"dli_01_0011.html", "doc_type":"usermanual", - "p_code":"57", - "code":"59" + "p_code":"58", + "code":"60" }, { "desc":"In practical use, developers create databases and tables, which are then handed over to testers for testing. Once testing is complete, the databases and tables are handed", @@ -536,8 +545,8 @@ "title":"Changing the Database Owner on the DLI Console", "uri":"dli_01_0376.html", "doc_type":"usermanual", - "p_code":"57", - "code":"60" + "p_code":"58", + "code":"61" }, { "desc":"A tag is a user-defined key-value pair used to identify cloud resources, helping you classify and search for them. A tag consists of a tag key and a tag value. If you use", @@ -545,8 +554,8 @@ "title":"Managing Tags for Database Tables", "uri":"dli_01_0552.html", "doc_type":"usermanual", - "p_code":"57", - "code":"61" + "p_code":"58", + "code":"62" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -554,8 +563,8 @@ "title":"Managing Table Resources on the DLI Console", "uri":"dli_01_0625.html", "doc_type":"usermanual", - "p_code":"51", - "code":"62" + "p_code":"52", + "code":"63" }, { "desc":"By setting permissions, you can assign varying table permissions to different users.The administrator and table owner have all permissions, which cannot be set or modifie", @@ -563,8 +572,8 @@ "title":"Configuring Table Permissions on the DLI Console", "uri":"dli_01_0448.html", "doc_type":"usermanual", - "p_code":"62", - "code":"63" + "p_code":"63", + "code":"64" }, { "desc":"You can delete an unused data table from the DLI console: when a data table is no longer needed, such as after a test data table has completed testing; if a data table ha", @@ -572,8 +581,8 @@ "title":"Deleting a Table on the DLI Console", "uri":"dli_01_0626.html", "doc_type":"usermanual", - "p_code":"62", - "code":"64" + "p_code":"63", + "code":"65" }, { "desc":"In practical use, developers create databases and tables, which are then handed over to testers for testing. Once testing is complete, the databases and tables are handed", @@ -581,8 +590,8 @@ "title":"Changing the Table Owner on the DLI Console", "uri":"dli_01_0627.html", "doc_type":"usermanual", - "p_code":"62", - "code":"65" + "p_code":"63", + "code":"66" }, { "desc":"This section describes how to import data stored in OBS to a table on the DLI console. Data can be imported to both DLI tables (table type: MANAGED) and OBS tables (table", @@ -590,8 +599,8 @@ "title":"Importing OBS Data to a DLI Table", "uri":"dli_01_0253.html", "doc_type":"usermanual", - "p_code":"62", - "code":"66" + "p_code":"63", + "code":"67" }, { "desc":"You can export data from a DLI table to OBS. During the export, a folder is created in OBS or the content in the existing folder is overwritten.The exported file can be i", @@ -599,8 +608,8 @@ "title":"Exporting DLI Table Data to OBS", "uri":"dli_01_0010.html", "doc_type":"usermanual", - "p_code":"62", - "code":"67" + "p_code":"63", + "code":"68" }, { "desc":"The Preview page displays the first 10 records in the table.You can preview data on either the Data Management page or the SQL Editor page.To preview data on the Data Man", @@ -608,8 +617,8 @@ "title":"Previewing Table Data on the DLI Console", "uri":"dli_01_0007.html", "doc_type":"usermanual", - "p_code":"62", - "code":"68" + "p_code":"63", + "code":"69" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -618,7 +627,7 @@ "uri":"dli_01_0661.html", "doc_type":"usermanual", "p_code":"", - "code":"69" + "code":"70" }, { "desc":"DLI enables direct access to data stored in OBS for query and analysis, eliminating the need for data migration.To begin using DLI for data analysis, just import your loc", @@ -626,8 +635,8 @@ "title":"Overview", "uri":"dli_01_0662.html", "doc_type":"usermanual", - "p_code":"69", - "code":"70" + "p_code":"70", + "code":"71" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -635,8 +644,8 @@ "title":"Migrating Data from External Data Sources to DLI", "uri":"dli_01_0631.html", "doc_type":"usermanual", - "p_code":"69", - "code":"71" + "p_code":"70", + "code":"72" }, { "desc":"To centrally analyze and manage scattered data from various systems, you can utilize migration tools like Cloud Data Migration (CDM) to migrate the data to DLI. Once the ", @@ -644,8 +653,8 @@ "title":"Overview of Data Migration Scenarios", "uri":"dli_01_0632.html", "doc_type":"usermanual", - "p_code":"71", - "code":"72" + "p_code":"72", + "code":"73" }, { "desc":"On its GUI, CDM enables you to create data migration tasks from multiple data sources to a data lake.This section describes how to use CDM to migrate data from data sourc", @@ -653,8 +662,8 @@ "title":"Using CDM to Migrate Data to DLI", "uri":"dli_01_0633.html", "doc_type":"usermanual", - "p_code":"71", - "code":"73" + "p_code":"72", + "code":"74" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -663,7 +672,7 @@ "uri":"dli_01_0634.html", "doc_type":"usermanual", "p_code":"", - "code":"74" + "code":"75" }, { "desc":"To read and write external data sources when running DLI jobs, two conditions must be met:Establish network connectivity between DLI and the external data source to ensur", @@ -671,8 +680,8 @@ "title":"Configuring DLI to Read and Write External Data Sources", "uri":"dli_01_0635.html", "doc_type":"usermanual", - "p_code":"74", - "code":"75" + "p_code":"75", + "code":"76" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -680,8 +689,8 @@ "title":"Configuring the Network Connection Between DLI and Data Sources (Enhanced Datasource Connection)", "uri":"dli_01_0426.html", "doc_type":"usermanual", - "p_code":"74", - "code":"76" + "p_code":"75", + "code":"77" }, { "desc":"Imagine DLI elastic resource pools and data sources as two isolated islands surrounded by water.To enable traffic between these two islands, you need to first construct a", @@ -689,17 +698,17 @@ "title":"Overview of Enhanced Datasource Connections", "uri":"dli_01_0003.html", "doc_type":"usermanual", - "p_code":"76", - "code":"77" + "p_code":"77", + "code":"78" }, { - "desc":"Create an enhanced datasource connection for DLI to access, import, query, and analyze data of other data sources.For example, to connect DLI to the MRS, RDS, CSS, Kafka,", + "desc":"Before using DLI to access data from other sources, you need to create an enhanced datasource connection to enable network communication between DLI and the destination d", "product_code":"dli", "title":"Creating an Enhanced Datasource Connection", "uri":"dli_01_0006.html", "doc_type":"usermanual", - "p_code":"76", - "code":"78" + "p_code":"77", + "code":"79" }, { "desc":"If DLI needs to access external data sources, you need to establish enhanced datasource connections to enable the network between DLI and the data sources, and then devel", @@ -707,8 +716,8 @@ "title":"Common Development Methods for DLI Cross-Source Analysis", "uri":"dli_01_0410.html", "doc_type":"usermanual", - "p_code":"76", - "code":"79" + "p_code":"77", + "code":"80" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -716,17 +725,44 @@ "title":"Managing Data Source Access Credentials Using DEW", "uri":"dli_01_0636.html", "doc_type":"usermanual", - "p_code":"74", - "code":"80" + "p_code":"75", + "code":"81" }, { - "desc":"When submitting Flink or Spark jobs through DLI to access external data sources (such as OBS and Kafka), there is a risk of plaintext exposure if AK/SK, usernames/passwor", + "desc":"When you use DLI to perform big data processing or cross-service data queries, authentication is a critical prerequisite to ensure legitimate access and data security. Em", "product_code":"dli", - "title":"Overview", + "title":"Using an Agency to Obtain Access Credentials in DLI", "uri":"dli_01_0687.html", "doc_type":"usermanual", - "p_code":"80", - "code":"81" + "p_code":"81", + "code":"82" + }, + { + "desc":"When DLI writes the output data of Flink jobs to MySQL or DWS, you need to set sensitive parameters such as the username and password in the connector. Storing this infor", + "product_code":"dli", + "title":"Flink OpenSource SQL Jobs Using DEW to Manage Access Credentials", + "uri":"dli_09_0210.html", + "doc_type":"usermanual", + "p_code":"81", + "code":"83" + }, + { + "desc":"When writing output data from Flink Jar jobs to OBS, you need to configure an AK/SK for accessing OBS. To ensure the security of AK/SK data, you can use DEW and CSMS for ", + "product_code":"dli", + "title":"Flink Jar Jobs Using DEW to Acquire Access Credentials for Reading and Writing Data from and to OBS", + "uri":"dli_09_0211.html", + "doc_type":"usermanual", + "p_code":"81", + "code":"84" + }, + { + "desc":"A temporary security credential grants temporary access rights. It includes a temporary AK/SK and a security token, both of which must be used together.When writing outpu", + "product_code":"dli", + "title":"Spark Jar Jobs Using DEW to Acquire Access Credentials for Reading and Writing Data from and to OBS", + "uri":"dli_09_0215.html", + "doc_type":"usermanual", + "p_code":"81", + "code":"85" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -734,8 +770,8 @@ "title":"Using DLI Datasource Authentication to Manage Access Credentials for Data Sources", "uri":"dli_01_0422.html", "doc_type":"usermanual", - "p_code":"74", - "code":"82" + "p_code":"75", + "code":"86" }, { "desc":"When analyzing across multiple sources, you are advised not to configure authentication information directly in a job as it can lead to password leakage. Instead, you are", @@ -743,8 +779,8 @@ "title":"Overview", "uri":"dli_01_0561.html", "doc_type":"usermanual", - "p_code":"82", - "code":"83" + "p_code":"86", + "code":"87" }, { "desc":"Create a CSS datasource authentication on the DLI console to store the authentication information of the CSS security cluster to DLI. This will allow you to access to the", @@ -752,8 +788,8 @@ "title":"Creating a CSS Datasource Authentication", "uri":"dli_01_0427.html", "doc_type":"usermanual", - "p_code":"82", - "code":"84" + "p_code":"86", + "code":"88" }, { "desc":"Create a Kerberos datasource authentication on the DLI console to store the authentication information of the data source to DLI. This will allow you to access to the dat", @@ -761,8 +797,8 @@ "title":"Creating a Kerberos Datasource Authentication", "uri":"dli_01_0558.html", "doc_type":"usermanual", - "p_code":"82", - "code":"85" + "p_code":"86", + "code":"89" }, { "desc":"Create a Kafka_SSL datasource authentication on the DLI console to store the Kafka authentication information to DLI. This will allow you to access to Kafka instances wit", @@ -770,17 +806,17 @@ "title":"Creating a Kafka_SSL Datasource Authentication", "uri":"dli_01_0560.html", "doc_type":"usermanual", - "p_code":"82", - "code":"86" + "p_code":"86", + "code":"90" }, { - "desc":"Create a password datasource authentication on the DLI console to store passwords of the GaussDB(DWS), RDS, DCS, and DDS data sources to DLI. This will allow you to acces", + "desc":"Create a password datasource authentication on the DLI console to store passwords of the DWS, RDS, DCS, and DDS data sources to DLI. This will allow you to access to the ", "product_code":"dli", "title":"Creating a Password Datasource Authentication", "uri":"dli_01_0559.html", "doc_type":"usermanual", - "p_code":"82", - "code":"87" + "p_code":"86", + "code":"91" }, { "desc":"Grant permissions on a datasource authentication to users so multiple user jobs can use the datasource authentication without affecting each other.The administrator and t", @@ -788,8 +824,8 @@ "title":"Datasource Authentication Permission Management", "uri":"dli_01_0480.html", "doc_type":"usermanual", - "p_code":"82", - "code":"88" + "p_code":"86", + "code":"92" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -797,8 +833,8 @@ "title":"Managing Enhanced Datasource Connections", "uri":"dli_01_0509.html", "doc_type":"usermanual", - "p_code":"74", - "code":"89" + "p_code":"75", + "code":"93" }, { "desc":"After creating an enhanced datasource connection, you can view and manage it on the management console.This section describes how to view basic information about an enhan", @@ -806,8 +842,8 @@ "title":"Viewing Basic Information About an Enhanced Datasource Connection", "uri":"dli_01_0623.html", "doc_type":"usermanual", - "p_code":"89", - "code":"90" + "p_code":"93", + "code":"94" }, { "desc":"Enhanced connections support user authorization by project. After authorization, users in the project have the permission to perform operations on the enhanced connection", @@ -815,8 +851,8 @@ "title":"Enhanced Connection Permission Management", "uri":"dli_01_0018.html", "doc_type":"usermanual", - "p_code":"89", - "code":"91" + "p_code":"93", + "code":"95" }, { "desc":"To connect other resource pools to data sources through enhanced datasource connections, bind enhanced datasource connections to resource pools on the Enhanced tab page.T", @@ -824,8 +860,8 @@ "title":"Binding an Enhanced Datasource Connection to an Elastic Resource Pool", "uri":"dli_01_0009.html", "doc_type":"usermanual", - "p_code":"89", - "code":"92" + "p_code":"93", + "code":"96" }, { "desc":"Unbind an enhanced datasource connection from an elastic resource pool that does not need to access a data source through an enhanced datasoruce connection.If the status ", @@ -833,8 +869,8 @@ "title":"Unbinding an Enhanced Datasource Connection from an Elastic Resource Pool", "uri":"dli_01_0555.html", "doc_type":"usermanual", - "p_code":"89", - "code":"93" + "p_code":"93", + "code":"97" }, { "desc":"Enhanced datasource connections establish peering connections between DLI elastic resource pools and data sources to interconnect two VPC networks. If we liken these peer", @@ -842,8 +878,8 @@ "title":"Adding a Route for an Enhanced Datasource Connection", "uri":"dli_01_0014.html", "doc_type":"usermanual", - "p_code":"89", - "code":"94" + "p_code":"93", + "code":"98" }, { "desc":"Delete a route that is no longer used.A custom route table cannot be deleted if it is associated with a subnet.Log in to the DLI management console.In the navigation pane", @@ -851,8 +887,8 @@ "title":"Deleting the Route for an Enhanced Datasource Connection", "uri":"dli_01_0556.html", "doc_type":"usermanual", - "p_code":"89", - "code":"95" + "p_code":"93", + "code":"99" }, { "desc":"Host information is the mapping between host IP addresses and domain names. After you configure host information, jobs can only use the configured domain names to access ", @@ -860,8 +896,8 @@ "title":"Modifying Host Information in an Elastic Resource Pool", "uri":"dli_01_0013.html", "doc_type":"usermanual", - "p_code":"89", - "code":"96" + "p_code":"93", + "code":"100" }, { "desc":"A tag is a user-defined key-value pair used to identify cloud resources, helping you classify and search for them. A tag consists of a tag key and a tag value.If you use ", @@ -869,8 +905,8 @@ "title":"Enhanced Datasource Connection Tag Management", "uri":"dli_01_0019.html", "doc_type":"usermanual", - "p_code":"89", - "code":"97" + "p_code":"93", + "code":"101" }, { "desc":"Delete an enhanced datasource connection that is no longer used on the console.Log in to the DLI management console.In the navigation pane on the left, choose Datasource ", @@ -878,8 +914,8 @@ "title":"Deleting an Enhanced Datasource Connection", "uri":"dli_01_0553.html", "doc_type":"usermanual", - "p_code":"89", - "code":"98" + "p_code":"93", + "code":"102" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -888,16 +924,16 @@ "uri":"dli_01_0486.html", "doc_type":"usermanual", "p_code":"", - "code":"99" + "code":"103" }, { - "desc":"Cloud services often interact with each other, with some of which dependent on other services. You can create an agency to delegate DLI to use other cloud services and pe", + "desc":"Cloud services often need to interact with each other for business operations. In some cases, one cloud service may require collaboration with another. To facilitate this", "product_code":"dli", "title":"DLI Agency Overview", "uri":"dli_01_0419.html", "doc_type":"usermanual", - "p_code":"99", - "code":"100" + "p_code":"103", + "code":"104" }, { "desc":"When Flink 1.15, Spark 3.3, or a later version is used to execute jobs and the required agency is not included in the DLI system agency dli_management_agency, you need to", @@ -905,8 +941,8 @@ "title":"Creating a Custom DLI Agency", "uri":"dli_01_0616.html", "doc_type":"usermanual", - "p_code":"99", - "code":"101" + "p_code":"103", + "code":"105" }, { "desc":"This section provides agency permission policies for common scenarios, which can be used to configure agency permission policies when you customize your permissions. The ", @@ -914,8 +950,8 @@ "title":"Agency Permission Policies in Common Scenarios", "uri":"dli_01_0617.html", "doc_type":"usermanual", - "p_code":"99", - "code":"102" + "p_code":"103", + "code":"106" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -924,7 +960,7 @@ "uri":"dli_01_0402.html", "doc_type":"usermanual", "p_code":"", - "code":"103" + "code":"107" }, { "desc":"DLI offers a SQL editor for executing data query operations using SQL statements.This section describes how to create and submit a SQL job using the DLI SQL editor.Before", @@ -932,8 +968,8 @@ "title":"Creating and Submitting a SQL Job", "uri":"dli_01_0320.html", "doc_type":"usermanual", - "p_code":"103", - "code":"104" + "p_code":"107", + "code":"108" }, { "desc":"Store the data results of analyzed SQL jobs in a specified location in the desired format.By default, DLI stores SQL job results in its job bucket. You can also download ", @@ -941,8 +977,8 @@ "title":"Exporting SQL Job Results", "uri":"dli_01_0621.html", "doc_type":"usermanual", - "p_code":"103", - "code":"105" + "p_code":"107", + "code":"109" }, { "desc":"There are numerous SQL engines in the big data field, which bring diversity to the solutions but also expose some issues such as varying quality of SQL input statements, ", @@ -950,8 +986,8 @@ "title":"Creating a SQL Inspection Rule", "uri":"dli_01_0612.html", "doc_type":"usermanual", - "p_code":"103", - "code":"106" + "p_code":"107", + "code":"110" }, { "desc":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", @@ -959,8 +995,8 @@ "title":"Setting the Priority for a SQL Job", "uri":"dli_01_0535.html", "doc_type":"usermanual", - "p_code":"103", - "code":"107" + "p_code":"107", + "code":"111" }, { "desc":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", @@ -968,8 +1004,8 @@ "title":"Querying Logs for SQL Jobs", "uri":"dli_01_0637.html", "doc_type":"usermanual", - "p_code":"103", - "code":"108" + "p_code":"107", + "code":"112" }, { "desc":"The SQL Jobs page displays all SQL jobs, which may span multiple pages if there are many jobs. You can navigate to a specific page as needed. DLI allows you to view jobs ", @@ -977,8 +1013,8 @@ "title":"Managing SQL Jobs", "uri":"dli_01_0017.html", "doc_type":"usermanual", - "p_code":"103", - "code":"109" + "p_code":"107", + "code":"113" }, { "desc":"A SQL execution plan is a logical flowchart of a database query that shows how a database management system executes a specific SQL query. The execution plan details the ", @@ -986,8 +1022,8 @@ "title":"Viewing a SQL Execution Plan", "uri":"dli_01_0638.html", "doc_type":"usermanual", - "p_code":"103", - "code":"110" + "p_code":"107", + "code":"114" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -995,8 +1031,8 @@ "title":"Creating and Managing SQL Job Templates", "uri":"dli_01_05110.html", "doc_type":"usermanual", - "p_code":"103", - "code":"111" + "p_code":"107", + "code":"115" }, { "desc":"DLI allows you to create custom templates or save currently used SQL statements as templates for quick and convenient SQL operations. Once a template is saved, you can ex", @@ -1004,8 +1040,8 @@ "title":"Creating a SQL Job Template", "uri":"dli_01_0021.html", "doc_type":"usermanual", - "p_code":"111", - "code":"112" + "p_code":"115", + "code":"116" }, { "desc":"DLI allows you to create custom templates or save currently used SQL statements as templates for quick and convenient SQL operations. Once a template is saved, you can ex", @@ -1013,8 +1049,8 @@ "title":"Developing and Submitting a SQL Job Using a SQL Job Template", "uri":"dli_01_0639.html", "doc_type":"usermanual", - "p_code":"111", - "code":"113" + "p_code":"115", + "code":"117" }, { "desc":"TPC-H is a test set developed by the Transaction Processing Performance Council (TPC) to simulate decision-making support applications. It is widely used in academia and ", @@ -1022,8 +1058,8 @@ "title":"TPC-H Sample Data in the SQL Templates Preset on DLI", "uri":"dli_01_05111.html", "doc_type":"usermanual", - "p_code":"111", - "code":"114" + "p_code":"115", + "code":"118" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1032,7 +1068,7 @@ "uri":"dli_01_0389.html", "doc_type":"usermanual", "p_code":"", - "code":"115" + "code":"119" }, { "desc":"DLI supports two types of Flink jobs:Flink OpenSource SQL job:It is fully compatible with Flink of the community edition, ensuring that jobs can run smoothly on these Fli", @@ -1040,8 +1076,8 @@ "title":"Flink Job Overview", "uri":"dli_01_0403.html", "doc_type":"usermanual", - "p_code":"115", - "code":"116" + "p_code":"119", + "code":"120" }, { "desc":"This section describes how to create a Flink OpenSource SQL job.DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink provided by the community. In ", @@ -1049,8 +1085,8 @@ "title":"Creating a Flink OpenSource SQL Job", "uri":"dli_01_0498.html", "doc_type":"usermanual", - "p_code":"115", - "code":"117" + "p_code":"119", + "code":"121" }, { "desc":"A Flink Jar job involves developing a custom application Jar package based on Flink's capabilities and submitting it to a DLI queue for execution.To create a Flink Jar jo", @@ -1058,8 +1094,8 @@ "title":"Creating a Flink Jar Job", "uri":"dli_01_0457.html", "doc_type":"usermanual", - "p_code":"115", - "code":"118" + "p_code":"119", + "code":"122" }, { "desc":"You can isolate Flink jobs allocated to different users by setting permissions to ensure data query performance.The administrator and job creator have all permissions, wh", @@ -1067,8 +1103,8 @@ "title":"Configuring Flink Job Permissions", "uri":"dli_01_0479.html", "doc_type":"usermanual", - "p_code":"115", - "code":"119" + "p_code":"119", + "code":"123" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1076,8 +1112,8 @@ "title":"Managing Flink Jobs", "uri":"dli_01_0377.html", "doc_type":"usermanual", - "p_code":"115", - "code":"120" + "p_code":"119", + "code":"124" }, { "desc":"After creating a Flink job, you can check the basic information, job details, task list, and execution plan of the job on the DLI console.This section describes how to ch", @@ -1085,8 +1121,8 @@ "title":"Viewing Flink Job Details", "uri":"dli_01_0462.html", "doc_type":"usermanual", - "p_code":"120", - "code":"121" + "p_code":"124", + "code":"125" }, { "desc":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", @@ -1094,8 +1130,8 @@ "title":"Setting the Priority for a Flink Job", "uri":"dli_01_0650.html", "doc_type":"usermanual", - "p_code":"120", - "code":"122" + "p_code":"124", + "code":"126" }, { "desc":"In actual job operations, the compute resources required by a job vary depending on the data volume. As a result, compute resources are wasted when the volume is small an", @@ -1103,8 +1139,8 @@ "title":"Enabling Dynamic Scaling for Flink Jobs", "uri":"dli_01_0534.html", "doc_type":"usermanual", - "p_code":"120", - "code":"123" + "p_code":"124", + "code":"127" }, { "desc":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", @@ -1112,8 +1148,8 @@ "title":"Querying Logs for Flink Jobs", "uri":"dli_01_0651.html", "doc_type":"usermanual", - "p_code":"120", - "code":"124" + "p_code":"124", + "code":"128" }, { "desc":"After creating a job, you can manage it by performing various operations such as editing its basic information, starting or stopping it, and importing or exporting it.You", @@ -1121,8 +1157,8 @@ "title":"Common Operations of Flink Jobs", "uri":"dli_01_0461.html", "doc_type":"usermanual", - "p_code":"120", - "code":"125" + "p_code":"124", + "code":"129" }, { "desc":"Flink templates include sample templates and custom templates. You can modify an existing sample template to meet the actual job logic requirements and save time for edit", @@ -1130,8 +1166,8 @@ "title":"Managing Flink Job Templates", "uri":"dli_01_0464.html", "doc_type":"usermanual", - "p_code":"115", - "code":"126" + "p_code":"119", + "code":"130" }, { "desc":"A tag is a user-defined key-value pair used to identify cloud resources, helping you classify and search for them. A tag consists of a tag key and a tag value.DLI allows ", @@ -1139,8 +1175,8 @@ "title":"Adding Tags to a Flink Job", "uri":"dli_01_0463.html", "doc_type":"usermanual", - "p_code":"115", - "code":"127" + "p_code":"119", + "code":"131" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1149,7 +1185,7 @@ "uri":"dli_01_0465.html", "doc_type":"usermanual", "p_code":"", - "code":"128" + "code":"132" }, { "desc":"DLI Spark jobs provide fully managed Spark computing services.On the Overview page, click Create Job in the upper right corner of the Spark Jobs tab or click Create Job i", @@ -1157,8 +1193,8 @@ "title":"Creating a Spark Job", "uri":"dli_01_0384.html", "doc_type":"usermanual", - "p_code":"128", - "code":"129" + "p_code":"132", + "code":"133" }, { "desc":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", @@ -1166,8 +1202,8 @@ "title":"Setting the Priority for a Spark Job", "uri":"dli_01_0652.html", "doc_type":"usermanual", - "p_code":"128", - "code":"130" + "p_code":"132", + "code":"134" }, { "desc":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", @@ -1175,17 +1211,17 @@ "title":"Querying Logs for Spark Jobs", "uri":"dli_01_0379.html", "doc_type":"usermanual", - "p_code":"128", - "code":"131" + "p_code":"132", + "code":"135" }, { - "desc":"On the Overview page, click Spark Jobs to go to the SQL job management page. Alternatively, you can click Job Management > Spark Jobs. The page displays all Spark jobs. I", + "desc":"To access the Spark job management page, click Spark Jobs on the Overview page or choose Job Management > Spark Jobs in the navigation pane on the left. The page displays", "product_code":"dli", "title":"Managing Spark Jobs", "uri":"dli_01_0385.html", "doc_type":"usermanual", - "p_code":"128", - "code":"132" + "p_code":"132", + "code":"136" }, { "desc":"You can modify a sample template to meet the Spark job requirements, saving time for editing SQL statements.Currently, the cloud platform does not provide preset Spark te", @@ -1193,8 +1229,8 @@ "title":"Managing Spark Job Templates", "uri":"dli_01_0551.html", "doc_type":"usermanual", - "p_code":"128", - "code":"133" + "p_code":"132", + "code":"137" }, { "desc":"This section describes metrics reported by DLI to Cloud Eye as well as their namespaces and dimensions. You can use the management console or APIs provided by Cloud Eye t", @@ -1203,7 +1239,7 @@ "uri":"dli_01_0445.html", "doc_type":"usermanual", "p_code":"", - "code":"134" + "code":"138" }, { "desc":"With CTS, you can log operations related to DLI, making it easier to search, audit, and trace in the future.", @@ -1212,7 +1248,7 @@ "uri":"dli_01_0318.html", "doc_type":"usermanual", "p_code":"", - "code":"135" + "code":"139" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1221,7 +1257,7 @@ "uri":"dli_01_0408.html", "doc_type":"usermanual", "p_code":"", - "code":"136" + "code":"140" }, { "desc":"DLI features two sets of permission systems. These two permission control mechanisms are used in conjunction, with their permissions overlapping and complementing each ot", @@ -1229,8 +1265,8 @@ "title":"Overview of the DLI Permission System", "uri":"dli_01_0440.html", "doc_type":"usermanual", - "p_code":"136", - "code":"137" + "p_code":"140", + "code":"141" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1238,8 +1274,8 @@ "title":"DLI Permission Management", "uri":"dli_01_0689.html", "doc_type":"usermanual", - "p_code":"136", - "code":"138" + "p_code":"140", + "code":"142" }, { "desc":"The permission management function of DLI itself primarily applies to the permission control of internal DLI resources, rather than relying on the unified identity authen", @@ -1247,8 +1283,8 @@ "title":"Overview", "uri":"dli_01_0690.html", "doc_type":"usermanual", - "p_code":"138", - "code":"139" + "p_code":"142", + "code":"143" }, { "desc":"For details about authorization through the DLI console, see Managing Permissions.For details about authorization through the DLI console, see Queue Permission Management", @@ -1256,8 +1292,8 @@ "title":"DLI Permissions", "uri":"dli_01_0691.html", "doc_type":"usermanual", - "p_code":"138", - "code":"140" + "p_code":"142", + "code":"144" }, { "desc":"This section describes how to authorize resources on the DLI console.", @@ -1265,8 +1301,8 @@ "title":"DLI Authorization Operation Guide", "uri":"dli_01_0692.html", "doc_type":"usermanual", - "p_code":"138", - "code":"141" + "p_code":"142", + "code":"145" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1274,8 +1310,8 @@ "title":"IAM Permission Management", "uri":"dli_01_0475.html", "doc_type":"usermanual", - "p_code":"136", - "code":"142" + "p_code":"140", + "code":"146" }, { "desc":"IAM is a foundational service for permission management, offering functions like user identity authentication, permission allocation, and access control, enabling you to ", @@ -1283,8 +1319,8 @@ "title":"Overview", "uri":"dli_01_0417.html", "doc_type":"usermanual", - "p_code":"142", - "code":"143" + "p_code":"146", + "code":"147" }, { "desc":"In IAM, authorization entities are primarily categorized into users and user groups. Integration with enterprise projects enables resource isolation by group and facilita", @@ -1292,8 +1328,8 @@ "title":"IAM Authorization Entity", "uri":"dli_01_0693.html", "doc_type":"usermanual", - "p_code":"142", - "code":"144" + "p_code":"146", + "code":"148" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1301,8 +1337,8 @@ "title":"IAM Permissions", "uri":"dli_01_0441.html", "doc_type":"usermanual", - "p_code":"142", - "code":"145" + "p_code":"146", + "code":"149" }, { "desc":"The role/policy-based authorization model provided by Identity and Access Management (IAM) lets you control access to DLI resources. With IAM, you can:Based on the organi", @@ -1310,8 +1346,8 @@ "title":"Using IAM Roles or Policies to Grant Access to DLI", "uri":"dli_01_0451.html", "doc_type":"usermanual", - "p_code":"142", - "code":"146" + "p_code":"146", + "code":"150" }, { "desc":"An e-commerce company conducts data analysis operations in region A. The marketing department's employees, user 1 and user 2, along with the technical department's employ", @@ -1319,8 +1355,8 @@ "title":"Example Use Case: Creating a User Group and Granting Permissions for Using DLI Elastic Resource Pools and Databases", "uri":"dli_01_0695.html", "doc_type":"usermanual", - "p_code":"136", - "code":"147" + "p_code":"140", + "code":"151" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1329,7 +1365,7 @@ "uri":"dli_01_0567.html", "doc_type":"usermanual", "p_code":"", - "code":"148" + "code":"152" }, { "desc":"To enhance the functions and performance of Spark and Flink jobs, you can create custom images by downloading the base images provided by DLI and adding dependencies (fil", @@ -1337,8 +1373,8 @@ "title":"Enhancing the Job Runtime Environment Using a Custom Image", "uri":"dli_01_0494.html", "doc_type":"usermanual", - "p_code":"148", - "code":"149" + "p_code":"152", + "code":"153" }, { "desc":"DLI allows you to set variables that are frequently used during job development as global variables on the DLI management console. This avoids repeated definitions during", @@ -1346,8 +1382,8 @@ "title":"Managing DLI Global Variables", "uri":"dli_01_0476.html", "doc_type":"usermanual", - "p_code":"148", - "code":"150" + "p_code":"152", + "code":"154" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1355,8 +1391,8 @@ "title":"Managing Program Packages of Jar Jobs", "uri":"dli_01_0366.html", "doc_type":"usermanual", - "p_code":"148", - "code":"151" + "p_code":"152", + "code":"155" }, { "desc":"Before running DLI jobs, UDF JAR files or Jar job packages need to be uploaded to the cloud platform for unified management and maintenance.There are two ways to manage p", @@ -1364,8 +1400,8 @@ "title":"Package Management Overview", "uri":"dli_01_0407.html", "doc_type":"usermanual", - "p_code":"151", - "code":"152" + "p_code":"155", + "code":"156" }, { "desc":"DLI allows you to submit program packages in batches to the general-use queue for running.If you need to update a package, you can use the same package or file to upload ", @@ -1373,8 +1409,8 @@ "title":"Creating a DLI Package", "uri":"dli_01_0367.html", "doc_type":"usermanual", - "p_code":"151", - "code":"153" + "p_code":"155", + "code":"157" }, { "desc":"By configuring permissions, you can grant different package groups or packages to various users, ensuring that job efficiency remains unaffected and job performance is ma", @@ -1382,8 +1418,8 @@ "title":"Configuring DLI Package Permissions", "uri":"dli_01_0477.html", "doc_type":"usermanual", - "p_code":"151", - "code":"154" + "p_code":"155", + "code":"158" }, { "desc":"DLI allows you to change the owner of a package group or package.Log in to the DLI management console and choose Data Management > Package Management.On the Package Manag", @@ -1391,8 +1427,8 @@ "title":"Changing the DLI Package Owner", "uri":"dli_01_0478.html", "doc_type":"usermanual", - "p_code":"151", - "code":"155" + "p_code":"155", + "code":"159" }, { "desc":"Tags are key-value pairs that you can define to identify cloud resources. They assist you in categorizing and searching for cloud resources. A tag consists of a key and a", @@ -1400,8 +1436,8 @@ "title":"Managing DLI Package Tags", "uri":"dli_01_0369.html", "doc_type":"usermanual", - "p_code":"151", - "code":"156" + "p_code":"155", + "code":"160" }, { "desc":"DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar ", @@ -1409,17 +1445,17 @@ "title":"DLI Built-in Dependencies", "uri":"dli_01_0397.html", "doc_type":"usermanual", - "p_code":"151", - "code":"157" + "p_code":"155", + "code":"161" }, { - "desc":"A quota limits the quantity of a resource available to users, thereby preventing spikes in the usage of the resource.You can also request for an increased quota if your e", + "desc":"Quotas are enforced for service resources on the platform to prevent unforeseen spikes in resource usage. Quotas can limit the quantity and capacity of resources availabl", "product_code":"dli", "title":"Managing DLI Resource Quotas", "uri":"dli_01_0550.html", "doc_type":"usermanual", - "p_code":"148", - "code":"158" + "p_code":"152", + "code":"162" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1428,7 +1464,7 @@ "uri":"dli_01_0539.html", "doc_type":"usermanual", "p_code":"", - "code":"159" + "code":"163" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1436,8 +1472,8 @@ "title":"DLI Basics", "uri":"dli_03_0001.html", "doc_type":"usermanual", - "p_code":"159", - "code":"160" + "p_code":"163", + "code":"164" }, { "desc":"Both DLI and MRS support Spark, but there are some differences in service mode, interface, application scenarios, and performance characteristics.The Spark component of D", @@ -1445,8 +1481,8 @@ "title":"What Are the Differences Between MRS Spark and DLI Spark?", "uri":"dli_03_0115.html", "doc_type":"usermanual", - "p_code":"160", - "code":"161" + "p_code":"164", + "code":"165" }, { "desc":"Supported data formats:ParquetCSVORCJSONAvroOBS: Data used by SQL jobs, Spark jobs, and Flink jobs can be stored in OBS, reducing storage costs.DLI: The column-based Parq", @@ -1454,8 +1490,8 @@ "title":"Where Can Data Be Stored in DLI?", "uri":"dli_03_0029.html", "doc_type":"usermanual", - "p_code":"160", - "code":"162" + "p_code":"164", + "code":"166" }, { "desc":"DLI supports importing data from OBS buckets shared by IAM users under the same tenant, but not from OBS buckets shared by other tenants.This ensures data security and is", @@ -1463,8 +1499,8 @@ "title":"Can I Import OBS Bucket Data Shared by Other Tenants into DLI?", "uri":"dli_03_0129.html", "doc_type":"usermanual", - "p_code":"160", - "code":"163" + "p_code":"164", + "code":"167" }, { "desc":"No, a global variable can only be used by the user who created it. Global variables can be used to simplify complex parameters. For example, long and difficult variables ", @@ -1472,8 +1508,8 @@ "title":"Can a Member Account Use Global Variables Created by Other Member Accounts?", "uri":"dli_03_0263.html", "doc_type":"usermanual", - "p_code":"160", - "code":"164" + "p_code":"164", + "code":"168" }, { "desc":"No.The spark.acls.enable configuration item is not used in DLI. The Apache Spark command injection vulnerability (CVE-2022-33891) does not exist in DLI.This vulnerability", @@ -1481,8 +1517,8 @@ "title":"Is DLI Affected by the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?", "uri":"dli_03_0260.html", "doc_type":"usermanual", - "p_code":"160", - "code":"165" + "p_code":"164", + "code":"169" }, { "desc":"To manage a large number of DLI jobs, you can use the following methods:Manage jobs by group.Group tens of thousands of jobs by type and run each group on a queue.Group t", @@ -1490,8 +1526,8 @@ "title":"How Do I Manage Jobs Running on DLI?", "uri":"dli_03_0126.html", "doc_type":"usermanual", - "p_code":"160", - "code":"166" + "p_code":"164", + "code":"170" }, { "desc":"DLI does not support directly changing the field names of a table. However, you can solve this issue by migrating the table data using the following steps:Create a table:", @@ -1499,8 +1535,8 @@ "title":"How Do I Change the Field Names of an Existing Table on DLI?", "uri":"dli_03_0162.html", "doc_type":"usermanual", - "p_code":"160", - "code":"167" + "p_code":"164", + "code":"171" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1508,8 +1544,8 @@ "title":"DLI Elastic Resource Pools and Queues", "uri":"dli_03_0049.html", "doc_type":"usermanual", - "p_code":"159", - "code":"168" + "p_code":"163", + "code":"172" }, { "desc":"In daily big data analysis work, it is important to allocate and manage compute resources properly to provide a good job execution environment.You can allocate resources ", @@ -1517,8 +1553,8 @@ "title":"How Can I Check the Actual and Used CUs for an Elastic Resource Pool as Well as the Required CUs for a Job?", "uri":"dli_03_0276.html", "doc_type":"usermanual", - "p_code":"168", - "code":"169" + "p_code":"172", + "code":"173" }, { "desc":"You need to check the large number of jobs in the Submitting and Running states on the queue.Use Cloud Eye to view jobs in different states on the queue. The procedure is", @@ -1526,8 +1562,8 @@ "title":"How Do I Check for a Backlog of Jobs in the Current DLI Queue?", "uri":"dli_03_0183.html", "doc_type":"usermanual", - "p_code":"168", - "code":"170" + "p_code":"172", + "code":"174" }, { "desc":"To check the running status of the DLI queue and determine whether to run more jobs on that queue, you need to check the queue load.Search for Cloud Eye on the console.In", @@ -1535,8 +1571,8 @@ "title":"How Do I View the Load of a DLI Queue?", "uri":"dli_03_0095.html", "doc_type":"usermanual", - "p_code":"168", - "code":"171" + "p_code":"172", + "code":"175" }, { "desc":"DLI allows you to subscribe to an SMN topic for failed jobs.Log in to the DLI console.In the navigation pane on the left, choose Queue Management.On the Queue Management ", @@ -1544,8 +1580,8 @@ "title":"How Do I Monitor Job Exceptions on a DLI Queue?", "uri":"dli_03_0098.html", "doc_type":"usermanual", - "p_code":"168", - "code":"172" + "p_code":"172", + "code":"176" }, { "desc":"Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earl", @@ -1553,8 +1589,8 @@ "title":"How Do I Migrate an Old Version Spark Queue to a General-Purpose Queue?", "uri":"dli_03_0065.html", "doc_type":"usermanual", - "p_code":"168", - "code":"173" + "p_code":"172", + "code":"177" }, { "desc":"After a SQL job was submitted to the default queue, the job runs abnormally. The job log reported that the execution timed out. The exception logs are as follows:[ERROR] ", @@ -1562,8 +1598,8 @@ "title":"How Do I Do If I Encounter a Timeout Exception When Executing DLI SQL Statements on the default Queue?", "uri":"dli_03_0171.html", "doc_type":"usermanual", - "p_code":"168", - "code":"174" + "p_code":"172", + "code":"178" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1571,8 +1607,8 @@ "title":"DLI Databases and Tables", "uri":"dli_03_0211.html", "doc_type":"usermanual", - "p_code":"159", - "code":"175" + "p_code":"163", + "code":"179" }, { "desc":"A DLI table exists but cannot be queried on the DLI console.If a table exists but cannot be queried, there is a high probability that the current user does not have the p", @@ -1580,8 +1616,8 @@ "title":"Why Am I Unable to Query a Table on the DLI Console?", "uri":"dli_03_0184.html", "doc_type":"usermanual", - "p_code":"175", - "code":"176" + "p_code":"179", + "code":"180" }, { "desc":"When submitting a job to import data into a DLI table, if the compression rate of the Parquet/ORC file corresponding to the OBS table is high, exceeding 5 times the compr", @@ -1589,8 +1625,8 @@ "title":"How Do I Do If the Compression Rate of an OBS Table Is High?", "uri":"dli_03_0013.html", "doc_type":"usermanual", - "p_code":"175", - "code":"177" + "p_code":"179", + "code":"181" }, { "desc":"To avoid garbled characters caused by inconsistent character encoding, you are advised to unify the encoding format of your data source when executing jobs in DLI.DLI onl", @@ -1598,17 +1634,17 @@ "title":"How Do I Do If Inconsistent Character Encoding Leads to Garbled Characters?", "uri":"dli_03_0009.html", "doc_type":"usermanual", - "p_code":"175", - "code":"178" + "p_code":"179", + "code":"182" }, { "desc":"User A created the testTable table in a database through a SQL job and granted user B the permission to insert and delete table data. User A deleted the testTable table a", "product_code":"dli", - "title":"Do I Need to to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With the Same Name?", + "title":"Do I Need to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With the Same Name?", "uri":"dli_03_0175.html", "doc_type":"usermanual", - "p_code":"175", - "code":"179" + "p_code":"179", + "code":"183" }, { "desc":"A CSV file is imported to a DLI partitioned table, but the imported file data does not contain the data in the partitioning column. The partitioning column needs to be sp", @@ -1616,8 +1652,8 @@ "title":"How Do I Do If Files Imported Into a DLI Partitioned Table Lack Data for the Partition Columns, Causing Query Failures After the Import Is Completed?", "uri":"dli_03_0177.html", "doc_type":"usermanual", - "p_code":"175", - "code":"180" + "p_code":"179", + "code":"184" }, { "desc":"When an OBS foreign table is created, a field in the specified OBS file contains a carriage return line feed (CRLF) character. As a result, the data is incorrect.The stat", @@ -1625,8 +1661,8 @@ "title":"How Do I Fix Incorrect Data in an OBS Foreign Table Caused by Newline Characters in OBS File Fields?", "uri":"dli_03_0181.html", "doc_type":"usermanual", - "p_code":"175", - "code":"181" + "p_code":"179", + "code":"185" }, { "desc":"The on clause was not added to the SQL statement for joining tables. As a result, the Cartesian product query occurs due to multi-table association, and the queue resourc", @@ -1634,8 +1670,8 @@ "title":"How Do I Prevent a Cartesian Product Query and Resource Overload Due to Missing \"ON\" Conditions in Table Joins?", "uri":"dli_03_0187.html", "doc_type":"usermanual", - "p_code":"175", - "code":"182" + "p_code":"179", + "code":"186" }, { "desc":"Partition data is manually uploaded to a partition of an OBS table. However, the data cannot be queried using DLI SQL editor.After manually adding partition data, you nee", @@ -1643,8 +1679,8 @@ "title":"How Do I Do If I Can't Query Data After Manually Adding It to the Partition Directory of an OBS Table?", "uri":"dli_03_0190.html", "doc_type":"usermanual", - "p_code":"175", - "code":"183" + "p_code":"179", + "code":"187" }, { "desc":"When using the INSERT OVERWRITE statement to overwrite data in a partitioned table, if you find that it overwrites all data instead of the expected partitioned data, it m", @@ -1652,8 +1688,8 @@ "title":"Why Does the \"insert overwrite\" Operation Affect All Data in a Partitioned Table Instead of Just the Targeted Partition?", "uri":"dli_03_0212.html", "doc_type":"usermanual", - "p_code":"175", - "code":"184" + "p_code":"179", + "code":"188" }, { "desc":"Spark does not have the datetime type and uses the TIMESTAMP type instead.You can use a function to convert data types.The following is an example.select cast(create_date", @@ -1661,8 +1697,8 @@ "title":"Why Does the \"create_date\" Field in an RDS Table (Datetime Data Type) Appear as a Timestamp in DLI Queries?", "uri":"dli_03_0214.html", "doc_type":"usermanual", - "p_code":"175", - "code":"185" + "p_code":"179", + "code":"189" }, { "desc":"Changing a table name immediately after executing a SQL job may result in an incorrect data size for the table.This is because DLI updates the metadata of the table when ", @@ -1670,8 +1706,8 @@ "title":"How Do I Do If Renaming a Table After a SQL Job Causes Incorrect Data Size?", "uri":"dli_03_0215.html", "doc_type":"usermanual", - "p_code":"175", - "code":"186" + "p_code":"179", + "code":"190" }, { "desc":"When DLI is used to insert data into an OBS temporary table, only part of data is imported.Possible causes are as follows:The amount of data read during job execution is ", @@ -1679,8 +1715,8 @@ "title":"How Can I Resolve Data Inconsistencies When Importing Data from DLI to OBS?", "uri":"dli_03_0231.html", "doc_type":"usermanual", - "p_code":"175", - "code":"187" + "p_code":"179", + "code":"191" }, { "desc":"The Hudi table created using Flink SQL statements is not displayed on the DLI console, making it impossible to manage or perform query operations on the table through the", @@ -1688,8 +1724,8 @@ "title":"Why Is a Hudi Table Not Displayed on the DLI Console?", "uri":"dli_03_0277.html", "doc_type":"usermanual", - "p_code":"175", - "code":"188" + "p_code":"179", + "code":"192" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1697,8 +1733,8 @@ "title":"Enhanced Datasource Connections", "uri":"dli_03_0022.html", "doc_type":"usermanual", - "p_code":"159", - "code":"189" + "p_code":"163", + "code":"193" }, { "desc":"The outbound rule had been configured for the security group of the queue associated with the enhanced datasource connection. The datasource authentication used a passwor", @@ -1706,8 +1742,8 @@ "title":"How Do I Resolve a Failure in Connecting DLI to GaussDB(DWS) Through an Enhanced Datasource Connection?", "uri":"dli_03_0238.html", "doc_type":"usermanual", - "p_code":"189", - "code":"190" + "p_code":"193", + "code":"194" }, { "desc":"A datasource connection is created and bound to a queue. The connectivity test fails and the following error information is displayed:failed to connect to specified addre", @@ -1715,8 +1751,8 @@ "title":"How Do I Do If the Datasource Connection Is Successfully Created but the Network Connectivity Test Fails?", "uri":"dli_03_0179.html", "doc_type":"usermanual", - "p_code":"189", - "code":"191" + "p_code":"193", + "code":"195" }, { "desc":"Configuring the Connection Between a DLI Queue and a Data Source in a Private NetworkIf your DLI job needs to connect to a data source, for example, MRS, RDS, CSS, Kafka,", @@ -1724,8 +1760,8 @@ "title":"How Do I Configure Network Connectivity Between a DLI Queue and a Data Source?", "uri":"dli_03_0186.html", "doc_type":"usermanual", - "p_code":"189", - "code":"192" + "p_code":"193", + "code":"196" }, { "desc":"The main reason for creating a VPC peering connection for DLI's enhanced datasource connection is to establish network connectivity between DLI and data sources in differ", @@ -1733,8 +1769,8 @@ "title":"Why Is Creating a VPC Peering Connection Necessary for Enhanced Datasource Connections in DLI?", "uri":"dli_03_0128.html", "doc_type":"usermanual", - "p_code":"189", - "code":"193" + "p_code":"193", + "code":"197" }, { "desc":"The possible causes and solutions are as follows:If you have created a queue, do not bind it to a datasource connection immediately. Wait for 5 to 10 minutes. After the c", @@ -1742,8 +1778,8 @@ "title":"How Do I Do If Creating a Datasource Connection in DLI Gets Stuck in the \"Creating\" State When Binding It to a Queue?", "uri":"dli_03_0257.html", "doc_type":"usermanual", - "p_code":"189", - "code":"194" + "p_code":"193", + "code":"198" }, { "desc":"The network connectivity is abnormal. Check whether the security group is correctly selected and whether the VPC is correctly configured.Example: When you create an RDS d", @@ -1751,8 +1787,8 @@ "title":"How Do I Resolve the \"communication link failure\" Error When Using a Newly Created Datasource Connection That Appears to Be Activated?", "uri":"dli_03_0047.html", "doc_type":"usermanual", - "p_code":"189", - "code":"195" + "p_code":"193", + "code":"199" }, { "desc":"If you have not added the cluster host information to the datasource connection, it can lead to KRB authentication failure, resulting in a connection timeout. In this cas", @@ -1760,8 +1796,8 @@ "title":"How Do I Troubleshoot a Connection Timeout Issue That Isn't Recorded in Logs When Accessing MRS HBase Through a Datasource Connection?", "uri":"dli_03_0080.html", "doc_type":"usermanual", - "p_code":"189", - "code":"196" + "p_code":"193", + "code":"200" }, { "desc":"A datasource RDS table was created in the DataArts Studio, and the insert overwrite statement was executed to write data into RDS. DLI.0999: BatchUpdateException: Incorre", @@ -1769,8 +1805,8 @@ "title":"How Do I Do If I Encounter the \"Incorrect string value\" Error When Executing insert overwrite on a Datasource RDS Table?", "uri":"dli_03_0239.html", "doc_type":"usermanual", - "p_code":"189", - "code":"197" + "p_code":"193", + "code":"201" }, { "desc":"The system failed to create a datasource RDS table, and null pointer error was reported.The following table creation statement was used:The RDS database is in a PostGre c", @@ -1778,8 +1814,8 @@ "title":"How Do I Resolve the Null Pointer Error When Creating an RDS Datasource Table?", "uri":"dli_03_0250.html", "doc_type":"usermanual", - "p_code":"189", - "code":"198" + "p_code":"193", + "code":"202" }, { "desc":"The system failed to execute insert overwrite on the datasource GaussDB(DWS) table, and org.postgresql.util.PSQLException: ERROR: tuple concurrently updated was displayed", @@ -1787,8 +1823,8 @@ "title":"Error Message \"org.postgresql.util.PSQLException: ERROR: tuple concurrently updated\" Is Displayed When the System Executes insert overwrite on a Datasource GaussDB(DWS) Table", "uri":"dli_03_0251.html", "doc_type":"usermanual", - "p_code":"189", - "code":"199" + "p_code":"193", + "code":"203" }, { "desc":"A datasource table was used to import data to a CloudTable HBase table. This HBase table contains a column family and a rowkey for 100 million simulating data records. Th", @@ -1796,8 +1832,8 @@ "title":"RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Datasource Table", "uri":"dli_03_0252.html", "doc_type":"usermanual", - "p_code":"189", - "code":"200" + "p_code":"193", + "code":"204" }, { "desc":"A table was created on GaussDB(DWS) and then a datasource connection was created on DLI to read and write data. An error message was displayed during data writing, indica", @@ -1805,8 +1841,8 @@ "title":"How Do I Do If A Null Value Is Written Into a Non-Null Field When Using a DLI Datasource Connection to Connect to a GaussDB(DWS) Table?", "uri":"dli_03_0253.html", "doc_type":"usermanual", - "p_code":"189", - "code":"201" + "p_code":"193", + "code":"205" }, { "desc":"A datasource GaussDB(DWS) table and the datasource connection were created in DLI, and the schema of the source table in GaussDB(DWS) were updated. During the job executi", @@ -1814,8 +1850,8 @@ "title":"How Do I Do If an Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updated?", "uri":"dli_03_0254.html", "doc_type":"usermanual", - "p_code":"189", - "code":"202" + "p_code":"193", + "code":"206" }, { "desc":"When creating an associated RDS table in DLI, if the RDS table contains an auto-increment primary key or other auto-populated fields, you can take the following measures ", @@ -1823,8 +1859,8 @@ "title":"How Do I Insert Data into an RDS Table with an Auto-Increment Primary Key Using DLI?", "uri":"dli_03_0028.html", "doc_type":"usermanual", - "p_code":"189", - "code":"203" + "p_code":"193", + "code":"207" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1832,8 +1868,8 @@ "title":"SQL Jobs", "uri":"dli_03_0020.html", "doc_type":"usermanual", - "p_code":"159", - "code":"204" + "p_code":"163", + "code":"208" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1841,8 +1877,8 @@ "title":"SQL Job Development", "uri":"dli_03_0204.html", "doc_type":"usermanual", - "p_code":"204", - "code":"205" + "p_code":"208", + "code":"209" }, { "desc":"A temporary table is used to store intermediate results. When a transaction or session ends, the data in the temporary table can be automatically deleted. For example, in", @@ -1850,17 +1886,17 @@ "title":"SQL Jobs", "uri":"dli_03_0200.html", "doc_type":"usermanual", - "p_code":"205", - "code":"206" + "p_code":"209", + "code":"210" }, { - "desc":"If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.You a", + "desc":"In distributed file systems, data is stored in blocks. A small file refers to a file whose size is significantly smaller than the block size of the storage system. Large ", "product_code":"dli", "title":"How Do I Merge Small Files?", "uri":"dli_03_0086.html", "doc_type":"usermanual", - "p_code":"205", - "code":"207" + "p_code":"209", + "code":"211" }, { "desc":"When creating an OBS table, you must specify a table path in the database.The path format is as follows: obs://xxx/database name/table name.If the specified path is akdc.", @@ -1868,8 +1904,8 @@ "title":"How Do I Specify an OBS Path When Creating an OBS Table?", "uri":"dli_03_0092.html", "doc_type":"usermanual", - "p_code":"205", - "code":"208" + "p_code":"209", + "code":"212" }, { "desc":"To associate JSON data nested in an OBS bucket, you can create a table in asynchronous mode.The following is an example of a table creation statement that shows how to us", @@ -1877,8 +1913,8 @@ "title":"How Do I Create a Table Using JSON Data in an OBS Bucket?", "uri":"dli_03_0108.html", "doc_type":"usermanual", - "p_code":"205", - "code":"209" + "p_code":"209", + "code":"213" }, { "desc":"The correct method for using the count function to perform aggregation is as follows:OrIf an incorrect method is used, an error will be reported.", @@ -1886,8 +1922,8 @@ "title":"How Can I Use the count Function to Perform Aggregation?", "uri":"dli_03_0069.html", "doc_type":"usermanual", - "p_code":"205", - "code":"210" + "p_code":"209", + "code":"214" }, { "desc":"You can use the cross-region replication function of OBS. The procedure is as follows:Export the DLI table data in region 1 to the user-defined OBS bucket.Use the OBS cro", @@ -1895,8 +1931,8 @@ "title":"How Do I Synchronize DLI Table Data Across Regions?", "uri":"dli_03_0072.html", "doc_type":"usermanual", - "p_code":"205", - "code":"211" + "p_code":"209", + "code":"215" }, { "desc":"If you need to insert data into a table but only want to specify certain fields, you can use the INSERT INTO statement combined with the SELECT clause.However, DLI curren", @@ -1904,8 +1940,8 @@ "title":"How Do I Insert Table Data into Specific Fields of a Table Using a SQL Job?", "uri":"dli_03_0191.html", "doc_type":"usermanual", - "p_code":"205", - "code":"212" + "p_code":"209", + "code":"216" }, { "desc":"If the job runs slowly, perform the following steps to find the causes and rectify the fault:Check whether the problem is caused by FullGC.Log in to the DLI console. In t", @@ -1913,8 +1949,8 @@ "title":"How Do I Troubleshoot Slow SQL Jobs?", "uri":"dli_03_0196.html", "doc_type":"usermanual", - "p_code":"205", - "code":"213" + "p_code":"209", + "code":"217" }, { "desc":"You can view SQL job logs for routine O&M.Obtain the ID of the DLI job executed on the DataArts Studio console.Job IDOn the DLI console, choose Job Management > SQL Jobs.", @@ -1922,8 +1958,8 @@ "title":"How Do I View DLI SQL Logs?", "uri":"dli_03_0091.html", "doc_type":"usermanual", - "p_code":"205", - "code":"214" + "p_code":"209", + "code":"218" }, { "desc":"You can view the job execution records when a job is running.Log in to the DLI management console.In the navigation pane on the left, choose Job Management > SQL Jobs.Ent", @@ -1931,8 +1967,8 @@ "title":"How Do I View SQL Execution Records in DLI?", "uri":"dli_03_0116.html", "doc_type":"usermanual", - "p_code":"205", - "code":"215" + "p_code":"209", + "code":"219" }, { "desc":"Data skew is a common issue during the execution of SQL jobs. When data is unevenly distributed, some compute nodes process significantly more data than others, which can", @@ -1940,8 +1976,8 @@ "title":"How Do I Do When Data Skew Occurs During the Execution of a SQL Job?", "uri":"dli_03_0093.html", "doc_type":"usermanual", - "p_code":"205", - "code":"216" + "p_code":"209", + "code":"220" }, { "desc":"A SQL job contains join operations. After the job is submitted, it is stuck in the Running state and no result is returned.When a Spark SQL job has join operations on sma", @@ -1949,8 +1985,8 @@ "title":"Why Does a SQL Job That Has Join Operations Stay in the Running State?", "uri":"dli_03_0182.html", "doc_type":"usermanual", - "p_code":"205", - "code":"217" + "p_code":"209", + "code":"221" }, { "desc":"The possible causes and solutions are as follows:After you purchase a DLI queue and submit a SQL job for the first time, wait for 5 to 10 minutes. After the cluster is st", @@ -1958,8 +1994,8 @@ "title":"Why Is a SQL Job Stuck in the Submitting State?", "uri":"dli_03_0213.html", "doc_type":"usermanual", - "p_code":"205", - "code":"218" + "p_code":"209", + "code":"222" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1967,8 +2003,8 @@ "title":"SQL Job O&M", "uri":"dli_03_0206.html", "doc_type":"usermanual", - "p_code":"204", - "code":"219" + "p_code":"208", + "code":"223" }, { "desc":"This message indicates that you are exporting data to an existing OBS path.Solution:Create an OBS directory.You can create an OBS directory that does not exist to store t", @@ -1976,8 +2012,8 @@ "title":"Why Is Error \"path obs://xxx already exists\" Reported When Data Is Exported to OBS?", "uri":"dli_03_0014.html", "doc_type":"usermanual", - "p_code":"219", - "code":"220" + "p_code":"223", + "code":"224" }, { "desc":"This message indicates that the two tables to be joined contain the same column, but the owner of the column is not specified when the command is executed.For example, ta", @@ -1985,8 +2021,8 @@ "title":"Why Is Error \"SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;\" Displayed When Two Tables Are Joined?", "uri":"dli_03_0066.html", "doc_type":"usermanual", - "p_code":"219", - "code":"221" + "p_code":"223", + "code":"225" }, { "desc":"This message indicates that your operation is restricted because your account is in arrears or there is insufficient balance in your account.Solution:Check the account st", @@ -1994,8 +2030,8 @@ "title":"Why Is Error \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget.\" Reported when a SQL Statement Is Executed?", "uri":"dli_03_0071.html", "doc_type":"usermanual", - "p_code":"219", - "code":"222" + "p_code":"223", + "code":"226" }, { "desc":"When you query the partitioned table XX.YYY, the partition column is not specified in the search criteria.A partitioned table can be queried only when the query condition", @@ -2003,8 +2039,8 @@ "title":"Why Is Error \"There should be at least one partition pruning predicate on partitioned table XX.YYY\" Reported When a Query Statement Is Executed?", "uri":"dli_03_0145.html", "doc_type":"usermanual", - "p_code":"219", - "code":"223" + "p_code":"223", + "code":"227" }, { "desc":"The following error message is displayed when the LOAD DATA command is executed by a Spark SQL job to import data to a DLI table:In some cases ,the following error messag", @@ -2012,8 +2048,8 @@ "title":"Why Is Error \"IllegalArgumentException: Buffer size too small. size\" Reported When Data Is Loaded to an OBS Foreign Table?", "uri":"dli_03_0169.html", "doc_type":"usermanual", - "p_code":"219", - "code":"224" + "p_code":"223", + "code":"228" }, { "desc":"An error is reported during SQL job execution:Please contact DLI service. DLI.0002: FileNotFoundException: getFileStatus on obs://xxx: status [404]Check whether there is ", @@ -2021,8 +2057,8 @@ "title":"Why Is Error \"DLI.0002 FileNotFoundException\" Reported During SQL Job Running?", "uri":"dli_03_0189.html", "doc_type":"usermanual", - "p_code":"219", - "code":"225" + "p_code":"223", + "code":"229" }, { "desc":"Currently, DLI supports the creation of TEXTFILE, SEQUENCEFILE, RCFILE, ORC, AVRO, and PARQUET tables using the Hive syntax.If you create a table using CTAS statements, s", @@ -2030,8 +2066,8 @@ "title":"Why Is a Schema Parsing Error Reported When I Create a Hive Table Using CTAS?", "uri":"dli_03_0046.html", "doc_type":"usermanual", - "p_code":"219", - "code":"226" + "p_code":"223", + "code":"230" }, { "desc":"When you run a DLI SQL script on DataArts Studio, the log shows that the statements fail to be executed. The error information is as follows:DLI.0999: RuntimeException: o", @@ -2039,8 +2075,8 @@ "title":"Why Is Error \"org.apache.hadoop.fs.obs.OBSIOException\" Reported When I Run DLI SQL Scripts on DataArts Studio?", "uri":"dli_03_0173.html", "doc_type":"usermanual", - "p_code":"219", - "code":"227" + "p_code":"223", + "code":"231" }, { "desc":"Error message \"File not Found\" is displayed when a SQL job is accessed.The system may not be able to locate the specified file path or file due to an incorrect file path ", @@ -2048,8 +2084,8 @@ "title":"Why Is Error \"File not Found\" Reported When I Access a SQL Job?", "uri":"dli_03_0207.html", "doc_type":"usermanual", - "p_code":"219", - "code":"228" + "p_code":"223", + "code":"232" }, { "desc":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.Check the permissions of the OBS bucket to ensure that the account has access ", @@ -2057,8 +2093,8 @@ "title":"Why Is Error \"DLI.0003: AccessControlException XXX\" Reported When I Access a SQL Job?", "uri":"dli_03_0208.html", "doc_type":"usermanual", - "p_code":"219", - "code":"229" + "p_code":"223", + "code":"233" }, { "desc":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Ch", @@ -2066,8 +2102,8 @@ "title":"Why Is Error \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" Reported When I Access a SQL Job?", "uri":"dli_03_0209.html", "doc_type":"usermanual", - "p_code":"219", - "code":"230" + "p_code":"223", + "code":"234" }, { "desc":"Error message \"The current account does not have permission to perform this operation,the current account was restricted.\" is reported during SQL statement execution.Chec", @@ -2075,8 +2111,8 @@ "title":"Why Am I Seeing the Error Message \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget\" When Executing a SQL Statement?", "uri":"dli_03_0210.html", "doc_type":"usermanual", - "p_code":"219", - "code":"231" + "p_code":"223", + "code":"235" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2084,8 +2120,8 @@ "title":"Flink Jobs", "uri":"dli_03_0037.html", "doc_type":"usermanual", - "p_code":"159", - "code":"232" + "p_code":"163", + "code":"236" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2093,8 +2129,8 @@ "title":"Flink Job Consulting", "uri":"dli_03_0137.html", "doc_type":"usermanual", - "p_code":"232", - "code":"233" + "p_code":"236", + "code":"237" }, { "desc":"A sub-user can view queues but cannot view Flink jobs. You can authorize the sub-user using DLI or IAM.Authorization on DLILog in to the DLI console using a tenant accoun", @@ -2102,8 +2138,8 @@ "title":"How Do I Authorize a Subuser to View Flink Jobs?", "uri":"dli_03_0139.html", "doc_type":"usermanual", - "p_code":"233", - "code":"234" + "p_code":"237", + "code":"238" }, { "desc":"DLI Flink jobs are highly available. You can enable the automatic restart function to automatically restart your jobs after short-time faults of peripheral services are r", @@ -2111,8 +2147,8 @@ "title":"How Do I Configure Auto Restart upon Exception for a Flink Job?", "uri":"dli_03_0090.html", "doc_type":"usermanual", - "p_code":"233", - "code":"235" + "p_code":"237", + "code":"239" }, { "desc":"When you create a Flink SQL job or Flink Jar job, you can select Save Job Log on the job editing page to save job running logs to OBS.To set the OBS bucket for storing th", @@ -2120,8 +2156,8 @@ "title":"How Do I Save Logs for Flink Jobs?", "uri":"dli_03_0099.html", "doc_type":"usermanual", - "p_code":"233", - "code":"236" + "p_code":"237", + "code":"240" }, { "desc":"Choose Job Management > Flink Jobs. In the Operation column of the target job, choose More > Permissions. When a new user is authorized, No such user. userName:xxxx. is d", @@ -2129,8 +2165,8 @@ "title":"Why Is Error \"No such user. userName:xxxx.\" Reported on the Flink Job Management Page When I Grant Permission to a User?", "uri":"dli_03_0160.html", "doc_type":"usermanual", - "p_code":"233", - "code":"237" + "p_code":"237", + "code":"241" }, { "desc":"Checkpoint was enabled when a Flink job is created, and the OBS bucket for storing checkpoints was specified. I am not sure how to restore a Flink job from a specific che", @@ -2138,8 +2174,8 @@ "title":"How Do I Restore a Flink Job from a Specific Checkpoint After Manually Stopping the Job?", "uri":"dli_03_0180.html", "doc_type":"usermanual", - "p_code":"233", - "code":"238" + "p_code":"237", + "code":"242" }, { "desc":"When you set running parameters of a DLI Flink job, you can enable Alarm Generation upon Job Exception to receive alarms when the job runs abnormally or is in arrears.If ", @@ -2147,8 +2183,8 @@ "title":"Why Is a Message Displayed Indicating That the SMN Topic Does Not Exist When I Use the SMN Topic in DLI?", "uri":"dli_03_0036.html", "doc_type":"usermanual", - "p_code":"233", - "code":"239" + "p_code":"237", + "code":"243" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2156,8 +2192,8 @@ "title":"Flink SQL Jobs", "uri":"dli_03_0131.html", "doc_type":"usermanual", - "p_code":"232", - "code":"240" + "p_code":"236", + "code":"244" }, { "desc":"When using a Flink SQL job, you need to create an OBS partition table for subsequent batch processing.In the following example, the day field is used as the partition fie", @@ -2165,8 +2201,8 @@ "title":"How Do I Map an OBS Table to a DLI Partitioned Table?", "uri":"dli_03_0089.html", "doc_type":"usermanual", - "p_code":"240", - "code":"241" + "p_code":"244", + "code":"245" }, { "desc":"You used Flink 1.10 to run a Flink Opensource SQL job. You set the number of Kafka partitions for the job a small value at the beginning and need to increase the number n", @@ -2174,8 +2210,8 @@ "title":"How Do I Change the Number of Kafka Partitions in a Flink SQL Job Without Stopping It?", "uri":"dli_03_0120.html", "doc_type":"usermanual", - "p_code":"240", - "code":"242" + "p_code":"244", + "code":"246" }, { "desc":"When I run the creation statement with an EL expression in the table name in a Flink SQL job, the following error message is displayed:DLI.0005: AnalysisException: t_user", @@ -2183,8 +2219,8 @@ "title":"How Do I Fix the DLI.0005 Error When Using EL Expressions to Create a Table in a Flink SQL Job?", "uri":"dli_03_0167.html", "doc_type":"usermanual", - "p_code":"240", - "code":"243" + "p_code":"244", + "code":"247" }, { "desc":"After data is written to OBS through the Flink job output stream, data cannot be queried from the DLI table created in the OBS file path.For example, use the following Fl", @@ -2192,8 +2228,8 @@ "title":"Why Is No Data Queried in the DLI Table Created Using the OBS File Path When Data Is Written to OBS by a Flink Job Output Stream?", "uri":"dli_03_0168.html", "doc_type":"usermanual", - "p_code":"240", - "code":"244" + "p_code":"244", + "code":"248" }, { "desc":"After a Flink SQL job is submitted on DLI, the job fails to be executed. The following error information is displayed in the job log:connect to DIS failed java.lang.Illeg", @@ -2201,8 +2237,8 @@ "title":"Why Does a Flink SQL Job Fails to Be Executed, and Is \"connect to DIS failed java.lang.IllegalArgumentException: Access key cannot be null\" Displayed in the Log?", "uri":"dli_03_0174.html", "doc_type":"usermanual", - "p_code":"240", - "code":"245" + "p_code":"244", + "code":"249" }, { "desc":"After a Flink SQL job consumed Kafka and sent data to the Elasticsearch cluster, the job was successfully executed, but no data is available.Possible causes are as follow", @@ -2210,8 +2246,8 @@ "title":"Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster", "uri":"dli_03_0232.html", "doc_type":"usermanual", - "p_code":"240", - "code":"246" + "p_code":"244", + "code":"250" }, { "desc":"You used Flink 1.10 to run a Flink Opensource SQL job. The job failed after the following error was reported when Flink Sink wrote data to Kafka.The CPU usage is too high", @@ -2219,8 +2255,8 @@ "title":"How Do I Configure Connection Retries for Kafka Sink If it is Disconnected?", "uri":"dli_03_0121.html", "doc_type":"usermanual", - "p_code":"240", - "code":"247" + "p_code":"244", + "code":"251" }, { "desc":"In a Flink job, you can use CREATE statements to define source and sink tables, and specify their connector types and related attributes.If you need to write data to diff", @@ -2228,8 +2264,8 @@ "title":"How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?", "uri":"dli_03_0048.html", "doc_type":"usermanual", - "p_code":"240", - "code":"248" + "p_code":"244", + "code":"252" }, { "desc":"To rectify this fault, perform the following steps:Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statemen", @@ -2237,8 +2273,8 @@ "title":"Why Does DIS Stream Not Exist During Job Semantic Check?", "uri":"dli_03_0040.html", "doc_type":"usermanual", - "p_code":"240", - "code":"249" + "p_code":"244", + "code":"253" }, { "desc":"If the Flink JobManager prompts \"Timeout expired while fetching topic metadata\", it means that the Flink job timed out while trying to fetch metadata for the Kafka topic.", @@ -2246,8 +2282,8 @@ "title":"Why Is Error \"Timeout expired while fetching topic metadata\" Repeatedly Reported in Flink JobManager Logs?", "uri":"dli_03_0265.html", "doc_type":"usermanual", - "p_code":"240", - "code":"250" + "p_code":"244", + "code":"254" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2255,8 +2291,8 @@ "title":"Flink Jar Jobs", "uri":"dli_03_0132.html", "doc_type":"usermanual", - "p_code":"232", - "code":"251" + "p_code":"236", + "code":"255" }, { "desc":"You can upload configuration files for custom jobs (Jar).Upload the configuration file to DLI through Package Management.In the Other Dependencies area of the Flink Jar j", @@ -2264,8 +2300,8 @@ "title":"Can I Upload Configuration Files for Flink Jar Jobs?", "uri":"dli_03_0044.html", "doc_type":"usermanual", - "p_code":"251", - "code":"252" + "p_code":"255", + "code":"256" }, { "desc":"The dependency of your Flink job conflicts with a built-in dependency of the DLI Flink platform. As a result, the job submission fails.Check whether there are conflicting", @@ -2273,8 +2309,8 @@ "title":"Why Does a Flink Jar Package Conflict Result in Job Submission Failure?", "uri":"dli_03_0119.html", "doc_type":"usermanual", - "p_code":"251", - "code":"253" + "p_code":"255", + "code":"257" }, { "desc":"When a Flink Jar job is submitted to access GaussDB(DWS), an error message is displayed indicating that the job fails to be started. The job log contains the following er", @@ -2282,8 +2318,8 @@ "title":"Why Does a Flink Jar Job Fail to Access GaussDB(DWS) and a Message Is Displayed Indicating Too Many Client Connections?", "uri":"dli_03_0161.html", "doc_type":"usermanual", - "p_code":"251", - "code":"254" + "p_code":"255", + "code":"258" }, { "desc":"The storage path of the Flink Jar job checkpoints was set to an OBS bucket. The job failed to be submitted, and an error message indicating an invalid OBS bucket name was", @@ -2291,8 +2327,8 @@ "title":"Why Is Error Invalid OBS Bucket Name Reported After a Flink Job Submission Failed?", "uri":"dli_03_0233.html", "doc_type":"usermanual", - "p_code":"251", - "code":"255" + "p_code":"255", + "code":"259" }, { "desc":"Flink Job submission failed. The exception information is as follows:Flink JAR files conflicted. The submitted Flink JAR file conflicted with the HDFS JAR file of the DLI", @@ -2300,8 +2336,8 @@ "title":"Why Does the Flink Submission Fail Due to Hadoop JAR File Conflict?", "uri":"dli_03_0234.html", "doc_type":"usermanual", - "p_code":"251", - "code":"256" + "p_code":"255", + "code":"260" }, { "desc":"On the Flink job management page, hover the cursor on the status of the job that fails to be submitted to view the brief information about the failure.The possible causes", @@ -2309,8 +2345,8 @@ "title":"How Do I Locate a Flink Job Submission Error?", "uri":"dli_03_0103.html", "doc_type":"usermanual", - "p_code":"251", - "code":"257" + "p_code":"255", + "code":"261" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2318,8 +2354,8 @@ "title":"Flink Job Performance Tuning", "uri":"dli_03_0133.html", "doc_type":"usermanual", - "p_code":"232", - "code":"258" + "p_code":"236", + "code":"262" }, { "desc":"Data Stacking in a Consumer GroupThe accumulated data of a consumer group can be calculated by the following formula: Total amount of data to be consumed by the consumer ", @@ -2327,8 +2363,8 @@ "title":"Flink Job Performance Tuning", "uri":"dli_03_0106.html", "doc_type":"usermanual", - "p_code":"258", - "code":"259" + "p_code":"262", + "code":"263" }, { "desc":"The DLI Flink checkpoint/savepoint mechanism is complete and reliable. You can use this mechanism to prevent data loss when a job is manually restarted or restarted due t", @@ -2336,8 +2372,8 @@ "title":"How Do I Prevent Data Loss After Flink Job Restart?", "uri":"dli_03_0096.html", "doc_type":"usermanual", - "p_code":"258", - "code":"260" + "p_code":"262", + "code":"264" }, { "desc":"On the Flink job management, click Edit in the Operation column of the target job. On the displayed page, check whether Save Job Log in the Running Parameters tab is enab", @@ -2345,8 +2381,8 @@ "title":"How Do I Locate a Flink Job Running Error?", "uri":"dli_03_0105.html", "doc_type":"usermanual", - "p_code":"258", - "code":"261" + "p_code":"262", + "code":"265" }, { "desc":"Flink's checkpointing is a fault tolerance and recovery mechanism. This mechanism ensures that real-time programs can self-recover in case of exceptions or machine issues", @@ -2354,8 +2390,8 @@ "title":"How Can I Check if a Flink Job Can Be Restored From a Checkpoint After Restarting It?", "uri":"dli_03_0136.html", "doc_type":"usermanual", - "p_code":"258", - "code":"262" + "p_code":"262", + "code":"266" }, { "desc":"Mode for storing generated job logs when a DLI Flink job fails to be submitted or executed. The options are as follows:If the submission fails, a submission log is genera", @@ -2363,8 +2399,8 @@ "title":"Why Are Logs Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?", "uri":"dli_03_0064.html", "doc_type":"usermanual", - "p_code":"258", - "code":"263" + "p_code":"262", + "code":"267" }, { "desc":"JobManager and TaskManager heartbeats timed out. As a result, the Flink job is abnormal.Check whether the network is intermittently disconnected and whether the cluster l", @@ -2372,8 +2408,8 @@ "title":"Why Is the Flink Job Abnormal Due to Heartbeat Timeout Between JobManager and TaskManager?", "uri":"dli_03_0236.html", "doc_type":"usermanual", - "p_code":"258", - "code":"264" + "p_code":"262", + "code":"268" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2381,8 +2417,8 @@ "title":"Spark Jobs", "uri":"dli_03_0021.html", "doc_type":"usermanual", - "p_code":"159", - "code":"265" + "p_code":"163", + "code":"269" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2390,8 +2426,8 @@ "title":"Spark Job Development", "uri":"dli_03_0217.html", "doc_type":"usermanual", - "p_code":"265", - "code":"266" + "p_code":"269", + "code":"270" }, { "desc":"DLI Spark does not support job scheduling. You can use other services, such as DataArts Studio, or use APIs or SDKs to customize job schedule.The Spark SQL syntax does no", @@ -2399,8 +2435,8 @@ "title":"Spark Jobs", "uri":"dli_03_0201.html", "doc_type":"usermanual", - "p_code":"266", - "code":"267" + "p_code":"270", + "code":"271" }, { "desc":"To use Spark to write data into a DLI table, configure the following parameters:fs.obs.access.keyfs.obs.secret.keyfs.obs.implfs.obs.endpointThe following is an example:", @@ -2408,8 +2444,8 @@ "title":"How Do I Use Spark to Write Data into a DLI Table?", "uri":"dli_03_0107.html", "doc_type":"usermanual", - "p_code":"266", - "code":"268" + "p_code":"270", + "code":"272" }, { "desc":"To obtain the AK/SK, set the parameters as follows:Create a SparkContext using code.val sc: SparkContext = new SparkContext()\nsc.hadoopConfiguration.set(\"fs.obs.access.ke", @@ -2417,8 +2453,8 @@ "title":"How Do I Set Up AK/SK So That a General Queue Can Access Tables Stored in OBS?", "uri":"dli_03_0017.html", "doc_type":"usermanual", - "p_code":"266", - "code":"269" + "p_code":"270", + "code":"273" }, { "desc":"Log in to the DLI console. In the navigation pane, choose Job Management > Spark Jobs. In the job list, locate the target job and click next to Job ID to view the parame", @@ -2426,8 +2462,8 @@ "title":"How Do I View the Resource Usage of DLI Spark Jobs?", "uri":"dli_03_0102.html", "doc_type":"usermanual", - "p_code":"266", - "code":"270" + "p_code":"270", + "code":"274" }, { "desc":"If the pymysql module is missing, check whether the corresponding EGG package exists. If the package does not exist, upload the pyFile package on the Package Management p", @@ -2435,8 +2471,8 @@ "title":"How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the Spark Job Results Stored in MySQL?", "uri":"dli_03_0076.html", "doc_type":"usermanual", - "p_code":"266", - "code":"271" + "p_code":"270", + "code":"275" }, { "desc":"DLI natively supports PySpark.For most cases, Python is preferred for data analysis, and PySpark is the best choice for big data analysis. Generally, JVM programs are pac", @@ -2444,8 +2480,8 @@ "title":"How Do I Run a Complex PySpark Program in DLI?", "uri":"dli_03_0082.html", "doc_type":"usermanual", - "p_code":"266", - "code":"272" + "p_code":"270", + "code":"276" }, { "desc":"When shuffle statements, such as GROUP BY and JOIN, are executed in Spark jobs, data skew occurs, which slows down the job execution.To solve this problem, you can config", @@ -2453,8 +2489,8 @@ "title":"How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?", "uri":"dli_03_0068.html", "doc_type":"usermanual", - "p_code":"266", - "code":"273" + "p_code":"270", + "code":"277" }, { "desc":"You can use SparkFiles to read the file submitted using –-file form a local path: SparkFiles.get(\"Name of the uploaded file\").The file path in the Driver is different fro", @@ -2462,8 +2498,8 @@ "title":"How Do I Read Uploaded Files for a Spark Jar Job?", "uri":"dli_03_0118.html", "doc_type":"usermanual", - "p_code":"266", - "code":"274" + "p_code":"270", + "code":"278" }, { "desc":"When running a job with Spark 3.3.1, certain fields have empty default attribute values when queried in the view attribute on the client. In contrast, when using Spark 3.", @@ -2471,8 +2507,8 @@ "title":"Why Are View Attributes Empty in Spark 3.3.1 Client?", "uri":"dli_03_0279.html", "doc_type":"usermanual", - "p_code":"266", - "code":"275" + "p_code":"270", + "code":"279" }, { "desc":"I cannot find the specified Python environment after adding the Python 3 package.Set spark.yarn.appMasterEnv.PYSPARK_PYTHON to python3 in the conf file to specify the Pyt", @@ -2480,8 +2516,8 @@ "title":"Why Can't I Find the Specified Python Environment After Adding the Python Package?", "uri":"dli_03_0077.html", "doc_type":"usermanual", - "p_code":"266", - "code":"276" + "p_code":"270", + "code":"280" }, { "desc":"The remaining CUs in the queue may be insufficient. As a result, the job cannot be submitted.To view the remaining CUs of a queue, perform the following steps:Check the C", @@ -2489,8 +2525,8 @@ "title":"Why Is a Spark Jar Job Stuck in the Submitting State?", "uri":"dli_03_0220.html", "doc_type":"usermanual", - "p_code":"266", - "code":"277" + "p_code":"270", + "code":"281" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2498,8 +2534,8 @@ "title":"Spark Job O&M", "uri":"dli_03_0218.html", "doc_type":"usermanual", - "p_code":"265", - "code":"278" + "p_code":"269", + "code":"282" }, { "desc":"The following error is reported when a Spark job accesses OBS data:Set the AK/SK to enable Spark jobs to access OBS data.For details, see How Do I Set Up AK/SK So That a ", @@ -2507,8 +2543,8 @@ "title":"Why Do I Get \"ResponseCode: 403\" and \"ResponseStatus: Forbidden\" Errors When a Spark Job Accesses OBS Data?", "uri":"dli_03_0156.html", "doc_type":"usermanual", - "p_code":"278", - "code":"279" + "p_code":"282", + "code":"283" }, { "desc":"This error message may be due to the OBS bucket being set as the DLI log bucket, which cannot be used for other purposes.You can follow these steps to check:Check if the ", @@ -2516,8 +2552,8 @@ "title":"Why Do I Encounter the Error \"verifyBucketExists on XXXX: status [403]\" When Using a Spark Job to Access an OBS Bucket That I Have Permission to Access?", "uri":"dli_03_0164.html", "doc_type":"usermanual", - "p_code":"278", - "code":"280" + "p_code":"282", + "code":"284" }, { "desc":"When running large amounts of data in a Spark job, if a timeout exception error occurs, it is usually due to insufficient resource configuration, data skew, network issue", @@ -2525,8 +2561,8 @@ "title":"Why Does a Job Running Timeout Occur When Processing a Large Amount of Data with a Spark Job?", "uri":"dli_03_0157.html", "doc_type":"usermanual", - "p_code":"278", - "code":"281" + "p_code":"282", + "code":"285" }, { "desc":"Spark jobs cannot access SFTP. Upload the files you want to access to OBS and then you can analyze the data using Spark jobs.Upload data to an OBS bucket: Upload data sto", @@ -2534,8 +2570,8 @@ "title":"Why Does a Spark Job Fail to Execute with an Abnormal Access Directory Error When Accessing Files in SFTP?", "uri":"dli_03_0188.html", "doc_type":"usermanual", - "p_code":"278", - "code":"282" + "p_code":"282", + "code":"286" }, { "desc":"When a Spark job is running, an error message is displayed, indicating that the user does not have the database permission. The error information is as follows:org.apache", @@ -2543,8 +2579,8 @@ "title":"Why Does the Job Fail to Be Executed Due to Insufficient Database and Table Permissions?", "uri":"dli_03_0192.html", "doc_type":"usermanual", - "p_code":"278", - "code":"283" + "p_code":"282", + "code":"287" }, { "desc":"I cannot find the global_temp database in the Spark 3.x job log.The global_temp database is the default built-in database of Spark 3.x and is Spark's global temporary vie", @@ -2552,8 +2588,8 @@ "title":"Why Is the global_temp Database Missing in the Job Log of Spark 3.x?", "uri":"dli_03_0272.html", "doc_type":"usermanual", - "p_code":"278", - "code":"284" + "p_code":"282", + "code":"288" }, { "desc":"I failed to use the DataSource syntax to create an OBS table in Avro format when selecting Spark to access metadata.Spark 2.3.x does not support creating OBS tables in Av", @@ -2561,8 +2597,8 @@ "title":"Why Does Using DataSource Syntax to Create an OBS Table of Avro Type Fail When Accessing Metadata With Spark 2.3.x?", "uri":"dli_03_0275.html", "doc_type":"usermanual", - "p_code":"278", - "code":"285" + "p_code":"282", + "code":"289" }, { "desc":"When the rand function is used in SQL statements of Spark 3.3.1, the error message \"Input argument to rand must be a constant\" is displayed.In Spark 3.3.1, the rand funct", @@ -2570,8 +2606,8 @@ "title":"How Do I Resolve the \"Input argument to rand must be a constant\" Error in Spark 3.3.1 SQL Statements?", "uri":"dli_03_0278.html", "doc_type":"usermanual", - "p_code":"278", - "code":"286" + "p_code":"282", + "code":"290" }, { "desc":"When using Spark 3.3.1 client to create a subquery view with aggregate functions, an error occurs when attempting to write the data, stating \"Not allowed to create a perm", @@ -2579,8 +2615,8 @@ "title":"When Using Spark 3.3.1 Client to Create a View and Perform a Join Query, an Error Occurs Stating \"Not allowed to create a permanent view\" When Attempting to Write the Data", "uri":"dli_03_0273.html", "doc_type":"usermanual", - "p_code":"278", - "code":"287" + "p_code":"282", + "code":"291" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2588,26 +2624,26 @@ "title":"DLI Resource Quotas", "uri":"dli_03_0053.html", "doc_type":"usermanual", - "p_code":"159", - "code":"288" + "p_code":"163", + "code":"292" }, { - "desc":"Log in to the management console.Click in the upper left corner and select a region and a project.Click the My Quota icon in the upper right corner of the page.The Serv", + "desc":"Log in to the management console.Click in the upper left corner and select a region and a project.Click the My Quota icon in the upper right corner of the page.This wil", "product_code":"dli", "title":"How Do I View My Quotas?", "uri":"dli_03_0031.html", "doc_type":"usermanual", - "p_code":"288", - "code":"289" + "p_code":"292", + "code":"293" }, { - "desc":"The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your applica", + "desc":"The system currently does not support online quota adjustments. If you need to modify your quota, please contact our customer service team via phone or email. They will p", "product_code":"dli", "title":"How Do I Apply for a Higher Quota?", "uri":"dli_03_0032.html", "doc_type":"usermanual", - "p_code":"288", - "code":"290" + "p_code":"292", + "code":"294" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2615,8 +2651,8 @@ "title":"DLI Permissions Management", "uri":"dli_03_0054.html", "doc_type":"usermanual", - "p_code":"159", - "code":"291" + "p_code":"163", + "code":"295" }, { "desc":"You are unable to perform permission operations on the partition columns of partitioned tables.However, when you grant the permission of any non-partition column in a par", @@ -2624,8 +2660,8 @@ "title":"What Is Column-Level Authorization for DLI Partitioned Tables?", "uri":"dli_03_0008.html", "doc_type":"usermanual", - "p_code":"291", - "code":"292" + "p_code":"295", + "code":"296" }, { "desc":"When the user update an existing program package, the following error information is displayed:\"error_code\"*DLI.0003\",\"error_msg\":\"Permission denied for resource 'resourc", @@ -2633,8 +2669,8 @@ "title":"How Do I Do If I Encounter Insufficient Permissions While Updating Packages?", "uri":"dli_03_0195.html", "doc_type":"usermanual", - "p_code":"291", - "code":"293" + "p_code":"295", + "code":"297" }, { "desc":"When the SQL query statement is executed, the system displays a message indicating that the user does not have the permission to query resources.Error information: DLI.00", @@ -2642,8 +2678,8 @@ "title":"Why Is Error \"DLI.0003: Permission denied for resource...\" Reported When I Run a SQL Statement?", "uri":"dli_03_0227.html", "doc_type":"usermanual", - "p_code":"291", - "code":"294" + "p_code":"295", + "code":"298" }, { "desc":"If you have already granted authorization to a table and a test query was successful, but you encounter an error when trying to query it again after some time, you should", @@ -2651,8 +2687,8 @@ "title":"How Do I Do If I Can't Query Table Data After Being Granted Table Permissions?", "uri":"dli_03_0228.html", "doc_type":"usermanual", - "p_code":"291", - "code":"295" + "p_code":"295", + "code":"299" }, { "desc":"If a table inherits database permissions, you do not need to regrant the inherited permissions to the table.Re-authorizing may cause confusion in table permission managem", @@ -2660,17 +2696,17 @@ "title":"Will Granting Duplicate Permissions to a Table After Inheriting Database Permissions Cause an Error?", "uri":"dli_03_0057.html", "doc_type":"usermanual", - "p_code":"291", - "code":"296" + "p_code":"295", + "code":"300" }, { - "desc":"User A created Table1.User B created View1 based on Table1.After the Select Table permission on Table1 is granted to user C, user C fails to query View1.User B does not h", + "desc":"User A created a table named Table1.User B created a view named View1 based on Table1.After granting user C the select permission on Table1, user C failed to query the vi", "product_code":"dli", - "title":"Why Can't I Query a View After I'm Granted the Select Table Permission on the View?", + "title":"Why Can't I Query a View Despite Having the Select Permission?", "uri":"dli_03_0067.html", "doc_type":"usermanual", - "p_code":"291", - "code":"297" + "p_code":"295", + "code":"301" }, { "desc":"Despite configuring a job bucket and authorizing DLI to access it, I still receive an error message stating that DLI is not authorized to access the bucket when attemptin", @@ -2678,8 +2714,8 @@ "title":"How Do I Do If I Receive a Message Saying I Don't Have Sufficient Permissions to Submit My Jobs to the Job Bucket?", "uri":"dli_03_0225.html", "doc_type":"usermanual", - "p_code":"291", - "code":"298" + "p_code":"295", + "code":"302" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2687,17 +2723,17 @@ "title":"DLI APIs", "uri":"dli_03_0056.html", "doc_type":"usermanual", - "p_code":"159", - "code":"299" + "p_code":"163", + "code":"303" }, { "desc":"In the REST API provided by DLI, the request header can be added to the request URI, for example, Content-Type.Content-Type indicates the request body type or format. The", "product_code":"dli", - "title":"Why Is Error \"unsupported media Type\" Reported When I Subimt a SQL Job?", + "title":"Why Am I Getting an \"unsupported media Type\" Error When Submitting a SQL Job?", "uri":"dli_03_0060.html", "doc_type":"usermanual", - "p_code":"299", - "code":"300" + "p_code":"303", + "code":"304" }, { "desc":"When the API call for submitting a SQL job times out, and the following error information is displayed:There are currently no resources tracked in the state, so there is ", @@ -2705,8 +2741,8 @@ "title":"What Can I Do If an Error Is Reported When the Execution of the API for Creating a SQL Job Times Out?", "uri":"dli_03_0178.html", "doc_type":"usermanual", - "p_code":"299", - "code":"301" + "p_code":"303", + "code":"305" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2715,6 +2751,6 @@ "uri":"dli_01_00006.html", "doc_type":"usermanual", "p_code":"", - "code":"302" + "code":"306" } ] \ No newline at end of file diff --git a/docs/dli/umn/dli_01_00006.html b/docs/dli/umn/dli_01_00006.html index 6c20f8be5..9d109b960 100644 --- a/docs/dli/umn/dli_01_00006.html +++ b/docs/dli/umn/dli_01_00006.html @@ -4,11 +4,17 @@
- - + + + @@ -144,7 +144,7 @@ @@ -176,7 +176,7 @@ - @@ -212,7 +212,7 @@

Released On

What's New

+

Change Description

2025-09-25

+

2025-02-10

+

Modified the following sections:

+ +

2025-09-25

Modified the following sections:

@@ -43,7 +49,7 @@

2023-11-01

Modified the following content:

- +

2023-10-08

@@ -56,8 +62,8 @@

Added the following sections:

-

Deleted the following section:

-
  • Deleted the content related to Flink job debugging.
+

Taken offline the following content:

+
  • Taken offline the relevant content for debugging Flink jobs.

Modified the following section:

  • Common Operations of Flink Jobs: Modified the description of the Auto Restart upon Exception parameter in the runtime configuration. Flink SQL jobs and Flink OpenSource SQL jobs cannot be restored from checkpoints.
Optimized the following sections: diff --git a/docs/dli/umn/dli_01_0001.html b/docs/dli/umn/dli_01_0001.html index f251d8bc4..840054d70 100644 --- a/docs/dli/umn/dli_01_0001.html +++ b/docs/dli/umn/dli_01_0001.html @@ -2,11 +2,11 @@

DLI Job Development Process

This chapter walks you through on how to develop a DLI job.

-

Creating an IAM User and Granting Permissions

  • To manage fine-grained permissions for your DLI resources using IAM, create an IAM user and grant them permissions to DLI if you are an enterprise user. For details, see Creating an IAM User and Granting Permissions.
  • When using DLI for the first time, you need to update the DLI agency according to the console's guidance so that DLI can use other cloud services and perform resource O&M operations on your behalf. The agency includes permissions to obtain IAM user information, access and use VPCs, CIDR blocks, routes, and peering connections, and send notifications via SMN in case of job execution failure.

    For more information on the specific permissions included in the agency, refer to Configuring DLI Agency Permissions.

    +

    Creating an IAM User and Granting Permissions

    • To manage fine-grained permissions for your DLI resources using IAM, create an IAM user and grant them permissions to DLI if you are an enterprise user. For details, see Creating an IAM User and Granting Permissions.
    • When using DLI for the first time, you need to update the DLI agency according to the console's guidance so that DLI can use other cloud services and perform resource O&M operations on your behalf. The agency includes permissions to obtain IAM user information, access and use VPCs, CIDR blocks, routes, and peering connections, and send notifications via SMN in case of job execution failure.

      For more information on the specific permissions included in the agency, see Configuring DLI Agency Permissions.

    -

    Creating Compute Resources and Metadata Required for Running Jobs

    • Before submitting a job using DLI, you need to create an elastic resource pool and create queues within it. This will provide the necessary compute resources for running the job. For how to create an elastic resource pool and create queues within it, see Overview of DLI Elastic Resource Pools and Queues.

      Alternatively, you can enhance DLI's computing environment by creating custom images. Specifically, to enhance the functions and performance of Spark and Flink jobs, you can create custom images by downloading the base images provided by DLI and adding dependencies (files, JAR files, or software) and private capabilities required for job execution. This changes the container runtime environment for the jobs.

      -

      For example, you can add a Python package or C library related to machine learning to a custom image to help you extend functions. For how to create a custom image, see Enhancing the Job Runtime Environment Using a Custom Image.

      +

      Creating Compute Resources and Metadata Required for Running Jobs

      • Before submitting a job using DLI, you need to create an elastic resource pool and create queues within it. This will provide the necessary compute resources for running the job. For details about how to create an elastic resource pool and create queues within it, see Overview of DLI Elastic Resource Pools and Queues.

        Alternatively, you can enhance DLI's computing environment by creating custom images. Specifically, to enhance the functions and performance of Spark and Flink jobs, you can create custom images by downloading the base images provided by DLI and adding dependencies (files, JAR files, or software) and private capabilities required for job execution. This changes the container runtime environment for the jobs.

        +

        For example, you can add a Python package or C library related to machine learning to a custom image to help you extend functions. For details about how to create a custom image, see Enhancing the Job Runtime Environment Using a Custom Image.

      • DLI metadata is the basis for developing SQL and Spark jobs. Before executing a job, you need to define databases and tables based on your business scenario.

        Flink allows for dynamic data types, enabling the definition of data structures at runtime without the need for predefined metadata.

        @@ -67,7 +67,7 @@

        Using Cloud Eye to Monitor DLI

        You can query DLI monitoring metrics and alarms through Cloud Eye management console or APIs.

        For example, you can monitor the resource usage and job status of a DLI queue. For details about DLI metrics, see Monitoring DLI Using Cloud Eye.

        -

        Using CTS to Audit DLI

        With CTS, you can log operations related to DLI, making it easier to search, audit, and trace in the future. For the supported operations, see Using CTS to Audit DLI.

        +

        Using CTS to Audit DLI

        With CTS, you can log operations related to DLI, making it easier to search, audit, and trace in the future. For details about the supported operations, see Using CTS to Audit DLI.

        diff --git a/docs/dli/umn/dli_01_0002.html b/docs/dli/umn/dli_01_0002.html index c5415bad0..68961d2c8 100644 --- a/docs/dli/umn/dli_01_0002.html +++ b/docs/dli/umn/dli_01_0002.html @@ -51,7 +51,7 @@ 22,book_2 29,book_9

        After the file is uploaded successfully, the file path is obs://obs1/sampledata.csv.

        -

        For more operations on the OBS console, see Object Storage Service User Guide.

        +

        For more operations on the OBS console, see Object Storage Service User Guide.

      Step 2: Create an Elastic Resource Pool and Create Queues Within It

      In this example, the elastic resource pool dli_resource_pool and queue dli_queue_01 are created.
      1. Log in to the DLI management console.
      2. In the navigation pane on the left, choose Resources > Resource Pool.
      3. On the displayed page, click Buy Resource Pool in the upper right corner.
      4. On the displayed page, set the parameters.
        Table 2 describes the parameters. diff --git a/docs/dli/umn/dli_01_0005.html b/docs/dli/umn/dli_01_0005.html index ef2c0e7dd..f29737fbf 100644 --- a/docs/dli/umn/dli_01_0005.html +++ b/docs/dli/umn/dli_01_0005.html @@ -51,9 +51,9 @@

      Creating a Table

      Before creating a table, ensure that a database has been created.

      -
      1. You can create a table on either the Databases and Tables page or the SQL Editor page.

        Datasource connection tables, such as View tables, HBase (MRS) tables, OpenTSDB (MRS) tables, GaussDB(DWS) tables, RDS tables, and CSS tables, cannot be created. You can use SQL to create views and datasource connection tables. For details, see "Creating a View" and "Creating a Datasource Connection Table" in Data Lake Insight SQL Syntax Reference.

        +
        1. You can create a table on either the Databases and Tables page or the SQL Editor page.

          Datasource connection tables, such as View tables, HBase (MRS) tables, OpenTSDB (MRS) tables, DWS tables, RDS tables, and CSS tables, cannot be created. You can use SQL to create views and datasource connection tables. For details, see "Creating a View" and "Creating a Datasource Connection Table" in the Data Lake Insight SQL Syntax Reference.

          -
          • To create a table on the Data Management page:
            1. In the navigation pane on the left of the console, choose Data Management > Databases and Tables.
            2. On the Databases and Tables page, select the database for which you want to create a table. In the Operation column, click More > Create Table to create a table in the current database.
            +
            • To create a table on the Data Management page:
              1. In the navigation pane on the left of the console, choose Data Management > Databases and Tables.
              2. On the Databases and Tables page, select the database for which you want to create a table. In the Operation column, click More > Create Table to create a table in the current database.
            • To create a table on the SQL Editor page:
              1. In the navigation pane on the left of the management console, choose SQL Editor.
              2. In the navigation pane of the displayed SQL Editor page, click Databases. You can create a table in either of the following ways:
                • Click a database name. In the Tables area, click on the right to create a table in the current database.
                • Click on the right of the database and choose Create Table from the shortcut menu to create a table in the current database.
            @@ -108,7 +108,7 @@

Type

Data type of a column. This parameter corresponds to Column Name.

-
  • string: The data is of the string type.
  • int: Each integer is stored on four bytes.
  • date: The value ranges from 0000-01-01 to 9999-12-31.
  • double: Each number is stored on eight bytes.
  • boolean: Each value is stored on one byte.
  • decimal: The valid bits are positive integers between 1 to 38, including 1 and 38. The decimal digits are integers less than 10.
  • smallint/short: The number is stored on two bytes.
  • bigint/long: The number is stored on eight bytes.
  • timestamp: The data indicates a date and time. The value can be accurate to six decimal points.
  • float: Each number is stored on four bytes.
  • tinyint: Each number is stored on one byte. Only OBS tables support this data type.
+
  • string: The data is of the string type.
  • int: Each integer is stored on four bytes.
  • date: The value ranges from 0000-01-01 to 9999-12-31.
  • double: Each number is stored on eight bytes.
  • boolean: Each value is stored on one byte.
  • decimal: The valid bits are positive integers ranging from 1 to 38. The decimal digits are integers less than 10.
  • smallint/short: The number is stored on two bytes.
  • bigint/long: The number is stored on eight bytes.
  • timestamp: The data indicates a date and time. The value can be accurate to six decimal points.
  • float: Each number is stored on four bytes.
  • tinyint: Each number is stored on one byte. Only OBS tables support this data type.

string

Data Format

DLI supports the following data formats:

-
  • Parquet: DLI can read non-compressed data or data that is compressed using Snappy and gzip.
  • CSV: DLI can read non-compressed data or data that is compressed using gzip.
  • ORC: DLI can read non-compressed data or data that is compressed using Snappy.
  • JSON: DLI can read non-compressed data or data that is compressed using gzip.
  • Avro: DLI can read uncompressed Avro data.
+
  • Parquet: DLI can read non-compressed data or data that is compressed using Snappy and gzip.
  • CSV: DLI can read non-compressed data or data that is compressed using gzip.
  • ORC: DLI can read non-compressed data or data that is compressed using Snappy.
  • JSON: DLI can read non-compressed data or data that is compressed using gzip.
  • Avro: DLI can read uncompressed Avro data.

CSV

User-defined Quotation Character

This parameter is valid only when Data Format is set to CSV and you select User-defined Quotation Character.

+

This parameter is valid only when Data Format is set to CSV and you select User-defined Quotation Character.

The following quotation characters are supported:

  • Single quotation mark (')
  • Double quotation marks (")
  • Others: Enter a user-defined quotation character.
-

  • Click OK.

    After a table is created, you can view and select the table for use on the Data Management page or SQL Editor page.

    +

  • Click OK.

    After a table is created, you can view and select the table for use on the Data Management page or SQL Editor page.

    • Related Operations

    After a table is created, you can import data from other OBS buckets to the table.

    diff --git a/docs/dli/umn/dli_01_0006.html b/docs/dli/umn/dli_01_0006.html index 96c36cb4e..a50f81dc0 100644 --- a/docs/dli/umn/dli_01_0006.html +++ b/docs/dli/umn/dli_01_0006.html @@ -1,9 +1,9 @@

    Creating an Enhanced Datasource Connection

    -

    Scenario

    Create an enhanced datasource connection for DLI to access, import, query, and analyze data of other data sources.

    -

    For example, to connect DLI to the MRS, RDS, CSS, Kafka, or GaussDB(DWS) data source, you need to enable the network between DLI and the VPC of the data source.

    -

    Create an enhanced datasource connection on the console.

    +

    Scenario

    Before using DLI to access data from other sources, you need to create an enhanced datasource connection to enable network communication between DLI and the destination data source. This allows DLI to seamlessly access, import, query, and analyze data stored in external systems.

    +

    For example, when connecting DLI to services such as MRS, RDS, CSS, Kafka, or DWS, you must first configure the network connectivity between DLI and the corresponding VPC of these data sources to facilitate smooth data exchange.

    +

    This section provides a step-by-step guide on how to create an enhanced datasource connection via the management console.

    Notes and Constraints

    + + +
    Table 1 Notes and constraints on enhanced datasource connections

    Item

    @@ -137,6 +137,7 @@

    Subnet

    Subnet used by the data source.

    +

    If the subnet of the selected data source has IPv6 enabled, the enhanced datasource connection you create will also support IPv6. For more information on using IPv6 for cross-source access, see How Do I Configure a Network Connection with IPv6 Address Enabled?.

    Host Information

    @@ -185,6 +186,15 @@

    Name of a custom route, which is unique in the same enhanced datasource connection. The name can contain up to 64 characters. Only digits, letters, underscores (_), and hyphens (-) are allowed.

    IP Address Type

    +

    The options are IPv4 and IPv6.

    +

    If your data source has IPv6 enabled and the current enhanced datasource connection supports IPv6, you can select IPv6 routes when adding a route table.

    +

    You can check whether the current enhanced datasource connection supports IPv6 in its basic information. For details, see Viewing Basic Information About an Enhanced Datasource Connection.

    +

    The route IP address example is as follows:

    +
    • IPv4 address: 192.168.2.0/24.
    • IPv6 address: 2407:c080:802:be7::/64.
    +

    IP Address

    Custom route CIDR block. The CIDR blocks of different routes can overlap but cannot be identical.

    @@ -196,12 +206,17 @@
  • After adding a route, you can view the route information on the route details page.
    • -

    Step 3: Test the Connectivity Between the Queue in the Elastic Resource Pool and the Data Source Address

    1. Log in to the DLI management console. In the navigation pane on the left, choose Resources > Queue Management.
    2. On the Queue Management page, locate the row containing the target queue, click More in the Operation column, and select Test Address Connectivity.
    3. On the Test Address Connectivity page, enter the address to be tested. The domain name and IP address are supported, and the port number can be specified.

      You can input the data source address in the following formats: IPv4 address; IPv4 address + Port number; Domain name; Domain name + Port number.

      -
      • IPv4 address: 192.168.x.x
      • IPv4 + Port number: 192.168.x.x:8080
      • Domain name: domain-xxxxxx.com
      • Domain name + Port number: domain-xxxxxx.com:8080
      +

      Step 3: Test the Connectivity Between the Queue in the Elastic Resource Pool and the Data Source Address

      1. Log in to the DLI management console. In the navigation pane on the left, choose Resources > Queue Management.
      2. On the Queue Management page, locate the row containing the target queue, click More in the Operation column, and select Test Address Connectivity.
      3. On the Test Address Connectivity page, enter the address to be tested. The domain name and IP address are supported, and the port number can be specified.

        You can input the data source address in the following formats: IPv4 address; IPv4 address + Port number; Domain name; Domain name + Port number.

        +
        • IPv4 address: 192.168.x.x
        • IPv4 + Port number: 192.168.x.x:8080
        • Domain name: domain-xxxxxx.com
        • Domain name + Port number: domain-xxxxxx.com:8080
        • IPv6 address: 2001:0db8:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX
        • [IPv6] + Port number: [2001:0db8:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX]:8080

      4. Click Test.
        • If the test address is reachable, you will receive a message.
        • If the test address is unreachable, you will also receive a message. Check the network configurations and retry. Network configurations include the VPC peering and the datasource connection. Check whether they have been activated.
      +

      How Do I Configure a Network Connection with IPv6 Address Enabled?

      DLI resource networks support IPv4/IPv6 dual stack. When creating an enhanced datasource connection, you can choose to use an IPv6 address for communication to enhance network compatibility and security.

      +

      A prerequisite for using IPv6 in datasource scenarios is that both the DLI elastic resource pool and the data source must have IPv6 enabled.

      + +

      To verify if IPv6 communication is successful, use an IPv6 address to test the network connectivity between the queue and the data source by referring to Step 3: Test the Connectivity Between the Queue in the Elastic Resource Pool and the Data Source Address.

      +
    diff --git a/docs/dli/umn/dli_01_0010.html b/docs/dli/umn/dli_01_0010.html index 7c4c9e409..69e00398b 100644 --- a/docs/dli/umn/dli_01_0010.html +++ b/docs/dli/umn/dli_01_0010.html @@ -4,8 +4,8 @@

    You can export data from a DLI table to OBS. During the export, a folder is created in OBS or the content in the existing folder is overwritten.

    Precautions

    • The exported file can be in JSON format, and the text format can only be UTF-8.
    • Data in DLI tables (table type: MANAGED) can only be exported to OBS buckets, and the export path must be a folder.
    • Data can be exported across accounts. That is, after account B authorizes account A, account A can export data to the OBS path of account B if account A has the permission to read the metadata and permission information about the OBS bucket of account B and read and write the path.
    -

    Procedure

    1. You can export data on either the Data Management page or the SQL Editor page.

      • To export data on the Data Management page:
        1. In the navigation pane on the left of the console, choose Data Management > Databases and Tables.
        2. Click the name of the database corresponding to the table where data is to be exported to switch to the Manage Tables page.
        3. Select the corresponding table (DLI table) and choose More > Export in the Operation column. The Export Data page is displayed.
        -
      • To export data on the SQL Editor page:
        1. In the navigation pane on the left of the management console, choose SQL Editor.
        2. In the navigation tree on the left, click Databases to see all databases. Click the database name corresponding to the table to which data is to be exported. The tables are displayed.
        3. Click on the right of the managed table (DLI table) whose data is to be exported, and choose Export from the shortcut menu.
        +

        Procedure

        1. You can export data on either the Data Management page or the SQL Editor page.

          • To export data on the Data Management page:
            1. In the navigation pane on the left of the console, choose Data Management > Databases and Tables.
            2. Click the name of the database corresponding to the table where data is to be exported to switch to the Manage Tables page.
            3. Select the corresponding table (DLI table) and choose More > Export in the Operation column. The Export Data page is displayed.
            +
          • To export data on the SQL Editor page:
            1. In the navigation pane on the left of the management console, choose SQL Editor.
            2. In the navigation tree on the left, click Databases to see all databases. Click the database name corresponding to the table to which data is to be exported. The tables are displayed.
            3. Click next to the managed table (DLI table) whose data is to be exported, and choose Export from the shortcut menu.

        2. In the displayed Export Data dialog box, specify parameters by referring to Table 1.

          + + +
          Table 1 Parameter descriptions

          Parameter

          diff --git a/docs/dli/umn/dli_01_0014.html b/docs/dli/umn/dli_01_0014.html index 6afc8a5ed..d9dc173ca 100644 --- a/docs/dli/umn/dli_01_0014.html +++ b/docs/dli/umn/dli_01_0014.html @@ -22,6 +22,15 @@

          Name of a custom route, which is unique in the same enhanced datasource connection. The name can contain up to 64 characters. Only digits, letters, underscores (_), and hyphens (-) are allowed.

          IP Address Type

          +

          The options are IPv4 and IPv6.

          +

          If your data source has IPv6 enabled and the current enhanced datasource connection supports IPv6, you can select IPv6 routes when adding a route table.

          +

          You can check whether the current enhanced datasource connection supports IPv6 in its basic information. For details, see Viewing Basic Information About an Enhanced Datasource Connection.

          +

          The route IP address example is as follows:

          +
          • IPv4 address: 192.168.2.0/24.
          • IPv6 address: 2407:c080:802:be7::/64.
          +

          IP Address

          Custom route CIDR block. The CIDR blocks of different routes can overlap but cannot be identical.

          diff --git a/docs/dli/umn/dli_01_0016.html b/docs/dli/umn/dli_01_0016.html index fadc18c45..ab7881a08 100644 --- a/docs/dli/umn/dli_01_0016.html +++ b/docs/dli/umn/dli_01_0016.html @@ -4,7 +4,7 @@

          You can delete a queue based on actual conditions.

          • This operation will fail if there are jobs in the Submitting or Running state on this queue.
          • Deleting a queue does not cause table data loss in your database.
          -

          Procedure

          1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
          2. Locate the row where the target queue locates and click Delete in the Operation column.

            If Delete in the Operation column is gray, the current user does not have the permission of deleting the queue. You can apply to the administrator for the permission.

            +

            Procedure

            1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
            2. Locate the row containing the queue to delete and click Delete in its Operation column.

              If Delete in the Operation column is gray, the current user does not have the permission of deleting the queue. You can apply to the administrator for the permission.

            3. In the displayed dialog box, click OK.
            diff --git a/docs/dli/umn/dli_01_0017.html b/docs/dli/umn/dli_01_0017.html index 4384ae05f..384bc8395 100644 --- a/docs/dli/umn/dli_01_0017.html +++ b/docs/dli/umn/dli_01_0017.html @@ -64,9 +64,9 @@
          -

          Viewing Job Details

          On the SQL Jobs page, you can click in front of a job record to view details about the job.

          -

          Job details vary with job types. The job details vary depending on the job types, status, and configuration options. The following describes how to load data, create a table, and select a job. For details about other job types, see the information on the management console.

          -
          • Load data (job type: IMPORT) include the following information: queue, job ID, username, type, status, execution statement, running duration, creation time, end time, parameter settings, label, number of results, scanned data, number of scanned data, number of error records, storage path, data format, database, table, table header, separator, reference character, escape character, date format, timestamp format, total CPU used, and output bytes.
          • Create table (job type: DDL) include the following information: queue, job ID, username, type, status, execution statement, running duration, creation time, end time, parameter settings, tags, number of results, scanned data, and database.
          • Select (job type: QUERY) include the following information: queue, job ID, username, type, status, execution statement, running duration, creation time, end time, parameter setting, label, number of results (results of successful executions can be exported), and scanned data, username, result status (results of successful tasks can be viewed. Failure causes of failed tasks are displayed), database, total CPU used, and output bytes.
            • Total CPU Used (Core x ms): total CPU used during job execution.
            • Output Bytes: number of output bytes after the job is executed.
            +

            Viewing Job Details

            On the SQL Jobs page, click next to a job to view its details.

            +

            The details displayed vary depending on the job type, status, and configuration options. The console will show the precise information based on these factors. Below are examples for three common job types: load data, create table, and select jobs. For other job types, refer to the console for supported details.

            +
            • For load data jobs (job type: IMPORT), the following details are included: queue, job ID, username, type, status, execution statement, runtime duration, creation time, end time, parameter settings, tags, result count, scanned data volume, scan record count, error record count, storage path, data format, database, table, table headers, delimiter, quote character, escape character, date format, timestamp format, cumulative CPU usage, and output bytes.
            • For create table jobs (job type: DDL), the details include: queue, job ID, username, type, status, execution statement, runtime duration, creation time, end time, parameter settings, tags, result count, scanned data volume, and database.
            • For select jobs (job type: QUERY), the details consist of: queue, job ID, username, type, status, execution statement, runtime duration, creation time, end time, parameter settings, tags, result count (exportable if successful), scanned data volume, executing user, result status (view results if successful; failure reason if failed), database, cumulative CPU usage, and output bytes.
              • Total CPU Used (Core x ms): total CPU used during job execution.
              • Output Bytes: number of output bytes after the job is executed.
            diff --git a/docs/dli/umn/dli_01_0019.html b/docs/dli/umn/dli_01_0019.html index b9fc34744..1e810e3d8 100644 --- a/docs/dli/umn/dli_01_0019.html +++ b/docs/dli/umn/dli_01_0019.html @@ -8,7 +8,7 @@

            DLI allows you to add, modify, or delete tags for datasource connections.

            -

            Procedure

            1. In the left navigation pane of the DLI management console, choose Datasource Connections.
            2. In the Operation column of the link, choose More > Tags.
            3. The tag management page is displayed, showing the tag information about the current connection.
            4. Click Add/Edit Tag. The Add/Edit Tag dialog is displayed. Add or edit tag keys and values and click OK. +

              Procedure

              1. In the navigation pane of the DLI management console, choose Datasource Connections.
              2. In the Operation column of the link, choose More > Tags.
              3. The tag management page is displayed, showing the tag information about the current connection.
              4. Click Add/Edit Tag. The Add/Edit Tag dialog is displayed. Add or edit tag keys and values and click OK.
                diff --git a/docs/dli/umn/dli_01_0320.html b/docs/dli/umn/dli_01_0320.html index b3bf585b8..cd7dada6a 100644 --- a/docs/dli/umn/dli_01_0320.html +++ b/docs/dli/umn/dli_01_0320.html @@ -8,13 +8,13 @@

                Notes

                • Before submitting a SQL job, configure a DLI job bucket. The bucket is used to store job results and temporary data generated by DLI, such as job logs.
                  • For details about how to configure a job bucket, see Configuring a DLI Job Bucket. The job bucket name is set by default.
                  • If your SQL queue has been configured to save job results to a DLI job bucket, the system will write the results directly to the specified OBS bucket when executing a SQL job. Ensure the following preparations are completed before submitting the job:
                    • Configure DLI job bucket information before submitting a SQL job. Otherwise, the SQL job may fail to be submitted.
                    • Ensure that the user executing the SQL job has read and write permissions for the DLI job bucket. Otherwise, the job results cannot be properly saved or retrieved.

                      When executing a job, the system accesses the job bucket using the identity credentials of the user who submitted the job. If permissions are insufficient, it results in failure to save or retrieve the job results.

                    -

                    For details, refer to How Do I Check if Job Result Saving to a DLI Job Bucket Is Enabled for a SQL Queue?

                    -
                  • On the OBS management console, you can configure lifecycle rules for a bucket to automatically delete objects within it or change object storage classes on a regular basis.
                  +

                  For details, see How Do I Check if Job Result Saving to a DLI Job Bucket Is Enabled for a SQL Queue?

                  +
                • On the OBS management console, you can configure lifecycle rules for a bucket to automatically delete objects within it or change object storage classes on a regular basis.

                Creating and Submitting a SQL Job Using the SQL Editor

                1. Log in to the DLI management console. In the navigation pane on the left, choose SQL Editor.

                  On the SQL Editor page, the system prompts you to create a temporary DLI data bucket to store temporary data generated by DLI. For details about how to configure a job bucket, see Configuring a DLI Job Bucket.

                  -
                2. Above the SQL job editing window, set the parameters required for running a SQL job, such as the queue and database. For how to set the parameters, refer to Table 1. -
                Table 1 Tag parameters

                Parameter

                Description

                diff --git a/docs/dli/umn/dli_01_0020.html b/docs/dli/umn/dli_01_0020.html index 96cab4a97..5bbff6f15 100644 --- a/docs/dli/umn/dli_01_0020.html +++ b/docs/dli/umn/dli_01_0020.html @@ -143,7 +143,7 @@

                VPC

                Select an existing VPC.

                -

                For how to recreate a VPC and subnet, refer to "Creating a VPC and Subnet" in the Virtual Private Cloud User Guide.

                +

                For details about how to recreate a VPC and subnet, see "Creating a VPC with a Subnet" in the Virtual Private Cloud User Guide.

                NOTE:

                In datasource scenarios, the CIDR block of the data source cannot overlap that of the elastic resource pool.
                Table 1 Setting SQL job parameters

                Button & Drop-Down List

                +
              5. Above the SQL job editing window, set the parameters required for running a SQL job, such as the queue and database. Configure the parameters based on Table 1. +
                @@ -34,7 +34,7 @@ @@ -54,7 +54,7 @@

                Alternatively, you can double-click the table name qw. The query statement is automatically entered in the SQL job editing window.

                DLI offers a range of SQL templates that come with use cases, code examples, and usage guides. You can also use these templates to quickly implement your service logic. For more information about templates, see Creating a SQL Job Template.

                -
              6. On top of the editing window, click More > Verify Syntax to check whether the SQL statement is correct.
                1. If the verification fails, check the SQL statement syntax by referring to Data Lake Insight SQL Syntax Reference.
                2. If the syntax verification is successful, click Execute. Read and agree to the privacy agreement. Click OK to execute the SQL statement.
                +
              7. On top of the editing window, click More > Verify Syntax to check whether the SQL statement is correct.
                1. If the verification fails, check the SQL statement syntax by referring to Data Lake Insight SQL Syntax Reference.
                2. If the syntax verification is successful, click Execute. Read and agree to the privacy agreement. Click OK to execute the SQL statement.

                Once successfully executed, you can check the execution result on the View Result tab below the SQL job editing window.

              8. View job execution results.

                On the View Result tab, click to display execution results in a chart. Click to switch back to the table view.

                You can view a maximum of 1,000 data records on this View Result tab. To view more or full data, click to export the data to OBS.
                • If no column of the numeric type is displayed in the execution result, the result cannot be represented in charts.
                • You can view the data in a bar chart, line chart, or fan chart.
                • In the bar chart and line chart, the X axis can be any column, while the Y axis can only be columns of the numeric type. The fan chart displays the corresponding legends and indicators.
                diff --git a/docs/dli/umn/dli_01_0363.html b/docs/dli/umn/dli_01_0363.html index f409712a7..b78b24ce3 100644 --- a/docs/dli/umn/dli_01_0363.html +++ b/docs/dli/umn/dli_01_0363.html @@ -2,7 +2,7 @@

                Creating a Non-Elastic Resource Pool Queue (Deprecated, Not Recommended)

                Queues in the non-elastic resource pool mode are the previous-gen of resource management for DLI. It involved purchasing and releasing resources based on usage demands, requiring estimation of resource needs before making purchases.

                -

                Queues in an elastic resource pool are recommended, as they offer the flexibility to use resources with high utilization as needed. For how to buy an elastic resource pool and create queues within it, see Creating an Elastic Resource Pool and Creating Queues Within It.

                +

                Queues in an elastic resource pool are recommended, as they offer the flexibility to use resources with high utilization as needed. For details about how to buy an elastic resource pool and create queues within it, see Creating an Elastic Resource Pool and Creating Queues Within It.

                • If you use a sub-account to create a queue for the first time, log in to the DLI management console using the main account and keep records in the DLI database before creating a queue.
                • It takes 6 to 10 minutes for a job running on a new queue for the first time.
                • After a queue is created, if no job is run within one hour, the system releases the queue.
                • Queues with 16 CUs do not support scale-out or scale-in.
                • Queues with 64 CUs do not support scale-in.

                Notes and Constraints

                @@ -32,10 +32,10 @@
                9. Table 1 Configuring SQL job parameters

                Button & Drop-Down List

                Description

                Select a database from the drop-down list box.

                If no database is available, the default database is displayed.

                -

                For how to create a database, see Creating a Data Catalog, Database, and Table on the DLI Console.

                +

                For details about how to create a database, see Creating a Data Catalog, Database, and Table on the DLI Console.

                If you have specified a database where tables are located in SQL statements, the database you choose here does not apply.
                -

                Procedure

                1. You can create a queue on the Overview, SQL Editor, or Queue Management page.
                  • In the upper right corner of the Overview page, click Create Queue.
                  • To create a queue on the Queue Management page:
                    1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
                    2. In the upper right corner of the Queue Management page, click Create Queue to create a queue.
                    +

                    Procedure

                    1. You can create a queue on the Overview, SQL Editor, or Queue Management page.
                      • In the upper right corner of the Overview page, click Create Queue.
                      • To create a queue on the Queue Management page:
                        1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
                        2. In the upper right corner of the Queue Management page, click Create Queue to create a queue.
                      • To create a queue on the SQL Editor page:
                        1. In the navigation pane on the left of the DLI management console, choose SQL Editor.
                        2. Click Queues. On the tab page displayed, click on the right to create a queue.
                      -
                    2. On the Create Queue page displayed, set the parameters according to Table 2. +
                    3. On the displayed Create Queue page, configure the parameters according to Table 2.
                      Table 2 Parameter descriptions

                      Parameter

                      Description

                      @@ -91,7 +91,7 @@
                      -
                    4. Click Create Now to create a queue.

                      Once created, you can check and use it on the Queue Management page.

                      +
                    5. Click Create Now to create a queue.

                      Once created, you can check and use it on the Queue Management page.

                      It takes 6 to 10 minutes for a job running on a new queue for the first time.

                    diff --git a/docs/dli/umn/dli_01_0375.html b/docs/dli/umn/dli_01_0375.html index 7162d0fce..90013733e 100644 --- a/docs/dli/umn/dli_01_0375.html +++ b/docs/dli/umn/dli_01_0375.html @@ -47,7 +47,7 @@
              9. Click Create Now.
              10. In the bucket list, click the name of the dli-test-obs01 bucket you just created to access its Objects tab.
              11. Click Upload Object. In the dialog box displayed, drag or add files or folders, for example, spark-examples.jar, to the upload area. Then, click Upload.

                In this example, the path after upload is obs://dli-test-obs01/spark-examples.jar.

                -

                For more operations on the OBS console, see Object Storage Service User Guide.

                +

                For more operations on the OBS console, see Object Storage Service User Guide.

                12. Step 2: Create an Elastic Resource Pool and Create Queues Within It

                In this example, the elastic resource pool dli_resource_pool and queue dli_queue_01 are created.
                1. Log in to the DLI management console.
                2. In the navigation pane on the left, choose Resources > Resource Pool.
                3. On the displayed page, click Buy Resource Pool in the upper right corner.
                4. On the displayed page, set the parameters.
                  Table 2 describes the parameters. @@ -192,7 +192,7 @@
                5. Click OK.
                -

                Step 3: Use DEW to Manage Access Credentials

                To write the output data of a Spark Jar job to OBS, AK/SK is required for accessing OBS. To ensure the security of AK/SK data, you can use DEW and CSMS for unified management of AK/SK, effectively avoiding sensitive information leakage and business risks caused by hard-coded or plaintext configuration of programs.

                +

                Step 3: Use DEW to Manage Access Credentials

                When writing output data from Spark Jar jobs to OBS, you need to configure an AK/SK for accessing OBS. To ensure the security of AK/SK data, you can use DEW and CSMS for centralized management of AK/SK. This approach effectively mitigates risks such as sensitive information leakage caused by hardcoding in programs or plaintext configurations, as well as potential business disruptions due to unauthorized access.

                This part introduces how to create a shared secret in DEW.
                1. Log in to the DEW management console.
                2. In the navigation pane on the left, choose Cloud Secret Management Service > Secrets.
                3. On the displayed page, click Create Secret. Set basic secret information.

                  Set the AK and SK credential key-value pairs.

                  • In this example, the key in the first line is the user's access key ID (AK).
                  • In this example, the key in the second line is the user's secret access key (SK).
                4. Set access credential parameters on the DLI Spark Jar job editing page.
                  spark.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.access.key= USER_AK_CSMS_KEY_obstest1 
@@ -224,7 +224,7 @@ spark.dli.job.agency.name=agency

                Step 5: Submit a Spark Job

                1. On the DLI management console, choose Job Management > Spark Jobs in the navigation pane on the left. On the displayed page, click Create Job in the upper right corner.
                2. Set the following Spark job parameters: -

                  For other parameters, refer to the description about the Spark job editing page in "Creating a Spark Job" in the Data Lake Insight User Guide.

                  +

                  For other parameters, refer to the description about the Spark job editing page in "Creating a Spark Job" in the Data Lake Insight User Guide.

                3. Click Execute in the upper right corner of the Spark job editing window, read and agree to the privacy agreement, and click OK. Submit the job. A message is displayed, indicating that the job is submitted.

                4. (Optional) Switch to the Job Management > Spark Jobs page to view the status and logs of the submitted Spark job.
                diff --git a/docs/dli/umn/dli_01_0378.html b/docs/dli/umn/dli_01_0378.html index 810717b70..50f53ac63 100644 --- a/docs/dli/umn/dli_01_0378.html +++ b/docs/dli/umn/dli_01_0378.html @@ -1,57 +1,58 @@

                What Is Data Lake Insight

                -

                DLI Introduction

                Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark and Apache Flink ecosystems. It frees you from managing any servers.

                -

                DLI supports multiple querying methods including standard SQL, Spark SQL, and Flink SQL, with compatibility with mainstream data formats. DLI supports SQL statements and Spark applications for heterogeneous data sources, including CloudTable, RDS, GaussDB(DWS), CSS, OBS, custom databases on ECSs, and offline databases.

                +

                DLI Introduction

                Data Lake Insight (DLI) is a fully-managed, serverless service for data processing and analytics. It seamlessly combines stream processing, batch processing, and interactive analysis into a single platform. DLI provides full support for Apache Spark and Apache Flink. No server management is required. You can get started with DLI right away.

                +

                DLI works with standard SQL, Spark SQL, and Flink SQL. It offers multiple access options and supports mainstream data formats. With DLI, you can explore diverse data sources such as CloudTable, RDS, DWS, CSS, OBS, ECS-hosted databases, or on-premises databases. Eliminate complex extraction, transformation, or loading processes. Analyze your data directly using SQL or programs.

                -

                Core Functions

                -
                Table 1 DLI core functions

                Function

                +

                Core Functions

                For details about DLI functions, see Features.

                + +
                - - - - - - - - - - - - - - - @@ -74,17 +75,17 @@ - - - - - - diff --git a/docs/dli/umn/dli_01_0385.html b/docs/dli/umn/dli_01_0385.html index 7166d620e..2212ea72c 100644 --- a/docs/dli/umn/dli_01_0385.html +++ b/docs/dli/umn/dli_01_0385.html @@ -1,7 +1,7 @@

                Managing Spark Jobs

                -

                Viewing Basic Information

                On the Overview page, click Spark Jobs to go to the SQL job management page. Alternatively, you can click Job Management > Spark Jobs. The page displays all Spark jobs. If there are a large number of jobs, they will be displayed on multiple pages. DLI allows you to view jobs in all statuses.

                +

                Viewing Basic Information

                To access the Spark job management page, click Spark Jobs on the Overview page or choose Job Management > Spark Jobs in the navigation pane on the left. The page displays all Spark jobs. If there are a large number of jobs, they will be displayed on multiple pages. DLI allows you to view jobs in all statuses.

                Table 1 DLI core functions

                Function

                Description

                +

                Description

                DLI is a data processing and analytics service built on the serverless architecture.

                +

                DLI is a data processing and analytics service built on the serverless architecture.

                DLI is a serverless big data query and analytics service. With DLI, you only pay for the actual compute resources used, with no need to maintain or manage cloud servers.

                +

                DLI is a serverless big data query and analytics service. With DLI, you only pay for the actual compute resources used, with no need to maintain or manage cloud servers.

                • Auto scaling: DLI ensures you always have enough capacity on hand to deal with any traffic spikes.

                DLI supports multiple compute engines.

                +

                DLI supports multiple compute engines.

                DLI is fully compatible with ecosystems like Apache Spark and Apache Flink, and supports standard SQL, Spark SQL, and Flink SQL. It is compatible with mainstream data formats such as CSV, JSON, Parquet, and ORC.

                +

                DLI provides full support for ecosystems like Apache Spark and Apache Flink. DLI works with standard SQL, Spark SQL, and Flink SQL. It supports mainstream data formats such as CSV, JSON, Parquet, and ORC.

                • Spark is a unified analytics engine designed for large-scale data processing, focusing on query, compute, and analysis. DLI has undergone extensive performance optimization and service-oriented enhancements over the open-source Spark, maintaining compatibility with the Apache Spark ecosystem and APIs while boosting performance by 2.5 times, enabling exabyte-scale data queries and analyses within hours.
                • Flink is a distributed compute engine that can be used for batch processing, which involves handling static datasets and historical datasets. It can also be used for stream processing, enabling the real-time processing of live data streams and the immediate generation of data results. DLI has enhanced features and security based on the open-source Flink and offers the Stream SQL feature needed for data processing.

                DLI supports multiple connection methods.

                +

                DLI supports multiple connection methods.

                DLI provides multiple connection methods to meet diverse user needs and scenarios.

                +

                DLI provides multiple connection methods to meet diverse user needs and scenarios.

                Connection methods:

                • Web-based console
                • APIs
                • SDKs
                • Client tools
                • Submitting DLI jobs using DataArts Studio
                • Connecting to BI tools for visual analysis

                DLI can connect to multiple data sources for cross-source data analysis.

                +

                DLI can connect to multiple data sources for cross-source data analysis.

                • Spark datasource connection: Data sources such as GaussDB(DWS), RDS, and CSS can be accessed through DLI.
                • Flink supports cross-source connectivity with various cloud services, forming a rich streaming ecosystem. DLI's streaming ecosystem is divided into cloud service ecosystems and open-source ecosystems:
                  • Cloud service ecosystem: DLI supports connectivity with other services in Flink SQL. You can directly use SQL to read and write data from these cloud services.
                  • Open-source ecosystem: By establishing network connections with other VPCs through enhanced datasource connections, you can access all Flink and Spark-supported data sources and output sources, such as Kafka, Hbase, Elasticsearch, in the tenant-authorized DLI queues.
                  +
                • Spark datasource connection: Data sources such as DWS, RDS, and CSS can be accessed through DLI.
                • Flink supports cross-source connectivity with various cloud services, forming a rich streaming ecosystem. DLI's streaming ecosystem is divided into cloud service ecosystems and open-source ecosystems:
                  • Cloud service ecosystem: DLI supports connectivity with other services in Flink SQL. You can directly use SQL to read and write data from these cloud services.
                  • Open-source ecosystem: By establishing network connections with other VPCs through enhanced datasource connections, you can access all Flink and Spark-supported data sources and output sources, such as Kafka, Hbase, Elasticsearch, in the tenant-authorized DLI queues.

                Three basic job types supported by DLI

                +

                Three basic job types supported by DLI

                • SQL jobs allow you to query data using standard SQL statements.
                • Flink jobs support Flink SQL online analysis capabilities: supporting aggregation functions such as Window and Join, using SQL to express service logic, and achieving service implementation conveniently and quickly.
                • Spark jobs provide fully managed Spark computing features: You can submit computing tasks through interactive sessions and batch processing, and perform data analysis on fully managed Spark queues.
                +
                • SQL jobs allow you to query data using standard SQL statements.
                • Flink jobs support Flink SQL online analysis capabilities: supporting aggregation functions such as Window and Join, using SQL to express service logic, and achieving service implementation conveniently and quickly.
                • Spark jobs provide fully managed Spark computing features: You can submit computing tasks through interactive sessions and batch processing, and perform data analysis on fully managed Spark queues.

                DLI supports decoupled storage and compute.

                +

                DLI supports decoupled storage and compute.

                After storing data in OBS, you can connect DLI to OBS for data analysis. Under the decoupled storage and compute architecture, storage resources and compute resources can be requested and billed separately, reducing costs and improving resource utilization.

                +

                After storing data in OBS, you can connect DLI to OBS for data analysis. Under the decoupled storage and compute architecture, storage resources and compute resources can be requested and billed separately, reducing costs and improving resource utilization.

                You can choose single-AZ or multi-AZ storage when creating an OBS bucket for storing redundant data on the DLI console. The differences between the two storage policies are as follows:

                • Multi-AZ storage means data will be redundantly stored across multiple AZs, offering higher reliability. Buckets with multi-AZ storage will store data in multiple different AZs within the same region. If one AZ becomes unavailable, data can still be accessed normally from other AZs, making it suitable for data storage scenarios requiring high reliability. You are advised to use this policy.
                • Single-AZ storage means data is only stored in a single AZ, but it is more cost-effective compared to multi-AZ storage.

                DLI manages and schedules resources in a unified manner using elastic resource pools.

                +

                DLI manages and schedules resources in a unified manner using elastic resource pools.

                The backend of elastic resource pools adopts a CCE cluster architecture, supporting heterogeneous resources, so you can manage and schedule resources in a unified manner.

                +

                The backend of elastic resource pools adopts a CCE cluster architecture, supporting heterogeneous resources, so you can manage and schedule resources in a unified manner.

                For details, see Creating an Elastic Resource Pool and Creating Queues Within It.

                Compute engine

                • Spark: supports batch processing and interactive analysis of large-scale data and provides high-performance distributed computing capabilities.
                • Flink: supports real-time stream processing, capable of handling large-scale real-time data streams, with support for event time processing and state management.
                • HetuEngine: supports interactive data analysis, swiftly handles complex SQL queries, and facilitates connections and queries across various data sources.
                +
                • Spark: supports batch processing and interactive analysis of large-scale data and provides high-performance distributed computing capabilities.
                • Flink: supports real-time stream processing, capable of handling large-scale real-time data streams, with support for event time processing and state management.

                Unified resource management

                • Resource decoupling: DLI adopts a decoupled compute and storage architecture, decoupling compute resources from storage resources. This allows for flexible adjustment of the ratio between compute and storage resources based on actual needs, enhancing resource utilization and reducing costs.
                • Elastic scaling: DLI compute resources are based on containerized Kubernetes and possess ultimate elastic scaling capabilities. Resources can be automatically adjusted based on job demands.
                • Multi-tenant support: Compute resources can be isolated by tenant to ensure independence among different tenants. Each tenant can independently manage their own compute resources, enabling fine-grained resource management and facilitating inter-departmental data sharing and permissions management within enterprises.
                • Pay-per-use compute resources: You only pay for the compute resources you actually use, with no need to pre-purchase or manage servers, enhancing usage efficiency.
                +
                • Resource decoupling: DLI adopts a decoupled compute and storage architecture, decoupling compute resources from storage resources. This allows for flexible adjustment of the ratio between compute and storage resources based on actual needs, enhancing resource utilization and reducing costs.
                • Elastic scaling: DLI compute resources are built upon containerized Kubernetes and possess elastic scaling capabilities. Resources can be automatically adjusted based on job demands.
                • Multi-tenant support: Compute resources can be isolated by tenant to ensure independence among different tenants. Each tenant can independently manage their own compute resources, enabling fine-grained resource management and facilitating inter-departmental data sharing and permissions management within enterprises.
                • Pay-per-use compute resources: You only pay for the compute resources you actually use, with no need to pre-purchase or manage servers, enhancing usage efficiency.

                Unified metadata management

                • Multi-source metadata integration: DLI supports centralized management of metadata from various data sources, including cloud-based data sources (such as OBS, RDS, GaussDB(DWS), and CSS) and on-premises data sources (such as self-built databases and Redis). You can manage and analyze metadata across different data sources without the need to migrate data to a unified data lake.
                • Metadata synchronization: DLI provides metadata management to ensure the timeliness and consistency of metadata.
                • Metadata query and management: DLI offers standard SQL APIs, enabling you to query and manage metadata using SQL statements. You can add, delete, modify, and query metadata to facilitate data governance and analysis.
                • Data security and permission management: Permissions on data catalogs, databases, and tables can be managed. You can assign different permissions to various tenants and user groups to ensure data security and compliance.
                +
                • Multi-source metadata integration: DLI supports centralized management of metadata from various data sources, including cloud-based data sources (such as OBS, RDS, DWS, and CSS) and on-premises data sources (such as self-built databases and Redis). You can manage and analyze metadata across different data sources without the need to migrate data to a unified data lake.
                • Metadata synchronization: DLI provides metadata management to ensure the timeliness and consistency of metadata.
                • Metadata query and management: DLI offers standard SQL APIs, enabling you to query and manage metadata using SQL statements. You can add, delete, modify, and query metadata to facilitate data governance and analysis.
                • Data security and permission management: Permissions on data catalogs, databases, and tables can be managed. You can assign different permissions to various tenants and user groups to ensure data security and compliance.

                Storage service

                @@ -94,7 +95,7 @@

                Data source connection

                • Cloud data sources can be connected. For example, OBS can be used to store and manage unstructured data. Relational database service (RDS) can be used to store and manage structured data. GaussDB(DWS) can be used to efficiently query and analyze data.
                • On-premises data sources, such as self-built databases (MySQL, PostgreSQL, and HDFS), can be connected.
                +
                • Cloud data sources can be connected. For example, OBS can be used to store and manage unstructured data. Relational database service (RDS) can be used to store and manage structured data. DWS can be used to efficiently query and analyze data.
                • On-premises data sources, such as self-built databases (MySQL, PostgreSQL, and HDFS), can be connected.

                Data applications

                diff --git a/docs/dli/umn/dli_01_0384.html b/docs/dli/umn/dli_01_0384.html index 0c2734a93..592fd9d08 100644 --- a/docs/dli/umn/dli_01_0384.html +++ b/docs/dli/umn/dli_01_0384.html @@ -5,12 +5,12 @@

                On the Overview page, click Create Job in the upper right corner of the Spark Jobs tab or click Create Job in the upper right corner of the Spark Jobs page. The Spark job editing page is displayed.

                On the Spark job editing page, a message is displayed, indicating that a temporary DLI data bucket will be created. The created bucket is used to store temporary data generated by DLI, such as job logs and job results. You cannot view job logs if you choose not to create the bucket. The bucket will be created and the default bucket name is used.

                If you do not need to create a DLI temporary data bucket and do not want to receive this message, select Do not show again and click Cancel.

                -

                Prerequisites

                • You have uploaded the dependencies to the corresponding OBS bucket on the Data Management > Package Management page.
                • Before creating a Spark job to access other external data sources, such as OpenTSDB, HBase, Kafka, GaussDB(DWS), RDS, CSS, CloudTable, DCS Redis, and DDS, you need to create a datasource connection to enable the network between the job running queue and external data sources.
                  • For details about the external data sources that can be accessed by Spark jobs, see Common Development Methods for DLI Cross-Source Analysis.
                  • For how to create a datasource connection, see Configuring the Network Connection Between DLI and Data Sources (Enhanced Datasource Connection).

                    On the Resources > Queue Management page, locate the queue you have created, click More in the Operation column, and select Test Address Connectivity to check if the network connection between the queue and the data source is normal. For details, see Testing the Network Connectivity Between a Queue and a Data Source.

                    +

                    Prerequisites

                    -

                    Procedure

                    1. In the left navigation pane of the DLI management console, choose Job Management > Spark Jobs. The Spark Jobs page is displayed.

                      Click Create Job in the upper right corner. In the job editing window, you can set parameters in Fill Form mode or Write API mode.

                      -

                      The following uses the Fill Form as an example. In Write API mode, refer to the Data Lake Insight API Reference for parameter settings.

                      +

                      Procedure

                      1. In the navigation pane of the DLI management console, choose Job Management > Spark Jobs.

                        Click Create Job in the upper right corner. In the job editing window, you can set parameters in Fill Form mode or Write API mode.

                        +

                        The following uses the Fill Form as an example. In Write API mode, see Data Lake Insight API Reference for parameter settings.

                      1. Select a queue.
                        1. Queues: Select a queue from the drop-down list.
                        2. Spark Version: Select a Spark version from the drop-down list. The latest version is recommended.

                          You are advised not to use Spark of different versions for a long time.

                          • Doing so can lead to code incompatibility, which can negatively impact job execution efficiency.
                          • Doing so may result in job execution failures due to conflicts in dependencies. Jobs rely on specific versions of libraries or components.
                          @@ -103,7 +103,7 @@

                          spark.executor.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/css/*

                GaussDB(DWS)

                +

                DWS

                spark.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/dws/*

                spark.executor.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/dws/*

                @@ -264,7 +264,7 @@

                Group Name

                If you select a group when creating a package, you can select all the packages and files in the group. For how to create a package, see Creating a DLI Package.

                +

                If you select a group when creating a package, you can select all the packages and files in the group. For details about how to create a package, see Creating a DLI Package.

                Spark 3.3.x or later does not support group information configuration.
                diff --git a/docs/dli/umn/dli_01_0403.html b/docs/dli/umn/dli_01_0403.html index 6ac2e8a2e..e24f6c329 100644 --- a/docs/dli/umn/dli_01_0403.html +++ b/docs/dli/umn/dli_01_0403.html @@ -2,7 +2,7 @@

                Flink Job Overview

                DLI supports two types of Flink jobs:

                -
                • Flink OpenSource SQL job:
                  • It is fully compatible with Flink of the community edition, ensuring that jobs can run smoothly on these Flink versions.
                  • DLI Flink has expanded the support for connectors based on Flink of the community edition, supporting Redis and GaussDB(DWS) as new data source types. With this expansion, you can now utilize a wider range of data source types, providing greater flexibility and convenience when working with datasets.
                  • Flink OpenSource SQL jobs are ideal for scenarios where stream processing logic can be defined and executed through SQL statements. This simplifies stream processing, allowing developers to focus more on implementing service logic.
                  +
                  • Flink OpenSource SQL job:
                    • It is fully compatible with Flink of the community edition, ensuring that jobs can run smoothly on these Flink versions.
                    • DLI Flink has expanded the support for connectors based on Flink of the community edition, supporting Redis and DWS as new data source types. With this expansion, you can now utilize a wider range of data source types, providing greater flexibility and convenience when working with datasets.
                    • Flink OpenSource SQL jobs are ideal for scenarios where stream processing logic can be defined and executed through SQL statements. This simplifies stream processing, allowing developers to focus more on implementing service logic.

                    For how to create a Flink OpenSource SQL job, see Creating a Flink OpenSource SQL Job.

                  • Flink Jar job:
                    • DLI allows you to submit Flink jobs compiled into JAR files, providing higher flexibility and customization capabilities. It is applicable to scenarios where complex data processing is required.
                    • If the connectors provided by Flink of the community edition cannot meet specific needs, you can use Jar jobs to implement custom connectors or data processing logic.
                    • It is ideal for scenarios where user-defined functions (UDFs) or specific library integration are required. You can use the Flink ecosystem to implement advanced stream processing logic and status management.

                    For how to create a Flink Jar job, see Creating a Flink Jar Job.

                    diff --git a/docs/dli/umn/dli_01_0410.html b/docs/dli/umn/dli_01_0410.html index 51ac7cead..b14e78c72 100644 --- a/docs/dli/umn/dli_01_0410.html +++ b/docs/dli/umn/dli_01_0410.html @@ -6,7 +6,7 @@

                Notes

                • Flink jobs can directly access DIS, OBS, and SMN data sources without using datasource connections.
                • You are advised to use enhanced datasource connections to connect DLI to data sources.
                -

                DLI Supported Data Sources

                Table 1 lists the data sources supported by DLI. For how to use the data sources, see Data Lake Insight SQL Syntax Reference.

                +

                DLI Supported Data Sources

                Table 1 lists the data sources supported by DLI. For details about how to use the data sources, see Data Lake Insight SQL Syntax Reference.

                Table 1 Job management parameters

                Parameter
                @@ -75,7 +75,7 @@ - diff --git a/docs/dli/umn/dli_01_0417.html b/docs/dli/umn/dli_01_0417.html index d458eadea..02ecf5920 100644 --- a/docs/dli/umn/dli_01_0417.html +++ b/docs/dli/umn/dli_01_0417.html @@ -4,43 +4,20 @@

                IAM is a foundational service for permission management, offering functions like user identity authentication, permission allocation, and access control, enabling you to securely manage access to cloud resources and perform fine-grained permission management.

                IAM Authorization Types and Use Cases

                IAM can authorize different enterprise users to access cloud service resources. For example, if there are employees responsible for data analysis within your organization whom you wish to have usage permissions for DLI compute resources but not the permission to perform high-risk operations such as deleting DLI resources, you can use IAM for permission allocation. By granting users the ability to only use DLI resources without allowing them to delete these resources, you can control their access to DLI resources.

                For newly created users, they must first log in to DLI once to record metadata before being able to use DLI.

                - -
                Table 1 Supported data sources

                Service

                √

                GaussDB(DWS)

                +

                DWS

                √

                - - - - - - - - - - - - - -
                Table 1 IAM authorization types

                Type

                -

                Core Relationship

                -

                Permission

                -

                Authorization Method

                -

                Use Case

                -

                Role/Policy-based authorization

                -

                User-permission-authorization scope

                -
                • System-defined role
                • System-defined policy
                • Custom policy
                -

                Assigning roles or policies to principals

                -

                To authorize a user, you need to add it to a user group first and then specify the scope of authorization. It provides a limited number of condition keys and cannot meet the requirements of fine-grained permissions control. This method is suitable for small- and medium-sized enterprises.

                -
                -

                IAM is free to use, and you only need to pay for the resources in your account.

                If your account does not need individual IAM users for permission management, skip over this section.

                -

                DLI System Permissions

                Table 2 lists all system-defined permissions for DLI.

                +

                DLI System Permissions

                Table 1 lists all system-defined permissions for DLI.

                -
                Table 2 DLI system permissions

                Type

                +
                + - + diff --git a/docs/dli/umn/dli_01_0419.html b/docs/dli/umn/dli_01_0419.html index af99864eb..b2de591b5 100644 --- a/docs/dli/umn/dli_01_0419.html +++ b/docs/dli/umn/dli_01_0419.html @@ -1,15 +1,12 @@

                DLI Agency Overview

                -

                What Is an Agency?

                Cloud services often interact with each other, with some of which dependent on other services. You can create an agency to delegate DLI to use other cloud services and perform resource O&M on your behalf.

                -

                For example, the AK/SK required by DLI Flink jobs is stored in DEW. To allow DLI to access DEW data during job execution, you need to provide an IAM agency to delegate the permissions to perform operations on DEW data to DLI.

                +

                What Is an Agency?

                Cloud services often need to interact with each other for business operations. In some cases, one cloud service may require collaboration with another. To facilitate this, you can create a cloud service agency, which grants operational permissions to DLI. This allows DLI to act on your behalf when using other cloud services, enabling it to perform resource management tasks for you.

                +

                For example, when creating a Flink job in DLI, the required AK/SK is stored in DEW. If you want DLI to access DEW data during job execution, you must provide an IAM agency. This grants the necessary permissions to DLI, allowing it to access DEW on your behalf.

                Figure 1 DLI service agency

                DLI Agencies

                Before using DLI, you are advised to set up DLI agency permissions to ensure the proper functioning of DLI.

                -
                • By default, DLI provides the following agencies: dli_admin_agency, dli_management_agency, and dli_data_clean_agency. The names of these agencies are fixed, but the permissions contained in them can be customized. In other scenarios, you need to create custom agencies. For details about the agencies, see Table 1.
                • DLI upgrades dli_admin_agency to dli_management_agency to meet the demand for fine-grained agency permissions management. The new agency has the necessary permissions for datasource operations, notifications, and user authorization operations. For details, see Configuring DLI Agency Permissions.
                • To use Flink 1.15, Spark 3.3.1 (Spark general queue scenario), or a later version to execute jobs, perform the following operations:
                  Create an agency on the IAM console and add the agency information to the job configuration. For details, see Creating a Custom DLI Agency.
                  • Common scenarios for creating an agency: DLI is allowed to read and write data from and to OBS, dump logs, and read and write Flink checkpoints. DLI is allowed to access DEW to obtain data access credentials and access catalogs to obtain metadata.
                  • You cannot use the default agency names dli_admin_agency, dli_management_agency, or dli_data_clean_agency. It must be unique.
                  -
                  -
                • If the engine version is earlier than Flink 1.15, dli_admin_agency is used by default during job execution. If the engine version is earlier than Spark 3.3.1, user authentication information (AK/SK and security token) is used during job execution.

                  This means that jobs whose engine versions are earlier than Flink 1.15 or Spark 3.3.1 are not affected by the update of agency permissions and do not require custom agencies.

                  -
                • To maintain compatibility with existing job agency permission requirements, dli_admin_agency will still be listed in the IAM agency list even after the update.
                +
                • By default, DLI provides the following agencies: dli_admin_agency, dli_management_agency, and dli_data_clean_agency. The names of these agencies are fixed, but the permissions contained in them can be customized. In other scenarios, you need to create custom agencies. For details about the agencies, see Table 1.
                • DLI upgrades dli_admin_agency to dli_management_agency to meet the demand for fine-grained agency permissions management. The new agency has the necessary permissions for datasource operations, notifications, and user authorization operations. For details, see Configuring DLI Agency Permissions.
                • To maintain compatibility with existing job agency permission requirements, dli_admin_agency will still be listed in the IAM agency list even after the update.
                • Only the tenant account or a member account of user group admin can authorize the service.
                • Do not delete the agency created by the system by default.
                @@ -84,6 +81,11 @@
                Table 1 DLI system permissions

                Type

                +

                Link

                System-defined permissions for role/policy-based authorization

                +

                System-defined permissions for role/policy-based authorization

                +
                +

                Notes and Constraints

                • To use Flink 1.15, Spark 3.3.1 (Spark general queue scenario), or a later version to execute jobs, perform the following operations:
                  Create an agency on the IAM console and add the agency information to the job configuration. For details, see Creating a Custom DLI Agency.
                  • Common scenarios for creating an agency: DLI is allowed to read and write data from and to OBS, dump logs, and read and write Flink checkpoints. DLI is allowed to access DEW to obtain data access credentials and access catalogs to obtain metadata.
                  • You cannot use the default agency names dli_admin_agency, dli_management_agency, or dli_data_clean_agency. It must be unique.
                  +
                  +
                • If the engine version is earlier than Flink 1.15, dli_admin_agency is used by default during job execution. If the engine version is earlier than Spark 3.3.1, user authentication information (AK/SK and security token) is used during job execution.

                  This means that jobs whose engine versions are earlier than Flink 1.15 or Spark 3.3.1 are not affected by the update of agency permissions and do not require custom agencies.

                  +
                +
                diff --git a/docs/dli/umn/dli_01_0445.html b/docs/dli/umn/dli_01_0445.html index c42ff9f6f..d7e821616 100644 --- a/docs/dli/umn/dli_01_0445.html +++ b/docs/dli/umn/dli_01_0445.html @@ -6,494 +6,550 @@

                Namespace

                SYS.DLI

                Metric

                -
                Table 1 DLI metrics

                Metric ID

                +
                - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - - - + - - - - - - - + - - - - - - - - + - - - - - - - - - + - - - - - - - + - - - - - - - - - + - - - - - - - - + - - - - - - - - + - - - - - - - + - - - - - - - - - + - - - - - - - + - - - - - - - - - + @@ -512,6 +568,11 @@ + + + - - - -
                Table 1 DLI metrics

                Metric ID

                Name

                +

                Name

                Description

                +

                Description

                Value Range

                +

                Value Range

                Unit

                +

                Unit

                Conversion Rule

                +

                Conversion Rule

                Monitored Object

                +

                Dimension

                Monitoring Period (Raw Data)

                +

                Monitored Object

                +

                Monitoring Period (Raw Data)

                queue_cu_num

                +

                queue_cu_num

                Queue CU Usage

                +

                Queue CU Usage

                Displays the number of CUs applied by the user queue

                +

                Displays the number of CUs applied by the user queue

                ≥ 0

                +

                ≥ 0

                Count

                +

                Count

                N/A

                +

                N/A

                Queues

                +

                queue_id

                5 minutes

                +

                Queues

                +

                5 minutes

                queue_job_launching_num

                +

                queue_job_launching_num

                Number of Jobs Being Submitted

                +

                Number of Jobs Being Submitted

                Displays the number of jobs in the Submitting state in the user queue.

                +

                Displays the number of jobs in the Submitting state in the user queue.

                ≥ 0

                +

                ≥ 0

                Count

                +

                Count

                N/A

                +

                N/A

                Queues

                +

                queue_id

                5 minutes

                +

                Queues

                +

                5 minutes

                queue_job_running_num

                +

                queue_job_running_num

                Number of Running Jobs

                +

                Number of Running Jobs

                Displays the number of running jobs in the user queue.

                +

                Displays the number of running jobs in the user queue.

                ≥ 0

                +

                ≥ 0

                Count

                +

                Count

                N/A

                +

                N/A

                Queues

                +

                queue_id

                5 minutes

                +

                Queues

                +

                5 minutes

                queue_job_succeed_num

                +

                queue_job_succeed_num

                Number of Finished Jobs

                +

                Number of Finished Jobs

                Displays the number of completed jobs in the user queue.

                +

                Displays the number of completed jobs in the user queue.

                ≥ 0

                +

                ≥ 0

                Count

                +

                Count

                N/A

                +

                N/A

                Queues

                +

                queue_id

                5 minutes

                +

                Queues

                +

                5 minutes

                queue_job_failed_num

                +

                queue_job_failed_num

                Failed Jobs

                +

                Failed Jobs

                Displays the number of failed jobs in the user queue.

                +

                Displays the number of failed jobs in the user queue.

                ≥ 0

                +

                ≥ 0

                Count

                +

                Count

                N/A

                +

                N/A

                Queues

                +

                queue_id

                5 minutes

                +

                Queues

                +

                5 minutes

                queue_job_cancelled_num

                +

                queue_job_cancelled_num

                Number of Canceled Jobs

                +

                Number of Canceled Jobs

                Displays the number of canceled jobs in the user queue.

                +

                Displays the number of canceled jobs in the user queue.

                ≥ 0

                +

                ≥ 0

                Count

                +

                Count

                N/A

                +

                N/A

                Queues

                +

                queue_id

                5 minutes

                +

                Queues

                +

                5 minutes

                queue_alloc_cu_num

                +

                queue_alloc_cu_num

                Allocated CUs (queue)

                +

                Allocated CUs (queue)

                Displays the CU allocation for user queues.

                +

                Displays the CU allocation for user queues.

                ≥ 0

                +

                ≥ 0

                Count

                +

                Count

                N/A

                +

                N/A

                Queues

                +

                queue_id

                5 minutes

                +

                Queues

                +

                5 minutes

                queue_min_cu_num

                +

                queue_min_cu_num

                Minimum CUs for Queue

                +

                Minimum CUs for Queue

                Displays the minimum number of CUs for a user queue.

                +

                Displays the minimum number of CUs for a user queue.

                ≥ 0

                +

                ≥ 0

                Count

                +

                Count

                N/A

                +

                N/A

                Queues

                +

                queue_id

                5 minutes

                +

                Queues

                +

                5 minutes

                queue_max_cu_num

                +

                queue_max_cu_num

                Maximum CUs for Queue

                +

                Maximum CUs for Queue

                Displays the maximum number of CUs for a user queue.

                +

                Displays the maximum number of CUs for a user queue.

                ≥ 0

                +

                ≥ 0

                Count

                +

                Count

                N/A

                +

                N/A

                Queues

                +

                queue_id

                5 minutes

                +

                Queues

                +

                5 minutes

                queue_priority

                +

                queue_priority

                Queue Priority

                +

                Queue Priority

                Displays the priority of a user queue.

                +

                Displays the priority of a user queue.

                1–100

                +

                1–100

                N/A

                +

                N/A

                N/A

                +

                N/A

                Queues

                +

                queue_id

                5 minutes

                +

                Queues

                +

                5 minutes

                queue_cpu_usage

                +

                queue_cpu_usage

                Queue CPU Usage

                +

                Queue CPU Usage

                Displays the CPU usage of user queues.

                +

                Displays the CPU usage of user queues.

                0–100

                +

                0–100

                %

                +

                %

                N/A

                +

                N/A

                Queues

                +

                queue_id

                +

                Queues

                This metric applies only to queues in non-elastic resource pools.

                5 minutes

                +

                5 minutes

                queue_disk_usage

                +

                queue_disk_usage

                Queue Disk Usage

                +

                Queue Disk Usage

                Displays the disk usage of user queues.

                +

                Displays the disk usage of user queues.

                0–100

                +

                0–100

                %

                +

                %

                N/A

                +

                N/A

                Queues

                +

                queue_id

                +

                Queues

                This metric applies only to queues in non-elastic resource pools.

                5 minutes

                +

                5 minutes

                queue_disk_used

                +

                queue_disk_used

                Max Disk Usage

                +

                Max Disk Usage

                Displays the maximum disk usage of user queues.

                +

                Displays the maximum disk usage of user queues.

                0–100

                +

                0–100

                %

                +

                %

                N/A

                +

                N/A

                Queues

                +

                queue_id

                +

                Queues

                This metric applies only to queues in non-elastic resource pools.

                5 minutes

                +

                5 minutes

                queue_mem_usage

                +

                queue_mem_usage

                Queue Memory Usage

                +

                Queue Memory Usage

                Displays the memory usage of user queues.

                +

                Displays the memory usage of user queues.

                0–100

                +

                0–100

                %

                +

                %

                N/A

                +

                N/A

                Queues

                +

                queue_id

                +

                Queues

                This metric applies only to queues in non-elastic resource pools.

                5 minutes

                +

                5 minutes

                queue_mem_used

                +

                queue_mem_used

                Used Memory

                +

                Used Memory

                Displays the memory usage rate of the user queues.

                +

                Displays the memory usage rate of the user queues.

                ≥ 0

                +

                ≥ 0

                MB

                +

                MB

                N/A

                +

                N/A

                Queues

                +

                queue_id

                +

                Queues

                This metric applies only to queues in non-elastic resource pools.

                5 minutes

                +

                5 minutes

                flink_read_records_per_second

                +

                flink_read_records_per_second

                Flink Job Data Read Rate

                +

                Flink Job Data Read Rate

                Displays the data input rate of a Flink job for monitoring and debugging.

                +

                Displays the data input rate of a Flink job for monitoring and debugging.

                ≥ 0

                +

                ≥ 0

                record/s

                +

                record/s

                N/A

                +

                N/A

                Flink jobs

                +

                flink_job_id

                10 seconds

                +

                Flink jobs

                +

                10 seconds

                flink_write_records_per_second

                +

                flink_write_records_per_second

                Flink Job Data Write Rate

                +

                Flink Job Data Write Rate

                Displays the data output rate of a Flink job for monitoring and debugging.

                +

                Displays the data output rate of a Flink job for monitoring and debugging.

                ≥ 0

                +

                ≥ 0

                record/s

                +

                record/s

                N/A

                +

                N/A

                Flink jobs

                +

                flink_job_id

                +

                Flink jobs

                10 seconds

                +

                10 seconds

                flink_read_records_total

                +

                flink_read_records_total

                Flink Job Total Data Read

                +

                Flink Job Total Data Read

                Displays the total number of data inputs of a Flink job for monitoring and debugging.

                +

                Displays the total number of data inputs of a Flink job for monitoring and debugging.

                ≥ 0

                +

                ≥ 0

                record/s

                +

                record/s

                N/A

                +

                N/A

                Flink jobs

                +

                flink_job_id

                +

                Flink jobs

                10 seconds

                +

                10 seconds

                flink_write_records_total

                +

                flink_write_records_total

                Flink Job Total Data Write

                +

                Flink Job Total Data Write

                Displays the total number of output data records of a Flink job for monitoring and debugging.

                +

                Displays the total number of output data records of a Flink job for monitoring and debugging.

                ≥ 0

                +

                ≥ 0

                record/s

                +

                record/s

                N/A

                +

                N/A

                Flink jobs

                +

                flink_job_id

                10 seconds

                +

                Flink jobs

                +

                10 seconds

                flink_read_bytes_per_second

                +

                flink_read_bytes_per_second

                Flink Job Byte Read Rate

                +

                Flink Job Byte Read Rate

                Displays the number of input bytes per second of a Flink job.

                +

                Displays the number of input bytes per second of a Flink job.

                ≥ 0

                +

                ≥ 0

                byte/s

                +

                byte/s

                1024(IEC)

                +

                1024(IEC)

                Flink jobs

                +

                flink_job_id

                +

                Flink jobs

                10 seconds

                +

                10 seconds

                flink_write_bytes_per_second

                +

                flink_write_bytes_per_second

                Flink Job Byte Write Rate

                +

                Flink Job Byte Write Rate

                Displays the number of output bytes per second of a Flink job.

                +

                Displays the number of output bytes per second of a Flink job.

                ≥ 0

                +

                ≥ 0

                byte/s

                +

                byte/s

                1024(IEC)

                +

                1024(IEC)

                Flink jobs

                +

                flink_job_id

                10 seconds

                +

                Flink jobs

                +

                10 seconds

                flink_read_bytes_total

                +

                flink_read_bytes_total

                Flink Job Total Read Byte

                +

                Flink Job Total Read Byte

                Displays the total number of input bytes of a Flink job.

                +

                Displays the total number of input bytes of a Flink job.

                ≥ 0

                +

                ≥ 0

                byte/s

                +

                byte/s

                1024(IEC)

                +

                1024(IEC)

                Flink jobs

                +

                flink_job_id

                10 seconds

                +

                Flink jobs

                +

                10 seconds

                flink_write_bytes_total

                +

                flink_write_bytes_total

                Flink Job Total Write Byte

                +

                Flink Job Total Write Byte

                Displays the total number of output bytes of a Flink job.

                +

                Displays the total number of output bytes of a Flink job.

                ≥ 0

                +

                ≥ 0

                byte/s

                +

                byte/s

                1024(IEC)

                +

                1024(IEC)

                Flink jobs

                +

                flink_job_id

                10 seconds

                +

                Flink jobs

                +

                10 seconds

                flink_cpu_usage

                +

                flink_cpu_usage

                Flink Job CPU Usage

                +

                Flink Job CPU Usage

                Displays the CPU usage of Flink jobs.

                +

                Displays the CPU usage of Flink jobs.

                0–100

                +

                0–100

                %

                +

                %

                N/A

                +

                N/A

                Flink jobs

                +

                flink_job_id

                +

                Flink jobs

                10 seconds

                +

                10 seconds

                flink_mem_usage

                +

                flink_mem_usage

                Flink Job Memory Usage

                +

                Flink Job Memory Usage

                Displays the memory usage of Flink jobs.

                +

                Displays the memory usage of Flink jobs.

                0–100

                +

                0–100

                %

                +

                %

                N/A

                +

                N/A

                Flink jobs

                +

                flink_job_id

                10 seconds

                +

                Flink jobs

                +

                10 seconds

                flink_max_op_latency

                +

                flink_max_op_latency

                Flink Job Max Operator Latency

                +

                Flink Job Max Operator Latency

                Displays the maximum operator delay of a Flink job. The unit is ms.

                +

                Displays the maximum operator delay of a Flink job. The unit is ms.

                ≥ 0

                +

                ≥ 0

                ms

                +

                ms

                N/A

                +

                N/A

                Flink jobs

                +

                flink_job_id

                +

                Flink jobs

                10 seconds

                +

                10 seconds

                flink_max_op_backpressure_level

                +

                flink_max_op_backpressure_level

                Flink Job Maximum Operator Backpressure

                +

                Flink Job Maximum Operator Backpressure

                Displays the maximum operator backpressure value of a Flink job. A larger value indicates severer backpressure.

                +

                Displays the maximum operator backpressure value of a Flink job. A larger value indicates severer backpressure.

                0: OK

                50: low

                100: high

                0–100

                +

                0–100

                N/A

                +

                N/A

                N/A

                +

                N/A

                Flink jobs

                +

                flink_job_id

                10 seconds

                +

                Flink jobs

                +

                10 seconds

                Queue

                elastic_resource_pool_id

                +

                Elastic resource pool

                +

                flink_job_id

                Flink job

                diff --git a/docs/dli/umn/dli_01_0448.html b/docs/dli/umn/dli_01_0448.html index 89453b441..3b210b659 100644 --- a/docs/dli/umn/dli_01_0448.html +++ b/docs/dli/umn/dli_01_0448.html @@ -55,21 +55,21 @@

                Authorization Object

                Select User or Project.

                +

                Select User or Project.

                Username/Project

                • If you select User, enter the IAM username when granting table permissions to the user.
                  NOTE:

                  The username must be an existing IAM username and has been used to log in to the DLI management console.

                  +
                • If you select User, enter the IAM username when granting table permissions to the user.
                  NOTE:

                  The username must be an existing IAM username and has been used to log in to the DLI management console.

                  -
                • If you select Project, select the project to be authorized in the current region.
                  NOTE:

                  If you select Project, you can only view information about the authorized tables and their databases.

                  +
                • If you select Project, select the project to be authorized in the current region.
                  NOTE:

                  If you select Project, you can only view information about the authorized tables and their databases.

                Non-inheritable Permissions

                Select a permission to grant it to the user, or deselect a permission to revoke it.
                • The following permissions are applicable to both user and project authorization:
                  • View Table Creation Statement: This permission allows you to view the statement for creating the current table.
                  • View Table Information: This permission allows you to view information about the current table.
                  • Select Table: This permission allows you to query data of the current table.
                  • Drop Table: This permission allows you to delete the current table.
                  • Rename Table: Rename the current table.
                  • Insert: This permission allows you to insert data into the current table.
                  • Overwrite: This permission allows you to insert data to overwrite the data in the current table.
                  • Add Column: This permission allows you to add columns to the current table.
                  • Grant Permission: This permission allows you to grant table permissions to other users or projects.
                  • Revoke Permission: This permission allows you to revoke the table's permissions that other users or projects have but cannot revoke the table owner's permissions.
                  • View Other Users' Permissions: This permission allows you to query other users' permission on the current table.
                  +
                Select a permission to grant it to the user, or deselect a permission to revoke it.
                • The following permissions are applicable to both user and project authorization:
                  • View Table Creation Statement: This permission allows you to view the statement for creating the current table.
                  • View Table Information: This permission allows you to view information about the current table.
                  • Select Table: This permission allows you to query data of the current table.
                  • Drop Table: This permission allows you to delete the current table.
                  • Rename Table: Rename the current table.
                  • Insert: This permission allows you to insert data into the current table.
                  • Overwrite: This permission allows you to insert data to overwrite the data in the current table.
                  • Add Column: This permission allows you to add columns to the current table.
                  • Grant Permission: This permission allows you to grant table permissions to other users or projects.
                  • Revoke Permission: This permission allows you to revoke the table's permissions that other users or projects have but cannot revoke the table owner's permissions.
                  • View Other Users' Permissions: This permission allows you to query other users' permission on the current table.
                  The partition table also has the following permissions:
                  • Add Partition: This permission allows you to add a partition to a partition table.
                  • Delete Partition: This permission allows you to delete existing partitions from a partition table.
                  • Configure Path for Partition: This permission allows you to set the path of a partition in a partition table to a specified OBS path.
                  • Rename Table Partition: This permission allows you to rename partitions in a partition table.
                  • Restore Table Partition: This permission allows you to export partition information from the file system and save the information to metadata.
                  • View All Partitions: This permission allows you to view all partitions in a partition table.
                @@ -92,7 +92,7 @@

                Authorization Object

                Select User or Project.

                +

                Select User or Project.

                Username/Project

                diff --git a/docs/dli/umn/dli_01_0451.html b/docs/dli/umn/dli_01_0451.html index 2ed803f45..16dad91f7 100644 --- a/docs/dli/umn/dli_01_0451.html +++ b/docs/dli/umn/dli_01_0451.html @@ -2,10 +2,10 @@

                Using IAM Roles or Policies to Grant Access to DLI

                The role/policy-based authorization model provided by Identity and Access Management (IAM) lets you control access to DLI resources. With IAM, you can:

                -
                • Based on the organizational structure of your enterprise, create IAM users in you master account for employees from different departments within the company. This ensures that each employee has unique security credentials and can use DLI resources.
                • Grant users the minimum permissions required to perform specific tasks based on their job responsibilities.
                • Entrust an account or cloud service to perform professional, efficient O&M on your DLI resources.
                +
                • Based on the organizational structure of your enterprise, create IAM users in your master account for employees from different departments within the company. This ensures that each employee has unique security credentials and can use DLI resources.
                • Grant users the minimum permissions required to perform specific tasks based on their job responsibilities.
                • Entrust an account or cloud service to perform professional, efficient O&M on your DLI resources.

                If your account does not need individual IAM users, you may skip over this section.

                Process Flow shows the process flow of role/policy-based authorization.

                -

                Prerequisites

                Before granting permissions to a user group, familiarize yourself with the DLI permissions that can be added to the user group and select them accordingly based on actual needs.

                +

                Prerequisites

                Before granting permissions to a user group, familiarize yourself with the DLI permissions that can be added to the user group and select them as needed.

                For details about the system permissions of other services, see Permissions.

                Process Flow

                Figure 1 Process for granting DLI permissions
                @@ -52,7 +52,7 @@
                -

                Creating a Custom DLI Policy

                If the predefined DLI permissions in the system do not meet your authorization requirements, you can create custom policies. For the actions that can be added to custom policies, refer to "Permission Policies and Supported Actions" in Data Lake Insight API Reference.

                +

                Creating a Custom DLI Policy

                If the predefined DLI permissions in the system do not meet your authorization requirements, you can create custom policies. For details about the actions that can be added to custom policies, see "Permission Policies and Supported Actions" in the Data Lake Insight API Reference.

                You can create custom policies in either of the following two ways:

                • Visual editor: Select cloud services, actions, resources, and request conditions. This does not require knowledge of policy syntax.
                • JSON: Create a JSON policy or edit an existing one.

                @@ -79,7 +79,7 @@
              12. Condition

                Determines when a policy is in effect. A condition consists of a condition key and a condition operator.

                A key in the Condition element of a statement. There are global and service-specific condition keys.

                -
                • Global-level condition key: The prefix is g:, which applies to all actions. For details, see the condition key description in Policy Syntax.
                • Service-level condition key: applies only to actions of the specific service.
                +
                • Global-level condition key: The prefix is g:, which applies to all actions. For details, see the condition key description in "Policy Syntax".
                • Service-level condition key: applies only to actions of the specific service.

                An operator must be used together with a condition key to form a complete condition statement. For details, see Table 2.

                IAM provides a set of DLI predefined condition keys. The following table lists the DLI predefined condition keys.

                @@ -184,7 +184,7 @@
                13. -
              13. Define a resource.

                The format is Service name:Region:Domain ID:Resource type:Resource path. The wildcard (*) indicates all resources. You can flexibly set these five fields. The Resource path field can be set with varying levels of access control based on the specific scenario. If you need to set permissions for all resources under this service, you can leave this field unspecified.

                +
              14. Define a resource.

                The format is Service name:Region:Domain ID:Resource type:Resource path. The wildcard (*) indicates all resources. You can flexibly set these five fields. The Resource path field can be set with varying levels of access control based on the specific scenario. If you need to set permissions for all resources under this service, you can leave this field unspecified.

                For details about how to define a resource, see Table 4.

                For details about the resource types and resource paths, see Table 5.

                @@ -218,103 +218,143 @@
                15. -
                Table 5 DLI resources and their paths

                Type

                +
                - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + @@ -412,110 +452,150 @@

                DLI Resources

                Resources are objects that exist within a service. In DLI, resources include the following, and you can select specific resources when creating custom policies by specifying the resource path.

                -
                Table 5 DLI resources and their paths

                Type

                Resource

                +

                Resource

                Path

                +

                Path

                +

                Description

                elasticresourcepool

                +

                elasticresourcepool

                DLI elastic resource pool

                +

                DLI elastic resource pool

                elasticresourcepools.name

                +

                DLI:*:*:elasticresourcepool:elasticresourcepools.name

                +
                • The path prefix for elasticresourcepool is fixed as DLI:*:*:elasticresourcepool:.
                • Supports wildcard (*): DLI:*:*:elasticresourcepool:elasticresourcepools.* indicates any DLI elastic resource pool.

                  DLI:*:*:elasticresourcepool:elasticresourcepools.pool01 indicates a specific DLI elastic resource pool named pool01.

                  +

                queue

                +

                queue

                DLI queue

                +

                DLI queue

                queues.queuename

                +

                DLI:*:*:queue:queues.queuename

                +
                • The path prefix for queue is fixed as DLI:*:*:queue:.
                • Supports wildcard (*): DLI:*:*:queue:queues.* indicates any DLI queue.

                  DLI:*:*:queue:queues.queue01 indicates a specific DLI queue named queue01.

                  +

                database

                +

                database

                DLI database

                +

                DLI database

                databases.dbname

                +

                DLI:*:*:database:databases.dbname

                +
                • The path prefix for database is fixed as DLI:*:*:database:.
                • Supports wildcard (*): DLI:*:*:database:databases.* indicates any DLI database.

                  DLI:*:*:database:databases.db01 indicates a specific DLI database named db01.

                  +

                table

                +

                table

                DLI table

                +

                DLI table

                databases.dbname.tables.tbname

                +

                DLI:*:*:table:databases.dbname.tables.tbname

                +
                • The path prefix for table is fixed as DLI:*:*:table:.
                • Supports wildcard (*): DLI:*:*:able:databases..tables.* indicates any DLI table.

                  DLI:*:*:table:databases.db01.tables.tb01 indicates a DLI table named tb01 in the db01 database.

                  +

                column

                +

                column

                DLI column

                +

                DLI column

                databases.dbname.tables.tbname.columns.colname

                +

                DLI:*:*:column:databases.dbname.tables.tbname.columns.colname

                +
                • The path prefix for column is fixed as DLI:*:*:column:.
                • Supports wildcard (*): DLI:*:*:column:databases..tables..columns.* indicates any DLI column.

                  DLI:*:*:column:databases.db01.tables.tb01.columns.col01 indicates a DLI column named col01 in the tb01 table of the db01 database.

                  +

                jobs

                +

                jobs

                DLI Flink job

                +

                DLI Flink job

                jobs.flink.jobid

                +

                DLI:*:*:jobs:jobs.flink.jobid

                +
                • The path prefix for jobs is fixed as DLI:*:*:jobs:.
                • Supports wildcard (*): DLI:*:*:jobs:jobs.flink.* indicates any DLI Flink job.

                  DLI:*:*:jobs:jobs.flink.123456 indicates a DLI Flink job with ID of 123456.

                  +

                resource

                +

                resource

                DLI package

                +

                DLI package

                resources.resourcename

                +

                DLI:*:*:resource:resources.resourcename

                +
                • The path prefix for resource is fixed as DLI:*:*:resource:.
                • Supports wildcard (*): DLI:*:*:resource:resources.* indicates any DLI package.

                  DLI:*:*:resource:resources.jar01 indicates a DLI package named jar01.

                  +

                group

                +

                group

                DLI package group

                +

                DLI package group

                groups.groupname

                +

                DLI:*:*:group:groups.groupname

                +
                • The path prefix for group is fixed as DLI:*:*:group:.
                • Supports wildcard (*): DLI:*:*:group:groups.* indicates any DLI package group.

                  DLI:*:*:group:groups.group01 indicates a DLI package group named group01.

                  +

                datasourceauth

                +

                datasourceauth

                DLI datasource authentication information

                +

                DLI datasource authentication information

                datasourceauth.name

                +

                DLI:*:*:datasourceauth:datasourceauth.name

                +
                • The path prefix for datasourceauth is fixed as DLI:*:*:datasourceauth:.
                • Supports wildcard (*): DLI:*:*:datasourceauth:datasourceauth.* indicates any DLI datasource authentication information.

                  DLI:*:*:datasourceauth:datasourceauth.auth01 indicates DLI datasource authentication information named auth01.

                  +

                edsconnections

                +

                edsconnections

                Enhanced datasource connection

                +

                Enhanced datasource connection

                edsconnections.Connection ID

                +

                DLI:*:*:edsconnections:edsconnections.Connection ID

                +
                • The path prefix for edsconnections is fixed as DLI:*:*:edsconnections:.
                • Supports wildcard (*): DLI:*:*:edsconnections:edsconnections.* indicates any DLI enhanced datasource connection.

                  DLI:*:*:edsconnections:edsconnections.conn01 indicates a DLI enhanced datasource connection with connection ID of conn01.

                  +

                variable

                +

                variable

                DLI global variable

                +

                DLI global variable

                variables.name

                +

                DLI:*:*:variable:variables.name

                +
                • The path prefix for variable is fixed as DLI:*:*:variable:.
                • Supports wildcard (*): DLI:*:*:variable:variables.* indicates any DLI global variable.

                  DLI:*:*:variable:variables.var01 indicates a DLI global variable named var01.

                  +

                sqldefendrule

                +

                sqldefendrule

                SQL inspection rule

                +

                SQL inspection rule

                sqldefendes.*

                +

                DLI:*:*:sqldefendrule:sqldefendes.*

                +
                • The path prefix for sqldefendrule is fixed as DLI:*:*:sqldefendrule:.
                • Supports wildcard (*): DLI:*:*:sqldefendrule:sqldefendes.* indicates all SQL inspection rules. (The sqldefendrule resource path inherently includes wildcard characters, so there is no need to specify them additionally.)

                catalog

                +

                catalog

                DLI data catalog

                +

                DLI data catalog

                catalogs.name

                +

                DLI:*:*:catalog:catalogs.name

                +
                • The path prefix for catalog is fixed as DLI:*:*:catalog:.
                • Supports wildcard (*): DLI:*:*:catalog:catalogs.* indicates any DLI data catalog.

                  DLI:*:*:catalog:catalogs.cat01 indicates a DLI data catalog named cat01.

                  +
                - - - - @@ -253,16 +257,16 @@ - - @@ -310,6 +314,7 @@ - - @@ -357,16 +363,16 @@ - - diff --git a/docs/dli/umn/dli_01_0504.html b/docs/dli/umn/dli_01_0504.html index 841279e30..45732867b 100644 --- a/docs/dli/umn/dli_01_0504.html +++ b/docs/dli/umn/dli_01_0504.html @@ -14,7 +14,7 @@

                The default queue does not support specifying the size of resources and resources are allocated on-demand during job execution, with billing based on the actual amount of data scanned.

                As resources of the default queue are shared among all users, there may be resource contention during use, and it cannot be guaranteed that resources will be available for every job execution.

                The default queue is suitable for small-scale or temporary data processing needs. For important jobs or jobs that require guaranteed resources, you are advised to buy an elastic resource pool and create queues within it to execute jobs.

                -
              15. For SQL:

                For SQL queues are used to execute SQL jobs and supports specifying engine types including Spark and HetuEngine.

                +
              16. For SQL:

                For SQL queues are designed to execute SQL jobs. They support the Spark engine.

                This type of queues is suitable for businesses that require fast data query and analysis, as well as regular cache clearing or environment resetting.

                • For general purpose:

                  For general purpose queues are used to execute Spark jobs, Flink OpenSource SQL jobs, and Flink Jar jobs.

                  @@ -27,7 +27,7 @@

                  DLI Compute Resource Modes

                  DLI offers three compute resource management modes, each with unique advantages and use cases.

                  Figure 1 DLI compute resource modes
                  -
                  • Elastic resource pool mode:
                    A pooled management mode for compute resources that provides dynamic scaling capabilities. Queues within the same elastic resource pool share compute resources. By setting up a reasonable compute resource allocation policy for queues, you can improve compute resource utilization and meet resource demands during peak hours.
                    • Use cases: suitable for scenarios with significant fluctuations in business volume, such as periodic data batch processing tasks or real-time data processing needs.
                    • Supported queue types: for SQL (Spark), for SQL (HetuEngine), and for general purpose. For details about DLI queue types, see Queue Types.

                      General-purpose queues and SQL queues in elastic resource pool mode do not support cross-AZ deployment.

                      +
                      • Elastic resource pool mode:
                        A pooled management mode for compute resources that provides dynamic scaling capabilities. Queues within the same elastic resource pool share compute resources. By setting up a reasonable compute resource allocation policy for queues, you can improve compute resource utilization and meet resource demands during peak hours.
                        • Use cases: suitable for scenarios with significant fluctuations in business volume, such as periodic data batch processing tasks or real-time data processing needs.
                        • Supported queue types: for SQL (Spark) and for general purpose. For details about DLI queue types, see Queue Types.

                          General-purpose queues and SQL queues in elastic resource pool mode do not support cross-AZ deployment.

                        • Usage: first create an elastic resource pool, then create queues within the pool and allocate compute resources. Queues are associated with specific jobs and data processing tasks.

                          For how to buy an elastic resource pool and create queues within it, see Creating an Elastic Resource Pool and Creating Queues Within It.

                        @@ -59,7 +59,6 @@
                + + + @@ -94,6 +99,13 @@

                192.168.0.0–192.168.0.0/16–19

                + + + -
                Table 6 DLI resources and their paths

                Type

                +
                - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - + - - - +
                Table 6 DLI resources and their paths

                Type

                Resource

                +

                Resource

                Path

                +

                Path

                +

                Description

                elasticresourcepool

                +

                elasticresourcepool

                DLI elastic resource pool

                +

                DLI elastic resource pool

                elasticresourcepools.name

                +

                DLI:*:*:elasticresourcepool:elasticresourcepools.name

                +
                • The path prefix for elasticresourcepool is fixed as DLI:*:*:elasticresourcepool:.
                • Supports wildcard (*): DLI:*:*:elasticresourcepool:elasticresourcepools.* indicates any DLI elastic resource pool.

                  DLI:*:*:elasticresourcepool:elasticresourcepools.pool01 indicates a specific DLI elastic resource pool named pool01.

                  +

                queue

                +

                queue

                DLI queue

                +

                DLI queue

                queues.queuename

                +

                DLI:*:*:queue:queues.queuename

                +
                • The path prefix for queue is fixed as DLI:*:*:queue:.
                • Supports wildcard (*): DLI:*:*:queue:queues.* indicates any DLI queue.

                  DLI:*:*:queue:queues.queue01 indicates a specific DLI queue named queue01.

                  +

                database

                +

                database

                DLI database

                +

                DLI database

                databases.dbname

                +

                DLI:*:*:database:databases.dbname

                +
                • The path prefix for database is fixed as DLI:*:*:database:.
                • Supports wildcard (*): DLI:*:*:database:databases.* indicates any DLI database.

                  DLI:*:*:database:databases.db01 indicates a specific DLI database named db01.

                  +

                table

                +

                table

                DLI table

                +

                DLI table

                databases.dbname.tables.tbname

                +

                DLI:*:*:table:databases.dbname.tables.tbname

                +
                • The path prefix for table is fixed as DLI:*:*:table:.
                • Supports wildcard (*): DLI:*:*:able:databases..tables.* indicates any DLI table.

                  DLI:*:*:table:databases.db01.tables.tb01 indicates a DLI table named tb01 in the db01 database.

                  +

                column

                +

                column

                DLI column

                +

                DLI column

                databases.dbname.tables.tbname.columns.colname

                +

                DLI:*:*:column:databases.dbname.tables.tbname.columns.colname

                +
                • The path prefix for column is fixed as DLI:*:*:column:.
                • Supports wildcard (*): DLI:*:*:column:databases..tables..columns.* indicates any DLI column.

                  DLI:*:*:column:databases.db01.tables.tb01.columns.col01 indicates a DLI column named col01 in the tb01 table of the db01 database.

                  +

                jobs

                +

                jobs

                DLI Flink job

                +

                DLI Flink job

                jobs.flink.jobid

                +

                DLI:*:*:jobs:jobs.flink.jobid

                +
                • The path prefix for jobs is fixed as DLI:*:*:jobs:.
                • Supports wildcard (*): DLI:*:*:jobs:jobs.flink.* indicates any DLI Flink job.

                  DLI:*:*:jobs:jobs.flink.123456 indicates a DLI Flink job with ID of 123456.

                  +

                resource

                +

                resource

                DLI package

                +

                DLI package

                resources.resourcename

                +

                DLI:*:*:resource:resources.resourcename

                +
                • The path prefix for resource is fixed as DLI:*:*:resource:.
                • Supports wildcard (*): DLI:*:*:resource:resources.* indicates any DLI package.

                  DLI:*:*:resource:resources.jar01 indicates a DLI package named jar01.

                  +

                group

                +

                group

                DLI package group

                +

                DLI package group

                groups.groupname

                +

                DLI:*:*:group:groups.groupname

                +
                • The path prefix for group is fixed as DLI:*:*:group:.
                • Supports wildcard (*): DLI:*:*:group:groups.* indicates any DLI package group.

                  DLI:*:*:group:groups.group01 indicates a DLI package group named group01.

                  +

                datasourceauth

                +

                datasourceauth

                DLI datasource authentication information

                +

                DLI datasource authentication information

                datasourceauth.name

                +

                DLI:*:*:datasourceauth:datasourceauth.name

                +
                • The path prefix for datasourceauth is fixed as DLI:*:*:datasourceauth:.
                • Supports wildcard (*): DLI:*:*:datasourceauth:datasourceauth.* indicates any DLI datasource authentication information.

                  DLI:*:*:datasourceauth:datasourceauth.auth01 indicates DLI datasource authentication information named auth01.

                  +

                edsconnections

                +

                edsconnections

                Enhanced datasource connection

                +

                Enhanced datasource connection

                edsconnections.Connection ID

                +

                DLI:*:*:edsconnections:edsconnections.Connection ID

                +
                • The path prefix for edsconnections is fixed as DLI:*:*:edsconnections:.
                • Supports wildcard (*): DLI:*:*:edsconnections:edsconnections.* indicates any DLI enhanced datasource connection.

                  DLI:*:*:edsconnections:edsconnections.conn01 indicates a DLI enhanced datasource connection with connection ID of conn01.

                  +

                variable

                +

                variable

                DLI global variable

                +

                DLI global variable

                variables.name

                +

                DLI:*:*:variable:variables.name

                +
                • The path prefix for variable is fixed as DLI:*:*:variable:.
                • Supports wildcard (*): DLI:*:*:variable:variables.* indicates any DLI global variable.

                  DLI:*:*:variable:variables.var01 indicates a DLI global variable named var01.

                  +

                sqldefendrule

                +

                sqldefendrule

                SQL inspection rule

                +

                SQL inspection rule

                sqldefendes.*

                +

                DLI:*:*:sqldefendrule:sqldefendes.*

                +
                • The path prefix for sqldefendrule is fixed as DLI:*:*:sqldefendrule:.
                • Supports wildcard (*): DLI:*:*:sqldefendrule:sqldefendes.* indicates all SQL inspection rules. (The sqldefendrule resource path inherently includes wildcard characters, so there is no need to specify them additionally.)

                catalog

                +

                catalog

                DLI data catalog

                +

                DLI data catalog

                catalogs.name

                +

                DLI:*:*:catalog:catalogs.name

                +
                • The path prefix for catalog is fixed as DLI:*:*:catalog:.
                • Supports wildcard (*): DLI:*:*:catalog:catalogs.* indicates any DLI data catalog.

                  DLI:*:*:catalog:catalogs.cat01 indicates a DLI data catalog named cat01.

                  +
                -

                DLI Request Condition

                Request conditions are useful in determining when a custom policy is in effect. A request condition consists of condition keys and operators. Condition keys are either global or service-level and are used in the Condition element of a policy statement. Global condition keys (starting with g:) are available for operations of all services, while service-level condition keys (starting with a service name such as dli) are available only for operations of a specific service. An operator must be used together with a condition key to form a complete condition statement.

                +

                DLI Request Condition

                Request conditions are useful in determining when a custom policy is in effect. A request condition consists of condition keys and operators. Condition keys are either global or service-level and are used in the Condition element of a policy statement. Global condition keys (starting with g:) are available for operations of all services, while service-level condition keys (starting with a service name such as dli) are available only for operations of a specific service. An operator must be used together with a condition key to form a complete condition statement.

                IAM provides a set of DLI predefined condition keys. The following table lists the DLI predefined condition keys.

                Table 7 DLI request conditions

                Condition Key

                diff --git a/docs/dli/umn/dli_01_0457.html b/docs/dli/umn/dli_01_0457.html index 54b5ac2c6..eeacc8443 100644 --- a/docs/dli/umn/dli_01_0457.html +++ b/docs/dli/umn/dli_01_0457.html @@ -4,14 +4,14 @@

                A Flink Jar job involves developing a custom application Jar package based on Flink's capabilities and submitting it to a DLI queue for execution.

                To create a Flink Jar job, you need to write and build your own application Jar package. This is suitable for users who require stream data processing and are proficient in Flink's secondary development capabilities.

                This section describes how to create a Flink Jar job on the DLI management console.

                -

                Prerequisites

                • When you use a Flink Jar job to access other external data sources, such as OpenTSDB, HBase, Kafka, GaussDB(DWS), RDS, CSS, CloudTable, DCS Redis, and DDS, you need to create a datasource connection to connect the job running queue to the external data source.
                  • For details about the external data sources that can be accessed by Flink jobs, see Common Development Methods for DLI Cross-Source Analysis.
                  • For how to create a datasource connection, see Configuring the Network Connection Between DLI and Data Sources (Enhanced Datasource Connection).

                    On the Resources > Queue Management page, locate the queue you have created, click More in the Operation column, and select Test Address Connectivity to check if the network connection between the queue and the data source is normal. For details, see Testing the Network Connectivity Between a Queue and a Data Source.

                    +

                    Prerequisites

                    • When you use a Flink Jar job to access other external data sources, such as OpenTSDB, HBase, Kafka, DWS, RDS, CSS, CloudTable, DCS Redis, and DDS, you need to create a datasource connection to connect the job running queue to the external data source.
                    • To run a Flink Jar job, you need to build your custom application code into a JAR file and upload it to the OBS bucket that has already been created.
                    • Flink dependencies have been built in the DLI server and security hardening has been performed based on the open-source community version. To avoid dependency package compatibility issues or log output and dump issues, be careful to exclude the following files when packaging:
                      • Built-in dependencies (or set the package dependency scope to provided in Maven or SBT)
                      • Log configuration files (example, log4j.properties/logback.xml)
                      • JAR package for log output implementation (example, log4j).

                    Precautions

                    Before creating and submitting jobs, you are advised to enable CTS to record DLI operations for queries, audits, and tracking. To view the DLI operations that can be recorded by CTS, see Using CTS to Audit DLI.

                    -

                    Creating a Flink Jar Job

                    1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                    2. In the upper right corner of the Flink Jobs page, click Create Job.

                      +

                      Creating a Flink Jar Job

                      1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                      2. In the upper right corner of the Flink Jobs page, click Create Job.

                      3. Specify job parameters.

                        @@ -77,7 +77,7 @@ @@ -99,7 +99,7 @@ - - - - - - - - - @@ -312,6 +289,7 @@ - - - - - - - - - - - -
                        Table 1 Job configuration information

                        Parameter

                        Select a Jar job package.

                        There are the following ways to manage JAR files:

                        -
                        • Upload packages to OBS: Upload Jar packages to an OBS bucket in advance and select the corresponding OBS path.
                        • Upload packages to DLI: Upload JAR files to an OBS bucket in advance and create a package on the Data Management > Package Management page of the DLI management console. For details, see Creating a DLI Package.
                        +
                        • Upload packages to OBS: Upload JAR files to an OBS bucket in advance and select the corresponding OBS path.
                        • Upload packages to DLI: Upload JAR files to an OBS bucket in advance and create a package on the Data Management > Package Management page of the DLI management console. For details, see Creating a DLI Package.

                        For Flink 1.15 or later, you can only select packages from OBS, instead of DLI.

                        Select a user-defined package dependency. The dependent program packages are stored in the classpath directory of the cluster.

                        There are the following ways to manage JAR files:

                        -
                        • Upload packages to OBS: Upload Jar packages to an OBS bucket in advance and select the corresponding OBS path.
                        • Upload packages to DLI: Upload JAR files to an OBS bucket in advance and create a package on the Data Management > Package Management page of the DLI management console. For details, see Creating a DLI Package.
                        +
                        • Upload packages to OBS: Upload JAR files to an OBS bucket in advance and select the corresponding OBS path.
                        • Upload packages to DLI: Upload JAR files to an OBS bucket in advance and create a package on the Data Management > Package Management page of the DLI management console. For details, see Creating a DLI Package.

                        For Flink 1.15 or later, you can only select packages from OBS, instead of DLI.

                        When creating a JAR file for a Flink Jar job, you do not need to upload existing built-in dependency packages to avoid package information conflicts.

                        For details about built-in dependency packages, see DLI Built-in Dependencies.

                        @@ -118,7 +118,7 @@

                        Agency

                        If you choose Flink 1.15 or later to execute your job, you can create a custom agency to allow DLI to access other services.

                        -

                        For how to create a custom agency, see Creating a Custom DLI Agency.

                        +

                        For details about how to create a custom agency, see Creating a Custom DLI Agency.

                        Runtime Configuration

                        @@ -154,6 +154,7 @@

                        Job Manager CUs

                        Number of management unit CUs.

                        +

                        If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the JobManager's CU count does not exceed 2 to avoid resource scheduling failures during job execution.

                        Parallelism

                        @@ -166,7 +167,8 @@

                        Task Manager Config

                        Whether TaskManager resource parameters are set

                        -
                        • If this option is selected, you need to set the following parameters:
                          • CU(s) per TM: Number of resources occupied by each TaskManager.
                          • Slot(s) per TM: Number of slots contained in each TaskManager.
                          +
                          • If this option is selected, you need to set the following parameters:
                            • CU(s) per TM: Number of resources occupied by each TaskManager.

                              If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that a single TaskManager's CU count does not exceed 2 to avoid resource scheduling failures during job execution.

                              +
                            • Slot(s) per TM: Number of slots contained in each TaskManager.
                          • If not selected, the system automatically uses the default values.
                            • CU(s) per TM: The default value is 1.
                            • Slot(s) per TM: The default value is (Parallelism x CU(s) per TM)/(CUs – Job Manager CUs).
                          @@ -204,7 +206,7 @@

                        Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                        If this option is selected, you need to set the following parameters:

                        SMN Topic

                        -

                        Select a custom SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                        +

                        Select a custom SMN topic. For details about how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                        Auto Restart upon Exception

                        @@ -222,8 +224,8 @@

                        DLI offers various resource configuration templates based on different Flink engine versions.

                        Compared with the v1 template, the v2 template does not support the setting of the number of CUs. The v2 template supports the setting of Job Manager Memory and Task Manager Memory.

                        -

                        v1: applicable to Flink 1.12, 1.13, and 1.15.

                        -

                        v2: applicable to Flink 1.13, 1.15, and 1.17.

                        +

                        V1: applicable to Flink 1.12 and 1.15.

                        +

                        V2: applicable to Flink 1.15 and 1.17.

                        You are advised to use the parameter settings of v2.

                        For details about the parameters of v1, see Table 4.

                        For details about the parameters of v2, see Table 5.

                        @@ -246,6 +248,7 @@

                        Job Manager CUs

                        Number of management unit CUs.

                        +

                        If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the JobManager's CU count does not exceed 2 to avoid resource scheduling failures during job execution.

                        Parallelism

                        @@ -258,36 +261,10 @@

                        Task Manager Config

                        Whether TaskManager resource parameters are set

                        -
                        • If this option is selected, you need to set the following parameters:
                          • CU(s) per TM: Number of resources occupied by each TaskManager.
                          • Slot(s) per TM: Number of slots contained in each TaskManager.
                          +
                          • If this option is selected, you need to set the following parameters:
                            • CU(s) per TM: Number of resources occupied by each TaskManager.

                              If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that a single TaskManager's CU count does not exceed 2 to avoid resource scheduling failures during job execution.

                              +
                            • Slot(s) per TM: Number of slots contained in each TaskManager.
                          -
                          • If not selected, the system automatically uses the default values.
                            • CU(s) per TM: The default value is 1.
                            • Slot(s) per TM: The default value is (Parallelism x CU(s) per TM)/(CUs – Job Manager CUs).
                            -
                          -

                        Save Job Log

                        -

                        Whether to save the job running logs to the OBS bucket.

                        -
                        CAUTION:

                        You are advised to select this parameter. Otherwise, no run logs will be generated after the job is executed. If the job runs abnormally later, you will be unable to obtain the run logs for troubleshooting.

                        -
                        -

                        If this option is selected, you need to set the following parameters:

                        -

                        OBS Bucket: Select an OBS bucket to store job logs. If the selected OBS bucket is not authorized, click Authorize.

                        -

                        Alarm on Job Exception

                        -

                        Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                        -

                        If this option is selected, you need to set the following parameters:

                        -

                        SMN Topic

                        -

                        Select a custom SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                        -

                        Auto Restart upon Exception

                        -

                        Whether automatic restart is enabled. If enabled, jobs will be automatically restarted and restored when exceptions occur.

                        -

                        If this option is selected, you need to set the following parameters:

                        -
                        • Max. Retry Attempts: maximum number of retries upon an exception. The unit is times/hour.
                          • Unlimited: The number of retries is unlimited.
                          • Limited: The number of retries is user-defined.
                          -
                        • Restore Job from Checkpoint: Restore the job from the saved checkpoint.

                          If you select this parameter, you also need to set Checkpoint Path.

                          -

                          Checkpoint Path: Select a path for storing checkpoints. This path must match that configured in the application package. Each job must have a unique checkpoint path, or, you will not be able to obtain the checkpoint.

                          +
                          • If not selected, the system automatically uses the default values.
                            • CU(s) per TM: The default value is 1.
                            • Slot(s) per TM: The default value is (Parallelism x CU(s) per TM)/(CUs – Job Manager CUs).

                        Number of vCPUs available for JobManager.

                        The default value is 1. The minimum value cannot be less than 0.5.

                        +

                        If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the JobManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

                        Job Manager Memory

                        @@ -324,6 +302,7 @@

                        Number of vCPUs available for TaskManager.

                        The default value is 1. The minimum value cannot be less than 0.5.

                        +

                        If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the TaskManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

                        Task Manager Memory

                        @@ -339,38 +318,6 @@

                        By default, a single TM slot is set to 1. The minimum parallelism must not be less than 1.

                        Save Job Log

                        -

                        Whether to save the job running logs to the OBS bucket.

                        -
                        CAUTION:

                        You are advised to select this parameter. Otherwise, no run logs will be generated after the job is executed. If the job runs abnormally later, you will be unable to obtain the run logs for troubleshooting.

                        -
                        -

                        If this option is selected, you need to set the following parameters:

                        -

                        OBS Bucket: Select an OBS bucket to store job logs. If the selected OBS bucket is not authorized, click Authorize.

                        -

                        OBS Bucket

                        -

                        OBS bucket to store job logs and checkpoint information. If the OBS bucket you selected is unauthorized, click Authorize.

                        -

                        Alarm on Job Exception

                        -

                        Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                        -

                        If this option is selected, you need to set the following parameters:

                        -

                        SMN Topic

                        -

                        Select a custom SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                        -

                        Auto Restart upon Exception

                        -

                        Whether automatic restart is enabled. If enabled, jobs will be automatically restarted and restored when exceptions occur.

                        -

                        If this option is selected, you need to set the following parameters:

                        -
                        • Max. Retry Attempts: maximum number of retries upon an exception. The unit is times/hour.
                          • Unlimited: The number of retries is unlimited.
                          • Limited: The number of retries is user-defined.
                          -
                        • Restore Job from Checkpoint: Restore the job from the saved checkpoint.

                          If you select this parameter, you also need to set Checkpoint Path.

                          -

                          Checkpoint Path: Select a path for storing checkpoints. This path must match that configured in the application package. Each job must have a unique checkpoint path, or, you will not be able to obtain the checkpoint.

                          -
                        -
                        @@ -433,6 +380,105 @@
                +

              17. Configure logs and status.

                +

                + + + + + + + + + + +
                Table 7 Configuring logs and status

                Parameter

                +

                Description

                +

                Save Job Log

                +

                Whether to save the job running logs to the OBS bucket.

                +
                CAUTION:

                You are advised to select this parameter. Otherwise, no run logs will be generated after the job is executed. If the job runs abnormally later, you will be unable to obtain the run logs for troubleshooting.

                +
                +

                If this option is selected, you need to set the following parameters:

                +

                OBS Bucket: Select an OBS bucket to store job logs. If the selected OBS bucket is not authorized, click Authorize.

                +

                OBS Bucket

                +

                OBS bucket to store job logs and checkpoint information. If the OBS bucket you selected is unauthorized, click Authorize.

                +
                +
                + +
                + + + + + + + + + + + + + + + + + + + + + + + + + +
                Table 8 Root Log Level

                Type

                +

                Description

                +

                Use Case

                +

                TRACE

                +

                The most granular level of logging, typically used during development and debugging. It captures all operational details, including variable values and function calls.

                +

                Primarily for the development phase to help developers understand code execution flow and state.

                +

                DEBUG

                +

                Less detailed than TRACE, intended for debugging purposes. It logs program runtime states and key variable values but omits exhaustive detail.

                +

                Mainly used during development and testing phases to assist developers in identifying issues.

                +

                INFO

                +

                Records important information during normal operations, useful for system operators and maintainers without impacting system functionality.

                +

                Tracks major system operations and statuses, such as startup, shutdown, and configuration changes.

                +

                WARN

                +

                Potential issues or anomalies that do not disrupt system operation but warrant attention from developers or operators.

                +

                Logs situations that may affect system performance or functionality, serving as a warning for possible problems.

                +

                ERROR

                +

                Highlights critical issues or exceptions that impair system functionality and require immediate resolution.

                +

                Captures errors and exceptions during system operation, enabling developers to quickly identify and address them.

                +
                +
                +

              18. Configures error handling parameters.

                +

                + + + + + + + + + + +
                Table 9 Configuring exception handling parameters

                Parameter

                +

                Description

                +

                Alarm on Job Exception

                +

                Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                +

                If this option is selected, you need to set the following parameters:

                +

                SMN Topic

                +

                Select a custom SMN topic. For details about how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                +

                Auto Restart upon Exception

                +

                Whether automatic restart is enabled. If enabled, jobs will be automatically restarted and restored when exceptions occur.

                +

                If this option is selected, you need to set the following parameters:

                +
                • Max. Retry Attempts: maximum number of retries upon an exception. The unit is times/hour.
                  • Unlimited: The number of retries is unlimited.
                  • Limited: The number of retries is user-defined.
                  +
                • Restore Job from Checkpoint: Restore the job from the saved checkpoint.

                  If you select this parameter, you also need to set Checkpoint Path.

                  +

                  Checkpoint Path: Select a path for storing checkpoints. This path must match that configured in the application package. Each job must have a unique checkpoint path, or, you will not be able to obtain the checkpoint.

                  +
                +
                +

              19. Click Save in the upper right of the page.
              20. Click Start in the upper right corner. On the displayed Start Flink Job page, confirm the job specifications, and click Start Now to start the job. After the job is started, the system automatically switches to the Flink Jobs page, and the created job is displayed in the job list. You can view the job status in the Status column.

                • Once a job is successfully submitted, its status changes from Submitting to Running. After the execution is complete, the status changes to Completed.
                • If the job status is Submission failed or Running exception, the job fails to submit or run. In this case, you can hover over the status icon in the Status column of the job list to view the error details. You can click to copy these details. Rectify the fault based on the error information and resubmit the job.

                21. diff --git a/docs/dli/umn/dli_01_0461.html b/docs/dli/umn/dli_01_0461.html index 8ebfcc601..99680d9e7 100644 --- a/docs/dli/umn/dli_01_0461.html +++ b/docs/dli/umn/dli_01_0461.html @@ -3,11 +3,11 @@

                Common Operations of Flink Jobs

                After creating a job, you can manage it by performing various operations such as editing its basic information, starting or stopping it, and importing or exporting it.

                Editing a Job

                You can edit a created job, for example, by modifying the SQL statement, job name, job description, or job configurations.

                -
                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                2. In the row where the job you want to edit locates, click Edit in the Operation column to switch to the editing page.
                3. Edit the job as required.

                  For details, see Creating a Flink OpenSource SQL Job and Creating a Flink Jar Job.

                  +
                  1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                  2. In the row where the job you want to edit locates, click Edit in the Operation column to switch to the editing page.
                  3. Edit the job as required.

                    For details, see Creating a Flink OpenSource SQL Job and Creating a Flink Jar Job.

                Starting a Job

                You can start a saved or stopped job.

                -
                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                2. Use either of the following methods to start jobs:

                  • Starting a single job

                    Select a job and click Start in the Operation column.

                    +
                    1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                    2. Use either of the following methods to start jobs:

                      • Starting a single job

                        Select a job and click Start in the Operation column.

                        Alternatively, you can select the row where the job you want to start locates and click Start in the upper left of the job list.

                      • Batch starting jobs

                        Select the rows where the jobs you want to start locate and click Start in the upper left of the job list.

                      @@ -16,8 +16,8 @@

                Stopping a Job

                You can stop a job in the Running or Submitting state.

                -
                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                2. Stop a job using either of the following methods:

                  • Stopping a job

                    Locate the row that contains the job to be stopped, click More in the Operation column, and select Stop.

                    -

                    Alternatively, you can select the row where the job you want to stop locates and click Stop in the upper left of the job list.

                    +
                    1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                    2. Stop a job using either of the following methods:

                      • Stopping a job

                        Locate the row that contains the job to be stopped, click More in the Operation column, and select Stop.

                        +

                        Alternatively, you can select the row where the job you want to stop locates and click Stop in the upper left of the job list.

                      • Batch stopping jobs

                        Locate the rows containing the jobs you want to stop and click Stop in the upper left of the job list.

                    3. In the displayed Stop Job dialog box, click OK to stop the job.

                      • Before stopping a job, you can trigger a savepoint to save the job status information. When you start the job again, you can choose whether to restore the job from the savepoint.
                      • If you select Trigger savepoint, a savepoint is created. If Trigger savepoint is not selected, no savepoint is created. By default, the savepoint function is disabled.
                      • The lifecycle of a savepoint starts when the savepoint is triggered and stops the job, and ends when the job is restarted. The savepoint is automatically deleted after the job is restarted.
                      @@ -28,7 +28,7 @@

                Deleting a Job

                If you do not need to use a job, perform the following operations to delete it. A deleted job cannot be restored. Therefore, exercise caution when deleting a job.

                -
                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                1. Perform either of the following methods to delete jobs:

                  • Deleting a single job

                    Locate the row containing the job you want to delete and click More > Delete in the Operation column.

                    +
                    1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                    1. Perform either of the following methods to delete jobs:

                      • Deleting a single job

                        Locate the row containing the job you want to delete and click More > Delete in the Operation column.

                        Alternatively, you can select the row containing the job you want to delete and click Delete in the upper left of the job list.

                      • Deleting jobs in batches

                        Select the rows containing the jobs you want to delete and click Delete in the upper left of the job list.

                      @@ -38,7 +38,7 @@

                      This mode is applicable to the scenario where a large number of jobs need to be created when you switch to another region, project, or user. In this case, you do not need to create a job. You only need to export the original job, log in to the system in a new region or project, or use a new user to import the job.

                      When switching to another project or user, you need to grant permissions to the new project or user. For details, see Configuring Flink Job Permissions.

                      -
                      1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                      1. Click Export Job in the upper right corner. The Export Job dialog box is displayed.
                      2. Select the OBS bucket where the job is stored. Click Next.
                      3. Select job information you want to export.

                        By default, configurations of all jobs are exported. You can enable the Custom Export function to export configurations of the desired jobs.

                        +
                        1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                        1. Click Export Job in the upper right corner. The Export Job dialog box is displayed.
                        2. Select the OBS bucket where the job is stored. Click Next.
                        3. Select job information you want to export.

                          By default, configurations of all jobs are exported. You can enable the Custom Export function to export configurations of the desired jobs.

                        4. Click Confirm to export the job.

                Importing a Job

                You can import the Flink job configuration file stored in the OBS bucket to the Flink Jobs page of DLI.

                @@ -47,11 +47,11 @@

                For details, see Creating a Flink OpenSource SQL Job and Creating a Flink Jar Job.

                • When switching to another project or user, you need to grant permissions to the new project or user. For details, see Configuring Flink Job Permissions.
                • Only jobs whose data format is the same as that of Flink jobs exported from DLI can be imported.
                -
                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                1. Click Import Job in the upper right corner. The Import Job dialog box is displayed.
                2. Select the complete OBS path of the job configuration file to be imported. Click Next.
                3. Configure the same-name job policy and click next. Click Next.

                  • Select Overwrite job of the same name. If the name of the job to be imported already exists, the existing job configuration will be overwritten and the job status switches to Draft.
                  • If Overwrite job of the same name is not selected and the name of the job to be imported already exists, the job will not be imported.
                  +
                  1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                  1. Click Import Job in the upper right corner. The Import Job dialog box is displayed.
                  2. Select the complete OBS path of the job configuration file to be imported. Click Next.
                  3. Configure the same-name job policy and click next. Click Next.

                    • Select Overwrite job of the same name. If the name of the job to be imported already exists, the existing job configuration will be overwritten and the job status switches to Draft.
                    • If Overwrite job of the same name is not selected and the name of the job to be imported already exists, the job will not be imported.

                  4. Ensure that Config File and Overwrite Same-Name Job are correctly configured. Click Confirm to import the job.

                Modifying the Name and Description of a Flink Job

                You can change the job name and description as required.

                -
                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                2. In the Operation column of the job whose name and description need to be modified, choose More > Modify Name and Description. The Modify Name and Description dialog box is displayed. Change the name or modify the description of a job.
                3. Click OK.
                +
                1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                2. In the Operation column of the job whose name and description need to be modified, choose More > Modify Name and Description. The Modify Name and Description dialog box is displayed. Change the name or modify the description of a job.
                3. Click OK.

                Triggering a Savepoint

                Before stopping a job, you can trigger a savepoint to save the status information of your job. When you restart the job, you can choose whether to quickly recover it from the most recent savepoint.

                1. In the navigation pane of the DLI console, choose Job Management > Flink Jobs.
                2. Locate the job you want to stop, click More in the Operation column, and select Trigger Savepoint. In the displayed dialog box, select a save path.
                3. Click OK.
                @@ -81,7 +81,7 @@

                Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                If this option is selected, you need to set the following parameters:

                SMN Topic

                -

                Select a custom SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                +

                Select a custom SMN topic. For details about how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                Auto Restart upon Exception

                diff --git a/docs/dli/umn/dli_01_0463.html b/docs/dli/umn/dli_01_0463.html index 969de7c5b..c5587a880 100644 --- a/docs/dli/umn/dli_01_0463.html +++ b/docs/dli/umn/dli_01_0463.html @@ -9,7 +9,7 @@

                This section includes the following content:

                Managing a Job Tag

                DLI allows you to add, modify, or delete tags for jobs.

                -
                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                2. Click the name of the job to be viewed. The Job Details page is displayed.
                3. Click Tags to display the tag information about the current job.
                4. On the page that appears, click Add/Edit Tag in the upper left corner.
                5. Configure the tag parameters in the Add/Edit Tag dialog box.

                  +

                  1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                  2. Click the name of the job to be viewed. The Job Details page is displayed.
                  3. Click Tags to display the tag information about the current job.
                  4. On the page that appears, click Add/Edit Tag in the upper left corner.
                  5. Configure the tag parameters in the Add/Edit Tag dialog box.

                    - @@ -98,7 +98,7 @@

                    Creating a Template

                    You can create a template using any of the following methods:

                    -
                    • Creating a template on the Template Management page
                      1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates.
                      2. Click Create Template in the upper right corner of the page. The Create Template dialog box is displayed.
                      3. Specify Name and Description. +
                        • Creating a template on the Template Management page
                          1. In the navigation pane of the DLI management console, choose Job Templates > Flink Templates.
                          2. Click Create Template in the upper right corner of the page. The Create Template dialog box is displayed.
                          3. Specify Name and Description.
                    Table 1 Tag parameters

                    Parameter

                    Description

                    @@ -43,7 +43,7 @@

                  6. Click OK.
                  7. (Optional) To delete a tag, locate the tag in the tag list and click Delete in its Operation column.

                    8. Searching for a Job by Tag

                    If tags have been added to a job, you can search for the job by setting tag filtering conditions to quickly find it.

                    -
                    1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                    2. In the upper right corner of the page, click the search box and select Tags.
                    3. Choose a tag key and value as prompted. If no tag key or value is available, create a tag for the job. For details, see Managing a Job Tag.
                    4. Choose other tags to generate a tag combination for job search.
                    5. Click search icon. The target job will be displayed in the job list.
                    +
                    1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                    2. In the upper right corner of the page, click the search box and select Tags.
                    3. Choose a tag key and value as prompted. If no tag key or value is available, create a tag for the job. For details, see Managing a Job Tag.
                    4. Choose other tags to generate a tag combination for job search.
                    5. Click search icon. The target job will be displayed in the job list.
                    diff --git a/docs/dli/umn/dli_01_0464.html b/docs/dli/umn/dli_01_0464.html index c420cab0d..6f73f92a2 100644 --- a/docs/dli/umn/dli_01_0464.html +++ b/docs/dli/umn/dli_01_0464.html @@ -52,7 +52,7 @@

                    Operation

                    Create Job: Create a job directly by using the template. After a job is created, the system switches to the Edit page under Job Management.

                    +

                    Create Job: Create a job directly by using the template. After a job is created, the system switches to the Edit page under Job Management.
                    -
                    Table 4 Template parameters

                    Parameter

                    Description

                    @@ -163,7 +163,7 @@

                    SQL statement editing area

                    In the area, you can enter detailed SQL statements to implement business logic. For how to compile SQL statements, see Data Lake Insight SQL Syntax Reference.

                    +

                    In the area, you can enter detailed SQL statements to implement business logic. For details about how to compile SQL statements, see Data Lake Insight SQL Syntax Reference.

                    Save

                    @@ -190,21 +190,21 @@
                -
              21. In the SQL statement editing area, enter SQL statements to implement service logic. For how to compile SQL statements, see Data Lake Insight SQL Syntax Reference.
              22. After the SQL statement is edited, click Save in the upper right corner to complete the template creation.
              23. (Optional) If you do not need to modify the template, click Create Job in the upper right corner to create a job based on the current template. For how to create a job, see Creating a Flink Jar Job.
                24. -
              24. Creating a template based on an existing job template
                1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates. Click the Custom Templates tab.
                2. In the row where the desired template is located in the custom template list, click Edit under Operation to enter the Edit page.
                3. After the modification is complete, set Saving Mode to Save as New.
                4. Click Save in the upper right corner to save the template as a new one.
                +
              25. In the SQL statement editing area, enter SQL statements to implement service logic. For details about how to compile SQL statements, see Data Lake Insight SQL Syntax Reference.
              26. After the SQL statement is edited, click Save in the upper right corner to complete the template creation.
              27. (Optional) If you do not need to modify the template, click Create Job in the upper right corner to create a job based on the current template. For details about how to create a job, see Creating a Flink Jar Job.
                28. +
              28. Creating a template based on an existing job template
                1. In the navigation pane of the DLI management console, choose Job Templates > Flink Templates. Click the Custom Templates tab.
                2. In the row where the desired template is located in the custom template list, click Edit under Operation to enter the Edit page.
                3. After the modification is complete, set Saving Mode to Save as New.
                4. Click Save in the upper right corner to save the template as a new one.
                29. -
                • Creating a template using a created job
                  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                  2. Click Create Job in the upper right corner. The Create Job page is displayed.
                  3. Specify parameters as required.
                  4. Click OK to enter the editing page.
                  5. After the SQL statement is compiled, click Set as Template.
                  6. In the Set as Template dialog box that is displayed, specify Name and Description and click OK.
                  -
                • Creating a template based on the existing job
                  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                  2. In the job list, locate the row where the job that you want to set as a template resides, and click Edit in the Operation column.
                  3. After the SQL statement is compiled, click Set as Template.
                  4. In the Set as Template dialog box that is displayed, specify Name and Description and click OK.
                  +
                  • Creating a template using a created job
                    1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                    2. Click Create Job in the upper right corner. The Create Job page is displayed.
                    3. Specify parameters as required.
                    4. Click OK to enter the editing page.
                    5. After the SQL statement is compiled, click Set as Template.
                    6. In the Set as Template dialog box that is displayed, specify Name and Description and click OK.
                    +
                  • Creating a template based on the existing job
                    1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                    2. In the job list, locate the row where the job that you want to set as a template resides, and click Edit in the Operation column.
                    3. After the SQL statement is compiled, click Set as Template.
                    4. In the Set as Template dialog box that is displayed, specify Name and Description and click OK.

                  Creating a Job Based on a Template

                  You can create jobs based on sample templates or custom templates.

                  -
                  1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates.
                  2. In the sample template list, click Create Job in the Operation column of the target template. For how to create a job, see Creating a Flink OpenSource SQL Job and Creating a Flink Jar Job.
                  +
                  1. In the navigation pane of the DLI management console, choose Job Templates > Flink Templates.
                  2. In the sample template list, click Create Job in the Operation column of the target template. For details about how to create a job, see Creating a Flink OpenSource SQL Job and Creating a Flink Jar Job.

                  Modifying a Template

                  After creating a custom template, you can modify it as required. The sample template cannot be modified, but you can view the template details.

                  -
                  1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates. Click the Custom Templates tab.
                  2. In the row where the template you want to modify is located in the custom template list, click Edit in the Operation column to enter the Edit page.
                  3. In the SQL statement editing area, modify the SQL statements as required.
                  4. Set Saving Mode to Save Here.
                  5. Click Save in the upper right corner to save the modification.
                  +
                  1. In the navigation pane of the DLI management console, choose Job Templates > Flink Templates. Click the Custom Templates tab.
                  2. In the row where the template you want to modify is located in the custom template list, click Edit in the Operation column to enter the Edit page.
                  3. In the SQL statement editing area, modify the SQL statements as required.
                  4. Set Saving Mode to Save Here.
                  5. Click Save in the upper right corner to save the modification.

                  Deleting a Template

                  You can delete a custom template as required. The sample templates cannot be deleted. Deleted templates cannot be restored. Exercise caution when performing this operation.

                  -
                  1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates. Click the Custom Templates tab.
                  2. In the custom template list, select the templates you want to delete and click Delete in the upper left of the custom template list.

                    Alternatively, you can delete a template by performing the following operations: In the custom template list, locate the row where the template you want to delete resides, and click More > Delete in the Operation column.

                    +
                    1. In the navigation pane of the DLI management console, choose Job Templates > Flink Templates. Click the Custom Templates tab.
                    2. In the custom template list, select the templates you want to delete and click Delete in the upper left of the custom template list.

                      Alternatively, you can delete a template by performing the following operations: In the custom template list, locate the row where the template you want to delete resides, and click More > Delete in the Operation column.

                    3. In the displayed dialog box, click Yes.
                  diff --git a/docs/dli/umn/dli_01_0489.html b/docs/dli/umn/dli_01_0489.html index 3443d0865..86164efa9 100644 --- a/docs/dli/umn/dli_01_0489.html +++ b/docs/dli/umn/dli_01_0489.html @@ -4,7 +4,7 @@

                  DLI's address connectivity testing feature can be used to verify network connectivity between DLI queues and destination addresses.

                  This feature is typically utilized for reading and writing external data sources. Once a datasource connection is configured, the communication capability between the DLI queue and the bound peer address is verified.

                  Testing the Address Connectivity Between a Queue and the Data Source

                  1. Log in to the DLI management console. In the navigation pane on the left, choose Resources > Queue Management.
                  2. On the Queue Management page, locate the row containing the target queue, click More in the Operation column, and select Test Address Connectivity.
                  3. On the Test Address Connectivity page, enter the address to be tested. The domain name and IP address are supported, and the port number can be specified.

                    You can input the data source address in the following formats: IPv4 address; IPv4 address + Port number; Domain name; Domain name + Port number.

                    -
                    • IPv4 address: 192.168.x.x
                    • IPv4 + Port number: 192.168.x.x:8080
                    • Domain name: domain-xxxxxx.com
                    • Domain name + Port number: domain-xxxxxx.com:8080
                    +
                    • IPv4 address: 192.168.x.x
                    • IPv4 + Port number: 192.168.x.x:8080
                    • Domain name: domain-xxxxxx.com
                    • Domain name + Port number: domain-xxxxxx.com:8080
                    • IPv6 address: 2001:0db8:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX
                    • [IPv6] + Port number: [2001:0db8:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX]:8080

                  4. Click Test.
                    • If the test address is reachable, you will receive a message.
                    • If the test address is unreachable, you will also receive a message. Check the network configurations and retry. Network configurations include the VPC peering and the datasource connection. Check whether they have been activated.
                  diff --git a/docs/dli/umn/dli_01_0494.html b/docs/dli/umn/dli_01_0494.html index 64ece7483..3b5e94005 100644 --- a/docs/dli/umn/dli_01_0494.html +++ b/docs/dli/umn/dli_01_0494.html @@ -9,11 +9,11 @@

                  Notes and Constraints

                  • The base images provided by DLI must be used to create custom images.
                  • You cannot modify the DLI components and directories in the base images.
                  • Only Spark Jar and Flink Jar jobs are supported.

                  Use Process

                  Figure 1 Process of using a custom image
                  -
                  1. Obtain DLI base images.
                  2. Use Dockerfile to pack dependencies (files, JAR files, or software) required for job execution into the base image to create a custom image.
                  3. Publish the custom image to SoftWare Repository for Container (SWR).
                  4. On the DLI job editing page, select the created image and run the job.
                  5. Check the job execution status.
                  +
                  1. Obtain DLI base images.
                  2. Use Dockerfile to pack dependencies (files, JAR files, or software) required for job execution into the base image to create a custom image.
                  3. Publish the custom image to SWR.
                  4. On the DLI job editing page, select the created image and run the job.
                  5. Check the job execution status.

                  Obtaining DLI Base Images

                  Contact the administrator to obtain DLI base images.

                  Select the base image of the same type as the architecture of the queue.

                  -

                  For the CPU architecture type of a queue, see Viewing Basic Information About a Queue.

                  +

                  For details about the CPU architecture type of a queue, see Viewing Basic Information About a Queue.

                  Creating a Custom Image

                  The following describes how to package TensorFlow into an image to generate a custom image with TensorFlow installed. Then, you can use the image to run jobs in DLI.

                  1. Prepare the container environment.
                  2. Log in to the prepared container environment as user root and run a command to obtain the base image.

                    In this example, the Spark base image is used and downloaded to the container image environment in 1 by running the following command:

                    diff --git a/docs/dli/umn/dli_01_0498.html b/docs/dli/umn/dli_01_0498.html index 78cd5eca5..f6035b0a4 100644 --- a/docs/dli/umn/dli_01_0498.html +++ b/docs/dli/umn/dli_01_0498.html @@ -2,12 +2,12 @@

                    Creating a Flink OpenSource SQL Job

                    This section describes how to create a Flink OpenSource SQL job.

                    -

                    DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink provided by the community. In addition, Redis and GaussDB(DWS) data source types are added based on the community connector. For the syntax and constraints of Flink SQL DDL, DML, and functions, see Table API & SQL.

                    -

                    Prerequisites

                    • You have prepared the source and sink streams.
                    • A datasource connection has been created to enable the network between the queue where the job is about to run and external data sources.
                      • For details about the external data sources that can be accessed by Flink jobs, see Common Development Methods for DLI Cross-Source Analysis.
                      • For how to create a datasource connection, see Configuring the Network Connection Between DLI and Data Sources (Enhanced Datasource Connection).

                        On the Resources > Queue Management page, locate the queue you have created, click More in the Operation column, and select Test Address Connectivity to check if the network connection between the queue and the data source is normal. For details, see Testing the Network Connectivity Between a Queue and a Data Source.

                        +

                        DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink provided by the community. In addition, Redis and DWS data source types are added based on the community connector. For details about the syntax and constraints of Flink SQL DDL, DML, and functions, see Table API & SQL.

                        +

                        Prerequisites

                        -

                        Creating a Flink OpenSource SQL Job

                        1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                        2. In the upper right corner of the Flink Jobs page, click Create Job.

                          +

                          Creating a Flink OpenSource SQL Job

                          1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                          2. In the upper right corner of the Flink Jobs page, click Create Job.

                          3. Set job parameters.

                            @@ -52,7 +52,7 @@
                            Table 1 Job parameters

                            Parameter
                            -

                          4. Click OK to enter the editing page.
                          5. Edit an OpenSource SQL job.

                            Enter detailed SQL statements in the statement editing area. For details about SQL statements, see Data Lake Insight Flink OpenSource SQL Syntax Reference.

                            +

                          6. Click OK to enter the editing page.
                          7. Edit an OpenSource SQL job.

                            Enter detailed SQL statements in the statement editing area. For details about SQL statements, see Data Lake Insight Flink OpenSource SQL Syntax Reference.

                          8. Click Check Semantics.

                            • You can Start a job only after the semantic verification is successful.
                            • If verification is successful, the message "The SQL semantic verification is complete. No error." will be displayed.
                            • If verification fails, a red "X" mark will be displayed in front of each SQL statement that produced an error. You can move the cursor to the "X" mark to view error details and change the SQL statement as prompted.

                            Flink 1.15 does not support syntax verification.

                            @@ -83,22 +83,22 @@

                UDF JAR file, which contains UDFs that can be called in subsequent jobs.

                There are the following ways to manage UDF JAR files:

                -
                • Upload packages to OBS: Upload Jar packages to an OBS bucket in advance and select the corresponding OBS path.
                • Upload packages to DLI: Upload JAR files to an OBS bucket in advance and create a package on the Data Management > Package Management page of the DLI management console. For details, see Creating a DLI Package.
                +
                • Upload packages to OBS: Upload JAR files to an OBS bucket in advance and select the corresponding OBS path.
                • Upload packages to DLI: Upload JAR files to an OBS bucket in advance and create a package on the Data Management > Package Management page of the DLI management console. For details, see Creating a DLI Package.

                For Flink 1.15 or later, only OBS packages can be selected when creating jobs, and DLI packages are not supported.

                Agency

                If you choose Flink 1.15 or later to execute your job, you can create a custom agency to allow DLI to access other services.

                -

                For how to create a custom agency, see Creating a Custom DLI Agency.

                +

                For details about how to create a custom agency, see Creating a Custom DLI Agency.

                Resource Configuration Version

                DLI offers various resource configuration templates based on different Flink engine versions.

                Compared with the v1 template, the v2 template does not support the setting of the number of CUs. The v2 template supports the setting of Job Manager Memory and Task Manager Memory.

                -

                v1: applicable to Flink 1.12, 1.13, and 1.15.

                -

                v2: applicable to Flink 1.13, 1.15, and 1.17.

                +

                V1: applicable to Flink 1.12 and 1.15.

                +

                V2: applicable to Flink 1.15 and 1.17.

                You are advised to use the parameter settings of v2.

                For details about the parameters of v1, see Table 3.

                For details about the parameters of v2, see Table 4.

                @@ -106,7 +106,7 @@

                CUs

                Sum of the number of compute units and JobManager CUs of DLI. One CU equals 1 vCPU and 4 GB of memory.

                +

                Sum of the number of compute units and JobManager CUs of DLI. One CU equals one vCPU and 4 GB of memory.

                The value is the number of CUs required for job running and cannot exceed the number of CUs in the bound queue.

                NOTE:

                When Task Manager Config is selected, elastic resource pool queue management is optimized by automatically adjusting CUs to match Actual CUs after setting Slot(s) per TM.

                CUs = Actual number of CUs = max[Job Manager CPU + Task Manager CPU, (Job Manager Memory + Task Manager Memory/4)]

                @@ -117,6 +117,7 @@

                Job Manager CUs

                Number of JobManager CUs.

                +

                If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the JobManager's CU count does not exceed 2 to avoid resource scheduling failures during job execution.

                Parallelism

                @@ -129,7 +130,8 @@

                Task Manager Config

                Whether TaskManager resource parameters are set

                -
                • If selected, you need to set the following parameters:
                  • CU(s) per TM: Number of resources occupied by each TaskManager.
                  • Slot(s) per TM: Number of slots contained in each TaskManager.
                  +
                  • If selected, you need to set the following parameters:
                    • CU(s) per TM: Number of resources occupied by each TaskManager.

                      If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that a single TaskManager's CU count does not exceed 2 to avoid resource scheduling failures during job execution.

                      +
                    • Slot(s) per TM: Number of slots contained in each TaskManager.
                  • If not selected, the system automatically uses the default values.
                    • CU(s) per TM: The default value is 1.
                    • Slot(s) per TM: The default value is (Parallelism x CU(s) per TM)/(CUs – Job Manager CUs).
                  @@ -156,7 +158,7 @@

                Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                If this option is selected, you need to set the following parameters:

                SMN Topic

                -

                Select a custom SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                +

                Select a custom SMN topic. For details about how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                Enable Checkpointing

                @@ -204,7 +206,7 @@

                CUs

                Sum of the number of compute units and JobManager CUs of DLI. One CU equals 1 vCPU and 4 GB of memory.

                +

                Sum of the number of compute units and JobManager CUs of DLI. One CU equals one vCPU and 4 GB of memory.

                The value is the number of CUs required for job running and cannot exceed the number of CUs in the bound queue.

                NOTE:

                When Task Manager Config is selected, elastic resource pool queue management is optimized by automatically adjusting CUs to match Actual CUs after setting Slot(s) per TM.

                CUs = Actual number of CUs = max[Job Manager CPU + Task Manager CPU, (Job Manager Memory + Task Manager Memory/4)]

                @@ -215,6 +217,7 @@

                Job Manager CUs

                Number of JobManager CUs.

                +

                If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the JobManager's CU count does not exceed 2 to avoid resource scheduling failures during job execution.

                Parallelism

                @@ -227,24 +230,25 @@

                Task Manager Config

                Whether TaskManager resource parameters are set

                -
                • If selected, you need to set the following parameters:
                  • CU(s) per TM: Number of resources occupied by each TaskManager.
                  • Slot(s) per TM: Number of slots contained in each TaskManager.
                  +
                  • If selected, you need to set the following parameters:
                    • CU(s) per TM: Number of resources occupied by each TaskManager.

                      If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that a single TaskManager's CU count does not exceed 2 to avoid resource scheduling failures during job execution.

                      +
                    • Slot(s) per TM: Number of slots contained in each TaskManager.
                  -
                  • If not selected, the system automatically uses the default values.
                    • CU(s) per TM: The default value is 1.
                    • Slot(s) per TM: The default value is (Parallelism x CU(s) per TM)/(CUs – Job Manager CUs).
                    +
                    • If not selected, the system automatically uses the default values.
                      • CU(s) per TM: The default value is 1.
                      • Slot(s) per TM: The default value is (Parallelism x CU(s) per TM)/(CUs – Job Manager CUs).

                OBS Bucket

                OBS bucket to store job logs and checkpoint information. If the OBS bucket you selected is unauthorized, click Authorize.

                +

                OBS bucket to store job logs and checkpoint information. If the OBS bucket you selected is unauthorized, click Authorize.

                Save Job Log

                Whether job running logs are saved to OBS. The logs are saved in the following path: Bucket name/jobs/logs/Directory starting with the job ID.

                +

                Whether job running logs are saved to OBS. The logs are saved in the following path: Bucket name/jobs/logs/Directory starting with the job ID.

                CAUTION:

                You are advised to select this parameter. Otherwise, no run logs will be generated after the job is executed. If the job runs abnormally later, you will be unable to obtain the run logs for troubleshooting.

                If this option is selected, you need to set the following parameters:

                -
                OBS Bucket: Select an OBS bucket to store job logs. If the OBS bucket you selected is unauthorized, click Authorize.
                NOTE:

                If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                +
                OBS Bucket: Select an OBS bucket to store job logs. If the OBS bucket you selected is unauthorized, click Authorize.
                NOTE:

                If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                If this option is selected, you need to set the following parameters:

                -

                SMN Topic

                -

                Select a custom SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                +

                SMN Topic

                +

                Select a custom SMN topic. For details about how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                Enable Checkpointing

                Whether to enable job snapshots. If this function is enabled, jobs can be restored based on the checkpoints.

                -
                If this option is selected, you need to set the following parameters:
                • Checkpoint Interval: interval for creating checkpoints, in seconds. The value ranges from 1 to 999999, and the default value is 30.
                • Checkpoint Mode can be set to either of the following values:
                  • At least once: Events are processed at least once.
                  • Exactly once: Events are processed only once.
                  +
                  If this option is selected, you need to set the following parameters:
                  • Checkpoint Interval: interval for creating checkpoints, in seconds. The value ranges from 1 to 999999, and the default value is 30.
                  • Checkpoint Mode can be set to either of the following values:
                    • At least once: Events are processed at least once.
                    • Exactly once: Events are processed only once.
                  -
                  • OBS Bucket: Select an OBS bucket to store your checkpoints. If the OBS bucket you selected is unauthorized, click Authorize.
                    The checkpoint path is Bucket name/jobs/checkpoint/Directory starting with the job ID.
                    NOTE:

                    If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                    +
                    • OBS Bucket: Select an OBS bucket to store your checkpoints. If the OBS bucket you selected is unauthorized, click Authorize.
                      The checkpoint path is Bucket name/jobs/checkpoint/Directory starting with the job ID.
                      NOTE:

                      If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                    @@ -273,19 +277,19 @@

                Whether automatic restart is enabled. If enabled, jobs will be automatically restarted and restored when exceptions occur.

                If this option is selected, you need to set the following parameters:

                -
                • Max. Retry Attempts: maximum number of retries upon an exception. The unit is times/hour.
                  • Unlimited: The number of retries is unlimited.
                  • Limited: The number of retries is user-defined.
                  -
                • Restore Job from Checkpoint: This parameter is available only when Enable Checkpointing is selected.
                +
                • Max. Retry Attempts: maximum number of retries upon an exception. The unit is times/hour.
                  • Unlimited: The number of retries is unlimited.
                  • Limited: The number of retries is user-defined.
                  +
                • Restore Job from Checkpoint: This parameter is available only when Enable Checkpointing is selected.

                Idle State Retention Time

                Clears intermediate states of operators such as GroupBy, RegularJoin, Rank, and Depulicate that have not been updated after the maximum retention time. The default value is 1 hour.

                +

                Clears intermediate states of operators such as GroupBy, RegularJoin, Rank, and Depulicate that have not been updated after the maximum retention time. The default value is 1 hour.

                Dirty Data Policy

                Policy for processing dirty data. The following policies are supported: Ignore, Trigger a job exception, and Save.

                -

                If you set this field to Save, the Dirty Data Dump Address must be set. Click the address box to select the OBS path for storing dirty data.

                +

                Policy for processing dirty data. The following policies are supported: Ignore, Trigger a job exception, and Save.

                +

                If you set this field to Save, the Dirty Data Dump Address must be set. Click the address box to select the OBS path for storing dirty data.

                This parameter is available only when a DIS data source is used.

                Number of vCPUs available for JobManager.

                The default value is 1. The minimum value cannot be less than 0.5.

                +

                If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the JobManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

                Job Manager Memory

                @@ -322,6 +327,7 @@

                Number of vCPUs available for TaskManager.

                The default value is 1. The minimum value cannot be less than 0.5.

                +

                If the current job is running on a basic edition elastic resource pool (16–64 CUs), it is recommended that the TaskManager's CPU value does not exceed 2 to avoid resource scheduling failures during job execution.

                Task Manager Memory

                @@ -339,16 +345,16 @@

                OBS Bucket

                OBS bucket to store job logs and checkpoint information. If the OBS bucket you selected is unauthorized, click Authorize.

                +

                OBS bucket to store job logs and checkpoint information. If the OBS bucket you selected is unauthorized, click Authorize.

                Save Job Log

                Whether job running logs are saved to OBS. The logs are saved in the following path: Bucket name/jobs/logs/Directory starting with the job ID.

                +

                Whether job running logs are saved to OBS. The logs are saved in the following path: Bucket name/jobs/logs/Directory starting with the job ID.

                CAUTION:

                You are advised to select this parameter. Otherwise, no run logs will be generated after the job is executed. If the job runs abnormally later, you will be unable to obtain the run logs for troubleshooting.

                If this option is selected, you need to set the following parameters:

                -
                OBS Bucket: Select an OBS bucket to store job logs. If the OBS bucket you selected is unauthorized, click Authorize.
                NOTE:

                If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                +
                OBS Bucket: Select an OBS bucket to store job logs. If the OBS bucket you selected is unauthorized, click Authorize.
                NOTE:

                If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                If this option is selected, you need to set the following parameters:

                -

                SMN Topic

                -

                Select a custom SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                +

                SMN Topic

                +

                Select a custom SMN topic. For details about how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                Enable Checkpointing

                Whether to enable job snapshots. If this function is enabled, jobs can be restored based on the checkpoints.

                -
                If this option is selected, you need to set the following parameters:
                • Checkpoint Interval: interval for creating checkpoints, in seconds. The value ranges from 1 to 999999, and the default value is 30.
                • Checkpoint Mode can be set to either of the following values:
                  • At least once: Events are processed at least once.
                  • Exactly once: Events are processed only once.
                  +
                  If this option is selected, you need to set the following parameters:
                  • Checkpoint Interval: interval for creating checkpoints, in seconds. The value ranges from 1 to 999999, and the default value is 30.
                  • Checkpoint Mode can be set to either of the following values:
                    • At least once: Events are processed at least once.
                    • Exactly once: Events are processed only once.
                  -
                  • OBS Bucket: Select an OBS bucket to store your checkpoints. If the OBS bucket you selected is unauthorized, click Authorize.
                    The checkpoint path is Bucket name/jobs/checkpoint/Directory starting with the job ID.
                    NOTE:

                    If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                    +
                    • OBS Bucket: Select an OBS bucket to store your checkpoints. If the OBS bucket you selected is unauthorized, click Authorize.
                      The checkpoint path is Bucket name/jobs/checkpoint/Directory starting with the job ID.
                      NOTE:

                      If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                    @@ -377,19 +383,19 @@

                Whether automatic restart is enabled. If enabled, jobs will be automatically restarted and restored when exceptions occur.

                If this option is selected, you need to set the following parameters:

                -
                • Max. Retry Attempts: maximum number of retries upon an exception. The unit is times/hour.
                  • Unlimited: The number of retries is unlimited.
                  • Limited: The number of retries is user-defined.
                  -
                • Restore Job from Checkpoint: This parameter is available only when Enable Checkpointing is selected.
                +
                • Max. Retry Attempts: maximum number of retries upon an exception. The unit is times/hour.
                  • Unlimited: The number of retries is unlimited.
                  • Limited: The number of retries is user-defined.
                  +
                • Restore Job from Checkpoint: This parameter is available only when Enable Checkpointing is selected.

                Idle State Retention Time

                Clears intermediate states of operators such as GroupBy, RegularJoin, Rank, and Depulicate that have not been updated after the maximum retention time. The default value is 1 hour.

                +

                Clears intermediate states of operators such as GroupBy, RegularJoin, Rank, and Depulicate that have not been updated after the maximum retention time. The default value is 1 hour.

                Dirty Data Policy

                Policy for processing dirty data. The following policies are supported: Ignore, Trigger a job exception, and Save.

                -

                If you set this field to Save, the Dirty Data Dump Address must be set. Click the address box to select the OBS path for storing dirty data.

                +

                Policy for processing dirty data. The following policies are supported: Ignore, Trigger a job exception, and Save.

                +

                If you set this field to Save, the Dirty Data Dump Address must be set. Click the address box to select the OBS path for storing dirty data.

                This parameter is available only when a DIS data source is used.

                Elastic resource pool mode

                For SQL (Spark)

                -

                For SQL (HetuEngine)

                For general purpose

                Resources are shared among multiple queues for a single user.

                @@ -185,7 +184,7 @@

                Use Cases of Elastic Resource Pools

                Queues in an elastic resource pool are recommended, as they offer the flexibility to use resources with high utilization as needed. This part describes common use cases of elastic resource pools.

                Resources too fixed to meet a range of requirements.

                The quantities of compute resources required for jobs change in different time of a day. If the resources cannot be scaled based on service requirements, they may be wasted or insufficient. Figure 2 shows the resource usage during a day.

                -
                • During the data period from approximately 04:00 to 07:00 in the morning, there are no other jobs running after ETL jobs complete. Since the resources remain constantly occupied, it leads to significant resource wastage.
                • Between 09:00 a.m. to 12:00 p.m. and 14:00 p.m. to 16:00 p.m., the volume of ETL report and job query requests is high. Due to insufficient allocated resources, jobs end up queuing continuously.
                  Figure 2 Fixed resources
                  +
                  • During the data period from approximately 04:00 to 07:00 in the morning, there are no other jobs running after ETL jobs complete. Since the resources remain constantly occupied, it leads to significant resource wastage.
                  • Between 09:00 a.m. to 12:00 p.m. and 14:00 to 16:00, the volume of ETL report and job query requests is high. Due to insufficient allocated resources, jobs end up queuing continuously.
                    Figure 2 Fixed resources

                  Resources are isolated and cannot be shared.

                  A company has two departments with different jobs running on two queues of DLI. Department A experiences low activity from 8:00 a.m. to 12:00 p.m., leaving some resources unused, while department B faces high demand during this time, exceeding its current resource specifications. When scaling up is necessary, it cannot utilize the idle queue resources of department A, leading to resource wastage.
                  Figure 3 Resource waste due to resource isolation
                  diff --git a/docs/dli/umn/dli_01_0505.html b/docs/dli/umn/dli_01_0505.html index e487cb8c5..8793b9058 100644 --- a/docs/dli/umn/dli_01_0505.html +++ b/docs/dli/umn/dli_01_0505.html @@ -30,8 +30,13 @@

                  The scaling step may vary depending on the resource specifications, usually 16 CUs, 32 CUs, 48 CUs, 64 CUs, and more.

                  For example, if the elastic resource pool has a capacity of 192 CUs and the queues in the pool are using 68 CUs due to running jobs, the plan is to scale in to 64 CUs.

                  When executing a scaling in task, the system determines that there are 124 CUs remaining and scales in by the minimum step of 64 CUs. However, the remaining 60 CUs cannot be scaled in any further. Therefore, after the elastic resource pool executes the scaling in task, its capacity is reduced to 128 CUs.

                  +
                • If jobs such as Flink or Spark Streaming jobs are continuously running on a node, that node cannot be scaled in as scheduled.
                - +

                Constraints for setting elastic resource pool CUs

                +
                • The minimum CUs (minCU) of an elastic resource pool must be less than or equal to the actual CUs. If expanding minCU exceeds the current actual CUs, you must first increase the actual CUs. Otherwise, the modification will fail.
                • The sum of all queues' minimum CUs in an elastic resource pool must not exceed the pool's minCU.
                • Any single queue's maxCU cannot exceed the pool's maxCU.
                • Adjustments to a queue's CU range, changes to the pool's specifications, or modifications to the pool's CU settings take effect at the next full hour.
                • Increasing the number of queues to adjust the pool's actual CUs takes immediate effect.

                IPv6

                +

                After IPv6 is enabled, the elastic resource pool can communicate with data sources (the CIDR block of which have IPv6 enabled) using IPv6 addresses.

                +

                If IPv6 is enabled, an IPv6 CIDR block will be automatically allocated to the elastic resource pool.

                +
                • IPv6 can only be enabled during creation.
                • After enabling, both IPv4 and IPv6 addresses will be available, allowing for both private and public network access. IPv6 cannot be used independently.
                • The IPv6 CIDR block cannot be specified; it is allocated by the system.
                • IPv6 cannot be disabled after being enabled.
                +

                Enterprise Project

                If the created elastic resource pool belongs to an enterprise project, select the enterprise project.

                @@ -141,9 +153,7 @@

                Engine

                If Type is For SQL, the queue engine can be Spark or HetuEngine.

                -

                If HetuEngine is selected, the minimum number of CUs of the SQL queue cannot be fewer than 96 CUs.

                -

                To use HetuEngine to submit SQL jobs, you need to configure a DLI job bucket. For details, see Configuring a DLI Job Bucket.

                +

                If Type is For SQL, the queue engine can only be Spark.

                Enterprise Project

                @@ -200,8 +210,7 @@

                Min CU

                Minimum number of CUs allowed by the scaling policy.

                -
                • In any time segment of a day, the total minimum CUs of all queues in an elastic resource pool cannot be more than the minimum CUs of the pool.
                • If the minimum CUs of the queue are fewer than 16 CUs, both Max. Spark Driver Instances and Max. Prestart Spark Driver Instances set in the queue properties do not apply. Refer to Setting Queue Properties.
                -

                For a HetuEngine SQL queue, there must be at least 96 CUs.

                +
                • In any time segment of a day, the total minimum CUs of all queues in an elastic resource pool cannot be more than the minimum CUs of the pool.
                • If the minimum CUs of the queue are fewer than 16 CUs, both Max. Spark Driver Instances and Max. Prestart Spark Driver Instances set in the queue properties do not apply. See Setting Queue Properties.

                Max CU

                diff --git a/docs/dli/umn/dli_01_0506.html b/docs/dli/umn/dli_01_0506.html index 51c23b372..2adbdb63c 100644 --- a/docs/dli/umn/dli_01_0506.html +++ b/docs/dli/umn/dli_01_0506.html @@ -1,10 +1,10 @@

                Adjusting Scaling Policies for Queues in an Elastic Resource Pool

                -

                Multiple queues can be added to an elastic resource pool. For how to add a queue, see Creating an Elastic Resource Pool and Creating Queues Within It. You can configure the number of CUs you want based on the compute resources used by DLI queues during peaks and troughs and set priorities for the scaling policies to ensure stable running of jobs.

                -

                Precautions

                • You are advised to implement fine-grained management of resource pools for stream and batch processing jobs by placing Flink real-time stream jobs and SQL batch processing jobs in separate elastic resource pools.

                  Flink real-time stream jobs can run stably without forced scale-in, thus avoiding job interruption and system instability.

                  -

                  SQL batch processing jobs are placed in independent resource pools, which can scale out and in more flexibly, significantly enhancing the success rate and operational efficiency of scaling operations.

                  -
                • In any time segment of a day, the total minimum CUs of all queues in an elastic resource pool cannot be more than the minimum CUs of the pool.
                • In any time segment of a day, the maximum CUs of any queue in an elastic resource pool cannot be more than the maximum CUs of the pool.
                • The periods of scaling policies cannot overlap.
                • The period of a scaling policy can only be set by hour and specified by the start time and end time. For example, if you set the period to 00-09, the time range when the policy takes effect is [00:00, 09:00). The period of the default scaling policy cannot be modified.
                • In any period, compute resources are preferentially allocated to meet the minimum number of CUs of all queues. The remaining CUs (total CUs of the elastic resource pool – total minimum CUs of all queues) are allocated in accordance with the scaling policy priorities.
                • After the queue is scaled out, the system starts billing you for the added CUs. So, if you do not have sufficient requirements, scale in your queue to release unnecessary CUs to save cost. +

                  Elastic resource pools support multiple queues to manage job execution efficiently. For details about how to create queues in a pool, see Creating an Elastic Resource Pool and Creating Queues Within It. Once configured, you can define scaling policies based on each queue's compute resource usage patterns, such as peak and off-peak periods, and priority levels. This ensures optimal allocation of CUs to maintain stable and efficient job performance.

                  +

                  Precautions

                  • You are advised to implement fine-grained management by separating Flink real-time streaming jobs from SQL batch processing jobs into distinct elastic resource pools. This approach offers two key benefits:

                    Flink streaming jobs run continuously and require stability without forced scale-in, preventing interruptions and system instability.

                    +

                    SQL batch jobs benefit from flexible scaling within their dedicated pool, significantly improving scaling success rates and operational efficiency.

                    +
                  • At any time of day, the sum of minimum CUs across all queues in the elastic resource pool must not exceed the pool's minimum CUs.
                  • Similarly, at any given time, no single queue's maximum CU allocation should exceed the pool's maximum CUs.
                  • The time intervals for different scaling policies within the same queue must not overlap.
                  • Scaling policy periods are set in whole-hour increments, including the start time but excluding the end time. For example, a 00–09 interval covers [00:00, 09:00). Note that default scaling policies do not allow modifications to these time settings.
                  • During any period, the system first ensures all queues meet their minimum CU requirements. Any remaining CUs (calculated as the pool's maximum CUs minus the sum of all queues' minimum CUs) are allocated based on configured priorities until fully distributed.
                  • Once a queue successfully scales out, billing begins for the additional CUs and continues until scale-in occurs. To avoid unnecessary costs, promptly release resources when they are no longer needed, as unused CUs will still incur charges.
                    Table 1 CU allocation (without jobs)

                    Scenario

                    CUs

                    diff --git a/docs/dli/umn/dli_01_0512.html b/docs/dli/umn/dli_01_0512.html index 13d97f4a4..8c307d914 100644 --- a/docs/dli/umn/dli_01_0512.html +++ b/docs/dli/umn/dli_01_0512.html @@ -44,14 +44,14 @@

                    Step 1: Develop a JAR File and Upload It to OBS

                    Develop a JAR file offline as the DLI console does not support this capability. For development examples, refer to "Flink Jar Job Examples".

                    Develop a Flink Jar job program, compile it, and pack it into flink-examples.jar. Perform the following steps to upload the program:

                    Before submitting a Flink job, upload data files to OBS.

                    -
                    1. Log in to the DLI console.
                    2. In the service list, click Object Storage Service under Storage.
                    3. Create a bucket. In this example, name it dli-test-obs01.
                      1. On the displayed Buckets page, click Create Bucket in the upper right corner.
                      2. On the displayed Create Bucket page, set Bucket Name. Retain the default values for other parameters or modify them as needed.

                        Select a region that matches the location of the DLI console.

                        +
                        1. Log in to the management console.
                        2. In the service list, click Object Storage Service under Storage.
                        3. Create a bucket. In this example, name it dli-test-obs01.
                          1. On the displayed Buckets page, click Create Bucket in the upper right corner.
                          2. On the displayed Create Bucket page, set Bucket Name. Retain the default values for other parameters or modify them as needed.

                            Select a region that matches the location of the DLI console.

                          3. Click Create Now.
                        4. In the bucket list, click the name of the dli-test-obs01 bucket you just created to access its Objects tab.
                        5. Click Upload Object. In the displayed dialog box, drag or add files or folders, for example, flink-examples.jar, to the upload area. Then, click Upload.

                          In this example, the path after upload is obs://dli-test-obs01/flink-examples.jar.

                          -

                          For more operations on the OBS console, see Object Storage Service User Guide.

                          +

                          For more operations on the OBS console, see Object Storage Service User Guide.

                        -

                        Step 2: Buy an Elastic Resource Pool and Create Queues Within It

                        To execute SQL jobs in datasource scenarios, you must use your own SQL queue as the existing default queue cannot be used. In this example, create an elastic resource pool named dli_resource_pool and a queue named dli_queue_01.

                        +

                        Step 2: Buy an Elastic Resource Pool and Create Queues Within It

                        When running a Flink job in datasource scenarios, you cannot use the existing default queue. You need to create a general-purpose queue. In this example, the elastic resource pool dli_resource_pool and queue dli_queue_01 are created.

                        1. Log in to the DLI management console.
                        2. In the navigation pane on the left, choose Resources > Resource Pool.
                        3. On the displayed page, click Buy Resource Pool in the upper right corner.
                        4. On the displayed page, set the parameters.
                          Table 2 describes the parameters.
                          @@ -194,7 +194,7 @@
                        5. Click OK.

                          6. Step 3: Use DEW to Manage Access Credentials

                          In cross-source analysis scenarios, you need to set attributes such as the username and password in the connector. However, information such as usernames and passwords is highly sensitive and needs to be encrypted to ensure user data privacy.

                          -

                          DEW is a secure, reliable, and easy-to-use solution for encrypting and decrypting private data while ensuring its security.

                          +

                          DEW offers a secure, reliable, and easy-to-use solution for encrypting and decrypting sensitive data.

                          This example introduces how to create a shared secret in DEW.

                          1. Log in to the DEW management console.
                          2. In the navigation pane on the left, choose Cloud Secret Management Service > Secrets.
                          3. Click Create Secret. On the displayed page, configure basic secret information.
                            • In this example, the key in the first line is the user's access key ID (AK).
                            • In this example, the key in the second line is the user's secret access key (SK).
                          4. Set access credential parameters on the DLI Flink Jar job editing page.
                            flink.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.access.key=USER_AK_CSMS_KEY_obstest1
@@ -311,7 +311,7 @@ flink.dli.job.agency.name=agencyname
                            Yes
                          @@ -340,7 +340,7 @@ flink.dli.job.agency.name=agencyname

                          Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                          If this option is selected, you need to set the following parameters:

                          SMN Topic

                          -

                          Select a custom SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                          +

                          Select a custom SMN topic. For details about how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                          -
                          Table 2 Parameter descriptions

                          Parameter

                          Maximum number of parallel operators in a job

                          -
                          NOTE:
                          • The value cannot exceed four times the number of compute units (CUs – Job Manager CUs).
                          • You are advised to set this parameter to a value greater than that configured in the code. Otherwise, job submission may fail.
                          +
                          NOTE:
                          • The value cannot exceed four times the number of compute units (CUs – Job Manager CUs).
                          • Set this parameter to a value greater than that configured in the code. Otherwise, job submission may fail.

                          Auto Restart on Exception

                          diff --git a/docs/dli/umn/dli_01_0515.html b/docs/dli/umn/dli_01_0515.html index f8be039f9..edcc5c277 100644 --- a/docs/dli/umn/dli_01_0515.html +++ b/docs/dli/umn/dli_01_0515.html @@ -30,7 +30,7 @@

                          (Optional) Create an enhanced datasource connection.

                          If a job needs to access data from other data sources, for example, GaussDB(DWS) and RDS, you need to create a datasource connection.

                          +

                          If a job needs to access data from other data sources, for example, DWS and RDS, you need to create a datasource connection.

                          The created datasource connection must be bound to the elastic resource pool.

                          Creating an Enhanced Datasource Connection

                          @@ -52,7 +52,7 @@

                          For details about how to create an elastic resource pool, see Creating an Elastic Resource Pool and Creating Queues Within It.

                        6. Click Buy. Confirm the configuration and click Pay.
                        7. Go to the Resource Pool page to view the creation status. If the status is Available, the elastic resource pool is ready for use.
                          8. -

                          Step 2: Add a Queue to the Elastic Resource Pool

                          1. In the Operation column of the created elastic resource pool, click Add Queue.
                          2. Specify the basic information about the queue. The configuration parameters are as follows:
                            • Name: Queue name
                            • Type: Queue type In this example, select For general purpose.

                              For SQL: The queue is used to run Spark SQL jobs.

                              +

                              Step 2: Add a Queue to the Elastic Resource Pool

                              1. In the Operation column of the created elastic resource pool, click Add Queue.
                              2. Specify the basic information about the queue. The configuration parameters are as follows:
                                • Name: Queue name
                                • Type: Queue type In this example, select For general purpose.

                                  For SQL: The queue is used to run Spark SQL jobs.

                                  For general purpose: The queue is used to run Flink and Spark Jar jobs.

                                • Set other parameters as required.
                              3. Click Next. On the displayed page, set Min CU to 64 and Max CU to 64.
                              4. Click OK. The queue is added.
                              diff --git a/docs/dli/umn/dli_01_0525.html b/docs/dli/umn/dli_01_0525.html index 4fd6ed900..dfa6ba109 100644 --- a/docs/dli/umn/dli_01_0525.html +++ b/docs/dli/umn/dli_01_0525.html @@ -8,7 +8,7 @@
                              • Predefined tags: global tags created on Tag Management Service (TMS).

                                For details about predefined tags, see Tag Management Service User Guide.

                              DLI allows you to add, modify, or delete tags for queues.

                              -
                              1. In the left navigation pane of the DLI console, choose Resources > Resource Pool.
                              2. In the Operation column of the queue, choose More > Tags.
                              3. The tag management page is displayed, showing the tag information about the current queue.
                              4. On the page that appears, click Add/Edit Tag in the upper left corner. Enter a tag and a value, and click Add.

                                +

                                1. In the navigation pane of the DLI console, choose Resources > Resource Pool.
                                2. In the Operation column of the queue, choose More > Tags.
                                3. The tag management page is displayed, showing the tag information about the current queue.
                                4. On the page that appears, click Add/Edit Tag in the upper left corner. Enter a tag and a value, and click Add.

                                  Table 1 Tag parameters

                                  Parameter

                                  Description

                                  diff --git a/docs/dli/umn/dli_01_0531.html b/docs/dli/umn/dli_01_0531.html index 69c3dcd4e..51fb49d1e 100644 --- a/docs/dli/umn/dli_01_0531.html +++ b/docs/dli/umn/dli_01_0531.html @@ -57,10 +57,10 @@

                                  Step 1: Prepare a Source Stream

                                  In this example, Kafka is the source stream.

                                  -

                                  Enable DIS to import Kafka data to DLI. For details, see "Buying a Kafka Instance" in the Distributed Message Service Kafka User Guide.

                                  -
                                  1. Create the dependent Kafka resources.
                                    Before creating a Kafka instance, ensure the availability of resources, including a virtual private cloud (VPC), subnet, security group, and security group rules.
                                    • For how to create a VPC and subnet, see "Creating a VPC and Subnet" in the Virtual Private Cloud User Guide. For how to create and use a subnet in an existing VPC, see "Create a Subnet for the VPC" in the Virtual Private Cloud User Guide.
                                      • The created VPC and the Kafka instance you will create must be in the same region.
                                      • Retain the default settings unless otherwise specified.
                                      +

                                      Enable DIS to import Kafka data to DLI. For details, see "Buying a Kafka Instance" in the Distributed Message Service Kafka User Guide.

                                      +
                                      1. Create the dependent Kafka resources.
                                        Before creating a Kafka instance, ensure the availability of resources, including a virtual private cloud (VPC), subnet, security group, and security group rules.
                                        • For details about how to create a VPC and subnet, see "Creating a VPC with a Subnet" in the Virtual Private Cloud User Guide. For details about how to create and use a subnet in an existing VPC, see "Creating a Subnet for an Existing VPC" in the Virtual Private Cloud User Guide.
                                          • The created VPC and the Kafka instance you will create must be in the same region.
                                          • Retain the default settings unless otherwise specified.
                                          -
                                        • For how to create a security group, see "Creating a Security Group" in the Virtual Private Cloud User Guide. For how to add rules to a security group, see "Creating a Subnet for the VPC" in the Virtual Private Cloud User Guide.
                                        +
                                      2. For details about how to create a security group, see "Creating a Security Group" in the Virtual Private Cloud User Guide. For details about how to add rules to a security group, see "Adding a Security Group Rule" in the Virtual Private Cloud User Guide.
                                  2. Create a Kafka premium instance as the job source stream.
                                    1. Log in to the DMS for Kafka management console.
                                    2. Select a region in the upper left corner.
                                    3. On the DMS for Kafka page, click Buy Instance in the upper right corner and set related parameters. The required instance information is as follows:
                                      Table 2 Parameters for buying a DMS for Kafka instance

                                      Parameter

                                      @@ -180,7 +180,7 @@

                                      Step 2: Prepare a Sink Stream

                                      Use RDS for MySQL as the data sink stream and create an RDS for MySQL DB instance.

                                      -
                                      1. Log in to the RDS management console.
                                      2. Select a region in the upper left corner.
                                      3. Click Buy DB Instance in the upper right corner of the page and set related parameters. Retain the default values for other parameters.

                                        For the parameters, see "RDS for MySQL Getting Started" in the Relational Database Service Getting Started.

                                        +
                                        1. Log in to the RDS management console.
                                        2. Select a region in the upper left corner.
                                        3. Click Buy DB Instance in the upper right corner of the page and set related parameters. Retain the default values for other parameters.

                                          For details about the parameters, see "RDS for MySQL Getting Started" in the Relational Database Service Getting Started.

                                          @@ -275,7 +275,7 @@ @@ -364,8 +364,8 @@

                                          Step 3: Create an OBS Bucket to Store Output Data

                                          In this example, you need to enable OBS for job JobSample to provide DLI Flink jobs with the functions of checkpointing, saving job logs, and commissioning test data.

                                          -

                                          For how to create a bucket, see "Creating a Bucket" in the Object Storage Service Console Operation Guide.

                                          -
                                          1. In the navigation pane on the OBS management console, choose Object Storage.
                                          2. In the upper right corner of the page, click Create Bucket and set bucket parameters. +

                                            For details about how to create a bucket, see "Creating a Bucket" in the Object Storage Service User Guide.

                                            +
                                            1. In the navigation pane on the OBS management console, choose Object Storage.
                                            2. In the upper right corner of the page, click Create Bucket and configure bucket parameters.
                                          Table 3 RDS for MySQL instance parameters

                                          Parameter

                                          VPC and Subnet

                                          Select an existing VPC and subnet.

                                          -

                                          For how to recreate a VPC and subnet, refer to "Creating a VPC and Subnet" in the Virtual Private Cloud User Guide.

                                          +

                                          For details about how to recreate a VPC and subnet, see "Creating a VPC with a Subnet" in the Virtual Private Cloud User Guide.

                                          NOTE:

                                          In datasource scenarios, the CIDR block of the data source cannot overlap that of the elastic resource pool.
                                          Table 4 OBS bucket parameters

                                          Parameter

                                          Description

                                          @@ -415,7 +415,7 @@
                                        4. Click Create Now.
                                          5. -

                                          Step 4: Create an Elastic Resource Pool and Create Queues Within It

                                          To create a Flink OpenSource SQL job, you must use your own queue as the existing default queue cannot be used. In this example, create an elastic resource pool named dli_resource_pool and a queue named dli_queue_01.

                                          +

                                          Step 4: Create an Elastic Resource Pool and Create Queues Within It

                                          When creating a Flink OpenSource SQL job, you cannot use the existing default queue. You need to create a general-purpose queue. In this example, the elastic resource pool dli_resource_pool and queue dli_queue_01 are created.

                                          1. Log in to the DLI management console.
                                          2. In the navigation pane on the left, choose Resources > Resource Pool.
                                          3. On the displayed page, click Buy Resource Pool in the upper right corner.
                                          4. On the displayed page, set the parameters.
                                            Table 5 describes the parameters.
                                            @@ -557,12 +557,12 @@
                                          5. Click OK.
                                            6. -

                                            Step 5: Create an Enhanced Datasource Connection Between DLI and Kafka

                                            You need to create an enhanced datasource connection for the Flink OpenSource SQL job. For details, see "Datasource Connections" > "Creating an Enhanced Datasource Connection" in the Data Lake Insight User Guide.

                                            +

                                            Step 5: Create an Enhanced Datasource Connection Between DLI and Kafka

                                            You need to create an enhanced datasource connection for the Flink OpenSource SQL job. For details, see "Datasource Connections" > "Creating an Enhanced Datasource Connection" in the Data Lake Insight User Guide.

                                            • The CIDR block of the DLI queue bound with a datasource connection cannot overlap with the CIDR block of the data source.
                                            • Datasource connections cannot be created for the default queue.
                                            • To access a table across data sources, you need to use a queue bound to a datasource connection.
                                            1. Create a Kafka security group rule to allow access from the CIDR block of the DLI queue.

                                              1. On the Kafka management console, click an instance name on the DMS for Kafka page. Basic information of the Kafka instance is displayed.
                                              2. In the Connection pane, obtain the Instance Address (Private Network). In the Network pane, obtain the VPC and subnet of the instance.
                                              3. Click the security group name in the Network pane. On the displayed page, click the Inbound Rules tab and add a rule to allow access from the DLI queue.

                                                For example, if the CIDR block of the queue is 10.0.0.0/16, set Protocol to TCP, Type to IPv4, Source to 10.0.0.0/16, and click OK.

                                              -

                                            2. Create an enhanced datasource connection to Kafka.

                                              1. Log in to the DLI management console. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Create in the Enhanced tab.
                                              2. In the displayed dialog box, set the following parameters: For details, see the following section:
                                                • Connection Name: Name of the enhanced datasource connection For this example, enter dli_kafka.
                                                • Resource Pool: Select the elastic resource pool created in Step 4: Create an Elastic Resource Pool and Create Queues Within It.
                                                • VPC: Select the VPC of the Kafka instance.
                                                • Subnet: Select the subnet of Kafka instance.
                                                • Set other parameters as you need.
                                                +

                                              3. Create an enhanced datasource connection to Kafka.

                                                1. Log in to the DLI management console. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Create in the Enhanced tab.
                                                2. In the dialog box that appears, configure the parameters as follows:
                                                  • Connection Name: Name of the enhanced datasource connection For this example, enter dli_kafka.
                                                  • Resource Pool: Select the elastic resource pool created in Step 4: Create an Elastic Resource Pool and Create Queues Within It.
                                                  • VPC: Select the VPC of the Kafka instance.
                                                  • Subnet: Select the subnet of Kafka instance.
                                                  • Set other parameters as you need.

                                                  Click OK. Click the name of the created datasource connection to view its status. You can perform subsequent steps only after the connection status changes to Active.

                                                3. Choose Resources > Queue Management and locate the queue created in Step 4: Create an Elastic Resource Pool and Create Queues Within It. In the Operation column, click More and select Test Address Connectivity.
                                                4. In the displayed dialog box, enter Kafka instance address (private network):port in the Address box and click Test to check whether the instance is reachable. Note that multiple addresses must be tested separately.

                                              @@ -570,7 +570,7 @@

                                              Step 6: Create an Enhanced Datasource Connection Between DLI and RDS

                                              1. Create an RDS security group rule to allow access from CIDR block of the DLI queue.

                                                If the RDS DB instance and Kafka instance are in the same security group of the same VPC, skip this step. Access from the DLI queue has been allowed in 1.
                                                1. Go to the RDS console, click the name of the target RDS for MySQL DB instance on the Instances page. Basic information of the instance is displayed.
                                                2. In the Connection Information pane, obtain the floating IP address, database port, VPC, and subnet.
                                                3. Click the security group name. On the displayed page, click the Inbound Rules tab and add a rule to allow access from the DLI queue. For example, if the CIDR block of the queue is 10.0.0.0/16, set Priority to 1, Action to Allow, Protocol to TCP, Type to IPv4, Source to 10.0.0.0/16, and click OK.

                                              2. Create an enhanced datasource connection to RDS.

                                                If the RDS DB instance and Kafka instance are in the same VPC and subnet, skip this step. The enhanced datasource connection created in 2 has connected the subnet.

                                                -
                                                If the two instances are in different VPCs or subnets, perform the following steps to create an enhanced datasource connection:
                                                1. Log in to the DLI management console. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Create in the Enhanced tab.
                                                2. In the displayed dialog box, set the following parameters: For details, see the following section:
                                                  • Connection Name: Name of the enhanced datasource connection For this example, enter dli_rds.
                                                  • Resource Pool: Select the name of the queue created in Step 4: Create an Elastic Resource Pool and Create Queues Within It.
                                                  • VPC: Select the VPC of the RDS DB instance.
                                                  • Subnet: Select the subnet of RDS DB instance.
                                                  • Set other parameters as you need.
                                                  +
                                                  If the two instances are in different VPCs or subnets, perform the following steps to create an enhanced datasource connection:
                                                  1. Log in to the DLI management console. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Create in the Enhanced tab.
                                                  2. In the dialog box that appears, configure the parameters as follows:
                                                    • Connection Name: Name of the enhanced datasource connection For this example, enter dli_rds.
                                                    • Resource Pool: Select the name of the queue created in Step 4: Create an Elastic Resource Pool and Create Queues Within It.
                                                    • VPC: Select the VPC of the RDS DB instance.
                                                    • Subnet: Select the subnet of RDS DB instance.
                                                    • Set other parameters as you need.

                                                    Click OK. Click the name of the created datasource connection to view its status. You can perform subsequent steps only after the connection status changes to Active.

                                                  3. Choose Resources > Queue Management and locate the queue created in Step 4: Create an Elastic Resource Pool and Create Queues Within It. In the Operation column, click More and select Test Address Connectivity.
                                                  4. In the displayed dialog box, enter Floating IP address:Database port of the RDS for MySQL DB instance in the Address box and click Test to check if the instance is reachable.
                                                  @@ -578,14 +578,15 @@

                                                Step 7: Use DEW to Manage Access Credentials and Create an Agency That Allows DLI to Access DEW

                                                In cross-source analysis scenarios, you need to set attributes such as the username and password in the connector. However, information such as usernames and passwords is highly sensitive and needs to be encrypted to ensure user data privacy.

                                                Flink 1.15 allows for the use of DEW to manage credentials. Before running a job, create a custom agency and configure agency information within the job.

                                                -

                                                Data Encryption Workshop (DEW) and Cloud Secret Management Service (CSMS) joint form a secure, reliable, and easy-to-use privacy data encryption and decryption solution. This example describes how a Flink OpenSource SQL job uses DEW to manage RDS access credentials.

                                                +

                                                Data Encryption Workshop (DEW) and Cloud Secret Management Service (CSMS) offer a secure, reliable, and easy-to-use solution for encrypting and decrypting sensitive data.

                                                +

                                                This example describes how a Flink OpenSource SQL job uses DEW to manage RDS access credentials.

                                                1. Create an agency for DLI to access DEW and complete authorization.
                                                2. Create a shared secret in DEW.
                                                3. Log in to the DEW management console.
                                                4. In the navigation pane on the left, choose Cloud Secret Management Service > Secrets.
                                                5. Click Create Secret. On the displayed page, configure basic secret information.
                                                  • Secret Name: Enter a secret name. In this example, the name is secretInfo.
                                                  • Secret Value: Enter the username and password for logging in to the RDS for MySQL DB instance.
                                                    • The key in the first line is MySQLUsername, and the value is the username for logging in to the DB instance.
                                                    • The key in the second line is MySQLPassword, and the value is the password for logging in to the DB instance.
                                                6. Set other parameters as required and click OK.

                                                Step 8: Create a Flink OpenSource SQL Job

                                                After the source and sink streams are prepared, you can create a Flink OpenSource SQL job.

                                                -
                                                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                2. In the upper right corner of the Flink Jobs page, click Create Job.

                                                  Table 8 describes the parameters required for this example. For other parameters, retain the default values.

                                                  +
                                                  1. In the navigation pane of the DLI management console, choose Job Management > Flink Jobs.
                                                  2. In the upper right corner of the Flink Jobs page, click Create Job.

                                                    Table 8 describes the parameters required for this example. For other parameters, retain the default values.

                                            Table 5 Parameter descriptions

                                            Parameter
                                            diff --git a/docs/dli/umn/dli_01_0538.html b/docs/dli/umn/dli_01_0538.html index d56cc0fac..10ac26cec 100644 --- a/docs/dli/umn/dli_01_0538.html +++ b/docs/dli/umn/dli_01_0538.html @@ -14,13 +14,15 @@
                                          6. Use Cases
                                            7. -
                                          7. Notes and Constraints
                                            +
                                          8. Features
                                          9. Compute Resource Types and Specifications
                                          10. Permission Management
                                            11. -
                                          11. Quotas
                                            +
                                          12. Notes and Constraints
                                            +
                                            13. +
                                          13. Quota Management
                                          14. Related Services
                                            15. diff --git a/docs/dli/umn/dli_01_0550.html b/docs/dli/umn/dli_01_0550.html index 355e7e2fe..f61d8dc9e 100644 --- a/docs/dli/umn/dli_01_0550.html +++ b/docs/dli/umn/dli_01_0550.html @@ -1,17 +1,17 @@

                                            Managing DLI Resource Quotas

                                            -

                                            What Is a Quota?

                                            A quota limits the quantity of a resource available to users, thereby preventing spikes in the usage of the resource.

                                            -

                                            You can also request for an increased quota if your existing quota cannot meet your service requirements.

                                            +

                                            What Is a Quota?

                                            Quotas are enforced for service resources on the platform to prevent unforeseen spikes in resource usage. Quotas can limit the quantity and capacity of resources available to users.

                                            +

                                            If your current quota does not meet your needs, you can apply for an increase.

                                            -

                                            How Do I View My Quotas?

                                            1. Log in to the management console.
                                            2. Click in the upper left corner and select a region and a project.
                                            3. Click the My Quota icon in the upper right corner of the page.

                                              The Service Quota page is displayed.

                                              -
                                            4. View the used and total quota of each type of resources on the displayed page.

                                              If a quota cannot meet service requirements, increase a quota.

                                              +

                                              How Do I View My Quotas?

                                              1. Log in to the management console.
                                              2. Click in the upper left corner and select a region and a project.
                                              3. Click the My Quota icon in the upper right corner of the page.

                                                This will take you to the Service Quota page.

                                                +
                                              4. Here, you can view the total quota and usage details for each resource.

                                                If your current quota is insufficient, follow the steps below to request an increase.

                                              -

                                              How Do I Apply for a Higher Quota?

                                              The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your application and inform you of the progress by phone call or email.

                                              -

                                              Before dialing the hotline number or sending an email, ensure that the following information has been obtained:

                                              +

                                              How Do I Apply for a Quota Increase?

                                              The system currently does not support online quota adjustments. If you need to modify your quota, please contact our customer service team via phone or email. They will promptly process your request and keep you updated on its progress through either a call or email.

                                              +

                                              Before reaching out, ensure you have the following details ready:

                                              • Domain name, project name, and project ID

                                                To obtain the preceding information, log in to the management console, click the username in the upper right corner, and choose My Credentials from the drop-down list.

                                                -
                                              • Quota information, including:
                                                • Service name
                                                • Quota type
                                                • Required quota
                                                +
                                              • Quota information, including:
                                                • Service name
                                                • Quota type
                                                • Desired quota value

                                              Learn how to obtain the service hotline and email address.

                                              diff --git a/docs/dli/umn/dli_01_0552.html b/docs/dli/umn/dli_01_0552.html index 9b3b33211..00b5ff4bb 100644 --- a/docs/dli/umn/dli_01_0552.html +++ b/docs/dli/umn/dli_01_0552.html @@ -52,7 +52,7 @@
                                            - @@ -30,7 +30,7 @@

                                            Datasource Authentication Types

                                            DLI supports four types of datasource authentication. Select an authentication type specific to each data source.

                                            -
                                            • CSS: applies to 6.5.4 or later CSS clusters with the security mode enabled. During the configuration, you need to specify the username, password, and authentication certificate of the cluster and store the information in DLI through datasource authentication so that DLI can securely access CSS data sources. For details, see Creating a CSS Datasource Authentication.
                                            • Kerberos: applies to MRS security clusters with Kerberos authentication enabled. During the configuration, you need to specify MRS cluster authentication credentials, including the krb5.conf and user.keytab files. For details, see Creating a Kerberos Datasource Authentication.
                                            • Kafka_SSL: applies to Kafka with SSL enabled. During the configuration, you need to specify the KafkaTruststore path and password. For details, see Creating a Kafka_SSL Datasource Authentication.
                                            • Password: applies to GaussDB(DWS), RDS, DDS, and DCS data sources. During the configuration, you need to store the passwords of the data sources in DLI. For details, see Creating a Password Datasource Authentication.
                                            +
                                            • CSS: applies to 6.5.4 or later CSS clusters with the security mode enabled. During the configuration, you need to specify the username, password, and authentication certificate of the cluster and store the information in DLI through datasource authentication so that DLI can securely access CSS data sources. For details, see Creating a CSS Datasource Authentication.
                                            • Kerberos: applies to MRS security clusters with Kerberos authentication enabled. During the configuration, you need to specify MRS cluster authentication credentials, including the krb5.conf and user.keytab files. For details, see Creating a Kerberos Datasource Authentication.
                                            • Kafka_SSL: applies to Kafka with SSL enabled. During the configuration, you need to specify the KafkaTruststore path and password. For details, see Creating a Kafka_SSL Datasource Authentication.
                                            • Password: applies to DWS, RDS, DDS, and DCS data sources. During the configuration, you need to store the passwords of the data sources in DLI. For details, see Creating a Password Datasource Authentication.

                                            Jobs That Can Connect to Data Sources Through Datasource Authentication

                                            Different types of jobs can connect to data sources through different types of datasource authentication.

                                            • For details about the data sources that Spark SQL jobs can connect to through datasource authentication and their constraints, see Table 2.
                                            • For details about the data sources that Flink SQL jobs can connect to through datasource authentication and their constraints, see Table 3.
                                            @@ -53,7 +53,7 @@
                                            - @@ -115,7 +115,7 @@ - diff --git a/docs/dli/umn/dli_01_0566.html b/docs/dli/umn/dli_01_0566.html index 7003ab8ff..d087d1746 100644 --- a/docs/dli/umn/dli_01_0566.html +++ b/docs/dli/umn/dli_01_0566.html @@ -1,16 +1,16 @@

                                            Allocating to an Enterprise Project

                                            -

                                            You can create enterprise projects matching the organizational structure of your enterprises to centrally manage cloud resources across regions by project. Then you can create user groups and users with different permissions and add them to enterprise projects.

                                            -

                                            DLI allows you to select an enterprise project when creating an elastic resource pool. This section describes how to bind an elastic resource pool to and modify an enterprise project.

                                            -

                                            Modifying the enterprise project of an elastic resource pool will modify the enterprise projects of the queues in the elastic resource pool.

                                            -

                                            Only queues under the same enterprise project can be bound to an elastic resource pool.

                                            +

                                            An enterprise project is a cloud resource management approach that allows organizations to plan resources based on their organizational structure. It enables unified management of resources distributed across different regions under specific enterprise projects. Additionally, user groups and users with varying permissions can be assigned to each enterprise project.

                                            +

                                            DLI allows you to select an enterprise project when creating an elastic resource pool. This section explains how to bind or modify the enterprise project for a DLI elastic resource pool.

                                            +

                                            Changing the enterprise project of an elastic resource pool will also update the enterprise project of its associated queue resources.

                                            +

                                            Here, the elastic resource pool only supports adding queues from the same enterprise project.

                                            Prerequisites

                                            You have logged in to the Enterprise Project Management Service console and created an enterprise project.

                                            Binding an Enterprise Project

                                            When creating an elastic resource pool, you can select a created enterprise project for Enterprise Project.

                                            Alternatively, you can click Create Enterprise Project to go to the Enterprise Project Management Service console to create an enterprise project and check existing ones.

                                            -

                                            For how to create a queue, see Creating an Elastic Resource Pool and Creating Queues Within It.

                                            +

                                            For details about how to create a queue, see Creating an Elastic Resource Pool and Creating Queues Within It.

                                            Modifying an Enterprise Project

                                            You can modify the enterprise project bound to a created cluster as needed.

                                            1. Log in to the DLI management console.
                                            2. In the navigation pane on the left, choose Resources > Resource Pool.
                                            3. In the elastic resource pool list, locate the elastic resource pool for which you want to modify the enterprise project, click More in the Operation column, and select Allocate to Enterprise Project.
                                            4. In the Modify Enterprise Project dialog box displayed, select an enterprise project.

                                              Alternatively, you can click Create Enterprise Project to go to the Enterprise Project Management Service console to create an enterprise project and check existing ones.

                                              diff --git a/docs/dli/umn/dli_01_0612.html b/docs/dli/umn/dli_01_0612.html index ab80b14a5..45bd3b6eb 100644 --- a/docs/dli/umn/dli_01_0612.html +++ b/docs/dli/umn/dli_01_0612.html @@ -89,7 +89,6 @@
                                            - @@ -60,7 +60,7 @@

                                            Notes and Constraints

                                            • The custom agency name cannot be the default agency names dli_admin_agency, dli_management_agency, or dli_data_clean_agency. It must be unique.
                                            • The name of the agency that allows DLI to clear data according to the lifecycle of a table must be dli_data_clean_agency.
                                            • Custom agencies can be configured only for jobs executed by engines like Flink 1.15, Spark 3.3.1 (Spark general queue scenario), or later versions.
                                            • Once the agency permissions are updated, your dli_admin_agency will be upgraded to dli_management_agency. This new agency will have the necessary permissions for datasource operations, notifications, and user authorization operations. If you have other agency permission needs, you will need to create custom agencies. For details about dli_management_agency, see DLI Agency Overview.
                                            • Common scenarios for creating an agency: DLI is allowed to read and write data from and to OBS, dump logs, and read and write Flink checkpoints. DLI is allowed to access DEW to obtain data access credentials and access catalogs to obtain metadata. For details about agency permissions in these scenarios, see Agency Permission Policies in Common Scenarios.

                                            Step 1: Create a Cloud Service Agency on the IAM Console and Grant Permissions

                                            1. Log in to the management console.
                                            2. In the upper right corner of the page, hover over the username and select Identity and Access Management.
                                            3. In the navigation pane of the IAM console, choose Agencies.
                                            4. On the displayed page, click Create Agency.
                                            5. On the Create Agency page, set the following parameters:
                                              • Agency Name: Enter an agency name, for example, dli_obs_agency_access.
                                              • Agency Type: Select Cloud service.
                                              • Cloud Service: This parameter is available only when you select Cloud service for Agency Type. Select Data Lake Insight (DLI) from the drop-down list.
                                              • Validity Period: Select Unlimited.
                                              • Description: You can enter Agency with OBS OperateAccess permissions. This parameter is optional.
                                              -
                                            6. Click Next.
                                            7. Click the agency name. On the displayed page, click the Permissions tab. Click Authorize. On the displayed page, click Create Policy.
                                            8. Configure policy information.
                                              1. Enter a policy name, for example, dli-obs-agency.
                                              2. Select JSON.
                                              3. In the Policy Content area, paste a custom policy.

                                                In this example, the permissions allow access and usage of OBS in various scenarios. For DLI Flink jobs, this includes downloading OBS objects, obtaining OBS/GaussDB(DWS) data sources (foreign tables), transferring logs, using savepoints, and enabling checkpointing. For DLI Spark jobs, the permissions allow downloading OBS objects and reading/writing OBS foreign tables.

                                                +
                                              4. Click Next.
                                              5. Click the agency name. On the displayed page, click the Permissions tab. Click Authorize. On the displayed page, click Create Policy.
                                              6. Configure policy information.
                                                1. Enter a policy name, for example, dli-obs-agency.
                                                2. Select JSON.
                                                3. In the Policy Content area, paste a custom policy.

                                                  In this example, the permissions allow access and usage of OBS in various scenarios. For DLI Flink jobs, this includes downloading OBS objects, obtaining OBS/DWS data sources (foreign tables), transferring logs, using savepoints, and enabling checkpointing. For DLI Spark jobs, the permissions allow downloading OBS objects and reading/writing OBS foreign tables.

                                                  For how to configure common agency permissions for Flink jobs, see Agency Permission Policies in Common Scenarios.
                                                  {
     "Version": "1.1",
     "Statement": [
@@ -109,7 +109,7 @@

                                            Step 2: Set Agency Permissions for a Job

                                            When Spark 3.3.1, Flink 1.15, or a later version is used to execute jobs, you need to add information about the new agency to the job configuration.

                                            -

                                            Otherwise, If you do not specify an agency for Spark 3.3.1 jobs, the jobs cannot use OBS. If you do not specify an agency for a Flink 1.15 job, checkpointing cannot be enabled, savepoints cannot be used, logs cannot be transferred, and data sources such as OBS and GaussDB(DWS) cannot be used.

                                            +

                                            Otherwise, if you do not specify an agency for Spark 3.3.1 jobs, the jobs cannot use OBS. If you do not specify an agency for a Flink 1.15 job, checkpointing cannot be enabled, savepoints cannot be used, logs cannot be transferred, and data sources such as OBS and DWS cannot be used.

                                            • You can only specify an agency for Flink 1.15 and Spark 3.3.1 jobs running on queues in an elastic resource pool.
                                            • After specifying an agency for a job, be careful when modifying the permissions granted to the agency. Any changes made may impact the job's normal operation.
                                            • Specifying an agency for a Flink Jar job
                                              1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                                              2. Select a desired job and click Edit in the Operation column.
                                              3. In the job configuration area on the right, configure agency information.
                                                • Flink Version: Select 1.15.
                                                • Agency: Select the agency created in Step 1: Create a Cloud Service Agency on the IAM Console and Grant Permissions.

                                                  In this example, set it to dli_obs_agency_access.

                                                  diff --git a/docs/dli/umn/dli_01_0617.html b/docs/dli/umn/dli_01_0617.html index 3f8d3b83c..349f96747 100644 --- a/docs/dli/umn/dli_01_0617.html +++ b/docs/dli/umn/dli_01_0617.html @@ -31,13 +31,14 @@ "dli:table:showPartitions", "dli:table:select", "dli:table:dropTable", + "dli:table:alter", "dli:table:alterTableDropPartition" ] } ] }
                                            -

                                            Permission Policies for Accessing and Using OBS

                                            Application scenario: For DLI Flink jobs, the permissions include downloading OBS objects, obtaining OBS/GaussDB(DWS) data sources (foreign tables), transferring logs, using savepoints, and enabling checkpointing. For DLI Spark jobs, the permissions allow downloading OBS objects and reading/writing OBS foreign tables.

                                            +

                                            Permission Policies for Accessing and Using OBS

                                            Application scenario: For DLI Flink jobs, the permissions include downloading OBS objects, obtaining OBS/DWS data sources (foreign tables), transferring logs, using savepoints, and enabling checkpointing. For DLI Spark jobs, the permissions allow downloading OBS objects and reading/writing OBS foreign tables.

                                            {
     "Version": "1.1",
     "Statement": [
@@ -137,6 +138,7 @@
                 "dli:table:alterTableRecoverPartition",
                 "dli:table:dropTable",
                 "dli:table:update",
+                "dli:table:alter",
                 "dli:table:alterTableDropPartition"
             ]
         }
diff --git a/docs/dli/umn/dli_01_0618.html b/docs/dli/umn/dli_01_0618.html
index 7c21009f5..4aed93db5 100644
--- a/docs/dli/umn/dli_01_0618.html
+++ b/docs/dli/umn/dli_01_0618.html
@@ -57,7 +57,7 @@
 
                                            • When Flink 1.15, Spark 3.3.1 (Spark general queue scenario), or a later version is used to execute jobs, you need to create an agency on the IAM console.
                                            • If the engine version is earlier than Flink 1.15, dli_admin_agency is used by default during job execution. If the engine version is earlier than Spark 3.3.1, user authentication information (AK/SK and security token) is used during job execution.
                                              This means that jobs whose engine versions are earlier than Flink 1.15 or Spark 3.3.1 are not affected by the update of agency permissions and do not require custom agencies.
                                              
 
                                            
 
                                            Common service scenarios where you need to create an agency:
                                            
-
                                            • Data cleanup agency required for clearing data according to the lifecycle of a table and clearing lakehouse table data. You need to create a DLI agency named dli_data_clean_agency on IAM and grant permissions to it. You need to create an agency and customize permissions for it. However, the agency name is fixed to dli_data_clean_agency.
                                            • Tenant Administrator permissions are required to access data from OBS to execute Flink jobs on DLI, for example, obtaining OBS data sources, log dump (including bucket authorization), checkpointing enabling, and job import and export.
                                            • The AK/SK required by DLI Flink jobs is stored in DEW. To allow DLI to access DEW data during job execution, you need to create an agency to delegate the permissions to operate on DEW data to DLI.
                                            • To allow DLI to access DLI catalogs to retrieve metadata when executing jobs, you need to create an agency that grants DLI catalog data operation permissions to DLI. This will enable DLI to access DLI catalogs on your behalf.
                                            • Cloud data required by DLI Flink jobs is stored in LakeFormation. To allow DLI to access catalogs to retrieve metadata during job execution, you need to create an agency to delegate the permissions to operate on catalog data to DLI.
                                            
+
                                            • Data cleanup agency required for clearing data according to the lifecycle of a table and clearing lakehouse table data. You need to create a DLI agency named dli_data_clean_agency on IAM and grant permissions to it. You need to create an agency and customize permissions for it. However, the agency name is fixed to dli_data_clean_agency.
                                            • Tenant Administrator permissions are required to access data from OBS to execute Flink jobs on DLI, for example, obtaining OBS data sources, log dump (including bucket authorization), checkpointing enabling, and job import and export.
                                            • The AK and SK required for DLI Flink jobs are securely stored in DEW. To enable DLI to access DEW data during job execution, you need to create an agency that grants DLI the necessary permissions to operate on DEW data. This agency allows DLI to access DEW on your behalf.
                                            • To allow DLI to access DLI catalogs to retrieve metadata when executing jobs, you need to create an agency that grants DLI catalog data operation permissions to DLI. This will enable DLI to access DLI catalogs on your behalf.
                                            • Cloud data required for DLI Flink jobs is securely stored in LakeFormation. To enable DLI to access catalogs to retrieve metadata during job execution, you need to create an agency that grants DLI the necessary permissions to operate on catalog data. This agency allows DLI to access catalog metadata on your behalf.
                                            
 
                                            When creating an agency, you cannot use the default agency names dli_admin_agency, dli_management_agency, or dli_data_clean_agency. It must be unique.
                                            
 
                                            For more information about custom agency operations, see Creating a Custom DLI Agency and Agency Permission Policies in Common Scenarios.
                                            diff --git a/docs/dli/umn/dli_01_0622.html b/docs/dli/umn/dli_01_0622.html index 5deef41a4..de21c0d8a 100644 --- a/docs/dli/umn/dli_01_0622.html +++ b/docs/dli/umn/dli_01_0622.html @@ -4,12 +4,12 @@

                                            After creating an elastic resource pool, you can check and manage it on the management console.

                                            This section describes how to check basic information about an elastic resource pool on the management console, including the VPC CIDR block, IPv6 CIDR block, and creation time.

                                            Procedure

                                            1. Log in to the DLI management console.
                                            2. In the navigation pane on the left, choose Resources > Resource Pool.
                                            3. On the displayed page, select the elastic resource pool you want to check.
                                              • In the upper right corner of the list page, click to customize the columns to display and set the rules for displaying the table content and the Operation column.
                                              • In the search box above the list, you can filter the required elastic resource pool by name or tag.
                                              -
                                            4. Click to expand the basic information card of the elastic resource pool and view detailed information about the pool.

                                              These details include the name of the pool, the user who created it, the date it was created, and the VPC CIDR block.

                                              -

                                              For the definitions of actual CUs, used CUs, CU range, and specifications of the elastic resource pool, refer to Actual CUs, Used CUs, CU Range, and Specifications of an Elastic Resource Pool.

                                              +
                                            5. Click to expand the basic information card of the elastic resource pool and view detailed information about the pool.

                                              These details include the name of the pool, the user who created it, the date it was created, whether IPv6 is enabled, and the VPC CIDR block. If IPv6 is enabled, the subnet's IPv6 CIDR block will also be displayed.

                                              +

                                              For details about the definitions of actual CUs, used CUs, CU range, and yearly/monthly CUs (specifications) of the elastic resource pool, see Actual CUs, Used CUs, CU Range, and Yearly/Monthly CUs (Specifications) of an Elastic Resource Pool.

                                            -

                                            Actual CUs, Used CUs, CU Range, and Specifications of an Elastic Resource Pool

                                            • Actual CUs: actual size of resources currently allocated to the elastic resource pool (in CUs).
                                              • When there is no queue in the resource pool, the actual CUs are equal to the minimum CUs when the elastic resource pool is created.
                                              • When there are queues in the resource pool, the formula for calculating actual CUs is:
                                                • Actual CUs = max{(min[sum(maximum CUs of queues), maximum CUs of the elastic resource pool]), minimum CUs of the elastic resource pool}.
                                                • The calculation result must be a multiple of 16 CUs. If it cannot be exactly divided by 16 CUs, round up to the nearest multiple.
                                                -
                                              • Scaling out or in an elastic resource pool means adjusting the actual CUs of the resource pool. Refer to Scaling Out or In an Elastic Resource Pool.
                                              • Example of actual CU allocation:

                                                In Table 1, the calculation process for the actual allocation of CUs in an elastic resource pool is as follows:

                                                +

                                                Actual CUs, Used CUs, CU Range, and Yearly/Monthly CUs (Specifications) of an Elastic Resource Pool

                                                • Actual CUs: The current allocated resource size of the elastic resource pool, measured in CUs.
                                                  • When no queues exist in the resource pool: The actual CUs equal the minimum CUs set during its creation.
                                                  • When there are queues in the resource pool, the formula to calculate actual CUs is:
                                                    • Actual CUs = max{(min[sum(maximum CUs of queues), maximum CUs of the elastic resource pool]), minimum CUs of the elastic resource pool}.
                                                    • The result must be a multiple of 16 CUs. If not divisible by 16, round up to the nearest multiple.
                                                    +
                                                  • Scaling out or in an elastic resource pool means adjusting its actual CUs. See Scaling Out or In an Elastic Resource Pool.
                                                  • Example of actual CU allocation:

                                                    Consider Table 1 below, which illustrates the process of calculating actual CUs for an elastic resource pool:

                                                    1. Calculate the sum of maximum CUs of the queues: sum(maximum CUs) = 32 + 56 = 88 CUs.
                                                    2. Compare the sum of maximum CUs of the queues with the maximum CUs of the elastic resource pool and take the smaller value: min{88 CUs, 112 CUs} = 88 CUs.
                                                    3. Compare the value with the minimum CUs of the elastic resource pool and take the larger value: max(88 CUs, 64 CUs) = 88 CUs.
                                                    4. Check if 88 CUs is a multiple of 16 CUs. Since 88 is not divisible by 16, round up to 96 CUs.
                                            Table 8 Job parameters

                                            Parameter

                                            Tag key

                                            You can specify the tag key in either of the following ways:

                                            -
                                            • Click the text box for tag key and select a predefined tag key from the drop-down list.

                                              To add a predefined tag, you need to create one on TMS and then select it from the Tag key drop-down list. You can click View predefined tags to go to the Predefined Tags page of the TMS console. Then, click Create Tag in the upper corner of the page to create a predefined tag.

                                              +
                                              • Click the text box for tag key and select a predefined tag key from the drop-down list.

                                                To add a predefined tag, you need to create one on TMS and then select it from the Tag key drop-down list. You can click View predefined tags to go to the Predefined Tags page of the TMS console. Then, click Create Tag in the upper corner of the page to create a predefined tag.

                                              • Enter a tag key in the text box.
                                                NOTE:

                                                A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                diff --git a/docs/dli/umn/dli_01_0559.html b/docs/dli/umn/dli_01_0559.html index b86b6a072..3d343eda1 100644 --- a/docs/dli/umn/dli_01_0559.html +++ b/docs/dli/umn/dli_01_0559.html @@ -1,7 +1,7 @@

                                                Creating a Password Datasource Authentication

                                                -

                                                Scenario

                                                Create a password datasource authentication on the DLI console to store passwords of the GaussDB(DWS), RDS, DCS, and DDS data sources to DLI. This will allow you to access to the data sources without having to configure a username and password in SQL jobs.

                                                +

                                                Scenario

                                                Create a password datasource authentication on the DLI console to store passwords of the DWS, RDS, DCS, and DDS data sources to DLI. This will allow you to access to the data sources without having to configure a username and password in SQL jobs.

                                                Procedure

                                                1. Create a datasource authentication.
                                                  1. Log in to the DLI management console.
                                                  2. Choose Datasource Connections. On the page displayed, click Datasource Authentication.
                                                  3. Click Create.

                                                    Configure authentication parameters according to Table 1.

                                                    diff --git a/docs/dli/umn/dli_01_0561.html b/docs/dli/umn/dli_01_0561.html index 51650bae3..d689164ba 100644 --- a/docs/dli/umn/dli_01_0561.html +++ b/docs/dli/umn/dli_01_0561.html @@ -2,7 +2,7 @@

                                                    Overview

                                                    What Is Datasource Authentication?

                                                    When analyzing across multiple sources, you are advised not to configure authentication information directly in a job as it can lead to password leakage. Instead, you are advised to use either DEW or datasource authentication provided by DLI to securely store data source authentication information.

                                                    -
                                                    • DEW is a comprehensive cloud-based encryption service designed to address challenges related to data security, key security, and the complexities of key management. You are advised to use DEW to store authentication information for data sources.

                                                      You are advised to use DEW to store authentication information of data sources when Spark 3.3.1 or later and Flink 1.15 or later jobs access data sources using datasource connections. This will help you address issues related to data security, key security, and complex key management. For details, see Managing Data Source Access Credentials Using DEW.

                                                      +
                                                      • DEW is designed to address critical challenges such as data security, key security, and the complexities of key management. You are advised to use DEW to securely store authentication credentials for your data sources.

                                                        For cross-source access scenarios involving Spark 3.3.1 or later versions, as well as Flink 1.15 or later versions, we strongly advise using DEW to manage your data source authentication information. This approach ensures robust solutions for data security, key protection, and streamlined key management. For details, see Managing Data Source Access Credentials Using DEW.

                                                      • Datasource authentication is used to manage authentication information for accessing specified data sources. After datasource authentication is configured, you do not need to repeatedly configure data source authentication information in jobs, improving data source authentication security while enabling DLI to securely access data sources.

                                                        When SQL and Flink 1.12 jobs access data sources using datasource connections, use DLI's datasource authentication feature to manage data source access credentials.

                                                      This section describes how to use datasource authentication provided by DLI.

                                                      @@ -21,7 +21,7 @@

                                            Datasource authentication type

                                            • DLI supports four types of datasource authentication. Select an authentication type specific to each data source.
                                              • CSS: applies to 6.5.4 or later CSS clusters with the security mode enabled.
                                              • Kerberos: applies to MRS security clusters with Kerberos authentication enabled.
                                              • Kafka_SSL: applies to Kafka with SSL enabled.
                                              • Password: applies to GaussDB(DWS), RDS, DDS, and DCS.
                                              +
                                            • DLI supports four types of datasource authentication. Select an authentication type specific to each data source.
                                              • CSS: applies to 6.5.4 or later CSS clusters with the security mode enabled.
                                              • Kerberos: applies to MRS security clusters with Kerberos authentication enabled.
                                              • Kafka_SSL: applies to Kafka with SSL enabled.
                                              • Password: applies to DWS, RDS, DDS, and DCS.

                                            Password

                                            GaussDB(DWS), RDS, DDS, and Redis

                                            +

                                            DWS, RDS, DDS, and Redis

                                            -

                                            Password

                                            GaussDB(DWS), RDS, and CSS

                                            +

                                            DWS, RDS, and CSS

                                            -

                                            Dynamic

                                            Spark

                                            -

                                            HetuEngine

                                            Info

                                            Block

                                            diff --git a/docs/dli/umn/dli_01_0616.html b/docs/dli/umn/dli_01_0616.html index 3105f2db1..b094a7649 100644 --- a/docs/dli/umn/dli_01_0616.html +++ b/docs/dli/umn/dli_01_0616.html @@ -28,7 +28,7 @@

                                            Custom

                                            For DLI Flink jobs, the permissions include downloading OBS objects, obtaining OBS/GaussDB(DWS) data sources (foreign tables), transferring logs, using savepoints, and enabling checkpointing. For DLI Spark jobs, the permissions allow downloading OBS objects and reading/writing OBS foreign tables.

                                            +

                                            For DLI Flink jobs, the permissions include downloading OBS objects, obtaining OBS/DWS data sources (foreign tables), transferring logs, using savepoints, and enabling checkpointing. For DLI Spark jobs, the permissions allow downloading OBS objects and reading/writing OBS foreign tables.

                                            Permission Policies for Accessing and Using OBS
                                            @@ -43,8 +43,8 @@ -
                                          15. Used CUs: CUs that have been used by jobs or tasks. These resources may be executing computing tasks.
                                          16. CU range: CU settings are used to control the maximum and minimum CU ranges for elastic resource pools to avoid unlimited resource scaling.
                                            • The total minimum CUs of all queues in an elastic resource pool must be no more than the minimum CUs of the pool.
                                            • The maximum CUs of any queue in an elastic resource pool must be no more than the maximum CUs of the pool.
                                            • An elastic resource pool should at least ensure that all queues in it can run with the minimum CUs and should try to ensure that all queues in it can run with the maximum CUs.
                                            • When expanding the specifications of an elastic resource pool, the minimum value of the CU range is linked to the specifications of the elastic resource pool. After changing the specifications of the elastic resource pool, the minimum value of the CU range is modified to match the specifications.
                                            -
                                          17. Specifications: The minimum CUs selected during elastic resource pool purchase are elastic resource pool specifications.
                                            18. +
                                          18. Used CUs: The portion of CUs currently occupied by jobs or tasks, which may be actively performing computations.
                                          19. CU range: CU settings are used to control the maximum and minimum CU ranges for elastic resource pools to avoid unlimited resource scaling.
                                            • The sum of all queues' minimum CUs in an elastic resource pool must not exceed the pool's minCU.
                                            • Any single queue's maxCU cannot exceed the pool's maxCU.
                                            • The resource pool ensures it meets the minCU requirements across all queues while striving to accommodate their maxCU demands.
                                            • When expanding the specifications of an elastic resource pool, the minimum value of the CU range is linked to the yearly/monthly CUs (specifications) of the elastic resource pool. After changing the specifications of the elastic resource pool, the minimum value of the CU range is modified to match the yearly/monthly CUs (specifications).
                                            +
                                          20. Yearly/monthly CUs (specifications): The minimum value of the CU range selected when purchasing an elastic resource pool is the elastic resource pool specifications.
                                            21. diff --git a/docs/dli/umn/dli_01_0623.html b/docs/dli/umn/dli_01_0623.html index 6cf257324..c9a5b1c09 100644 --- a/docs/dli/umn/dli_01_0623.html +++ b/docs/dli/umn/dli_01_0623.html @@ -2,10 +2,10 @@

                                            Viewing Basic Information About an Enhanced Datasource Connection

                                            After creating an enhanced datasource connection, you can view and manage it on the management console.

                                            -

                                            This section describes how to view basic information about an enhanced datasource connection on the management console, including the enhanced datasource connection's host information and more.

                                            +

                                            This section describes how to view basic information about an enhanced datasource connection on the management console, including the enhanced datasource connection's host information, IPv6 support, and more.

                                            Procedure

                                            1. Log in to the DLI management console.
                                            2. In the navigation pane on the left, choose Datasource Connections.
                                            3. On the displayed Enhanced tab, locate the enhanced datasource connection whose basic information you want to view.
                                              • In the upper right corner of the list page, click to customize the columns to display and set the rules for displaying the table content and the Operation column.
                                              • In the search box above the list, you can filter the required enhanced datasource connection by name or tag.
                                            4. Click to expand details about the enhanced datasource connection.

                                              You can view the following information:

                                              - +
                                              • IPv6 Support: If you selected a subnet with IPv6 enabled when creating the enhanced datasource connection, then your enhanced datasource connection will support IPv6.
                                              • Host Information: When accessing an MRS HBase cluster, you need to configure the host name (domain name) and the corresponding IP address of the instance. For details, see Modifying Host Information in an Elastic Resource Pool.
                                            diff --git a/docs/dli/umn/dli_01_0632.html b/docs/dli/umn/dli_01_0632.html index a8a193ed0..942382a7e 100644 --- a/docs/dli/umn/dli_01_0632.html +++ b/docs/dli/umn/dli_01_0632.html @@ -10,7 +10,7 @@
                                            - diff --git a/docs/dli/umn/dli_01_0636.html b/docs/dli/umn/dli_01_0636.html index 8c837a32c..bc8618d81 100644 --- a/docs/dli/umn/dli_01_0636.html +++ b/docs/dli/umn/dli_01_0636.html @@ -4,7 +4,13 @@
                                            diff --git a/docs/dli/umn/dli_01_0638.html b/docs/dli/umn/dli_01_0638.html index 06d7b4d64..144498699 100644 --- a/docs/dli/umn/dli_01_0638.html +++ b/docs/dli/umn/dli_01_0638.html @@ -3,7 +3,7 @@

                                            Viewing a SQL Execution Plan

                                            A SQL execution plan is a logical flowchart of a database query that shows how a database management system executes a specific SQL query. The execution plan details the steps needed to execute the query, such as table scans, index lookups, join operations (for example, inner join, outer join), sorting, and aggregation. Viewing an execution plan can help analyze query performance, identify potential performance bottlenecks, understand the query's execution logic, and use this information to adjust the query or database structure to improve SQL query efficiency.

                                            This section describes how to view a SQL execution plan on the DLI management console.

                                            -

                                            Notes and Constraints

                                            • You can only view SQL execution plans for Spark 3.3.x or later queues and HetuEngine queues.
                                            • You can only view a SQL execution plan after a SQL job is executed.
                                            • You can only view the SQL execution plan for SQL jobs that have reached the Finished state.
                                            • Make sure you have authorized DLI to use OBS buckets for saving the SQL execution plans of user jobs.
                                            • SQL execution plans are stored in paid storage buckets for DLI jobs. The system does not automatically delete them. You are advised to configure the bucket lifecycle and specify rules to regularly delete or migrate unused SQL execution plans. Refer to Configuring a DLI Job Bucket.
                                            +

                                            Notes and Constraints

                                            • You can only view SQL execution plans for Spark 3.3.x or later queues.
                                            • You can only view a SQL execution plan after a SQL job is executed.
                                            • You can only view the SQL execution plan for SQL jobs that have reached the Finished state.
                                            • Make sure you have authorized DLI to use OBS buckets for saving the SQL execution plans of user jobs.
                                            • SQL execution plans are stored in paid storage buckets for DLI jobs. The system does not automatically delete them. You are advised to configure the bucket lifecycle and specify rules to regularly delete or migrate unused SQL execution plans. Refer to Configuring a DLI Job Bucket.

                                            Procedure

                                            1. Log in to the DLI management console.
                                            2. In the navigation pane on the left, choose Job Management > SQL Jobs.
                                            3. Select the SQL job you want to query.
                                            4. At the bottom of the page, click the name of the job you selected to view its details.

                                              In the details area, click Expand next to SQL Execution Plan. The system queries the SQL execution plan of the job from the DLI job bucket and displays the plan on the console.

                                              If the SQL execution plan in the DLI job bucket is deleted, the plan may not be displayed because the source file is missing.

                                              diff --git a/docs/dli/umn/dli_01_0639.html b/docs/dli/umn/dli_01_0639.html index c280cb78f..58e904e50 100644 --- a/docs/dli/umn/dli_01_0639.html +++ b/docs/dli/umn/dli_01_0639.html @@ -8,7 +8,7 @@
                                            5. Check the query result on the View Result tab below the editing window.

                                            This example uses the default queue and database preset in the system as an example. You can also run the command in a self-created queue and database.

                                            -

                                            For details, see "Creating a Queue" in the Data Lake Insight User Guide. For how to create a database, see "Data Management" > "Databases and Tables" > "Creating a Database" in the Data Lake Insight User Guide.

                                            +

                                            For details, see "Creating a Queue" in the Data Lake Insight User Guide. For details about how to create a database, see "Data Management" > "Databases and Tables" > "Creating a Database" in the Data Lake Insight User Guide.

                                            diff --git a/docs/dli/umn/dli_01_0687.html b/docs/dli/umn/dli_01_0687.html index 87c61ec9c..dc03d9c6f 100644 --- a/docs/dli/umn/dli_01_0687.html +++ b/docs/dli/umn/dli_01_0687.html @@ -1,13 +1,20 @@ -

                                            Overview

                                            -

                                            When submitting Flink or Spark jobs through DLI to access external data sources (such as OBS and Kafka), there is a risk of plaintext exposure if AK/SK, usernames/passwords are directly embedded in the job code or parameter configurations.

                                            -

                                            To securely store data source access credentials, ensure data source authentication safety, and facilitate secure access to data sources by DLI, you are advised to use DEW for managing data source access credentials. DLI employs "agency + temporary credentials" to safely retrieve data source access credentials.

                                            -

                                            DEW is a comprehensive cloud-based encryption service designed to address challenges related to data security, key security, and the complexities of key management.

                                            -

                                            This section describes how to use DEW to store data source authentication information across various job types.

                                            -

                                            -

                                            Notes and Constraints

                                            You are advised to use DEW for storing data source authentication information exclusively when Spark 3.3.1 or later and Flink 1.15 or later jobs access data sources using datasoure connections.

                                            -

                                            When SQL and Flink 1.12 jobs access data sources using datasource connections, use DLI's datasource authentication feature to manage data source access credentials. For details, see Using DLI Datasource Authentication to Manage Access Credentials for Data Sources.

                                            +

                                            Using an Agency to Obtain Access Credentials in DLI

                                            +

                                            When you use DLI to perform big data processing or cross-service data queries, authentication is a critical prerequisite to ensure legitimate access and data security. Embedding AK/SK or usernames and passwords directly in job code or configuration files introduces the risk of plaintext credential leakage.

                                            + +

                                            To address diverse access scenarios and authentication protocol requirements, DLI provides the following solutions:

                                            +

                                            DLI Agency with DEW Temporary Credentials and Permanent AK/SK

                                            +

                                            This solution is suitable for services that support only fine-grained authorization (version = 1.1) or workloads that require highly stable credentials without frequent rotation. It provides reliable authentication for these scenarios.

                                            +

                                            In this approach, DLI uses agencies and temporary credentials to access the DEW service. DEW then provides permanent AK/SK, which are used to securely access other cloud services.

                                            +

                                            Related guidance:

                                            + +

                                            Use Cases

                                            • Address security risks caused by hard-coded credentials.
                                            • Avoid embedding sensitive information, such as data source usernames and passwords, in job code.
                                            • Enable dynamic credential acquisition and periodic credential rotation.
                                            +
                                            +

                                            Notes and Constraints

                                            You are advised to use DEW for storing data source authentication information exclusively when Spark 3.3.1 or later and Flink 1.15 or later jobs access data sources using datasource connections.

                                            +

                                            When SQL and Flink 1.12 jobs access data sources using datasource connections, use DLI's datasource authentication feature to manage data source access credentials. For details, see Overview.

                                            +
                                            +

                                            Learn More: What Are Temporary Security Credentials?

                                            A temporary security credential grants temporary access rights. It includes a temporary AK/SK and a security token, both of which must be used together.

                                            diff --git a/docs/dli/umn/dli_01_0698.html b/docs/dli/umn/dli_01_0698.html new file mode 100644 index 000000000..2f185744e --- /dev/null +++ b/docs/dli/umn/dli_01_0698.html @@ -0,0 +1,51 @@ + + +

                                            Features

                                            +

                                            This section outlines the key features supported by DLI. For detailed information on regional availability of each feature, you can refer to the console.

                                            +

                                            Elastic Resource Pools and Queues

                                            Before submitting jobs using DLI, you need to prepare the necessary compute resources.

                                            +
                                            +
                                            • Elastic Resource Pool

                                              An elastic resource pool provides the required compute resources (CPU and memory) for running DLI jobs. It offers robust computing power, high availability, and flexible resource management capabilities. This makes it ideal for large-scale computing tasks and business use cases that require long-term resource planning. Additionally, it can dynamically adapt to changing demands for compute resources.

                                              +
                                            • Queue

                                              After creating an elastic resource pool, you can create multiple queues within it. Each queue is associated with specific jobs and data processing tasks, serving as the fundamental unit for allocating and utilizing resources in the pool. In other words, a queue represents the actual compute resources needed to execute a job.

                                              +

                                              Within the same elastic resource pool, compute resources are shared among queues. By configuring appropriate allocation policies for these queues, you can optimize the utilization of compute resources.

                                              +
                                            • default Queue

                                              DLI comes pre-configured with a default queue named default, where resources are allocated on-demand. If you are unsure about the required queue capacity or lack available space to create queues, you can use this default queue to run your jobs. However, since the default queue is shared among all users, there may be instances of resource contention. As a result, access to resources cannot always be guaranteed for every operation.

                                              +
                                            +

                                            DLI Metadata Management

                                            DLI metadata serves as the foundation for developing SQL jobs and Spark jobs. Before executing a job, you need to define databases and tables based on your business requirements.

                                            +

                                            In addition to managing its own metadata, DLI supports integration with LakeFormation for unified metadata management. This enables seamless connectivity with various compute engines and big data cloud services, making it efficient and convenient to build data lakes and operate related businesses.

                                            +
                                            • DLI Metadata

                                              DLI metadata serves as the foundation for developing SQL jobs and Spark jobs. Prior to running a job, you must define databases and tables according to your specific use case. Data catalog: A data catalog is a metadata management object that can contain multiple databases. You can create and manage multiple catalogs in DLI to isolate different sets of metadata.

                                              +
                                              • Database

                                                A database is a structured repository built on computer storage devices used to organize, store, and manage data. It typically stores, retrieves, and manages structured data, consisting of multiple interrelated data tables connected through keys and indexes.

                                                +
                                              • Table

                                                Tables are one of the most critical components of a database, composed of rows and columns. Each row represents a data entry, while each column defines an attribute or characteristic of the data. Tables are used to organize and store specific types of data, enabling efficient query and analysis. While a database provides the framework, tables constitute its actual content—a single database may include one or more tables.

                                                +
                                              • Metadata

                                                Metadata refers to data that describes other data. It primarily includes information about the source, size, format, or other characteristics of the data itself. In the context of database fields, metadata helps interpret the contents of a data warehouse. When creating a table, metadata is defined by specifying three elements: column names, data types, and column descriptions.

                                                +
                                              +
                                            +
                                            • Connecting DLI to LakeFormation for Metadata Management

                                              After creating an elastic resource pool, you can create multiple queues within it. Each queue is associated with specific jobs and data processing tasks, serving as the fundamental unit for allocating and utilizing resources in the pool. In other words, a queue represents the actual compute resources needed to execute a job. Within the same elastic resource pool, compute resources are shared among queues. By configuring appropriate allocation policies for these queues, you can optimize the utilization of compute resources.

                                              +
                                            +
                                            +

                                            DLI SQL Job

                                            DLI SQL jobs, also known as DLI Spark SQL jobs, allow you to execute data queries and other operations by executing SQL statements in the SQL editor. It supports SQL:2003 and is fully compatible with Spark SQL.

                                            +
                                            +

                                            DLI Spark Job

                                            Spark is a unified analytics engine designed for large-scale data processing, focusing on query, computation, and analysis. DLI has undergone extensive performance optimization and service-oriented enhancements over the open-source Spark, maintaining compatibility with the Apache Spark ecosystem and APIs while boosting performance by 2.5 times, enabling exabyte-scale data queries and analyses within hours.

                                            +
                                            +

                                            DLI Flink Job

                                            DLI Flink jobs are specifically designed for real-time data stream processing, making them ideal for scenarios that require low latency and quick response. They support cross-source connectivity with various cloud services, forming a robust streaming ecosystem. These jobs are ideal for applications such as real-time monitoring and online analytics.

                                            +
                                            • Flink OpenSource Job

                                              DLI provides standard connectors and a rich set of APIs, facilitating seamless integration with other data systems.

                                              +
                                            • Flink Jar Job

                                              You can submit Flink jobs compiled into JAR files, offering greater flexibility and customization capabilities. This option is well-suited for complex data processing scenarios requiring custom functions, user-defined functions (UDFs), or specific library integrations. Leveraging Flink's ecosystem, advanced stream processing logic and state management can be achieved.

                                              +
                                            • Flink Python Job

                                              Starting from Flink 1.17, DLI introduces support for PyFlink jobs, providing you with more flexible and powerful data processing tools. You can directly submit PyFlink jobs through the DLI job management page. Additionally, it allows specifying third-party Python and Java dependencies, customizing Python virtual environments, and uploading compressed data files, significantly enhancing the convenience of job submission and execution flexibility.

                                              +

                                              Additionally, the DLI Flink image comes pre-installed with default Python execution environments supporting versions 3.7, 3.8, 3.9, and 3.10, meeting diverse development needs. This empowers Python developers to efficiently use DLI for data processing and analysis tasks.

                                              +

                                              Flink Python jobs are particularly suitable for scenarios involving customized stream processing logic, complex state management, or specific library integrations. You are required to write and build Python job packages independently. Before submitting a Flink Python job, upload the Jar job package to OBS and submit it along with the data and job parameters to execute the job.

                                              +
                                            +
                                            +

                                            Datasource Connection

                                            Before performing cross-source analysis using DLI, you need to establish a datasource connection to enable network communication between data sources.

                                            +

                                            DLI's enhanced datasource connections use VPC peering connections to directly connect the VPC networks of DLI queues and destination data sources. This point-to-point approach facilitates seamless data exchange, offering more flexible use cases and superior performance compared to basic datasource connections.

                                            +

                                            Note: The system's default queue does not support creating datasource connections. Establishing such connections requires functionalities like VPCs, subnets, routing, and VPC peering connections. Therefore, you must have the VPC Administrator permission for VPC. You can set these permissions by referring to "Service Authorization".

                                            +
                                            +

                                            Permission Management

                                            DLI is equipped with a robust permission control mechanism. Additionally, DLI supports fine-grained authentication through Identity and Access Management (IAM). You can create IAM policies to manage access controls within DLI. Both permission control mechanisms can operate simultaneously without conflict.

                                            +
                                            +

                                            Custom DLI Agency

                                            To perform cross-source analysis, DLI requires agency permissions to access other cloud services. This allows DLI to act on behalf of users or services in other cloud services, enabling it to read/write data and execute specific operations during job execution. Custom DLI agency ensures secure and efficient access to other cloud services during cross-source analysis.

                                            +
                                            +

                                            Custom Image

                                            DLI supports containerized cluster deployments. In these clusters, components related to Spark and Flink jobs run within containers. By downloading custom images provided by DLI, you can modify the runtime environment of Spark and Flink containers. For example, adding Python packages or C libraries for machine learning into the custom image enables easy functional expansion tailored to your needs.

                                            +
                                            +
                                            +
                                            +
                                            +
                                            Parent topic: Service Overview
                                            +
                                            +
                                            + diff --git a/docs/dli/umn/dli_03_0031.html b/docs/dli/umn/dli_03_0031.html index c4e80e760..00ec98467 100644 --- a/docs/dli/umn/dli_03_0031.html +++ b/docs/dli/umn/dli_03_0031.html @@ -1,8 +1,8 @@

                                            How Do I View My Quotas?

                                            -
                                            1. Log in to the management console.
                                            2. Click in the upper left corner and select a region and a project.
                                            3. Click the My Quota icon in the upper right corner of the page.

                                              The Service Quota page is displayed.

                                              -
                                            4. View the used and total quota of each type of resources on the displayed page.

                                              If a quota cannot meet service requirements, increase a quota.

                                              +
                                              1. Log in to the management console.
                                              2. Click in the upper left corner and select a region and a project.
                                              3. Click the My Quota icon in the upper right corner of the page.

                                                This will take you to the Service Quota page.

                                                +
                                              4. Here, you can view the total quota and usage details for each resource.

                                                If your current quota is insufficient, follow the steps below to request an increase.

                                              diff --git a/docs/dli/umn/dli_03_0032.html b/docs/dli/umn/dli_03_0032.html index 62901e820..fe6c39fb3 100644 --- a/docs/dli/umn/dli_03_0032.html +++ b/docs/dli/umn/dli_03_0032.html @@ -1,10 +1,10 @@

                                              How Do I Apply for a Higher Quota?

                                              -

                                              How Do I Apply for a Higher Quota?

                                              The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your application and inform you of the progress by phone call or email.

                                              -

                                              Before dialing the hotline number or sending an email, ensure that the following information has been obtained:

                                              +

                                              How Do I Apply for a Quota Increase?

                                              The system currently does not support online quota adjustments. If you need to modify your quota, please contact our customer service team via phone or email. They will promptly process your request and keep you updated on its progress through either a call or email.

                                              +

                                              Before reaching out, ensure you have the following details ready:

                                              • Domain name, project name, and project ID

                                                To obtain the preceding information, log in to the management console, click the username in the upper right corner, and choose My Credentials from the drop-down list.

                                                -
                                              • Quota information, including:
                                                • Service name
                                                • Quota type
                                                • Required quota
                                                +
                                              • Quota information, including:
                                                • Service name
                                                • Quota type
                                                • Desired quota value

                                              Learn how to obtain the service hotline and email address.

                                              diff --git a/docs/dli/umn/dli_03_0040.html b/docs/dli/umn/dli_03_0040.html index 691b63673..80f114d22 100644 --- a/docs/dli/umn/dli_03_0040.html +++ b/docs/dli/umn/dli_03_0040.html @@ -2,7 +2,7 @@

                                              Why Does DIS Stream Not Exist During Job Semantic Check?

                                              To rectify this fault, perform the following steps:

                                              -
                                              1. Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statements to check whether the DIS stream exists.
                                              2. If the DIS stream was not created, create a DIS stream by referring to "Creating a DIS Stream" in the Data Ingestion Service User Guide.

                                                Ensure that the created DIS stream and Flink job are in the same region.

                                                +
                                                1. Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statements to check whether the DIS stream exists.
                                                2. If the DIS stream in the Flink job has not been created yet, create one by referring to "Creating a DIS Stream" in Data Lake Insight User Guide.

                                                  Ensure that the created DIS stream and Flink job are in the same region.

                                                3. If a DIS stream has been created, check whether the DIS stream and the Flink job are in the same region.
                                              diff --git a/docs/dli/umn/dli_03_0054.html b/docs/dli/umn/dli_03_0054.html index a6bd6e987..e512ecd32 100644 --- a/docs/dli/umn/dli_03_0054.html +++ b/docs/dli/umn/dli_03_0054.html @@ -14,7 +14,7 @@
                                            5. Will Granting Duplicate Permissions to a Table After Inheriting Database Permissions Cause an Error?
                                              6. -
                                            6. Why Can't I Query a View After I'm Granted the Select Table Permission on the View?
                                              +
                                            7. Why Can't I Query a View Despite Having the Select Permission?
                                            8. How Do I Do If I Receive a Message Saying I Don't Have Sufficient Permissions to Submit My Jobs to the Job Bucket?
                                              9. diff --git a/docs/dli/umn/dli_03_0056.html b/docs/dli/umn/dli_03_0056.html index 072be28b2..25cdd4135 100644 --- a/docs/dli/umn/dli_03_0056.html +++ b/docs/dli/umn/dli_03_0056.html @@ -8,7 +8,7 @@
                                                -
                                              • Why Is Error "unsupported media Type" Reported When I Subimt a SQL Job?
                                                +
                                              • Why Am I Getting an "unsupported media Type" Error When Submitting a SQL Job?
                                              • What Can I Do If an Error Is Reported When the Execution of the API for Creating a SQL Job Times Out?
                                                • diff --git a/docs/dli/umn/dli_03_0060.html b/docs/dli/umn/dli_03_0060.html index 21f17b638..a72688ef2 100644 --- a/docs/dli/umn/dli_03_0060.html +++ b/docs/dli/umn/dli_03_0060.html @@ -1,6 +1,6 @@ -

                                                Why Is Error "unsupported media Type" Reported When I Subimt a SQL Job?

                                                +

                                                Why Am I Getting an "unsupported media Type" Error When Submitting a SQL Job?

                                                In the REST API provided by DLI, the request header can be added to the request URI, for example, Content-Type.

                                                Content-Type indicates the request body type or format. The default value is application/json.

                                                URI for submitting a SQL job: POST /v1.0/{project_id}/jobs/submit-job

                                                diff --git a/docs/dli/umn/dli_03_0067.html b/docs/dli/umn/dli_03_0067.html index 1f6194540..2b704b049 100644 --- a/docs/dli/umn/dli_03_0067.html +++ b/docs/dli/umn/dli_03_0067.html @@ -1,13 +1,20 @@ -

                                                Why Can't I Query a View After I'm Granted the Select Table Permission on the View?

                                                -

                                                Symptom

                                                User A created Table1.

                                                -

                                                User B created View1 based on Table1.

                                                -

                                                After the Select Table permission on Table1 is granted to user C, user C fails to query View1.

                                                +

                                                Why Can't I Query a View Despite Having the Select Permission?

                                                +

                                                Symptom

                                                User A created a table named Table1.

                                                +

                                                User B created a view named View1 based on Table1.

                                                +

                                                After granting user C the select permission on Table1, user C failed to query the view.

                                                -

                                                Possible Causes

                                                User B does not have the Select Table permission on Table1.

                                                +

                                                Current Permission Assignments

                                                • User A already has: admin permission on Table1.
                                                • User B already has: admin permission on View1.
                                                • User C already has: select permission on Table1.
                                                -

                                                Solution

                                                Grant the Select Table permission on Table1 to user B. Then, query View1 as user C again.

                                                +

                                                Solution

                                                Different versions of the Spark engine have varying permission requirements for views:

                                                +
                                                • Spark 2.4.x: User C needs the view query permission, and user B requires the select permission on Table1.

                                                  Permission requirements:

                                                  +
                                                  • User B should have: admin permission on View1 and select permission on Table1 (currently missing).
                                                  • User C already has: select permission on Table1.
                                                  +

                                                  To resolve this, grant user B the select permission on Table1, then retry querying View1 as user C.

                                                  +
                                                • Spark 3.3.x: User C needs the view query permission, and user C requires the select permission on Table1.

                                                  Permission requirements:

                                                  +
                                                  • User C should have: select permission on Table1 and select permission on View1 (currently missing).
                                                  +

                                                  To resolve this, grant user C the select permission on View1, then retry querying View1 as user C.

                                                  +
                                                diff --git a/docs/dli/umn/dli_03_0076.html b/docs/dli/umn/dli_03_0076.html index eb276329e..8cbc8f12f 100644 --- a/docs/dli/umn/dli_03_0076.html +++ b/docs/dli/umn/dli_03_0076.html @@ -3,8 +3,8 @@

                                                How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the Spark Job Results Stored in MySQL?

                                                1. If the pymysql module is missing, check whether the corresponding EGG package exists. If the package does not exist, upload the pyFile package on the Package Management page. The procedure is as follows:
                                                  1. Upload the egg package to the specified OBS path.
                                                  2. Log in to the DLI management console and choose Data Management > Package Management.
                                                  3. On the Package Management page, click Create Package in the upper right corner to create a package.
                                                  4. In the Create Package dialog, set the following parameters:
                                                    • Type: Select PyFile.
                                                    • OBS Path: Select the OBS path where the egg package is stored.
                                                    • Set Group and Group Name as you need.
                                                  5. Click OK.
                                                  6. On the Spark job editing page where the error is reported, choose the uploaded egg package from the Python File Dependencies drop-down list and run the Spark job again.
                                                  -
                                                2. To interconnect PySpark jobs with MySQL, you need to create a datasource connection to enable the network between DLI and RDS.

                                                  For how to create a datasource connection on the management console, see "Enhanced Datasource Connections" in Data Lake Insight User Guide.

                                                  -

                                                  For how to call an API to create a datasource connection, see "Creating an Enhanced Datasource Connection" in Data Lake Insight API Reference.

                                                  +
                                                3. To interconnect PySpark jobs with MySQL, you need to create a datasource connection to enable the network between DLI and RDS.

                                                  For details about how to create a datasource connection on the management console, see "Enhanced Datasource Connections" in Data Lake Insight User Guide.

                                                  +

                                                  For details about how to call an API to create a datasource connection, see "Creating an Enhanced Datasource Connection" in Data Lake Insight API Reference.

                                                diff --git a/docs/dli/umn/dli_03_0086.html b/docs/dli/umn/dli_03_0086.html index c840f01f5..79bb610f8 100644 --- a/docs/dli/umn/dli_03_0086.html +++ b/docs/dli/umn/dli_03_0086.html @@ -1,11 +1,254 @@

                                                How Do I Merge Small Files?

                                                -

                                                If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.

                                                -

                                                You are advised to use temporary tables for data transfer. There is a risk of data loss in self-read and self-write operations during unexpected exceptional scenarios.

                                                -
                                                Run the following SQL statements:
                                                INSERT OVERWRITE TABLE tablename
-select  * FROM  tablename
-DISTRIBUTE BY floor(rand()*20)
                                                +

                                                What Are Small Files?

                                                In distributed file systems, data is stored in blocks. A small file refers to a file whose size is significantly smaller than the block size of the storage system. Large numbers of small files can cause significant performance and management issues for big data systems. Merging small files is one of the key strategies to optimize system performance.

                                                +

                                                This section explains how to use DISTRIBUTE BY in DLI to merge small files.

                                                +
                                                +

                                                Key Issues Caused by Small Files

                                                • Metadata overhead: In file systems like HDFS, the master node (NameNode) manages metadata for each file in memory. A massive number of small files drastically increases memory consumption, becoming a bottleneck for cluster scalability.
                                                • Poor computational performance: Compute engines like Spark typically launch a task for each file or file block. Processing numerous small files leads to:
                                                  • High task scheduling overhead: The time spent launching and scheduling thousands of tasks may exceed the actual data processing time.
                                                  • Resource wastage: Significant CPU and memory resources are consumed by task management rather than effective computation.
                                                  • Query latency: Overall job execution time increases, slowing down query responses.
                                                  +
                                                • Low storage efficiency: Small files often result in inefficient storage utilization, failing to leverage the advantages of distributed block sizes.
                                                • Risk of limits being reached: They may trigger limits on the number of files in a directory or the number of tasks per job in compute engines or file systems, causing job failures.
                                                +
                                                +

                                                Basic Principles of Small File Merging

                                                • Basic Principles

                                                  In MapReduce-based engines like Spark or Hive, the number of output files depends on the number of tasks in the final stage (usually the Reduce phase). Each Reduce task generates one file by default.

                                                  +

                                                  By using DISTRIBUTE BY, data is redistributed into a specified number of Reduce tasks, with each task producing one file, thereby merging files.

                                                  +
                                                  • Data redistribution: Data is sent to different Reduce tasks based on the hash value of the expression that follows (e.g., floor(rand()*N)).
                                                  • Controlling Reduce task count: By setting the value of N, you indirectly and probabilistically control the number of Reduce tasks, which determines the number of output files. Typically, N represents the desired number of merged files per partition.
                                                  +
                                                +
                                                + +

                                                Non-Partitioned Table Merging Example

                                                • Using a temporary table for intermediate storage (recommended)
                                            Table 1 Example of actual CU allocation of an elastic resource pool

                                            Scenario

                                            Hive

                                            GaussDB(DWS)

                                            +

                                            DWS

                                            Oracle

                                            		 
                                             1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
                                            -- 1. Create a temporary table with the same structure as the source table.
+CREATE TABLE temp_tablename LIKE tablename;
+
+-- 2. Write merged data into the temporary table.
+INSERT OVERWRITE TABLE temp_tablename
+SELECT * FROM tablename
+DISTRIBUTE BY floor(rand()*20); -- Generate approximately 20 files.
+
+-- 3. Verify the data.
+SELECT count(*) FROM tablename;       -- Source table
+SELECT count(*) FROM temp_tablename;  -- Temporary table
+
+-- 4. Write the data back to the source table.
+INSERT OVERWRITE TABLE tablename
+SELECT 
+*
+FROM temp_tablename;
+
+-- 5. (Optional) Delete the old table if no longer needed.
+DROP TABLE table_name_old;
+
                                            +
                                            +
                                          21. Self read-write method
                                            INSERT OVERWRITE TABLE tablename
+SELECT *
+FROM tablename
+DISTRIBUTE BY floor(rand() * 20)
                                            +
                                            • INSERT OVERWRITE TABLE tablename

                                              Overwrite the target table with query results.

                                              +

                                              This operation deletes the source table data, so consider using a temporary table to avoid data loss.

                                              +
                                              +
                                            • SELECT * FROM tablename

                                              Read all data from the source table.

                                              +
                                            • DISTRIBUTE BY floor(rand() * 20)
                                              • Random data distribution:
                                                • rand() generates a random float between [0, 1).
                                                • rand() * 20 generates a random float between [0, 20).
                                                • floor(...) rounds down to integers in [0, 19].
                                                +
                                              • Distribution result:
                                                • Data is randomly and evenly distributed across up to 20 Reduce tasks.
                                                • Up to 20 output files are generated (each Reduce task corresponds to one file).
                                                +
                                              +
                                            +
                                            22. +
                                          +

                                          Partitioned Table Merging

                                          For partitioned tables, we generally prefer to merge files within each partition separately rather than mixing data across partitions.
                                          • Parameter description:
                                            • N: The desired number of files per partition after merging. For example, set it to 5 if you wish for each partition to ultimately contain only five files.
                                            • pt: The partition field.
                                            +
                                          • Why do I need to add a partition field to DISTRIBUTE BY?
                                            • This ensures data from the same partition is sent to the same group of Reduce tasks for processing.
                                            • Using DISTRIBUTE BY pt, floor(rand() * N) first distributes data by the partition field (pt) and then further shuffles it within each partition based on randomness.

                                              Each partition's data is split into N files without affecting other partitions.

                                              +
                                            +
                                          +
                                          +
                                          • Using a temporary table (recommended)
                                             1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
                                            -- 1. Create a temporary table with the same structure as the source table.
+CREATE TABLE table_tmp LIKE table_name;
+
+-- 2. Merge partitions into the temporary table.
+INSERT OVERWRITE TABLE table_tmp PARTITION(pt)
+SELECT col1, col2, ..., pt 
+FROM tablename
+WHERE pt = *  -- Partition conditions
+DISTRIBUTE BY pt, floor(rand() * N); -- N indicates the number of target files per partition.
+
+-- 3. Verify the data.
+SELECT COUNT(*) FROM tablename;   -- Source table
+SELECT COUNT(*) FROM temp_tablename;    -- Temporary table
+
+-- 4. Write the data back to the source table.
+INSERT OVERWRITE TABLE tablename PARTITION (pt)
+SELECT 
+*
+FROM temp_tablename
+WHERE pt = *;
+
+-- 5. (Optional) Delete the old table if no longer needed.
+DROP TABLE temp_tablename;
+
                                            +
                                            +
                                          • Self read-write method
                                             1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
                                            INSERT OVERWRITE TABLE target_table 
+PARTITION (pt)
+SELECT      
+    col1,
+    col2, 
+    ...,    
+    pt -- The partition field must appear last in the SELECT statement.
+FROM
+    target_table
+WHERE pt = * 
+DISTRIBUTE BY pt, floor(rand() * N) 
+
                                            +
                                            +
                                          +
                                          +

                                          Example: Merging Partitions in a Student Table

                                          • Scenario

                                            Consider a partitioned table named student, with partition fields facultyNo (college ID) and classNo (class ID). The goal is to merge small files within each partition, limiting each partition to a maximum of 2 files.

                                            +

                                            - Table name: student

                                            +

                                            - Partition fields: facultyNo (college ID) and classNo (class ID)

                                            +

                                            - Goal: Merge each partition into 2 files.

                                            +
                                          +
                                          • Example code
                                            -- 1. Create a temporary table.
+CREATE TABLE student_tmp LIKE student;
+
+-- 2. Merge specific partitions.
+INSERT OVERWRITE TABLE student_tmp PARTITION(facultyNo, classNo)
+SELECT 
+    id,
+    name,
+    gender,
+    age,
+    birth_date,
+    phone,
+    email,
+    address,
+    enrollment_date,
+    major,
+    grade,
+    status,
+    facultyNo,
+    classNo
+FROM student
+WHERE facultyNo = 1 AND classNo = 7  -- Partition conditions
+DISTRIBUTE BY facultyNo, classNo, floor(rand() * 2);  -- Controlling the number of files
+
                                            +
                                            After execution, the data under facultyNo = 1, classNo = 7 will be consolidated into up to 2 files.
                                            -- 3. Verify the partition (e.g., facultyNo = 1, classNo = 7):
+SELECT count(*) FROM student 
+WHERE facultyNo=1 AND classNo=7;   --> Original data volume
+
+SELECT count(*) FROM student_tmp 
+WHERE facultyNo=1 AND classNo=7;   --> Merged data volume
+
+-- 4. Run INSERT OVERWRITE to write the merged data back to the source table.
+INSERT OVERWRITE TABLE student PARTITION(facultyNo, classNo)
+SELECT 
+*
+FROM student_temp 
+WHERE facultyNo > 0;
                                            +
                                            +
                                          +
                                          +

                                          More Operation Recommendations

                                          When directly overwriting the source table, data loss may occur if the task fails during execution.

                                          +

                                          To mitigate this risk, you are advised to use a temporary table as an intermediary:

                                          +
                                          1. Create a temporary table with the same structure as the source table.
                                          2. Insert merged data into the temporary table.
                                          3. After verifying the data in the temporary table, switch it to the production table using ALTER TABLE ... RENAME TO or INSERT OVERWRITE.
                                          +

                                          For the DISTRIBUTE BY clause: Instead of relying solely on random numbers, you can use a naturally high-cardinality field like user_id. This approach not only merges files but also sorts the data based on that field, potentially improving query performance.

                                          +
                                          -- 1. Create a temporary table with the same structure as the source table table_name.
+CREATE TABLE table_name_tmp LIKE table_name;
+
+-- 2. Merge the data from the source table and insert it into the temporary table.
+INSERT OVERWRITE TABLE table_name_tmp PARTITION (pt) -- For a partitioned table
+SELECT ... -- Selected fields
+FROM table_name
+WHERE pt = *
+DISTRIBUTE BY pt, floor(rand() * N); -- Partitioned table syntax
+-- For a non-partitioned table: DISTRIBUTE BY floor(rand() * N)
+
+-- 3. Verify data integrity (e.g., row counts, partitions).
+SELECT count(*) FROM table_name;
+SELECT count(*) FROM table_name_tmp;
+
+-- 4. Swap tables via renaming (a metadata operation completed instantly).
+ALTER TABLE table_name RENAME TO table_name_old;
+ALTER TABLE table_name_tmp RENAME TO table_name;
+
+-- 5. (Optional) Delete the old table if no longer needed.
+DROP TABLE table_name_old;
                                          +
                                          +

                                          Summary of Small File Merging Methods

                                          +
                                          + + + + + + + + + + + + + + + + + +

                                          Scenario

                                          +

                                          Recommended SQL Syntax

                                          +

                                          Key Point

                                          +

                                          Non-partitioned table

                                          +
                                          INSERT OVERWRITE TABLE table_tmp SELECT * FROM table DISTRIBUTE BY floor(rand()*N);
                                          +

                                          Use random numbers to control the number of files (N).

                                          +

                                          Partitioned table

                                          +
                                          INSERT OVERWRITE TABLE table_tmp PARTITION(pt) SELECT ..., pt FROM table WHERE pt = * DISTRIBUTE BY pt, floor(rand()*N);
                                          +

                                          Include the partition field in DISTRIBUTE BY to ensure merging occurs within partitions.

                                          +

                                          General strategy

                                          +

                                          Use a temporary table for intermediate storage. Verify data before swapping tables via renaming to avoid data loss risks.

                                          +

                                          Minimize the risk of data loss during the process.

                                          +
                                          +
                                          diff --git a/docs/dli/umn/dli_03_0098.html b/docs/dli/umn/dli_03_0098.html index 66e15f11f..835ac542d 100644 --- a/docs/dli/umn/dli_03_0098.html +++ b/docs/dli/umn/dli_03_0098.html @@ -2,7 +2,7 @@

                                          How Do I Monitor Job Exceptions on a DLI Queue?

                                          DLI allows you to subscribe to an SMN topic for failed jobs.

                                          -
                                          1. Log in to the DLI console.
                                          2. In the navigation pane on the left, choose Queue Management.
                                          3. On the Queue Management page, click Create SMN Topic in the upper left corner. .
                                          +
                                          1. Log in to the DLI console.
                                          2. In the navigation pane on the left, choose Queue Management.
                                          3. On the Queue Management page, click Create SMN Topic in the upper left corner.
                                          diff --git a/docs/dli/umn/dli_03_0156.html b/docs/dli/umn/dli_03_0156.html index 68fb70d32..2368f85e2 100644 --- a/docs/dli/umn/dli_03_0156.html +++ b/docs/dli/umn/dli_03_0156.html @@ -2,7 +2,7 @@

                                          Why Do I Get "ResponseCode: 403" and "ResponseStatus: Forbidden" Errors When a Spark Job Accesses OBS Data?

                                          Symptom

                                          The following error is reported when a Spark job accesses OBS data:

                                          -
                                          Caused by: com.obs.services.exception.ObsException: Error message:Request Error.OBS servcie Error Message. -- ResponseCode: 403, ResponseStatus: Forbidden
                                          +
                                          Caused by: com.obs.services.exception.ObsException: Error message:Request Error.OBS service Error Message. -- ResponseCode: 403, ResponseStatus: Forbidden

                                          Solution

                                          Set the AK/SK to enable Spark jobs to access OBS data.

                                          For details, see How Do I Set Up AK/SK So That a General Queue Can Access Tables Stored in OBS?

                                          diff --git a/docs/dli/umn/dli_03_0173.html b/docs/dli/umn/dli_03_0173.html index 7a4146eb3..db32cea3a 100644 --- a/docs/dli/umn/dli_03_0173.html +++ b/docs/dli/umn/dli_03_0173.html @@ -2,9 +2,9 @@

                                          Why Is Error "org.apache.hadoop.fs.obs.OBSIOException" Reported When I Run DLI SQL Scripts on DataArts Studio?

                                          Symptom

                                          When you run a DLI SQL script on DataArts Studio, the log shows that the statements fail to be executed. The error information is as follows:
                                          DLI.0999: RuntimeException: org.apache.hadoop.fs.obs.OBSIOException: initializing on obs://xxx.csv: status [-1] - request id
-[null] - error code [null] - error message [null] - trace :com.obs.services.exception.ObsException: OBS servcie Error Message. Request Error:
+[null] - error code [null] - error message [null] - trace :com.obs.services.exception.ObsException: OBS service Error Message. Request Error:
 ...
-Cause by: ObsException: com.obs.services.exception.ObsException: OBSs servcie Error Message. Request Error: java.net.UnknownHostException: xxx: Name or service not known
                                          +Cause by: ObsException: com.obs.services.exception.ObsException: OBSs service Error Message. Request Error: java.net.UnknownHostException: xxx: Name or service not known

                                          Possible Causes

                                          When you execute a DLI SQL script for the first time, you did not agree to the privacy agreement on the DLI console. As a result, the error is reported when the SQL script is executed on DataArts Studio.

                                          diff --git a/docs/dli/umn/dli_03_0175.html b/docs/dli/umn/dli_03_0175.html index 831e9c50b..c5c0dd25c 100644 --- a/docs/dli/umn/dli_03_0175.html +++ b/docs/dli/umn/dli_03_0175.html @@ -1,9 +1,9 @@ -

                                          Do I Need to to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With the Same Name?

                                          +

                                          Do I Need to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With the Same Name?

                                          Scenario

                                          User A created the testTable table in a database through a SQL job and granted user B the permission to insert and delete table data. User A deleted the testTable table and created a new testTable table. If user A wants user B to retain the insert and delete permission, user A needs to grant the permissions to user B again.

                                          -

                                          Possible Causes

                                          After a table is deleted, the table permissions are not retained. You need to grant permissions to a user or project.

                                          +

                                          Possible Causes

                                          After deleting a table and recreating one with the same name, table permissions do not automatically inherit in this scenario. You need to regrant permissions to the users or projects.

                                          Solution

                                          Operations to grant permissions to a user or project are as follows:

                                          1. On the left of the management console, choose Data Management > Databases and Tables.
                                          2. Click the database name whose table permission is to be granted. The table management page of the database is displayed.
                                          3. Locate the row of the target table and click Permissions in the Operation column.
                                          4. On the displayed page, click Grant Permission in the upper right corner.
                                          5. In the displayed Grant Permission dialog box, select the required permissions.
                                          6. Click OK.
                                          diff --git a/docs/dli/umn/dli_03_0184.html b/docs/dli/umn/dli_03_0184.html index b0e29c71c..7283f057c 100644 --- a/docs/dli/umn/dli_03_0184.html +++ b/docs/dli/umn/dli_03_0184.html @@ -6,7 +6,7 @@

                                          Possible Causes

                                          If a table exists but cannot be queried, there is a high probability that the current user does not have the permission to query or operate the table.

                                          Solution

                                          Contact the user who creates the table and obtain the required permissions. To assign permissions, perform the following steps:

                                          -
                                          1. Log in to the DLI management console as the user who creates the table. Choose Data Management > Databases and Tables form the navigation pane on the left.
                                          2. Click the database name. The table management page is displayed. In the Operation column of the target table, click Permissions. The table permission management page is displayed.
                                          3. Click Set Permission. In the displayed dialog box, set Authorization Object to User, set Username to the name of the user that requires the permission, and select the required permissions. For example, Select Table and Insert permissions.
                                          4. Click OK.
                                          5. Log in to the DLI console as the user that has been granted permission and check whether the table can be queried.
                                          +
                                          1. Log in to the DLI management console as the user who creates the table. In the navigation pane on the left, choose Data Management > Databases and Tables.
                                          2. Click the database name. The table management page is displayed. In the Operation column of the target table, click Permissions. The table permission management page is displayed.
                                          3. Click Set Permission. In the displayed dialog box, set Authorization Object to User, set Username to the name of the user that requires the permission, and select the required permissions. For example, Select Table and Insert permissions.
                                          4. Click OK.
                                          5. Log in to the DLI console as the user that has been granted permission and check whether the table can be queried.
                                          diff --git a/docs/dli/umn/dli_03_0211.html b/docs/dli/umn/dli_03_0211.html index 42643c967..fd58dab21 100644 --- a/docs/dli/umn/dli_03_0211.html +++ b/docs/dli/umn/dli_03_0211.html @@ -14,7 +14,7 @@
                                        5. How Do I Do If Inconsistent Character Encoding Leads to Garbled Characters?
                                          6. -
                                        6. Do I Need to to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With the Same Name?
                                          +
                                        7. Do I Need to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With the Same Name?
                                        8. How Do I Do If Files Imported Into a DLI Partitioned Table Lack Data for the Partition Columns, Causing Query Failures After the Import Is Completed?
                                          9. diff --git a/docs/dli/umn/dli_07_0003.html b/docs/dli/umn/dli_07_0003.html index ab00e3320..1fd940e1a 100644 --- a/docs/dli/umn/dli_07_0003.html +++ b/docs/dli/umn/dli_07_0003.html @@ -1,38 +1,34 @@

                                          Basic Concepts

                                          -

                                          Elastic Resource Pool

                                          An elastic resource pool consists of dedicated compute resources where the resources in different pools are completely isolated from each other. Within a single elastic resource pool, multiple queues can share the resources available in that pool. Additionally, you can configure policies based on the resource load of these queues to enable time-based elastic scaling to meet diverse business needs.

                                          -
                                          -

                                          DLI Storage Resource

                                          DLI storage resources are the internal storage capacities of the DLI service. They are utilized for storing databases and DLI tables, playing a crucial role in data import into DLI. These resources also indicate the volume of data that users have stored within DLI.

                                          -
                                          -

                                          Actual CUs, Used CUs, CU Range, and Specifications of an Elastic Resource Pool

                                          • Actual CUs: actual size of resources currently allocated to the elastic resource pool (in CUs).
                                            • When there is no queue in the resource pool, the actual CUs are equal to the minimum CUs when the elastic resource pool is created.
                                            • When there are queues in the resource pool, the formula for calculating actual CUs is:
                                              • Actual CUs = max{(min[sum(maximum CUs of queues), maximum CUs of the elastic resource pool]), minimum CUs of the elastic resource pool}.
                                              • The calculation result must be a multiple of 16 CUs. If it cannot be exactly divided by 16 CUs, round up to the nearest multiple.
                                              -
                                            • Scaling out or in an elastic resource pool means adjusting the actual CUs of the resource pool. Refer to Scaling Out or In an Elastic Resource Pool.
                                            • Example of actual CU allocation:

                                              In Table 1, the calculation process for the actual allocation of CUs in an elastic resource pool is as follows:

                                              +

                                              Actual CUs, Used CUs, CU Range, and Yearly/Monthly CUs (Specifications) of an Elastic Resource Pool

                                              • Actual CUs: The current allocated resource size of the elastic resource pool, measured in CUs.
                                                • When no queues exist in the resource pool: The actual CUs equal the minimum CUs set during its creation.
                                                • When there are queues in the resource pool, the formula to calculate actual CUs is:
                                                  • Actual CUs = max{(min[sum(maximum CUs of queues), maximum CUs of the elastic resource pool]), minimum CUs of the elastic resource pool}.
                                                  • The result must be a multiple of 16 CUs. If not divisible by 16, round up to the nearest multiple.
                                                  +
                                                • Scaling out or in an elastic resource pool means adjusting its actual CUs. See Scaling Out or In an Elastic Resource Pool.
                                                • Example of actual CU allocation:

                                                  Consider Table 1 below, which illustrates the process of calculating actual CUs for an elastic resource pool:

                                                  1. Calculate the sum of maximum CUs of the queues: sum(maximum CUs) = 32 + 56 = 88 CUs.
                                                  2. Compare the sum of maximum CUs of the queues with the maximum CUs of the elastic resource pool and take the smaller value: min{88 CUs, 112 CUs} = 88 CUs.
                                                  3. Compare the value with the minimum CUs of the elastic resource pool and take the larger value: max(88 CUs, 64 CUs) = 88 CUs.
                                                  4. Check if 88 CUs is a multiple of 16 CUs. Since 88 is not divisible by 16, round up to 96 CUs. -
                                                    Table 1 Example of actual CU allocation of an elastic resource pool

                                                    Scenario

                                                    +
                                                    - - - - - - - - - @@ -40,32 +36,33 @@ -
                                                  5. Used CUs: CUs that have been used by jobs or tasks. These resources may be executing computing tasks.
                                                  6. CU range: CU settings are used to control the maximum and minimum CU ranges for elastic resource pools to avoid unlimited resource scaling.
                                                    • The total minimum CUs of all queues in an elastic resource pool must be no more than the minimum CUs of the pool.
                                                    • The maximum CUs of any queue in an elastic resource pool must be no more than the maximum CUs of the pool.
                                                    • An elastic resource pool should at least ensure that all queues in it can run with the minimum CUs and should try to ensure that all queues in it can run with the maximum CUs.
                                                    • When expanding the specifications of an elastic resource pool, the minimum value of the CU range is linked to the specifications of the elastic resource pool. After changing the specifications of the elastic resource pool, the minimum value of the CU range is modified to match the specifications.
                                                    -
                                                  7. Specifications: The minimum CUs selected during elastic resource pool purchase are elastic resource pool specifications.
                                                    8. +
                                                  8. Used CUs: The portion of CUs currently occupied by jobs or tasks, which may be actively performing computations.
                                                  9. CU range: CU settings are used to control the maximum and minimum CU ranges for elastic resource pools to avoid unlimited resource scaling.
                                                    • The sum of all queues' minimum CUs in an elastic resource pool must not exceed the pool's minCU.
                                                    • Any single queue's maxCU cannot exceed the pool's maxCU.
                                                    • The resource pool ensures it meets the minCU requirements across all queues while striving to accommodate their maxCU demands.
                                                    • When expanding the specifications of an elastic resource pool, the minimum value of the CU range is linked to the yearly/monthly CUs (specifications) of the elastic resource pool. After changing the specifications of the elastic resource pool, the minimum value of the CU range is modified to match the yearly/monthly CUs (specifications).
                                                    +
                                                  10. Yearly/monthly CUs (specifications): The minimum value of the CU range selected when purchasing an elastic resource pool is the elastic resource pool specifications.
                                                    11. -

                                                    Database

                                                    A database is a warehouse where data is organized, stored, and managed based on the data structure. DLI management permissions are granted on a per database basis.

                                                    -

                                                    In DLI, tables and databases are metadata containers that define underlying data. The metadata in the table shows the location of the data and specifies the data structure, such as the column name, data type, and table name. A database is a collection of tables.

                                                    +

                                                    Database

                                                    A database is a structured repository designed to organize, store, and manage data efficiently. In DLI, databases serve as the fundamental unit for managing permissions, with access rights assigned at the database level.

                                                    +

                                                    Within DLI, both tables and databases act as metadata containers that define underlying data structures. Table metadata informs DLI about the location of the data and specifies its structure, such as column names, data types, and table names. Databases provide logical groupings for these tables.

                                                    -

                                                    OBS Table, DLI Table, and CloudTable Table

                                                    The table type indicates the storage location of data.

                                                    -
                                                    • OBS table indicates that data is stored in the OBS bucket.
                                                    • DLI table indicates that data is stored in the internal table of DLI.
                                                    • CloudTable table indicates that data is stored in CloudTable.
                                                    -

                                                    You can create a table on DLI and associate the table with other services to achieve querying data from multiple data sources.

                                                    +

                                                    OBS Tables, DLI Tables, CloudTable Tables

                                                    Different table types indicate distinct storage locations:

                                                    +
                                                    • OBS table: Data is stored in buckets within OBS.
                                                    • DLI table: Data is stored in tables internal to DLI.

                                                      DLI storage resources are internal resources used to house databases and DLI tables, essential for importing data into DLI and reflecting the volume of user data stored within the service.

                                                      +
                                                    • CloudTable table: Data is stored in tables managed by CloudTable.
                                                    +

                                                    Tables can be created through DLI to establish connections with other services, enabling federated query and analysis across diverse data sources.

                                                    -

                                                    Metadata

                                                    Metadata is used to define data types. It describes information about the data, including the source, size, format, and other data features. In database fields, metadata interprets data content in the data warehouse.

                                                    +

                                                    Metadata

                                                    Metadata refers to data that defines other data types. It primarily describes information about the data itself, including its source, size, format, or other characteristics. In database fields, metadata is used to interpret the contents of a data warehouse.

                                                    -

                                                    SQL Job

                                                    SQL job refers to the SQL statement executed in the SQL job editor. It serves as the execution entity used for performing operations, such as importing and exporting data, in the SQL job editor.

                                                    -

                                                    This type is suitable for scenarios where standard SQL statements are used for querying. It is typically used for querying and analyzing structured data.

                                                    +

                                                    SQL Jobs

                                                    A SQL job refers to the execution entity within the system that handles operations such as running SQL statements, importing data, and exporting data through the SQL job editor.

                                                    +

                                                    It is ideal for scenarios involving structured data queries and analysis using standard SQL.

                                                    -

                                                    Flink Job

                                                    This type is specifically designed for real-time data stream processing, making it ideal for scenarios that require low latency and quick response. It is well-suited for real-time monitoring and online analysis.

                                                    -
                                                    • Flink OpenSource job: When submitting jobs, you can quickly integrate with other data systems using DLI's standard connectors and various APIs.
                                                    • Flink Jar job: allows you to submit Flink jobs compiled into JAR files, providing greater flexibility and customization capabilities. It is suitable for complex data processing scenarios that require user-defined functions (UDFs) or specific library integration. The Flink ecosystem can be utilized to implement advanced stream processing logic and status management.
                                                    +

                                                    Flink Jobs

                                                    Designed for real-time stream processing, Flink jobs are suited for low-latency applications requiring rapid responses, such as real-time monitoring and online analytics.

                                                    +
                                                    • Flink OpenSource jobs: These allow you to use DLI-provided connectors and APIs for seamless integration with other data systems during job submission.
                                                    • Flink Jar jobs: You can submit pre-compiled JAR files containing Flink jobs, offering greater flexibility and customization. This type is ideal for complex data processing tasks involving custom functions, UDFs, or specific library integrations, enabling advanced stream processing logic and state management using Flink's ecosystem.
                                                    -

                                                    Spark Job

                                                    Spark jobs are those submitted by users through visualized interfaces and RESTful APIs. Full-stack Spark jobs are allowed, such as Spark Core, DataSet, MLlib, and GraphX jobs.

                                                    +

                                                    Spark Jobs

                                                    Spark jobs refer to those submitted via visual interfaces or RESTful APIs, supporting full-stack Spark functionalities including Spark Core, DataSet, MLlib, and GraphX.

                                                    -

                                                    CU

                                                    CU is the unit of compute resources in DLI, where 1 CU equals 1 vCPU and 4 GB of memory. The higher the specifications of compute resources, the better its computing power.

                                                    +

                                                    CU

                                                    CU represents the unit of compute resources in DLI. One CU equals one vCPU paired with 4 GB of memory. Higher specifications correspond to increased computational power.

                                                    Constants and Variables

                                                    The differences between constants and variables are as follows:

                                                    -
                                                    • During the running of a program, the value of a constant cannot be changed.
                                                    • Variables are readable and writable, whereas constants are read-only. A variable is a memory address that contains a segment of data that can be changed during program running. For example, in int a = 123, a is an integer variable.
                                                    +
                                                    • Constants retain their value throughout program execution and cannot be altered. They are strictly read-only.
                                                    • Variables are both readable and writable. A variable represents a specific memory address where the stored value can be updated at any time during runtime. For example, in int a = 123, a is an integer variable.
                                                    -

                                                    Table Lifecycle

                                                    The table lifecycle management feature in DLI refers to the automatic recycling of tables or partitions that have not been updated for a specified period of time since their last update. This specified period is known as the lifecycle. This feature simplifies the process of recycling data and frees up storage space. Additionally, it provides data backup and recovery functions to prevent data loss due to accidental operations.

                                                    +

                                                    Table Lifecycle

                                                    The table lifecycle management feature in DLI automatically reclaims table (or partition) data if it remains unchanged after a specified period from its last update. This duration is termed the lifecycle. The feature simplifies storage space reclamation and data recycling processes while providing backup and recovery options to prevent accidental data loss.

                                                    diff --git a/docs/dli/umn/dli_07_0004.html b/docs/dli/umn/dli_07_0004.html index ff799a519..3bafa0a14 100644 --- a/docs/dli/umn/dli_07_0004.html +++ b/docs/dli/umn/dli_07_0004.html @@ -29,27 +29,27 @@

                                                    RDS

                                                    Relational Database Service (RDS) works as the data source and data storage system for DLI, and delivers the following capabilities:
                                                    • Data source: DLI allows you to import RDS data using DataFrame or SQL.
                                                    • Query result storage: DLI uses the SQL INSERT syntax to store query result data to RDS tables.
                                                    -

                                                    For how to access RDS data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    +

                                                    For details about how to access RDS data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    -

                                                    GaussDB(DWS)

                                                    GaussDB(DWS) works as the data source and data storage system for DLI, and delivers the following capabilities:
                                                    • Data source: DLI allows you to import GaussDB(DWS) data using DataFrame or SQL.
                                                    • Query result storage: DLI uses the SQL INSERT syntax to store query result data to GaussDB(DWS) tables.
                                                    +

                                                    DWS

                                                    DWS works as the data source and data storage system for DLI, and delivers the following capabilities:
                                                    • Data source: DLI allows you to import DWS data using DataFrame or SQL.
                                                    • Query result storage: DLI uses the SQL INSERT syntax to store query result data to DWS tables.
                                                    -

                                                    For how to access GaussDB(DWS) data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    +

                                                    For details about how to access DWS data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    CSS

                                                    CSS works as the data source and data storage system for DLI, and delivers the following capabilities:

                                                    • Data source: DLI allows you to import CSS data using DataFrame or SQL.
                                                    • Query result storage: DLI uses the SQL INSERT syntax to store query result data to CSS tables.
                                                    -

                                                    For how to access CSS data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    +

                                                    For details about how to access CSS data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    DCS

                                                    Distributed Cache Service (DCS) works as the data source and data storage system for DLI, and delivers the following capabilities:

                                                    • Data source: DLI allows you to import DCS data using DataFrame or SQL.
                                                    • Query result storage: DLI uses the SQL INSERT syntax to store query result data to DCS tables.
                                                    -

                                                    For how to access DCS data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    +

                                                    For details about how to access DCS data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    DDS

                                                    Document Database Service (DDS) works as the data source and data storage system for DLI, and delivers the following capabilities:

                                                    • Data source: DLI allows you to import DDS data using DataFrame or SQL.
                                                    • Query result storage: DLI uses the SQL INSERT syntax to store query result data to DDS tables.
                                                    -

                                                    For how to access DDS data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    +

                                                    For details about how to access DDS data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    MRS

                                                    MapReduce Service (MRS) works as the data source and data storage system for DLI, and delivers the following capabilities:

                                                    • Data source: DLI allows you to import MRS data using DataFrame or SQL.
                                                    • Query result storage: DLI uses the SQL INSERT syntax to store query result data to MRS tables.
                                                    -

                                                    For how to access MRS data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    +

                                                    For details about how to access MRS data through a DLI datasource connection, see Common Development Methods for DLI Cross-Source Analysis.

                                                    DataArts Studio

                                                    DataArts Studio is a one-stop big data development platform for collaborations and provides fully-managed big data scheduling capabilities. It manages various big data services, making big data more accessible than ever before and helping you effortlessly build big data processing centers.

                                                    • The DLI SQL node in DataArts Studio transfers SQL statements to DLI for execution.
                                                    • The DLI Spark node in DataArts Studio is used to execute a predefined Spark job.
                                                    diff --git a/docs/dli/umn/dli_07_0005.html b/docs/dli/umn/dli_07_0005.html index 81e217b34..9a6670996 100644 --- a/docs/dli/umn/dli_07_0005.html +++ b/docs/dli/umn/dli_07_0005.html @@ -25,8 +25,13 @@

                                                    The scaling step may vary depending on the resource specifications, usually 16 CUs, 32 CUs, 48 CUs, 64 CUs, and more.

                                                    For example, if the elastic resource pool has a capacity of 192 CUs and the queues in the pool are using 68 CUs due to running jobs, the plan is to scale in to 64 CUs.

                                                    When executing a scaling in task, the system determines that there are 124 CUs remaining and scales in by the minimum step of 64 CUs. However, the remaining 60 CUs cannot be scaled in any further. Therefore, after the elastic resource pool executes the scaling in task, its capacity is reduced to 128 CUs.

                                                    +
                                                  11. If jobs such as Flink or Spark Streaming jobs are continuously running on a node, that node cannot be scaled in as scheduled.
                                                    12. - + + +
                                                    + @@ -188,7 +193,7 @@ - @@ -226,7 +231,7 @@ -
                                                    Table 1 Example of actual CU allocation of an elastic resource pool

                                                    Scenario

                                                    Resource Type

                                                    +

                                                    Resource Type

                                                    CU Range

                                                    +

                                                    CU Range

                                                    New elastic resource pool: 64–112 CUs

                                                    +

                                                    New elastic resource pool: 64–112 CUs

                                                    Queues A and B are created within the elastic resource pool. The CU ranges of the two queues are:

                                                    • CU range of queue A: 16–32 CUs
                                                    • CU range of queue B: 16–56 CUs

                                                    Elastic resource pool

                                                    +

                                                    Elastic resource pool

                                                    64–112 CUs

                                                    +

                                                    64–112 CUs

                                                    Queue A

                                                    +

                                                    Queue A

                                                    16–32 CUs

                                                    +

                                                    16–32 CUs

                                                    Queue B

                                                    +

                                                    Queue B

                                                    16-56CUS

                                                    +

                                                    16-56CUS

                                                    Constraints for setting elastic resource pool CUs

                                                    +
                                                    • The minimum CUs (minCU) of an elastic resource pool must be less than or equal to the actual CUs. If expanding minCU exceeds the current actual CUs, you must first increase the actual CUs. Otherwise, the modification will fail.
                                                    • The sum of all queues' minimum CUs in an elastic resource pool must not exceed the pool's minCU.
                                                    • Any single queue's maxCU cannot exceed the pool's maxCU.
                                                    • Adjustments to a queue's CU range, changes to the pool's specifications, or modifications to the pool's CU settings take effect at the next full hour.
                                                    • Increasing the number of queues to adjust the pool's actual CUs takes immediate effect.

                                                    Datasource authentication type

                                                    • DLI supports four types of datasource authentication. Select an authentication type specific to each data source.
                                                      • CSS: applies to 6.5.4 or later CSS clusters with the security mode enabled.
                                                      • Kerberos: applies to MRS security clusters with Kerberos authentication enabled.
                                                      • Kafka_SSL: applies to Kafka with SSL enabled.
                                                      • Password: applies to GaussDB(DWS), RDS, DDS, and DCS.
                                                      +
                                                    • DLI supports four types of datasource authentication. Select an authentication type specific to each data source.
                                                      • CSS: applies to 6.5.4 or later CSS clusters with the security mode enabled.
                                                      • Kerberos: applies to MRS security clusters with Kerberos authentication enabled.
                                                      • Kafka_SSL: applies to Kafka with SSL enabled.
                                                      • Password: applies to DWS, RDS, DDS, and DCS.

                                                    Quota

                                                    For quota notes and constraints, see Quotas.

                                                    +

                                                    For quota notes and constraints, see Quota Management.

                                                    Browser version

                                                    diff --git a/docs/dli/umn/dli_07_0006.html b/docs/dli/umn/dli_07_0006.html index d80fa810b..c27938591 100644 --- a/docs/dli/umn/dli_07_0006.html +++ b/docs/dli/umn/dli_07_0006.html @@ -1,12 +1,12 @@

                                                    Permission Management

                                                    -

                                                    If you need to assign different permissions to employees in your enterprise to access your DLI resources, IAM is a good choice for fine-grained permissions management. IAM provides identity authentication, permissions management, and access control, helping you securely access to your cloud resources.

                                                    -

                                                    With IAM, you can use your account to create IAM users for your employees, and assign permissions to the users to control their access to specific resource types. For example, some software developers in your enterprise need to use DLI resources but must not delete them or perform any high-risk operations. To achieve this result, you can create IAM users for the software developers and grant them only the permissions required for using DLI resources.

                                                    -

                                                    If your account does not require individual IAM users for permissions management, you may skip over this section.

                                                    -

                                                    DLI Permissions

                                                    New IAM users do not have any permissions assigned by default. You need to first add them to one or more groups and attach policies or roles to these groups. The users then inherit permissions from the groups and can perform specified operations on cloud services based on the permissions they have been assigned.

                                                    -

                                                    DLI is a project-level service deployed and accessed in specific physical regions. To assign ServiceStage permissions to a user group, specify the scope as region-specific projects and select projects for the permissions to take effect. If All projects is selected, the permissions will take effect for the user group in all region-specific projects. When accessing DLI, the users need to switch to a region where they have been authorized to use cloud services.

                                                    -
                                                    Permission types: Based on the granularity of authorization, they are divided into roles and policies.
                                                    • Roles: A coarse-grained authorization strategy that defines permissions by job responsibility. This strategy offers limited service-level roles for authorization. If one role has a dependency role required for accessing SA, assign both roles to the users. Roles are not suitable for fine-grained authorization and least privilege access.
                                                    • Policies: A fine-grained authorization strategy that defines permissions required to perform operations on specific cloud resources under certain conditions. This type of authorization is more flexible and is ideal for least privilege access. For example, you can grant DLI users only the permissions for managing a certain type of ECSs. For the actions supported by DLI APIs, see "Permissions Policies and Supported Actions" in the Data Lake Insight API Reference.
                                                    +

                                                    After purchasing DLI resources, you can use IAM to assign different access rights to employees, ensuring proper isolation between roles. IAM provides identity authentication, permission assignment, and access control, helping you securely manage access to cloud resources.

                                                    +

                                                    With IAM, you can create users under your account and apply policies to define their access scope. For example, developers may need permission to use DLI but should not be allowed to delete it. In this case, you can create IAM users for developers and assign a policy that grants usage rights while restricting deletion.

                                                    +

                                                    If your account already meets your needs, you can skip creating separate IAM users without affecting other DLI functions.

                                                    +

                                                    DLI Permissions

                                                    By default, newly created IAM users have no permissions. To enable access, you must add them to a user group and assign roles or policies. This process is called authorization. Once authorized, users can perform operations within the scope of their assigned permissions.

                                                    +

                                                    DLI is deployed by physical region and operates as a project-level service. When authorizing, select Region-specific projects for Scope, then assign permissions within the specific project. These permissions apply only to that project. If you assign permissions to All projects, they apply across all regions. To access DLI, users must first switch to the authorized region.

                                                    +
                                                    Permission types: Permissions are categorized into roles and policies.
                                                    • Roles: A coarse-grained authorization mechanism based on job responsibilities. This strategy offers limited service-level roles for authorization. Granting a role may require assigning additional dependency roles. Roles cannot fully meet fine-grained authorization or strict least-privilege access.
                                                    • Policies: A fine-grained authorization mechanism that defines permissions at the level of specific operations, resources, and conditions. Policies provide flexible control and support enterprise-level least-privilege security. For example, you can restrict IAM users to perform only certain management operations on specific DLI resources. For details about the actions supported by DLI APIs, see "Permissions Policies and Supported Actions" in the Data Lake Insight API Reference.
                                                    Table 1 DLI system permissions

                                                    Role/Policy Name

                                                    @@ -62,7 +62,7 @@
                                                    -

                                                    Table 2 lists the common operations supported by each system policy. You can choose required system policies according to this table.

                                                    +

                                                    Table 2 lists the common operations supported by each system policy. You can use this table to select appropriate system policies.

                                                    diff --git a/docs/dli/umn/dli_07_0007.html b/docs/dli/umn/dli_07_0007.html index 5159f0d71..0ce30d0ff 100644 --- a/docs/dli/umn/dli_07_0007.html +++ b/docs/dli/umn/dli_07_0007.html @@ -1,14 +1,14 @@

                                                    Advantages

                                                    -

                                                    Full SQL Compatibility

                                                    You do not need a background in big data to use DLI for data analysis. You only need to know SQL, and you are good to go. The SQL syntax is fully compatible with the standard ANSI SQL 2003.

                                                    +

                                                    Pure SQL Operations: Zero Learning Curve

                                                    • DLI offers standard SQL APIs, enabling you to perform massive data query and analysis using only SQL. Its syntax is fully compatible with ANSI SQL 2003 standards.
                                                    • This significantly lowers the barrier for data analysts and business professionals, enhancing overall efficiency in data analysis.
                                                    -

                                                    Decoupled Storage and Compute

                                                    DLI compute and storage loads are decoupled. This architecture allows you to flexibly configure storage and compute resources on demand, improving resource utilization and reducing costs.

                                                    -

                                                    DLI's elastic resource pool feature effectively boosts compute resource utilization. The same compute resources can support multiple compute engines like Flink and Spark simultaneously.

                                                    +

                                                    Decoupled Storage and Compute: Efficient Resource Utilization

                                                    • DLI decouples storage and compute workloads through its decoupling architecture, allowing flexible configuration of resources based on demand. This improves resource utilization and reduces costs.
                                                    • The elastic resource pool supports multiple engines like Flink and Spark, further optimizing resource allocation efficiency.
                                                    -

                                                    Serverless DLI

                                                    DLI is fully compatible with Apache Spark and Apache Flink ecosystems and APIs. It is a serverless big data computing and analysis service that integrates real-time, offline, and interactive analysis. Offline applications can be seamlessly migrated to the cloud, reducing the migration workload. DLI provides a highly-scalable framework integrating batch and stream processing, allowing you to handle data analysis requests with ease. With a deeply optimized kernel and architecture, DLI delivers 100-fold performance improvement compared with the MapReduce model. Your analysis is backed by an industry-vetted 99.95% SLA.

                                                    +

                                                    Serverless Architecture: Full-Scenario Adaptability

                                                    DLI is fully compatible with Apache Spark and Apache Flink ecosystems and APIs, providing a unified serverless big data computing service for real-time, offline, and interactive analytics.

                                                    +
                                                    • On-premises Spark/Flink applications can migrate to the cloud effortlessly, minimizing migration efforts and ensuring smooth transitions.
                                                    • A batch-stream fusion framework delivers scalable, high-performance processing for TB to EB-level data, meeting diverse big data needs.
                                                    • Deep optimizations in product core and architecture result in performance over 100x faster than traditional MapReduce models, with 99.95% SLA.
                                                    -

                                                    Cross-Source Analysis

                                                    Analyze your data across databases. No migration required. A unified view of your data gives you a comprehensive understanding of your data and helps you innovate faster. There are no restrictions on data formats, cloud data sources, or whether the database is created online or off.

                                                    +

                                                    Cross-Source Analysis: No Data Migration Required

                                                    • Supports multiple data formats and sources, including cloud-based (e.g., OBS, RDS, DWS, CSS, MongoDB, Redis), ECS-hosted databases, and on-premises databases.
                                                    • Enables unified cross-source analysis without data relocation, accelerating enterprise-wide data insights and innovation.
                                                    diff --git a/docs/dli/umn/dli_07_0009.html b/docs/dli/umn/dli_07_0009.html index e2eefe8fe..712f78045 100644 --- a/docs/dli/umn/dli_07_0009.html +++ b/docs/dli/umn/dli_07_0009.html @@ -1,17 +1,17 @@ -

                                                    Quotas

                                                    -

                                                    What Is a Quota?

                                                    A quota limits the quantity of a resource available to users, thereby preventing spikes in the usage of the resource.

                                                    -

                                                    You can also request for an increased quota if your existing quota cannot meet your service requirements.

                                                    +

                                                    Quota Management

                                                    +

                                                    What Is a Quota?

                                                    Quotas are enforced for service resources on the platform to prevent unforeseen spikes in resource usage. Quotas can limit the quantity and capacity of resources available to users.

                                                    +

                                                    If your current quota does not meet your needs, you can apply for an increase.

                                                    -

                                                    How Do I View My Quotas?

                                                    1. Log in to the management console.
                                                    2. Click in the upper left corner and select a region and a project.
                                                    3. Click the My Quota icon in the upper right corner of the page.

                                                      The Service Quota page is displayed.

                                                      -
                                                    4. View the used and total quota of each type of resources on the displayed page.

                                                      If a quota cannot meet service requirements, increase a quota.

                                                      +

                                                      How Do I View My Quotas?

                                                      1. Log in to the management console.
                                                      2. Click in the upper left corner and select a region and a project.
                                                      3. Click the My Quota icon in the upper right corner of the page.

                                                        This will take you to the Service Quota page.

                                                        +
                                                      4. Here, you can view the total quota and usage details for each resource.

                                                        If your current quota is insufficient, follow the steps below to request an increase.

                                                      -

                                                      How Do I Apply for a Higher Quota?

                                                      The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your application and inform you of the progress by phone call or email.

                                                      -

                                                      Before dialing the hotline number or sending an email, ensure that the following information has been obtained:

                                                      +

                                                      How Do I Apply for a Quota Increase?

                                                      The system currently does not support online quota adjustments. If you need to modify your quota, please contact our customer service team via phone or email. They will promptly process your request and keep you updated on its progress through either a call or email.

                                                      +

                                                      Before reaching out, ensure you have the following details ready:

                                                      • Domain name, project name, and project ID

                                                        To obtain the preceding information, log in to the management console, click the username in the upper right corner, and choose My Credentials from the drop-down list.

                                                        -
                                                      • Quota information, including:
                                                        • Service name
                                                        • Quota type
                                                        • Required quota
                                                        +
                                                      • Quota information, including:
                                                        • Service name
                                                        • Quota type
                                                        • Desired quota value

                                                      Learn how to obtain the service hotline and email address.

                                                      diff --git a/docs/dli/umn/dli_07_0027.html b/docs/dli/umn/dli_07_0027.html index a3f2c62e3..79e9fa32d 100644 --- a/docs/dli/umn/dli_07_0027.html +++ b/docs/dli/umn/dli_07_0027.html @@ -1,12 +1,12 @@

                                                      Compute Resource Types and Specifications

                                                      -

                                                      DLI compute resources are the foundation for job execution. Both DLI's elastic resource pools and queues fall under compute resources. This section introduces the types and specifications of DLI compute resources.

                                                      +

                                                      When performing big data analysis with DLI, you need to select the appropriate compute resources based on specific business needs. DLI offers a variety of resource types and product specifications, including elastic resource pools and queues. This section provides a detailed overview of these options to help you make informed decisions tailored to your requirements.

                                                      What Are Elastic Resource Pools and Queues?

                                                      Before we dive into the compute resource modes of DLI, let us first understand the basic concepts of elastic resource pools and queues.

                                                      • Elastic resource pool

                                                        An elastic resource pool is a pooled management mode for DLI compute resources, which can be seen as a collection of DLI compute resources. DLI supports the creation of multiple queues within an elastic resource pool, and these queues can share the resources in the elastic resource pool.

                                                        For details about the specifications of elastic resource pools, see Elastic Resource Pool Specifications.

                                                        -
                                                        • The physical resource layer of an elastic resource pool consists of compute nodes distributed in different AZs.
                                                        • Multiple queues within the same resource pool share physical resources but maintain logical isolation to enforce resource allocation policies such as priorities and quotas.
                                                        • Elastic resource pools can adjust resources in real time based on queue loads, achieving on-demand auto scaling in minutes.
                                                        • An elastic resource pool can simultaneously support SQL, Spark, and Flink jobs. The specific job types supported depend on the queue types created within the elastic resource pool.

                                                          Refer to DLI Compute Resource Modes and Supported Queue Types.

                                                          +
                                                          • The physical resource layer of an elastic resource pool consists of compute nodes distributed in different AZs.
                                                          • Multiple queues within the same resource pool share physical resources but maintain logical isolation to enforce resource allocation policies such as priorities and quotas.
                                                          • Elastic resource pools can adjust resources in real time based on queue loads, achieving on-demand auto scaling in minutes.
                                                          • An elastic resource pool can simultaneously support SQL, Spark, and Flink jobs. The specific job types supported depend on the queue types created within the elastic resource pool.

                                                            See DLI Compute Resource Modes and Supported Queue Types.

                                                        • Queue

                                                          Queues are the basic units of compute resources that are actually used and allocated in DLI. You can create different queues for different jobs or data processing tasks, and allocate and adjust resources for these queues as needed.

                                                          DLI is divided into three queue types: default queue, for SQL, and for general purpose. You can choose the most suitable queue type based on your business scenario and job characteristics.

                                                          @@ -14,7 +14,7 @@

                                                          The default queue does not support specifying the size of resources and resources are allocated on-demand during job execution, with billing based on the actual amount of data scanned.

                                                          As resources of the default queue are shared among all users, there may be resource contention during use, and it cannot be guaranteed that resources will be available for every job execution.

                                                          The default queue is suitable for small-scale or temporary data processing needs. For important jobs or jobs that require guaranteed resources, you are advised to buy an elastic resource pool and create queues within it to execute jobs.

                                                          -
                                                        • For SQL:

                                                          For SQL queues are used to execute SQL jobs and supports specifying engine types including Spark and HetuEngine.

                                                          +
                                                        • For SQL:

                                                          For SQL queues are designed to execute SQL jobs. They support the Spark engine.

                                                          This type of queues is suitable for businesses that require fast data query and analysis, as well as regular cache clearing or environment resetting.

                                                        • For general purpose:

                                                          For general purpose queues are used to execute Spark jobs, Flink OpenSource SQL jobs, and Flink Jar jobs.

                                                          @@ -24,7 +24,7 @@

                                                      DLI Compute Resource Modes

                                                      DLI offers three compute resource management modes, each with unique advantages and use cases.

                                                      Figure 1 DLI compute resource modes
                                                      -
                                                      • Elastic resource pool mode:
                                                        A pooled management mode for compute resources that provides dynamic scaling capabilities. Queues within the same elastic resource pool share compute resources. By appropriately setting up compute resource allocation policies for queues, you can enhance compute resource utilization and handle peak business demands efficiently.
                                                        • Use cases: suitable for scenarios with significant fluctuations in business volume, such as periodic data batch processing tasks or real-time data processing needs.
                                                        • Supported queue types: for SQL (Spark), for SQL (HetuEngine), and for general purpose. For details about DLI queue types, see Queue Types.

                                                          General-purpose queues and SQL queues in elastic resource pool mode do not support cross-AZ deployment.

                                                          +
                                                          • Elastic resource pool mode:
                                                            A pooled management mode for compute resources that provides dynamic scaling capabilities. Queues within the same elastic resource pool share compute resources. By appropriately setting up compute resource allocation policies for queues, you can enhance compute resource utilization and handle peak business demands efficiently.
                                                            • Use cases: suitable for scenarios with significant fluctuations in business volume, such as periodic data batch processing tasks or real-time data processing needs.
                                                            • Supported queue types: for SQL (Spark) and for general purpose. For details about DLI queue types, see Queue Types.

                                                              General-purpose queues and SQL queues in elastic resource pool mode do not support cross-AZ deployment.

                                                            • Usage: first create an elastic resource pool, then create queues within the pool and allocate compute resources. Queues are associated with specific jobs and data processing tasks.
                                                            @@ -55,7 +55,6 @@
                                                    Table 2 Common operations supported by each system permission

                                                    Resource

                                                    Elastic resource pool mode

                                                    For SQL (Spark)

                                                    -

                                                    For SQL (HetuEngine)

                                                    For general purpose

                                                    Resources are shared among multiple queues for a single user.

                                                    diff --git a/docs/dli/umn/dli_09_0210.html b/docs/dli/umn/dli_09_0210.html new file mode 100644 index 000000000..b6b98b16a --- /dev/null +++ b/docs/dli/umn/dli_09_0210.html @@ -0,0 +1,180 @@ + + +

                                                    Flink OpenSource SQL Jobs Using DEW to Manage Access Credentials

                                                    +

                                                    Scenario

                                                    When DLI writes the output data of Flink jobs to MySQL or DWS, you need to set sensitive parameters such as the username and password in the connector. Storing this information in plaintext poses significant security risks. To safeguard user data privacy, you are advised to encrypt these credentials.

                                                    +

                                                    Data Encryption Workshop (DEW) and Cloud Secret Management Service (CSMS) offer a secure, reliable, and easy-to-use solution for encrypting and decrypting sensitive data. By hosting database account details (for example, username and password) as managed secrets within CSMS, you can securely reference these credentials in your Flink jobs. This ensures that sensitive information is retrieved through a secure channel during runtime.

                                                    +

                                                    Additionally, CSMS offers comprehensive lifecycle management for credentials, enhancing both security and efficiency. It effectively mitigates risks associated with hardcoding sensitive information or storing it in plaintext configurations, thereby preventing unauthorized access and potential business disruptions.

                                                    +

                                                    This section walks you through on how to use DEW to manage access credentials for Flink OpenSource SQL jobs.

                                                    +
                                                    +

                                                    Notes and Constraints

                                                    DEW can be used to manage access credentials only in Flink 1.15. When creating a Flink job, select version 1.15 and configure the information of the agency that allows DLI to access DEW for the job.

                                                    +
                                                    +

                                                    Prerequisites

                                                    • A shared secret has been created on the DEW console and the secret value has been stored.
                                                    • An agency has been created and authorized for DLI to access DEW. The agency must have been granted the following permissions:
                                                      • Permission of the ShowSecretVersion interface for querying secret versions and secret values in DEW: csms:secretVersion:get.
                                                      • Permission of the ListSecretVersions interface for listing secret versions in DEW: csms:secretVersion:list.
                                                      • Permission to decrypt DEW secrets: kms:dek:decrypt
                                                      +
                                                    • On the DLI management console, create an enhanced datasource connection and configure the network connection between DLI and the data source.

                                                      +
                                                    +
                                                    +

                                                    Syntax

                                                    create table tableName(
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (',' WATERMARK FOR rowtime_column_name AS watermark-strategy_expression)
+)
+with (
+   ...
+  'dew.endpoint'='',
+  'dew.csms.secretName'='',
+  'dew.csms.decrypt.fields'='',
+  'dew.projectId'='',
+  'dew.csms.version'=''
+  
+);
                                                    +
                                                    +

                                                    Parameter Description

                                                    +
                                                    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                    Table 1 Parameter descriptions

                                                    Parameter

                                                    +

                                                    Mandatory

                                                    +

                                                    Default Value

                                                    +

                                                    Data Type

                                                    +

                                                    Description

                                                    +

                                                    dew.endpoint

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    Endpoint of the DEW service to be used.

                                                    +

                                                    dew.projectId

                                                    +

                                                    No

                                                    +

                                                    Yes

                                                    +

                                                    String

                                                    +

                                                    ID of the project DEW belongs to. The default value is the ID of the project where the Flink job is.

                                                    +

                                                    dew.csms.secretName

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    Name of the shared secret in DEW's secret management.

                                                    +

                                                    Configuration example: 'dew.csms.secretName'='secretInfo'

                                                    +

                                                    dew.csms.decrypt.fields

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    Specify which fields in the connector with attribute need to be decrypted using DEW's CSMS.

                                                    +

                                                    Separate the field attributes with commas, for example, 'dew.csms.decrypt.fields'='field1,field2,field3'

                                                    +

                                                    dew.csms.version

                                                    +

                                                    Yes

                                                    +

                                                    Latest version

                                                    +

                                                    String

                                                    +

                                                    Version number (secret version identifier) of the shared secret created in DEW CSMS.

                                                    +

                                                    If the latest version number (secret version identifier) is not specified, the system will not be able to directly retrieve the most recent version of the secret, potentially leading to application access failures or the use of outdated secrets, thereby compromising data security and service stability.

                                                    +

                                                    View the version information of the secret on the DEW management console and configure this parameter with the latest version number to ensure that applications can securely and reliably access the necessary data.

                                                    +

                                                    Configuration example: 'dew.csms.version'='v1'

                                                    +
                                                    +
                                                    +
                                                    +

                                                    Example

                                                    This example demonstrates how to configure Flink OpenSource SQL to manage access credentials using DEW by generating random data through a DataGen table and outputting it to a MySQL result table.

                                                    +
                                                    1. Create a shared secret in DEW.

                                                      1. Log in to the DEW management console.
                                                      2. In the navigation pane on the left, choose Cloud Secret Management Service > Secrets.
                                                      3. Click Create Secret. On the displayed page, configure basic secret information.
                                                        • Secret Name: Enter a secret name. In this example, the name is secretInfo.
                                                        • Secret Value: Enter the username and password for logging in to the RDS for MySQL DB instance.
                                                          • The key in the first line is MySQLUsername, and the value is the username for logging in to the DB instance.
                                                          • The key in the second line is MySQLPassword, and the value is the password for logging in to the DB instance.
                                                          +
                                                        +
                                                      4. Set other parameters as required and click OK.

                                                        +
                                                      +

                                                    2. Configure DEW to manage access credentials in the Flink OpenSource SQL job.

                                                      Ensure that an enhanced datasource connection between DLI and MySQL has been created.

                                                      +

                                                      Ensure that an agency has been created for DLI to access DEW and authorization has been completed.

                                                      +
                                                      Here is an example configuration for a Flink OpenSource SQL job:
                                                       1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
                                                      create table dataGenSource(
+  user_id string,
+  amount int
+) with (
+  'connector' = 'datagen',
+  'rows-per-second' = '1', --Generate a piece of data per second.
+  'fields.user_id.kind' = 'random', --Specify a random generator for the user_id field.
+  'fields.user_id.length' = '3' --Limit the length of user_id to 3.
+);
+
+CREATE TABLE jdbcSink ( 
+  user_id string,
+  amount int
+ ) 
+WITH (  
+ 'connector' = 'jdbc',  
+  'url? = 'jdbc:mysql://MySQLAddress:MySQLPort/flink',--flink is the MySQL database where the orders table locates.
+ 'table-name' = 'orders',  
+ 'username' = 'MySQLUsername',  -- Shared secret in DEW whose name is secretInfo and version is v1. The key MySQLUsername defines the secret value. The value is the user's sensitive information.
+ 'password' = 'MySQLPassword',  -- Shared secret in DEW whose name is secretInfo and version is v1. The key MySQLPassword defines the secret value. The value is the user's sensitive information.
+ 'sink.buffer-flush.max-rows' = '1',
+ 'dew.endpoint'='endpoint', --Endpoint information for the DEW service being used
+ 'dew.csms.secretName'='secretInfo', --Name of the DEW shared secret
+ 'dew.csms.decrypt.fields'='username,password', --The username and password field values must be decrypted and replaced using DEW secret management.
+ 'dew.csms.version'='v1'
+);
+
+insert into jdbcSink select * from dataGenSOurce;
+
                                                      +
                                                      +
                                                      +

                                                    +
                                                    +
                                                    + + diff --git a/docs/dli/umn/dli_09_0211.html b/docs/dli/umn/dli_09_0211.html new file mode 100644 index 000000000..520259e50 --- /dev/null +++ b/docs/dli/umn/dli_09_0211.html @@ -0,0 +1,421 @@ + + +

                                                    Flink Jar Jobs Using DEW to Acquire Access Credentials for Reading and Writing Data from and to OBS

                                                    +

                                                    Scenario

                                                    When writing output data from Flink Jar jobs to OBS, you need to configure an AK/SK for accessing OBS. To ensure the security of AK/SK data, you can use DEW and CSMS for centralized management of AK/SK. This approach effectively mitigates risks such as sensitive information leakage caused by hardcoding in programs or plaintext configurations, as well as potential business disruptions due to unauthorized access.

                                                    +

                                                    This section walks you through on how a Flink Jar job acquires an AK/SK to read and write data from and to OBS.

                                                    +
                                                    +

                                                    Notes and Constraints

                                                    DEW can be used to manage access credentials only in Flink 1.15. When creating a Flink job, select version 1.15 and configure the information of the agency that allows DLI to access DEW for the job.

                                                    +
                                                    +

                                                    Prerequisites

                                                    • A shared secret has been created on the DEW console and the secret value has been stored.
                                                    • An agency has been created and authorized for DLI to access DEW. The agency must have been granted the following permissions:
                                                      • Permission of the ShowSecretVersion interface for querying secret versions and secret values in DEW: csms:secretVersion:get.
                                                      • Permission of the ListSecretVersions interface for listing secret versions in DEW: csms:secretVersion:list.
                                                      • Permission to decrypt DEW secrets: kms:dek:decrypt
                                                      +
                                                    • To use this function, you need to configure AK/SK for all OBS buckets.
                                                    +
                                                    +

                                                    Syntax

                                                    On the Flink Jar job editing page, set Runtime Configuration as needed. The configuration information is as follows:

                                                    +

                                                    Different OBS buckets use different AK/SK authentication information. You can use the following configuration method to specify the AK/SK information based on the bucket. For details about the parameters, see Table 1.

                                                    +
                                                    1
+2
+3
+4
+5
+6
+7
+8
                                                    flink.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.access.key=USER_AK_CSMS_KEY
+flink.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.secret.key=USER_SK_CSMS_KEY
+flink.hadoop.fs.obs.security.provider=com.dli.provider.UserObsBasicCredentialProvider
+flink.hadoop.fs.dew.csms.secretName=CredentialName
+flink.hadoop.fs.dew.endpoint=ENDPOINT
+flink.hadoop.fs.dew.csms.version=VERSION_ID
+flink.hadoop.fs.dew.csms.cache.time.second=CACHE_TIME
+flink.dli.job.agency.name=USER_AGENCY_NAME
+
                                                    +
                                                    +
                                                    +

                                                    Parameter Description

                                                    +
                                                    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                    Table 1 Parameter descriptions

                                                    Parameter

                                                    +

                                                    Mandatory

                                                    +

                                                    Default Value

                                                    +

                                                    Data Type

                                                    +

                                                    Description

                                                    +

                                                    flink.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.access.key

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    USER_BUCKET_NAME needs to be replaced with the user's OBS bucket name.

                                                    +

                                                    The value of this parameter is the key defined by the user in the CSMS shared secret. The value corresponding to the key is the user's access key ID (AK). The user must have the permission to access the bucket on OBS.

                                                    +

                                                    flink.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.secret.key

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    USER_BUCKET_NAME needs to be replaced with the user's OBS bucket name.

                                                    +

                                                    The value of this parameter is the key defined by the user in the CSMS shared secret. The value corresponding to the key is the user's secret access key (SK). The user must have the permission to access the bucket on OBS.

                                                    +

                                                    flink.hadoop.fs.obs.security.provider

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    OBS AK/SK authentication mechanism, which uses DEW-CSMS' secret management to obtain the AK and SK for accessing OBS.

                                                    +

                                                    The default value is com.dli.provider.UserObsBasicCredentialProvider.

                                                    +

                                                    flink.hadoop.fs.dew.endpoint

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    Endpoint of the DEW service to be used.

                                                    +

                                                    flink.hadoop.fs.dew.projectId

                                                    +

                                                    No

                                                    +

                                                    Yes

                                                    +

                                                    String

                                                    +

                                                    ID of the project DEW belongs to. The default value is the ID of the project where the Flink job is.

                                                    +

                                                    flink.hadoop.fs.dew.csms.secretName

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    Name of the shared secret in DEW's secret management.

                                                    +

                                                    Configuration example: flink.hadoop.fs.dew.csms.secretName=secretInfo

                                                    +

                                                    flink.hadoop.fs.dew.csms.version

                                                    +

                                                    Yes

                                                    +

                                                    Latest version

                                                    +

                                                    String

                                                    +

                                                    Version number (secret version identifier) of the shared secret created in DEW CSMS.

                                                    +

                                                    If the latest version number (secret version identifier) is not specified, the system will not be able to directly retrieve the most recent version of the secret, potentially leading to application access failures or the use of outdated secrets, thereby compromising data security and service stability.

                                                    +

                                                    View the version information of the secret on the DEW management console and configure this parameter with the latest version number to ensure that applications can securely and reliably access the necessary data.

                                                    +

                                                    Configuration example: flink.hadoop.fs.dew.csms.version=v1

                                                    +

                                                    flink.hadoop.fs.dew.csms.cache.time.second

                                                    +

                                                    No

                                                    +

                                                    3600

                                                    +

                                                    Long

                                                    +

                                                    Cache duration after the CSMS shared secret is obtained during Flink job access.

                                                    +

                                                    The unit is second. The default value is 3600 seconds.

                                                    +

                                                    flink.dli.job.agency.name

                                                    +

                                                    Yes

                                                    +

                                                    -

                                                    +

                                                    String

                                                    +

                                                    Custom agency name.

                                                    +
                                                    +
                                                    +
                                                    +

                                                    Sample Code

                                                    This section describes how to write processed DataGen data to OBS. You need to modify the parameters in the sample Java code based on site requirements.

                                                    +
                                                    1. Create an agency for DLI to access DEW and complete authorization.
                                                    2. Create a shared secret in DEW.
                                                      1. Log in to the DEW management console.
                                                      2. In the navigation pane on the left, choose Cloud Secret Management Service > Secrets.
                                                      3. On the displayed page, click Create Secret. Set basic secret information.
                                                      +
                                                    3. Set job parameters on the DLI Flink Jar job editing page.
                                                      • Class name
                                                        1
                                                        com.dli.demo.dew.DataGen2FileSystemSink
+
                                                        +
                                                        +
                                                      • Parameters
                                                        1
+2
                                                        --checkpoint.path obs://test/flink/jobs/checkpoint/120891/ 
+--output.path obs://dli/flink.db/79914/DataGen2FileSystemSink
+
                                                        +
                                                        +
                                                      • Runtime configuration:
                                                        1
+2
+3
+4
+5
+6
+7
+8
                                                        flink.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.access.key=USER_AK_CSMS_KEY
+flink.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.secret.key=USER_SK_CSMS_KEY
+flink.hadoop.fs.obs.security.provider=com.dli.provider.UserObsBasicCredentialProvider
+flink.hadoop.fs.dew.csms.secretName=obsAksK
+flink.hadoop.fs.dew.endpoint=kmsendpoint
+flink.hadoop.fs.dew.csms.version=v6
+flink.hadoop.fs.dew.csms.cache.time.second=3600
+flink.dli.job.agency.name=***
+
                                                        +
                                                        +
                                                      +
                                                    4. Flink Jar job example.
                                                      • Environment preparation

                                                        Development tools such as IntelliJ IDEA and other development tools, JDK, and Maven have been installed and configured.

                                                        +

                                                        Dependency package in POM file configurations

                                                        +
                                                         1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
                                                         <properties>
+        <flink.version>1.15.0</flink.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-statebackend-rocksdb</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-streaming-java</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- fastjson -->
+        <dependency>
+            <artifactId>fastjson</artifactId>
+            <version>2.0.15</version>
+        </dependency>
+    </dependencies>
+
                                                        +
                                                        +
                                                      • Example code
                                                         1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
                                                        import org.apache.flink.api.common.serialization.SimpleStringEncoder;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.contrib.streaming.state.EmbeddedRocksDBStateBackend;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.CheckpointConfig;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink;
+import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.OnCheckpointRollingPolicy;
+import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Random;
+
+public class DataGen2FileSystemSink {
+    private static final Logger LOG = LoggerFactory.getLogger(DataGen2FileSystemSink.class);
+
+    public static void main(String[] args) {
+        ParameterTool params = ParameterTool.fromArgs(args);
+        LOG.info("Params: " + params.toString());
+        try {
+            StreamExecutionEnvironment streamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
+
+            // set checkpoint
+            String checkpointPath = params.get("checkpoint.path", "obs://bucket/checkpoint/jobId_jobName/");
+            LocalDateTime localDateTime = LocalDateTime.ofEpochSecond(System.currentTimeMillis() / 1000,
+                0, ZoneOffset.ofHours(8));
+            String dt = localDateTime.format(DateTimeFormatter.ofPattern("yyyyMMdd_HH:mm:ss"));
+            checkpointPath = checkpointPath + dt;
+
+            streamEnv.setStateBackend(new EmbeddedRocksDBStateBackend());
+            streamEnv.getCheckpointConfig().setCheckpointStorage(checkpointPath);
+            streamEnv.getCheckpointConfig().setExternalizedCheckpointCleanup(
+                CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
+            streamEnv.enableCheckpointing(30 * 1000);
+
+            DataStream<String> stream = streamEnv.addSource(new DataGen())
+                .setParallelism(1)
+                .disableChaining();
+
+            String outputPath = params.get("output.path", "obs://bucket/outputPath/jobId_jobName");
+
+            // Sink OBS
+            final StreamingFileSink<String> sinkForRow = StreamingFileSink
+                .forRowFormat(new Path(outputPath), new SimpleStringEncoder<String>("UTF-8"))
+                .withRollingPolicy(OnCheckpointRollingPolicy.build())
+                .build();
+
+            stream.addSink(sinkForRow);
+
+            streamEnv.execute("sinkForRow");
+        } catch (Exception e) {
+            LOG.error(e.getMessage(), e);
+        }
+    }
+}
+
+class DataGen implements ParallelSourceFunction<String> {
+
+    private boolean isRunning = true;
+
+    private Random random = new Random();
+
+    @Override
+    public void run(SourceContext<String> ctx) throws Exception {
+        while (isRunning) {
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("id", random.nextLong());
+            jsonObject.put("name", "Molly" + random.nextInt());
+            jsonObject.put("address", "hangzhou" + random.nextInt());
+            jsonObject.put("birthday", System.currentTimeMillis());
+            jsonObject.put("city", "hangzhou" + random.nextInt());
+            jsonObject.put("number", random.nextInt());
+            ctx.collect(jsonObject.toJSONString());
+            Thread.sleep(1000);
+        }
+    }
+
+    @Override
+    public void cancel() {
+        isRunning = false;
+    }
+}
+
                                                        +
                                                        +
                                                      +
                                                    +
                                                    +
                                                    + + diff --git a/docs/dli/umn/dli_09_0215.html b/docs/dli/umn/dli_09_0215.html new file mode 100644 index 000000000..778d63d71 --- /dev/null +++ b/docs/dli/umn/dli_09_0215.html @@ -0,0 +1,186 @@ + + +

                                                    Spark Jar Jobs Using DEW to Acquire Access Credentials for Reading and Writing Data from and to OBS

                                                    +

                                                    What Is a Temporary Credential?

                                                    A temporary security credential grants temporary access rights. It includes a temporary AK/SK and a security token, both of which must be used together.

                                                    +
                                                    +

                                                    Scenario

                                                    When writing output data from Spark Jar jobs to OBS, you need to configure an AK/SK for accessing OBS. To ensure the security of AK/SK data, you can use DEW and CSMS for centralized management of AK/SK. This approach effectively mitigates risks such as sensitive information leakage caused by hardcoding in programs or plaintext configurations, as well as potential business disruptions due to unauthorized access.

                                                    +

                                                    This section walks you through on how a Spark Jar job acquires an AK/SK to read and write data from and to OBS.

                                                    +
                                                    +

                                                    Notes and Constraints

                                                    • DEW can be used to manage access credentials only in Spark 3.3.1 (Spark general queue scenario) or later. When creating a Spark job, select version 3.3.1 and configure the information of the agency that allows DLI to access DEW for the job.

                                                      +
                                                    • To use this function, you need to configure AK/SK for all OBS buckets.
                                                    +
                                                    +

                                                    Prerequisites

                                                    • A shared secret has been created on the DEW console and the secret value has been stored.
                                                    • An agency has been created and authorized for DLI to access DEW. The agency must have been granted the following permissions:
                                                      • Permission of the ShowSecretVersion interface for querying secret versions and secret values in DEW: csms:secretVersion:get.
                                                      • Permission of the ListSecretVersions interface for listing secret versions in DEW: csms:secretVersion:list.
                                                      • Permission to decrypt DEW secrets: kms:dek:decrypt
                                                      +
                                                    +
                                                    +

                                                    Syntax

                                                    On the Spark Jar job editing page, configure the Spark Arguments(--conf) parameter as needed. The configuration information is as follows:

                                                    +

                                                    Different OBS buckets use different AK/SK authentication information. You can use the following configuration method to specify the AK/SK information based on the bucket. For details about the parameters, see Table 1.

                                                    +
                                                    1
+2
+3
+4
+5
+6
+7
+8
                                                    spark.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.access.key= USER_AK_CSMS_KEY
+spark.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.secret.key= USER_SK_CSMS_KEY
+spark.hadoop.fs.obs.security.provider = com.dli.provider.UserObsBasicCredentialProvider
+spark.hadoop.fs.dew.csms.secretName= CredentialName
+spark.hadoop.fs.dew.endpoint=ENDPOINT
+spark.hadoop.fs.dew.csms.version=VERSION_ID
+spark.hadoop.fs.dew.csms.cache.time.second =CACHE_TIME
+spark.dli.job.agency.name=USER_AGENCY_NAME
+
                                                    +
                                                    +
                                                    +

                                                    Parameter Description

                                                    +
                                                    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                    Table 1 Parameter descriptions

                                                    Parameter

                                                    +

                                                    Mandatory

                                                    +

                                                    Default Value

                                                    +

                                                    Data Type

                                                    +

                                                    Description

                                                    +

                                                    spark.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.access.key

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    USER_BUCKET_NAME needs to be replaced with the user's OBS bucket name.

                                                    +

                                                    The value of this parameter is the key defined by the user in the CSMS shared secret. The value corresponding to the key is the user's access key ID (AK). The user must have the permission to access the bucket on OBS.

                                                    +

                                                    spark.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.secret.key

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    USER_BUCKET_NAME needs to be replaced with the user's OBS bucket name.

                                                    +

                                                    The value of this parameter is the key defined by the user in the CSMS shared secret. The value corresponding to the key is the user's secret access key (SK). The user must have the permission to access the bucket on OBS.

                                                    +

                                                    spark.hadoop.fs.obs.security.provider

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    OBS AK/SK authentication mechanism, which uses DEW-CSMS' secret management to obtain the AK and SK for accessing OBS.

                                                    +

                                                    The default value is com.dli.provider.UserObsBasicCredentialProvider.

                                                    +

                                                    spark.hadoop.fs.dew.csms.secretName

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    Name of the shared secret in DEW's secret management.

                                                    +

                                                    Configuration example: spark.hadoop.fs.dew.csms.secretName=secretInfo

                                                    +

                                                    spark.hadoop.fs.dew.endpoint

                                                    +

                                                    Yes

                                                    +

                                                    None

                                                    +

                                                    String

                                                    +

                                                    Endpoint of the DEW service to be used.

                                                    +

                                                    spark.hadoop.fs.dew.csms.version

                                                    +

                                                    Yes

                                                    +

                                                    Latest version

                                                    +

                                                    String

                                                    +

                                                    Version number (secret version identifier) of the shared secret created in DEW CSMS.

                                                    +

                                                    If the latest version number (secret version identifier) is not specified, the system will not be able to directly retrieve the most recent version of the secret, potentially leading to application access failures or the use of outdated secrets, thereby compromising data security and service stability.

                                                    +

                                                    View the version information of the secret on the DEW management console and configure this parameter with the latest version number to ensure that applications can securely and reliably access the necessary data.

                                                    +

                                                    Configuration example: spark.hadoop.fs.dew.csms.version=v1

                                                    +

                                                    spark.hadoop.fs.dew.csms.cache.time.second

                                                    +

                                                    No

                                                    +

                                                    3600

                                                    +

                                                    Long

                                                    +

                                                    Cache duration after the CSMS shared secret is obtained during Spark job access.

                                                    +

                                                    The unit is second. The default value is 3600 seconds.

                                                    +

                                                    spark.hadoop.fs.dew.projectId

                                                    +

                                                    No

                                                    +

                                                    Yes

                                                    +

                                                    String

                                                    +

                                                    ID of the project DEW belongs to. The default value is the ID of the project where the Spark job is.

                                                    +

                                                    spark.dli.job.agency.name

                                                    +

                                                    Yes

                                                    +

                                                    -

                                                    +

                                                    String

                                                    +

                                                    Custom agency name.

                                                    +
                                                    +
                                                    +
                                                    +

                                                    Sample Code

                                                    This section describes how to write processed DataGen data to OBS. You need to modify the parameters in the sample Java code based on site requirements.

                                                    +
                                                    1. Create an agency for DLI to access DEW and complete authorization.

                                                      +
                                                    2. Create a shared secret in DEW.
                                                      1. Log in to the DEW management console.
                                                      2. In the navigation pane on the left, choose Cloud Secret Management Service > Secrets.
                                                      3. On the displayed page, click Create Secret. Set basic secret information.
                                                      +
                                                    3. Set job parameters on the DLI Spark Jar job editing page.
                                                      Spark Arguments
                                                      1
+2
+3
+4
+5
+6
+7
                                                      spark.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.access.key= USER_AK_CSMS_KEY
+spark.hadoop.fs.obs.bucket.USER_BUCKET_NAME.dew.secret.key= USER_SK_CSMS_KEY
+spark.hadoop.fs.obs.security.provider=com.dli.provider.UserObsBasicCredentialProvider
+spark.hadoop.fs.dew.csms.secretName=obsAkSk
+spark.hadoop.fs.dew.endpoint=kmsendpoint
+spark.hadoop.fs.dew.csms.version=v3
+spark.dli.job.agency.name=agency
+
                                                      +
                                                      +
                                                      +
                                                    +
                                                    +
                                                    + +