diff --git a/docs/dli/sqlreference/ALL_META.TXT.json b/docs/dli/sqlreference/ALL_META.TXT.json index 6313d6c5d..e15b31ea0 100644 --- a/docs/dli/sqlreference/ALL_META.TXT.json +++ b/docs/dli/sqlreference/ALL_META.TXT.json @@ -14,7 +14,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Spark SQL Syntax Reference", @@ -32,7 +34,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Common Configuration Items of Batch SQL Jobs", @@ -50,7 +54,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"SQL Syntax Overview of Batch Jobs", @@ -68,7 +74,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Spark Open Source Commands", @@ -86,7 +94,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Databases", @@ -104,7 +114,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Database", @@ -122,7 +134,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Deleting a Database", @@ -135,15 +149,17 @@ "code":"8", "des":"This syntax is used to view the information about a specified database, including the database name and database description.EXTENDED: Displays the database properties.If", "doc_type":"sqlreference", - "kw":"Viewing a Specified Database,Databases,SQL Syntax Reference", + "kw":"Checking a Specified Database,Databases,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Viewing a Specified Database", + "title":"Checking a Specified Database", "githuburl":"" }, { @@ -153,15 +169,17 @@ "code":"9", "des":"This syntax is used to query all current databases.NoneKeyword DATABASES is equivalent to SCHEMAS. You can use either of them in this statement.View all the current datab", "doc_type":"sqlreference", - "kw":"Viewing All Databases,Databases,SQL Syntax Reference", + "kw":"Checking All Databases,Databases,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Viewing All Databases", + "title":"Checking All Databases", "githuburl":"" }, { @@ -176,7 +194,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating an OBS Table", @@ -194,7 +214,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating an OBS Table Using the DataSource Syntax", @@ -212,7 +234,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating an OBS Table Using the Hive Syntax", @@ -230,7 +254,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a DLI Table", @@ -248,7 +274,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a DLI Table Using the DataSource Syntax", @@ -266,7 +294,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a DLI Table Using the Hive Syntax", @@ -284,7 +314,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Deleting a Table", @@ -297,15 +329,17 @@ "code":"17", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", - "kw":"Viewing Tables", + "kw":"Checking Tables", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Viewing Tables", + "title":"Checking Tables", "githuburl":"" }, { @@ -313,17 +347,19 @@ "node_id":"dli_08_0090.xml", "product_code":"dli", "code":"18", - "des":"This statement is used to view all tables and views in the current database.FROM/IN: followed by the name of a database whose tables and views will be displayed.NoneCreat", + "des":"This statement is used to check all tables and views in the current database.FROM/IN: followed by the name of a database whose tables and views will be displayed.NoneCrea", "doc_type":"sqlreference", - "kw":"Viewing All Tables,Viewing Tables,SQL Syntax Reference", + "kw":"Checking All Tables,Checking Tables,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Viewing All Tables", + "title":"Checking All Tables", "githuburl":"" }, { @@ -331,17 +367,19 @@ "node_id":"dli_08_0091.xml", "product_code":"dli", "code":"19", - "des":"This statement is used to show the statements for creating a table.CREATE TABLE: statement for creating a tableThe table specified in this statement must exist. Otherwise", + "des":"This statement is used to show the statements for creating a table.Use the following syntax to view table creation statements when using Spark 3.3.1 (this applies only to", "doc_type":"sqlreference", - "kw":"Viewing Table Creation Statements,Viewing Tables,SQL Syntax Reference", + "kw":"Checking Table Creation Statements,Checking Tables,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Viewing Table Creation Statements", + "title":"Checking Table Creation Statements", "githuburl":"" }, { @@ -351,15 +389,17 @@ "code":"20", "des":"Check the properties of a table.TBLPROPERTIES: This statement allows you to add a key/value property to a table.property_name is case sensitive. You cannot specify multip", "doc_type":"sqlreference", - "kw":"Viewing Table Properties,Viewing Tables,SQL Syntax Reference", + "kw":"Checking Table Properties,Checking Tables,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Viewing Table Properties", + "title":"Checking Table Properties", "githuburl":"" }, { @@ -369,15 +409,17 @@ "code":"21", "des":"This statement is used to query all columns in a specified table.COLUMNS: columns in the current tableFROM/IN: followed by the name of a database whose tables and views w", "doc_type":"sqlreference", - "kw":"Viewing All Columns in a Specified Table,Viewing Tables,SQL Syntax Reference", + "kw":"Checking All Columns in a Specified Table,Checking Tables,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Viewing All Columns in a Specified Table", + "title":"Checking All Columns in a Specified Table", "githuburl":"" }, { @@ -385,17 +427,19 @@ "node_id":"dli_08_0094.xml", "product_code":"dli", "code":"22", - "des":"This statement is used to view all partitions in a specified table.PARTITIONS: partitions in a specified tablePARTITION: a specified partitionThe table specified in this ", + "des":"This statement is used to check all partitions in a specified table.PARTITIONS: partitions in a specified tablePARTITION: a specified partitionThe table specified in this", "doc_type":"sqlreference", - "kw":"Viewing All Partitions in a Specified Table,Viewing Tables,SQL Syntax Reference", + "kw":"Checking All Partitions in a Specified Table,Checking Tables,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Viewing All Partitions in a Specified Table", + "title":"Checking All Partitions in a Specified Table", "githuburl":"" }, { @@ -403,17 +447,19 @@ "node_id":"dli_08_0105.xml", "product_code":"dli", "code":"23", - "des":"This statement is used to view the table statistics. The names and data types of all columns in a specified table will be returned.EXTENDED: displays all metadata of the ", + "des":"This statement is used to check table statistics. The names and data types of all columns in a specified table will be returned.EXTENDED: displays all metadata of the spe", "doc_type":"sqlreference", - "kw":"Viewing Table Statistics,Viewing Tables,SQL Syntax Reference", + "kw":"Checking Table Statistics,Checking Tables,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Viewing Table Statistics", + "title":"Checking Table Statistics", "githuburl":"" }, { @@ -428,7 +474,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Modifying a Table", @@ -446,7 +494,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Adding a Column", @@ -464,7 +514,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Modifying Column Comments", @@ -482,7 +534,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Enabling or Disabling Multiversion Backup", @@ -500,7 +554,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Syntax for Partitioning a Table", @@ -518,7 +574,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Adding Partition Data (Only OBS Tables Supported)", @@ -536,7 +594,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Renaming a Partition (Only OBS Tables Supported)", @@ -554,7 +614,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Deleting a Partition", @@ -572,7 +634,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Deleting Partitions by Specifying Filter Criteria (Only Supported on OBS Tables)", @@ -590,7 +654,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Altering the Partition Location of a Table (Only OBS Tables Supported)", @@ -608,7 +674,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Updating Partitioned Table Data (Only OBS Tables Supported)", @@ -626,7 +694,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Updating Table Metadata with REFRESH TABLE", @@ -644,7 +714,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Importing Data to the Table", @@ -655,24 +727,46 @@ "node_id":"dli_08_0095.xml", "product_code":"dli", "code":"37", - "des":"This statement is used to insert the SELECT query result or a certain data record into a table.Insert the SELECT query result into a table.INSERT INTO [TABLE] [db_name.]t", + "des":"This statement is used to insert the SELECT query result or a certain data record into a table.The INSERT OVERWRITE syntax is not suitable for \"read-write\" scenarios, whe", "doc_type":"sqlreference", "kw":"Inserting Data,Spark SQL Syntax Reference,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Inserting Data", "githuburl":"" }, + { + "uri":"en-us_topic_0000001873107668.html", + "node_id":"en-us_topic_0000001873107668.xml", + "product_code":"dli", + "code":"38", + "des":"To improve query performance, caching the results of a subquery and reusing them in different parts of the query can be done to avoid redundant calculations of the same s", + "doc_type":"sqlreference", + "kw":"Reusing Results of Subqueries,Spark SQL Syntax Reference,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Reusing Results of Subqueries", + "githuburl":"" + }, { "uri":"dli_08_0217.html", "node_id":"dli_08_0217.xml", "product_code":"dli", - "code":"38", + "code":"39", "des":"This statement is used to delete data from the DLI or OBS table.Only data in the DLI or OBS table can be deleted.", "doc_type":"sqlreference", "kw":"Clearing Data,Spark SQL Syntax Reference,SQL Syntax Reference", @@ -680,7 +774,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Clearing Data", @@ -690,7 +786,7 @@ "uri":"dli_08_0205.html", "node_id":"dli_08_0205.xml", "product_code":"dli", - "code":"39", + "code":"40", "des":"This statement is used to directly write query results to a specified directory. The query results can be stored in CSV, Parquet, ORC, JSON, or Avro format.USING: Specifi", "doc_type":"sqlreference", "kw":"Exporting Search Results,Spark SQL Syntax Reference,SQL Syntax Reference", @@ -698,7 +794,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Exporting Search Results", @@ -708,7 +806,7 @@ "uri":"dli_08_0349.html", "node_id":"dli_08_0349.xml", "product_code":"dli", - "code":"40", + "code":"41", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Backing Up and Restoring Data of Multiple Versions", @@ -716,7 +814,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Backing Up and Restoring Data of Multiple Versions", @@ -726,7 +826,7 @@ "uri":"dli_08_0350.html", "node_id":"dli_08_0350.xml", "product_code":"dli", - "code":"41", + "code":"42", "des":"After multiversion is enabled, backup data is retained for seven days by default. You can change the retention period by setting system parameterdli.multi.version.retenti", "doc_type":"sqlreference", "kw":"Setting the Retention Period for Multiversion Backup Data,Backing Up and Restoring Data of Multiple ", @@ -734,7 +834,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Setting the Retention Period for Multiversion Backup Data", @@ -744,25 +846,27 @@ "uri":"dli_08_0351.html", "node_id":"dli_08_0351.xml", "product_code":"dli", - "code":"42", + "code":"43", "des":"After the multiversion function is enabled, you can run the SHOW HISTORY command to view the backup data of a table. For details about the syntax for enabling or disablin", "doc_type":"sqlreference", - "kw":"Viewing Multiversion Backup Data,Backing Up and Restoring Data of Multiple Versions,SQL Syntax Refer", + "kw":"Checking Multiversion Backup Data,Backing Up and Restoring Data of Multiple Versions,SQL Syntax Refe", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Viewing Multiversion Backup Data", + "title":"Checking Multiversion Backup Data", "githuburl":"" }, { "uri":"dli_08_0352.html", "node_id":"dli_08_0352.xml", "product_code":"dli", - "code":"43", + "code":"44", "des":"After the multiversion function is enabled, you can run the RESTORE TABLE statement to restore a table or partition of a specified version. For details about the syntax f", "doc_type":"sqlreference", "kw":"Restoring Multiversion Backup Data,Backing Up and Restoring Data of Multiple Versions,SQL Syntax Ref", @@ -770,7 +874,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Restoring Multiversion Backup Data", @@ -780,7 +886,7 @@ "uri":"dli_08_0353.html", "node_id":"dli_08_0353.xml", "product_code":"dli", - "code":"44", + "code":"45", "des":"After the multiversion function is enabled, expired backup data will be directly deleted by the system when theinsert overwrite or truncate statement is executed. You can", "doc_type":"sqlreference", "kw":"Configuring the Trash Bin for Expired Multiversion Data,Backing Up and Restoring Data of Multiple Ve", @@ -788,7 +894,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Configuring the Trash Bin for Expired Multiversion Data", @@ -798,7 +906,7 @@ "uri":"dli_08_0355.html", "node_id":"dli_08_0355.xml", "product_code":"dli", - "code":"45", + "code":"46", "des":"The retention period of multiversion backup data takes effect each time the insert overwrite or truncate statement is executed. If neither statement is executed for the t", "doc_type":"sqlreference", "kw":"Deleting Multiversion Backup Data,Backing Up and Restoring Data of Multiple Versions,SQL Syntax Refe", @@ -806,17 +914,19 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Deleting Multiversion Backup Data", "githuburl":"" }, { - "uri":"en-us_topic_0000001571023676.html", - "node_id":"en-us_topic_0000001571023676.xml", + "uri":"dli_08_0465.html", + "node_id":"dli_08_0465.xml", "product_code":"dli", - "code":"46", + "code":"47", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Table Lifecycle Management", @@ -824,17 +934,19 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Table Lifecycle Management", "githuburl":"" }, { - "uri":"en-us_topic_0000001621263317.html", - "node_id":"en-us_topic_0000001621263317.xml", + "uri":"dli_08_0466.html", + "node_id":"dli_08_0466.xml", "product_code":"dli", - "code":"47", + "code":"48", "des":"DLI provides table lifecycle management to allow you to specify the lifecycle of a table when creating the table. DLI determines whether to reclaim a table based on the t", "doc_type":"sqlreference", "kw":"Specifying the Lifecycle of a Table When Creating the Table,Table Lifecycle Management,SQL Syntax Re", @@ -842,17 +954,19 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Specifying the Lifecycle of a Table When Creating the Table", "githuburl":"" }, { - "uri":"en-us_topic_0000001621382957.html", - "node_id":"en-us_topic_0000001621382957.xml", + "uri":"dli_08_0467.html", + "node_id":"dli_08_0467.xml", "product_code":"dli", - "code":"48", + "code":"49", "des":"This section describes how to modify the lifecycle of an existing partitioned or non-partitioned table.When the lifecycle function is enabled for the first time, the syst", "doc_type":"sqlreference", "kw":"Modifying the Lifecycle of a Table,Table Lifecycle Management,SQL Syntax Reference", @@ -860,17 +974,19 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Modifying the Lifecycle of a Table", "githuburl":"" }, { - "uri":"en-us_topic_0000001621542965.html", - "node_id":"en-us_topic_0000001621542965.xml", + "uri":"dli_08_0468.html", + "node_id":"dli_08_0468.xml", "product_code":"dli", - "code":"49", + "code":"50", "des":"This section describes how to disable or restore the lifecycle of a specified table or partition.You can disable or restore the lifecycle of a table in either of the foll", "doc_type":"sqlreference", "kw":"Disabling or Restoring the Lifecycle of a Table,Table Lifecycle Management,SQL Syntax Reference", @@ -878,7 +994,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Disabling or Restoring the Lifecycle of a Table", @@ -888,7 +1006,7 @@ "uri":"dli_08_0118.html", "node_id":"dli_08_0118.xml", "product_code":"dli", - "code":"50", + "code":"51", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Datasource Connection with an HBase Table", @@ -896,7 +1014,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Datasource Connection with an HBase Table", @@ -906,7 +1026,7 @@ "uri":"dli_08_0119.html", "node_id":"dli_08_0119.xml", "product_code":"dli", - "code":"51", + "code":"52", "des":"This statement is used to create a DLI table and associate it with an existing HBase table.In Spark cross-source development scenarios, there is a risk of password leakag", "doc_type":"sqlreference", "kw":"Creating a DLI Table and Associating It with HBase,Creating a Datasource Connection with an HBase Ta", @@ -914,7 +1034,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a DLI Table and Associating It with HBase", @@ -924,7 +1046,7 @@ "uri":"dli_08_0120.html", "node_id":"dli_08_0120.xml", "product_code":"dli", - "code":"52", + "code":"53", "des":"This statement is used to insert data in a DLI table to the associated HBase table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field", "doc_type":"sqlreference", "kw":"Inserting Data to an HBase Table,Creating a Datasource Connection with an HBase Table,SQL Syntax Ref", @@ -932,7 +1054,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Inserting Data to an HBase Table", @@ -942,7 +1066,7 @@ "uri":"dli_08_0121.html", "node_id":"dli_08_0121.xml", "product_code":"dli", - "code":"53", + "code":"54", "des":"This statement is used to query data in an HBase table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.The table to be querie", "doc_type":"sqlreference", "kw":"Querying an HBase Table,Creating a Datasource Connection with an HBase Table,SQL Syntax Reference", @@ -950,7 +1074,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Querying an HBase Table", @@ -960,7 +1086,7 @@ "uri":"dli_08_0220.html", "node_id":"dli_08_0220.xml", "product_code":"dli", - "code":"54", + "code":"55", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Datasource Connection with an OpenTSDB Table", @@ -968,7 +1094,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Datasource Connection with an OpenTSDB Table", @@ -978,7 +1106,7 @@ "uri":"dli_08_0122.html", "node_id":"dli_08_0122.xml", "product_code":"dli", - "code":"55", + "code":"56", "des":"Run the CREATE TABLE statement to create the DLI table and associate it with the existing metric in OpenTSDB. This syntax supports the OpenTSDB of CloudTable and MRS.Befo", "doc_type":"sqlreference", "kw":"Creating a DLI Table and Associating It with OpenTSDB,Creating a Datasource Connection with an OpenT", @@ -986,7 +1114,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a DLI Table and Associating It with OpenTSDB", @@ -996,7 +1126,7 @@ "uri":"dli_08_0123.html", "node_id":"dli_08_0123.xml", "product_code":"dli", - "code":"56", + "code":"57", "des":"Run the INSERT INTO statement to insert the data in the DLI table to the associated OpenTSDB metric.If no metric exists on the OpenTSDB, a new metric is automatically cre", "doc_type":"sqlreference", "kw":"Inserting Data to the OpenTSDB Table,Creating a Datasource Connection with an OpenTSDB Table,SQL Syn", @@ -1004,7 +1134,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Inserting Data to the OpenTSDB Table", @@ -1014,7 +1146,7 @@ "uri":"dli_08_0124.html", "node_id":"dli_08_0124.xml", "product_code":"dli", - "code":"57", + "code":"58", "des":"This SELECT command is used to query data in an OpenTSDB table.If no metric exists in OpenTSDB, an error will be reported when the corresponding DLI table is queried.If t", "doc_type":"sqlreference", "kw":"Querying an OpenTSDB Table,Creating a Datasource Connection with an OpenTSDB Table,SQL Syntax Refere", @@ -1022,7 +1154,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Querying an OpenTSDB Table", @@ -1032,7 +1166,7 @@ "uri":"dli_08_0192.html", "node_id":"dli_08_0192.xml", "product_code":"dli", - "code":"58", + "code":"59", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Datasource Connection with a DWS table", @@ -1040,7 +1174,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Datasource Connection with a DWS table", @@ -1050,25 +1186,27 @@ "uri":"dli_08_0193.html", "node_id":"dli_08_0193.xml", "product_code":"dli", - "code":"59", - "des":"This statement is used to create a DLI table and associate it with an existing DWS table.In Spark cross-source development scenarios, there is a risk of password leakage ", + "code":"60", + "des":"This statement is used to create a DLI table and associate it with an existing GaussDB(DWS) table.In Spark cross-source development scenarios, there is a risk of password", "doc_type":"sqlreference", - "kw":"Creating a DLI Table and Associating It with DWS,Creating a Datasource Connection with a DWS table,S", + "kw":"Creating a DLI Table and Associating It with GaussDB(DWS),Creating a Datasource Connection with a DW", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Creating a DLI Table and Associating It with DWS", + "title":"Creating a DLI Table and Associating It with GaussDB(DWS)", "githuburl":"" }, { "uri":"dli_08_0194.html", "node_id":"dli_08_0194.xml", "product_code":"dli", - "code":"60", + "code":"61", "des":"This statement is used to insert data in a DLI table to the associated DWS table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2.", "doc_type":"sqlreference", "kw":"Inserting Data to the DWS Table,Creating a Datasource Connection with a DWS table,SQL Syntax Referen", @@ -1076,7 +1214,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Inserting Data to the DWS Table", @@ -1086,7 +1226,7 @@ "uri":"dli_08_0195.html", "node_id":"dli_08_0195.xml", "product_code":"dli", - "code":"61", + "code":"62", "des":"This statement is used to query data in a DWS table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.The table to be queried m", "doc_type":"sqlreference", "kw":"Querying the DWS Table,Creating a Datasource Connection with a DWS table,SQL Syntax Reference", @@ -1094,7 +1234,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Querying the DWS Table", @@ -1104,7 +1246,7 @@ "uri":"dli_08_0196.html", "node_id":"dli_08_0196.xml", "product_code":"dli", - "code":"62", + "code":"63", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Datasource Connection with an RDS Table", @@ -1112,7 +1254,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Datasource Connection with an RDS Table", @@ -1122,7 +1266,7 @@ "uri":"dli_08_0197.html", "node_id":"dli_08_0197.xml", "product_code":"dli", - "code":"63", + "code":"64", "des":"This statement is used to create a DLI table and associate it with an existing RDS table. This function supports access to the MySQL and PostgreSQL clusters of RDS.In Spa", "doc_type":"sqlreference", "kw":"Creating a DLI Table and Associating It with RDS,Creating a Datasource Connection with an RDS Table,", @@ -1130,7 +1274,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a DLI Table and Associating It with RDS", @@ -1140,7 +1286,7 @@ "uri":"dli_08_0198.html", "node_id":"dli_08_0198.xml", "product_code":"dli", - "code":"64", + "code":"65", "des":"This statement is used to insert data in a DLI table to the associated RDS table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2.", "doc_type":"sqlreference", "kw":"Inserting Data to the RDS Table,Creating a Datasource Connection with an RDS Table,SQL Syntax Refere", @@ -1148,7 +1294,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Inserting Data to the RDS Table", @@ -1158,7 +1306,7 @@ "uri":"dli_08_0199.html", "node_id":"dli_08_0199.xml", "product_code":"dli", - "code":"65", + "code":"66", "des":"This statement is used to query data in an RDS table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.The table to be queried ", "doc_type":"sqlreference", "kw":"Querying the RDS Table,Creating a Datasource Connection with an RDS Table,SQL Syntax Reference", @@ -1166,7 +1314,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Querying the RDS Table", @@ -1176,7 +1326,7 @@ "uri":"dli_08_0200.html", "node_id":"dli_08_0200.xml", "product_code":"dli", - "code":"66", + "code":"67", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Datasource Connection with a CSS Table", @@ -1184,7 +1334,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Datasource Connection with a CSS Table", @@ -1194,7 +1346,7 @@ "uri":"dli_08_0201.html", "node_id":"dli_08_0201.xml", "product_code":"dli", - "code":"67", + "code":"68", "des":"This statement is used to create a DLI table and associate it with an existing CSS table.In Spark cross-source development scenarios, there is a risk of password leakage ", "doc_type":"sqlreference", "kw":"Creating a DLI Table and Associating It with CSS,Creating a Datasource Connection with a CSS Table,S", @@ -1202,7 +1354,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a DLI Table and Associating It with CSS", @@ -1212,7 +1366,7 @@ "uri":"dli_08_0202.html", "node_id":"dli_08_0202.xml", "product_code":"dli", - "code":"68", + "code":"69", "des":"This statement is used to insert data in a DLI table to the associated CSS table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2.", "doc_type":"sqlreference", "kw":"Inserting Data to the CSS Table,Creating a Datasource Connection with a CSS Table,SQL Syntax Referen", @@ -1220,7 +1374,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Inserting Data to the CSS Table", @@ -1230,7 +1386,7 @@ "uri":"dli_08_0203.html", "node_id":"dli_08_0203.xml", "product_code":"dli", - "code":"69", + "code":"70", "des":"This statement is used to query data in a CSS table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.The table to be queried m", "doc_type":"sqlreference", "kw":"Querying the CSS Table,Creating a Datasource Connection with a CSS Table,SQL Syntax Reference", @@ -1238,7 +1394,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Querying the CSS Table", @@ -1248,7 +1406,7 @@ "uri":"dli_08_0225.html", "node_id":"dli_08_0225.xml", "product_code":"dli", - "code":"70", + "code":"71", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Datasource Connection with a DCS Table", @@ -1256,7 +1414,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Datasource Connection with a DCS Table", @@ -1266,7 +1426,7 @@ "uri":"dli_08_0226.html", "node_id":"dli_08_0226.xml", "product_code":"dli", - "code":"71", + "code":"72", "des":"This statement is used to create a DLI table and associate it with an existing DCS key.In Spark cross-source development scenarios, there is a risk of password leakage if", "doc_type":"sqlreference", "kw":"Creating a DLI Table and Associating It with DCS,Creating a Datasource Connection with a DCS Table,S", @@ -1274,7 +1434,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a DLI Table and Associating It with DCS", @@ -1284,7 +1446,7 @@ "uri":"dli_08_0227.html", "node_id":"dli_08_0227.xml", "product_code":"dli", - "code":"72", + "code":"73", "des":"This statement is used to insert data in a DLI table to the DCS key.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2...\n [FROM DL", "doc_type":"sqlreference", "kw":"Inserting Data to a DCS Table,Creating a Datasource Connection with a DCS Table,SQL Syntax Reference", @@ -1292,7 +1454,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Inserting Data to a DCS Table", @@ -1302,7 +1466,7 @@ "uri":"dli_08_0228.html", "node_id":"dli_08_0228.xml", "product_code":"dli", - "code":"73", + "code":"74", "des":"This statement is used to query data in a DCS table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.Query data in the test_re", "doc_type":"sqlreference", "kw":"Querying the DCS Table,Creating a Datasource Connection with a DCS Table,SQL Syntax Reference", @@ -1310,7 +1474,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Querying the DCS Table", @@ -1320,7 +1486,7 @@ "uri":"dli_08_0229.html", "node_id":"dli_08_0229.xml", "product_code":"dli", - "code":"74", + "code":"75", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Datasource Connection with a DDS Table", @@ -1328,7 +1494,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Datasource Connection with a DDS Table", @@ -1338,7 +1506,7 @@ "uri":"dli_08_0230.html", "node_id":"dli_08_0230.xml", "product_code":"dli", - "code":"75", + "code":"76", "des":"This statement is used to create a DLI table and associate it with an existing DDS collection.In Spark cross-source development scenarios, there is a risk of password lea", "doc_type":"sqlreference", "kw":"Creating a DLI Table and Associating It with DDS,Creating a Datasource Connection with a DDS Table,S", @@ -1346,7 +1514,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a DLI Table and Associating It with DDS", @@ -1356,7 +1526,7 @@ "uri":"dli_08_0231.html", "node_id":"dli_08_0231.xml", "product_code":"dli", - "code":"76", + "code":"77", "des":"This statement is used to insert data in a DLI table to the associated DDS table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2.", "doc_type":"sqlreference", "kw":"Inserting Data to the DDS Table,Creating a Datasource Connection with a DDS Table,SQL Syntax Referen", @@ -1364,7 +1534,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Inserting Data to the DDS Table", @@ -1374,7 +1546,7 @@ "uri":"dli_08_0232.html", "node_id":"dli_08_0232.xml", "product_code":"dli", - "code":"77", + "code":"78", "des":"This statement is used to query data in a DDS table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.If schema information is ", "doc_type":"sqlreference", "kw":"Querying the DDS Table,Creating a Datasource Connection with a DDS Table,SQL Syntax Reference", @@ -1382,7 +1554,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Querying the DDS Table", @@ -1392,7 +1566,7 @@ "uri":"dli_08_0460.html", "node_id":"dli_08_0460.xml", "product_code":"dli", - "code":"78", + "code":"79", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Datasource Connection with an Oracle Table", @@ -1400,7 +1574,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Datasource Connection with an Oracle Table", @@ -1410,7 +1586,7 @@ "uri":"dli_08_0461.html", "node_id":"dli_08_0461.xml", "product_code":"dli", - "code":"79", + "code":"80", "des":"This statement is used to create a DLI table and associate it with an existing Oracle table.Before creating a DLI table and associating it with Oracle, you need to create", "doc_type":"sqlreference", "kw":"Creating a DLI Table and Associating It with Oracle,Creating a Datasource Connection with an Oracle ", @@ -1418,7 +1594,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a DLI Table and Associating It with Oracle", @@ -1428,7 +1606,7 @@ "uri":"dli_08_0462.html", "node_id":"dli_08_0462.xml", "product_code":"dli", - "code":"80", + "code":"81", "des":"This statement is used to insert data into an associated Oracle table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2...\n [FROM ", "doc_type":"sqlreference", "kw":"Inserting Data to an Oracle Table,Creating a Datasource Connection with an Oracle Table,SQL Syntax R", @@ -1436,7 +1614,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Inserting Data to an Oracle Table", @@ -1446,7 +1626,7 @@ "uri":"dli_08_0463.html", "node_id":"dli_08_0463.xml", "product_code":"dli", - "code":"81", + "code":"82", "des":"This statement is used to query data in an Oracle table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.If schema information", "doc_type":"sqlreference", "kw":"Querying an Oracle Table,Creating a Datasource Connection with an Oracle Table,SQL Syntax Reference", @@ -1454,7 +1634,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Querying an Oracle Table", @@ -1464,7 +1646,7 @@ "uri":"dli_08_0129.html", "node_id":"dli_08_0129.xml", "product_code":"dli", - "code":"82", + "code":"83", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Views", @@ -1472,7 +1654,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Views", @@ -1482,7 +1666,7 @@ "uri":"dli_08_0130.html", "node_id":"dli_08_0130.xml", "product_code":"dli", - "code":"83", + "code":"84", "des":"This statement is used to create views.CREATE VIEW: creates views based on the given select statement. The result of the select statement will not be written into the dis", "doc_type":"sqlreference", "kw":"Creating a View,Views,SQL Syntax Reference", @@ -1490,7 +1674,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a View", @@ -1500,7 +1686,7 @@ "uri":"dli_08_0131.html", "node_id":"dli_08_0131.xml", "product_code":"dli", - "code":"84", + "code":"85", "des":"This statement is used to delete views.DROP: Deletes the metadata of a specified view. Although views and tables have many common points, the DROP TABLE statement cannot ", "doc_type":"sqlreference", "kw":"Deleting a View,Views,SQL Syntax Reference", @@ -1508,7 +1694,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Deleting a View", @@ -1518,25 +1706,27 @@ "uri":"dli_08_0138.html", "node_id":"dli_08_0138.xml", "product_code":"dli", - "code":"85", + "code":"86", "des":"This statement returns the logical plan and physical execution plan for the SQL statement.EXTENDED: After this keyword is specified, the logical and physical plans are ou", "doc_type":"sqlreference", - "kw":"Viewing the Execution Plan,Spark SQL Syntax Reference,SQL Syntax Reference", + "kw":"Checking the Execution Plan,Spark SQL Syntax Reference,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Viewing the Execution Plan", + "title":"Checking the Execution Plan", "githuburl":"" }, { "uri":"dli_08_0139.html", "node_id":"dli_08_0139.xml", "product_code":"dli", - "code":"86", + "code":"87", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Data Permissions Management", @@ -1544,7 +1734,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Data Permissions Management", @@ -1554,7 +1746,7 @@ "uri":"dli_08_0140.html", "node_id":"dli_08_0140.xml", "product_code":"dli", - "code":"87", + "code":"88", "des":"Table 1 describes the SQL statement permission matrix in DLI in terms of permissions on databases, tables, and roles.For privilege granting or revocation on databases and", "doc_type":"sqlreference", "kw":"Data Permissions List,Data Permissions Management,SQL Syntax Reference", @@ -1562,7 +1754,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Data Permissions List", @@ -1572,7 +1766,7 @@ "uri":"dli_08_0141.html", "node_id":"dli_08_0141.xml", "product_code":"dli", - "code":"88", + "code":"89", "des":"This statement is used to create a role in the current database or a specified database.Only users with the CREATE_ROLE permission on the database can create roles. For e", "doc_type":"sqlreference", "kw":"Creating a Role,Data Permissions Management,SQL Syntax Reference", @@ -1580,7 +1774,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Role", @@ -1590,7 +1786,7 @@ "uri":"dli_08_0148.html", "node_id":"dli_08_0148.xml", "product_code":"dli", - "code":"89", + "code":"90", "des":"This statement is used to delete a role in the current database or a specified database.NoneThe role_name to be deleted must exist in the current database or the specifie", "doc_type":"sqlreference", "kw":"Deleting a Role,Data Permissions Management,SQL Syntax Reference", @@ -1598,7 +1794,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Deleting a Role", @@ -1608,7 +1806,7 @@ "uri":"dli_08_0142.html", "node_id":"dli_08_0142.xml", "product_code":"dli", - "code":"90", + "code":"91", "des":"This statement is used to bind a user with a role.NoneThe role_name and username must exist. Otherwise, an error will be reported.", "doc_type":"sqlreference", "kw":"Binding a Role,Data Permissions Management,SQL Syntax Reference", @@ -1616,7 +1814,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Binding a Role", @@ -1626,7 +1826,7 @@ "uri":"dli_08_0147.html", "node_id":"dli_08_0147.xml", "product_code":"dli", - "code":"91", + "code":"92", "des":"This statement is used to unbind the user with the role.Nonerole_name and user_name must exist and user_name has been bound to role_name.To unbind the user_name1 from rol", "doc_type":"sqlreference", "kw":"Unbinding a Role,Data Permissions Management,SQL Syntax Reference", @@ -1634,7 +1834,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Unbinding a Role", @@ -1644,7 +1846,7 @@ "uri":"dli_08_0143.html", "node_id":"dli_08_0143.xml", "product_code":"dli", - "code":"92", + "code":"93", "des":"This statement is used to display all roles or roles bound to the user_name in the current database.ALL: Displays all roles.Keywords ALL and user_name cannot coexist.To d", "doc_type":"sqlreference", "kw":"Displaying a Role,Data Permissions Management,SQL Syntax Reference", @@ -1652,7 +1854,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Displaying a Role", @@ -1662,7 +1866,7 @@ "uri":"dli_08_0144.html", "node_id":"dli_08_0144.xml", "product_code":"dli", - "code":"93", + "code":"94", "des":"This statement is used to grant permissions to a user or role.ROLE: The subsequent role_name must be a role.USER: The subsequent user_name must be a user.The privilege mu", "doc_type":"sqlreference", "kw":"Granting a Permission,Data Permissions Management,SQL Syntax Reference", @@ -1670,7 +1874,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Granting a Permission", @@ -1680,7 +1886,7 @@ "uri":"dli_08_0146.html", "node_id":"dli_08_0146.xml", "product_code":"dli", - "code":"94", + "code":"95", "des":"This statement is used to revoke permissions granted to a user or role.ROLE: The subsequent role_name must be a role.USER: The subsequent user_name must be a user.The pri", "doc_type":"sqlreference", "kw":"Revoking a Permission,Data Permissions Management,SQL Syntax Reference", @@ -1688,7 +1894,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Revoking a Permission", @@ -1698,7 +1906,7 @@ "uri":"dli_08_0145.html", "node_id":"dli_08_0145.xml", "product_code":"dli", - "code":"95", + "code":"96", "des":"This statement is used to show the permissions granted to a user on a resource.USER: The subsequent user_name must be a user.The resource can be a queue, database, table,", "doc_type":"sqlreference", "kw":"Showing Granted Permissions,Data Permissions Management,SQL Syntax Reference", @@ -1706,7 +1914,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Showing Granted Permissions", @@ -1716,7 +1926,7 @@ "uri":"dli_08_0149.html", "node_id":"dli_08_0149.xml", "product_code":"dli", - "code":"96", + "code":"97", "des":"This statement is used to display the binding relationship between roles and a user in the current database.NoneThe ROLE variable must exist.", "doc_type":"sqlreference", "kw":"Displaying the Binding Relationship Between All Roles and Users,Data Permissions Management,SQL Synt", @@ -1724,7 +1934,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Displaying the Binding Relationship Between All Roles and Users", @@ -1734,7 +1946,7 @@ "uri":"dli_08_0056.html", "node_id":"dli_08_0056.xml", "product_code":"dli", - "code":"97", + "code":"98", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Data Types", @@ -1742,7 +1954,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Data Types", @@ -1752,7 +1966,7 @@ "uri":"dli_08_0057.html", "node_id":"dli_08_0057.xml", "product_code":"dli", - "code":"98", + "code":"99", "des":"Data type is a basic attribute of data. It is used to distinguish different types of data. Different data types occupy different storage space and support different opera", "doc_type":"sqlreference", "kw":"Overview,Data Types,SQL Syntax Reference", @@ -1760,7 +1974,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Overview", @@ -1770,7 +1986,7 @@ "uri":"dli_08_0058.html", "node_id":"dli_08_0058.xml", "product_code":"dli", - "code":"99", + "code":"100", "des":"Table 1 lists the primitive data types supported by DLI.VARCHAR and CHAR data is stored in STRING type on DLI. Therefore, the string that exceeds the specified length wil", "doc_type":"sqlreference", "kw":"Primitive Data Types,Data Types,SQL Syntax Reference", @@ -1778,7 +1994,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Primitive Data Types", @@ -1788,7 +2006,7 @@ "uri":"dli_08_0059.html", "node_id":"dli_08_0059.xml", "product_code":"dli", - "code":"100", + "code":"101", "des":"Spark SQL supports complex data types, as shown in Table 1.When a table containing fields of the complex data type is created, the storage format of this table cannot be ", "doc_type":"sqlreference", "kw":"Complex Data Types,Data Types,SQL Syntax Reference", @@ -1796,7 +2014,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Complex Data Types", @@ -1806,7 +2026,7 @@ "uri":"dli_08_0282.html", "node_id":"dli_08_0282.xml", "product_code":"dli", - "code":"101", + "code":"102", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"User-Defined Functions", @@ -1814,7 +2034,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"User-Defined Functions", @@ -1824,7 +2046,7 @@ "uri":"dli_08_0283.html", "node_id":"dli_08_0283.xml", "product_code":"dli", - "code":"102", + "code":"103", "des":"DLI allows you to create and use user-defined functions (UDF) and user-defined table functions (UDTF) in Spark jobs.OrIf a function with the same name exists in the datab", "doc_type":"sqlreference", "kw":"Creating a Function,User-Defined Functions,SQL Syntax Reference", @@ -1832,7 +2054,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Function", @@ -1842,7 +2066,7 @@ "uri":"dli_08_0284.html", "node_id":"dli_08_0284.xml", "product_code":"dli", - "code":"103", + "code":"104", "des":"This statement is used to delete functions.TEMPORARY: Indicates whether the function to be deleted is a temporary function.IF EXISTS: Used when the function to be deleted", "doc_type":"sqlreference", "kw":"Deleting a Function,User-Defined Functions,SQL Syntax Reference", @@ -1850,7 +2074,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Deleting a Function", @@ -1860,7 +2086,7 @@ "uri":"dli_08_0281.html", "node_id":"dli_08_0281.xml", "product_code":"dli", - "code":"104", + "code":"105", "des":"Displays information about a specified function.EXTENDED: displays extended usage information.The metadata (implementation class and usage) of an existing function is ret", "doc_type":"sqlreference", "kw":"Displaying Function Details,User-Defined Functions,SQL Syntax Reference", @@ -1868,7 +2094,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Displaying Function Details", @@ -1878,7 +2106,7 @@ "uri":"dli_08_0285.html", "node_id":"dli_08_0285.xml", "product_code":"dli", - "code":"105", + "code":"106", "des":"View all functions in the current project.In the preceding statement, regex is a regular expression. For details about its parameters, see Table 1.For details about other", "doc_type":"sqlreference", "kw":"Displaying All Functions,User-Defined Functions,SQL Syntax Reference", @@ -1886,7 +2114,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Displaying All Functions", @@ -1896,7 +2126,7 @@ "uri":"dli_08_0064.html", "node_id":"dli_08_0064.xml", "product_code":"dli", - "code":"106", + "code":"107", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Built-in Functions", @@ -1904,7 +2134,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Built-in Functions", @@ -1914,7 +2146,7 @@ "uri":"dli_08_0471.html", "node_id":"dli_08_0471.xml", "product_code":"dli", - "code":"107", + "code":"108", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Date Functions", @@ -1922,7 +2154,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Date Functions", @@ -1932,7 +2166,7 @@ "uri":"dli_08_0066.html", "node_id":"dli_08_0066.xml", "product_code":"dli", - "code":"108", + "code":"109", "des":"Table 1 lists the date functions supported by DLI.", "doc_type":"sqlreference", "kw":"Overview,Date Functions,SQL Syntax Reference", @@ -1940,7 +2174,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Overview", @@ -1950,7 +2186,7 @@ "uri":"dli_spark_add_months.html", "node_id":"dli_spark_add_months.xml", "product_code":"dli", - "code":"109", + "code":"110", "des":"This function is used to calculate the date after a date value is increased by a specified number of months. That is, it calculates the data that is num_months after star", "doc_type":"sqlreference", "kw":"add_months,Date Functions,SQL Syntax Reference", @@ -1958,7 +2194,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"add_months", @@ -1968,7 +2206,7 @@ "uri":"dli_spark_current_date.html", "node_id":"dli_spark_current_date.xml", "product_code":"dli", - "code":"110", + "code":"111", "des":"This function is used to return the current date, in the yyyy-mm-dd format.Similar function: getdate. The getdate function is used to return the current system time, in t", "doc_type":"sqlreference", "kw":"current_date,Date Functions,SQL Syntax Reference", @@ -1976,7 +2214,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"current_date", @@ -1986,7 +2226,7 @@ "uri":"dli_spark_current_timestamp.html", "node_id":"dli_spark_current_timestamp.xml", "product_code":"dli", - "code":"111", + "code":"112", "des":"This function is used to return the current timestamp.NoneThe return value is of the TIMESTAMP type.The value 1692002816300 is returned.", "doc_type":"sqlreference", "kw":"current_timestamp,Date Functions,SQL Syntax Reference", @@ -1994,7 +2234,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"current_timestamp", @@ -2004,7 +2246,7 @@ "uri":"dli_spark_date_add.html", "node_id":"dli_spark_date_add.xml", "product_code":"dli", - "code":"112", + "code":"113", "des":"This function is used to calculate the number of days in which start_date is increased by days.To obtain the date with a specified change range based on the current date,", "doc_type":"sqlreference", "kw":"date_add,Date Functions,SQL Syntax Reference", @@ -2012,7 +2254,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"date_add", @@ -2022,7 +2266,7 @@ "uri":"dli_spark_dateadd.html", "node_id":"dli_spark_dateadd.xml", "product_code":"dli", - "code":"113", + "code":"114", "des":"This function is used to change a date based on datepart and delta.To obtain the date with a specified change range based on the current date, use this function together ", "doc_type":"sqlreference", "kw":"dateadd,Date Functions,SQL Syntax Reference", @@ -2030,7 +2274,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"dateadd", @@ -2040,7 +2286,7 @@ "uri":"dli_spark_date_sub.html", "node_id":"dli_spark_date_sub.xml", "product_code":"dli", - "code":"114", + "code":"115", "des":"This function is used to calculate the number of days in which start_date is subtracted by days.To obtain the date with a specified change range based on the current date", "doc_type":"sqlreference", "kw":"date_sub,Date Functions,SQL Syntax Reference", @@ -2048,7 +2294,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"date_sub", @@ -2058,7 +2306,7 @@ "uri":"dli_spark_date_format.html", "node_id":"dli_spark_date_format.xml", "product_code":"dli", - "code":"115", + "code":"116", "des":"This function is used to convert a date into a string based on the format specified by format.The return value is of the STRING type.If the value of date is not of the DA", "doc_type":"sqlreference", "kw":"date_format,Date Functions,SQL Syntax Reference", @@ -2066,7 +2314,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"date_format", @@ -2076,7 +2326,7 @@ "uri":"dli_spark_datediff.html", "node_id":"dli_spark_datediff.xml", "product_code":"dli", - "code":"116", + "code":"117", "des":"This function is used to calculate the difference between date1 and date2.Similar function: datediff1. The datediff1 function is used to calculate the difference between ", "doc_type":"sqlreference", "kw":"datediff,Date Functions,SQL Syntax Reference", @@ -2084,7 +2334,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"datediff", @@ -2094,7 +2346,7 @@ "uri":"dli_spark_datediff1.html", "node_id":"dli_spark_datediff1.xml", "product_code":"dli", - "code":"117", + "code":"118", "des":"This function is used to calculate the difference between date1 and date2 and return the difference in a specified datepart.Similar function: datediff. The datediff funct", "doc_type":"sqlreference", "kw":"datediff1,Date Functions,SQL Syntax Reference", @@ -2102,7 +2354,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"datediff1", @@ -2112,7 +2366,7 @@ "uri":"dli_spark_datepart.html", "node_id":"dli_spark_datepart.xml", "product_code":"dli", - "code":"118", + "code":"119", "des":"This function is used to calculate the value that meets the specified datepart in date.The return value is of the BIGINT type.If the value of date is not of the DATE or S", "doc_type":"sqlreference", "kw":"datepart,Date Functions,SQL Syntax Reference", @@ -2120,7 +2374,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"datepart", @@ -2130,7 +2386,7 @@ "uri":"dli_spark_datetrunc.html", "node_id":"dli_spark_datetrunc.xml", "product_code":"dli", - "code":"119", + "code":"120", "des":"This function is used to calculate the date otained through the truncation of a specified date based on a specified datepart.It truncates the date before the specified da", "doc_type":"sqlreference", "kw":"datetrunc,Date Functions,SQL Syntax Reference", @@ -2138,7 +2394,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"datetrunc", @@ -2148,7 +2406,7 @@ "uri":"dli_spark_daydayofmonth.html", "node_id":"dli_spark_daydayofmonth.xml", "product_code":"dli", - "code":"120", + "code":"121", "des":"This function is used to return the day of a specified date.The return value is of the INT type.If the value of date is not of the DATE or STRING type, the error message ", "doc_type":"sqlreference", "kw":"day/dayofmonth,Date Functions,SQL Syntax Reference", @@ -2156,7 +2414,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"day/dayofmonth", @@ -2166,7 +2426,7 @@ "uri":"dli_spark_from_unixtime.html", "node_id":"dli_spark_from_unixtime.xml", "product_code":"dli", - "code":"121", + "code":"122", "des":"This function is used to convert a timestamp represented by a numeric UNIX value to a date value.The return value is of the STRING type, in the yyyy-mm-dd hh:mi:ss format", "doc_type":"sqlreference", "kw":"from_unixtime,Date Functions,SQL Syntax Reference", @@ -2174,7 +2434,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"from_unixtime", @@ -2184,7 +2446,7 @@ "uri":"dli_spark_from_utc_timestamp.html", "node_id":"dli_spark_from_utc_timestamp.xml", "product_code":"dli", - "code":"122", + "code":"123", "des":"This function is used to convert a UTC timestamp to a UNIX timestamp in a given time zone.The return value is of the TIMESTAMP type.If the value of timestamp is not of th", "doc_type":"sqlreference", "kw":"from_utc_timestamp,Date Functions,SQL Syntax Reference", @@ -2192,7 +2454,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"from_utc_timestamp", @@ -2202,7 +2466,7 @@ "uri":"dli_spark_getdate.html", "node_id":"dli_spark_getdate.xml", "product_code":"dli", - "code":"123", + "code":"124", "des":"This function is used to return the current system time, in the yyyy-mm-dd hh:mi:ss format.Similar function: current_date. The current_date function is used to return the", "doc_type":"sqlreference", "kw":"getdate,Date Functions,SQL Syntax Reference", @@ -2210,7 +2474,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"getdate", @@ -2220,7 +2486,7 @@ "uri":"dli_spark_hour.html", "node_id":"dli_spark_hour.xml", "product_code":"dli", - "code":"124", + "code":"125", "des":"This function is used to return the hour (from 0 to 23) of a specified time.The return value is of the INT type.If the value of date is not of the DATE or STRING type, th", "doc_type":"sqlreference", "kw":"hour,Date Functions,SQL Syntax Reference", @@ -2228,7 +2494,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"hour", @@ -2238,7 +2506,7 @@ "uri":"dli_spark_isdate.html", "node_id":"dli_spark_isdate.xml", "product_code":"dli", - "code":"125", + "code":"126", "des":"This function is used to determine whether a date string can be converted into a date value based on a specified format.The return value is of the BOOLEAN type.If the val", "doc_type":"sqlreference", "kw":"isdate,Date Functions,SQL Syntax Reference", @@ -2246,7 +2514,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"isdate", @@ -2256,7 +2526,7 @@ "uri":"dli_spark_last_day.html", "node_id":"dli_spark_last_day.xml", "product_code":"dli", - "code":"126", + "code":"127", "des":"This function is used to return the last day of the month a date belongs to.Similar function: lastday. The lastday function is used to return the last day of the month a ", "doc_type":"sqlreference", "kw":"last_day,Date Functions,SQL Syntax Reference", @@ -2264,7 +2534,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"last_day", @@ -2274,7 +2546,7 @@ "uri":"dli_spark_lastday.html", "node_id":"dli_spark_lastday.xml", "product_code":"dli", - "code":"127", + "code":"128", "des":"This function is used to return the last day of the month a date belongs to. The hour, minute, and second part is 00:00:00.Similar function: last_day. The last_day functi", "doc_type":"sqlreference", "kw":"lastday,Date Functions,SQL Syntax Reference", @@ -2282,7 +2554,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"lastday", @@ -2292,7 +2566,7 @@ "uri":"dli_spark_minute.html", "node_id":"dli_spark_minute.xml", "product_code":"dli", - "code":"128", + "code":"129", "des":"This function is used to return the minute (from 0 to 59) of a specified time.The return value is of the INT type.If the value of date is not of the DATE or STRING type, ", "doc_type":"sqlreference", "kw":"minute,Date Functions,SQL Syntax Reference", @@ -2300,7 +2574,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"minute", @@ -2310,7 +2586,7 @@ "uri":"dli_spark_month.html", "node_id":"dli_spark_month.xml", "product_code":"dli", - "code":"129", + "code":"130", "des":"This function is used to return the month (from January to December) of a specified time.The return value is of the INT type.If the value of date is not of the DATE or ST", "doc_type":"sqlreference", "kw":"month,Date Functions,SQL Syntax Reference", @@ -2318,7 +2594,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"month", @@ -2328,7 +2606,7 @@ "uri":"dli_spark_months_between.html", "node_id":"dli_spark_months_between.xml", "product_code":"dli", - "code":"130", + "code":"131", "des":"This function returns the month difference between date1 and date2.The return value is of the DOUBLE type.If the values of date1 and date2 are not of the DATE or STRING t", "doc_type":"sqlreference", "kw":"months_between,Date Functions,SQL Syntax Reference", @@ -2336,7 +2614,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"months_between", @@ -2346,7 +2626,7 @@ "uri":"dli_spark_next_day.html", "node_id":"dli_spark_next_day.xml", "product_code":"dli", - "code":"131", + "code":"132", "des":"This function is used to return the date closest to day_of_week after start_date.The return value is of the DATE type, in the yyyy-mm-dd format.If the value of start_date", "doc_type":"sqlreference", "kw":"next_day,Date Functions,SQL Syntax Reference", @@ -2354,7 +2634,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"next_day", @@ -2364,7 +2646,7 @@ "uri":"dli_spark_quarter.html", "node_id":"dli_spark_quarter.xml", "product_code":"dli", - "code":"132", + "code":"133", "des":"This function is used to return the quarter of a date. The value ranges from 1 to 4.The return value is of the INT type.If the value of date is not of the DATE or STRING ", "doc_type":"sqlreference", "kw":"quarter,Date Functions,SQL Syntax Reference", @@ -2372,7 +2654,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"quarter", @@ -2382,7 +2666,7 @@ "uri":"dli_spark_second.html", "node_id":"dli_spark_second.xml", "product_code":"dli", - "code":"133", + "code":"134", "des":"This function is used to return the second (from 0 to 59) of a specified time.The return value is of the INT type.If the value of date is not of the DATE or STRING type, ", "doc_type":"sqlreference", "kw":"second,Date Functions,SQL Syntax Reference", @@ -2390,7 +2674,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"second", @@ -2400,7 +2686,7 @@ "uri":"dli_spark_to_char.html", "node_id":"dli_spark_to_char.xml", "product_code":"dli", - "code":"134", + "code":"135", "des":"This function is used to convert a date into a string in a specified format.The return value is of the STRING type.If the value of date is not of the DATE or STRING type,", "doc_type":"sqlreference", "kw":"to_char,Date Functions,SQL Syntax Reference", @@ -2408,7 +2694,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"to_char", @@ -2418,7 +2706,7 @@ "uri":"dli_spark_to_date.html", "node_id":"dli_spark_to_date.xml", "product_code":"dli", - "code":"135", + "code":"136", "des":"This function is used to return the year, month, and day in a time.Similar function: to_date1. The to_date1 function is used to convert a string in a specified format to ", "doc_type":"sqlreference", "kw":"to_date,Date Functions,SQL Syntax Reference", @@ -2426,7 +2714,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"to_date", @@ -2436,7 +2726,7 @@ "uri":"dli_spark_to_date1.html", "node_id":"dli_spark_to_date1.xml", "product_code":"dli", - "code":"136", + "code":"137", "des":"This function is used to convert a string in a specified format to a date value.Similar function: to_date. The to_date function is used to return the year, month, and day", "doc_type":"sqlreference", "kw":"to_date1,Date Functions,SQL Syntax Reference", @@ -2444,7 +2734,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"to_date1", @@ -2454,7 +2746,7 @@ "uri":"dli_spark_to_utc_timestamp.html", "node_id":"dli_spark_to_utc_timestamp.xml", "product_code":"dli", - "code":"137", + "code":"138", "des":"This function is used to convert a timestamp in a given time zone to a UTC timestamp.The return value is of the BIGINT type.If the value of timestamp is not of the DATE o", "doc_type":"sqlreference", "kw":"to_utc_timestamp,Date Functions,SQL Syntax Reference", @@ -2462,7 +2754,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"to_utc_timestamp", @@ -2472,7 +2766,7 @@ "uri":"dli_spark_trunc.html", "node_id":"dli_spark_trunc.xml", "product_code":"dli", - "code":"138", + "code":"139", "des":"This function is used to reset a date to a specific format.Resetting means returning to default values, where the default values for year, month, and day are 01, and the ", "doc_type":"sqlreference", "kw":"trunc,Date Functions,SQL Syntax Reference", @@ -2480,7 +2774,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"trunc", @@ -2490,7 +2786,7 @@ "uri":"dli_spark_unix_timestamp.html", "node_id":"dli_spark_unix_timestamp.xml", "product_code":"dli", - "code":"139", + "code":"140", "des":"This function is used to convert a date value to a numeric date value in UNIX format.The function returns the first ten digits of the timestamp in normal UNIX format.The ", "doc_type":"sqlreference", "kw":"unix_timestamp,Date Functions,SQL Syntax Reference", @@ -2498,7 +2794,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"unix_timestamp", @@ -2508,7 +2806,7 @@ "uri":"dli_spark_weekday.html", "node_id":"dli_spark_weekday.xml", "product_code":"dli", - "code":"140", + "code":"141", "des":"This function is used to return the day of the current week.The return value is of the INT type.If Monday is used as the first day of a week, the value 0 is returned. For", "doc_type":"sqlreference", "kw":"weekday,Date Functions,SQL Syntax Reference", @@ -2516,7 +2814,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"weekday", @@ -2526,7 +2826,7 @@ "uri":"dli_spark_weekofyear.html", "node_id":"dli_spark_weekofyear.xml", "product_code":"dli", - "code":"141", + "code":"142", "des":"This function is used to return the week number (from 0 to 53) of a specified date.The return value is of the INT type.If the value of date is not of the DATE or STRING t", "doc_type":"sqlreference", "kw":"weekofyear,Date Functions,SQL Syntax Reference", @@ -2534,7 +2834,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"weekofyear", @@ -2544,7 +2846,7 @@ "uri":"dli_spark_year.html", "node_id":"dli_spark_year.xml", "product_code":"dli", - "code":"142", + "code":"143", "des":"This function is used to return the year of a specified date.The return value is of the INT type.If the value of date is not of the DATE or STRING type, the error message", "doc_type":"sqlreference", "kw":"year,Date Functions,SQL Syntax Reference", @@ -2552,7 +2854,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"year", @@ -2562,7 +2866,7 @@ "uri":"dli_08_0472.html", "node_id":"dli_08_0472.xml", "product_code":"dli", - "code":"143", + "code":"144", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"String Functions", @@ -2570,7 +2874,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"String Functions", @@ -2580,7 +2886,7 @@ "uri":"dli_08_0067.html", "node_id":"dli_08_0067.xml", "product_code":"dli", - "code":"144", + "code":"145", "des":"Table 1 lists the string functions supported by DLI.", "doc_type":"sqlreference", "kw":"Overview,String Functions,SQL Syntax Reference", @@ -2588,7 +2894,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Overview", @@ -2598,7 +2906,7 @@ "uri":"dli_spark_ascii.html", "node_id":"dli_spark_ascii.xml", "product_code":"dli", - "code":"145", + "code":"146", "des":"This function is used to return the ASCII code of the first character in str.The return value is of the BIGINT type.If the value of str is not of the STRING, BIGINT, DOUB", "doc_type":"sqlreference", "kw":"ascii,String Functions,SQL Syntax Reference", @@ -2606,7 +2914,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"ascii", @@ -2616,7 +2926,7 @@ "uri":"dli_spark_concat.html", "node_id":"dli_spark_concat.xml", "product_code":"dli", - "code":"146", + "code":"147", "des":"This function is used to concatenate arrays or strings.If multiple arrays are used as the input, all elements in the arrays are connected to generate a new array.If multi", "doc_type":"sqlreference", "kw":"concat,String Functions,SQL Syntax Reference", @@ -2624,7 +2934,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"concat", @@ -2634,7 +2946,7 @@ "uri":"dli_spark_concat_ws.html", "node_id":"dli_spark_concat_ws.xml", "product_code":"dli", - "code":"147", + "code":"148", "des":"This function is used to return a string concatenated from multiple input strings that are separated by specified separators.orReturns the result of joining all the strin", "doc_type":"sqlreference", "kw":"concat_ws,String Functions,SQL Syntax Reference", @@ -2642,7 +2954,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"concat_ws", @@ -2652,7 +2966,7 @@ "uri":"dli_spark_char_matchcount.html", "node_id":"dli_spark_char_matchcount.xml", "product_code":"dli", - "code":"148", + "code":"149", "des":"This parameter is used to return the number of characters in str1 that appear in str2.The return value is of the BIGINT type.If the value of str1 or str2 is NULL, NULL is", "doc_type":"sqlreference", "kw":"char_matchcount,String Functions,SQL Syntax Reference", @@ -2660,7 +2974,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"char_matchcount", @@ -2670,7 +2986,7 @@ "uri":"dli_spark_encode.html", "node_id":"dli_spark_encode.xml", "product_code":"dli", - "code":"149", + "code":"150", "des":"This function is used to encode str in charset format.encode(string , string )The return value is of the BINARY type.If the value of str or charset is NULL,", "doc_type":"sqlreference", "kw":"encode,String Functions,SQL Syntax Reference", @@ -2678,7 +2994,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"encode", @@ -2688,7 +3006,7 @@ "uri":"dli_spark_find_in_set.html", "node_id":"dli_spark_find_in_set.xml", "product_code":"dli", - "code":"150", + "code":"151", "des":"This function is used to return the position (stating from 1) of str1 in str2 separated by commas (,).The return value is of the BIGINT type.If str1 cannot be matched in ", "doc_type":"sqlreference", "kw":"find_in_set,String Functions,SQL Syntax Reference", @@ -2696,7 +3014,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"find_in_set", @@ -2706,7 +3026,7 @@ "uri":"dli_spark_get_json_object.html", "node_id":"dli_spark_get_json_object.xml", "product_code":"dli", - "code":"151", + "code":"152", "des":"This function is used to parse the JSON object in a specified JSON path. The function will return NULL if the JSON object is invalid.The return value is of the STRING typ", "doc_type":"sqlreference", "kw":"get_json_object,String Functions,SQL Syntax Reference", @@ -2714,7 +3034,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"get_json_object", @@ -2724,7 +3046,7 @@ "uri":"dli_spark_instr.html", "node_id":"dli_spark_instr.xml", "product_code":"dli", - "code":"152", + "code":"153", "des":"This function is used to return the index of substr that appears earliest in str.It returns NULL if either of the arguments are NULL and returns 0 if substr does not exis", "doc_type":"sqlreference", "kw":"instr,String Functions,SQL Syntax Reference", @@ -2732,7 +3054,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"instr", @@ -2742,7 +3066,7 @@ "uri":"dli_spark_instr1.html", "node_id":"dli_spark_instr1.xml", "product_code":"dli", - "code":"153", + "code":"154", "des":"This function is used to return the position of substring str2 in string str1.Similar function: instr. The instr function is used to return the index of substr that appea", "doc_type":"sqlreference", "kw":"instr1,String Functions,SQL Syntax Reference", @@ -2750,7 +3074,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"instr1", @@ -2760,7 +3086,7 @@ "uri":"dli_spark_initcap.html", "node_id":"dli_spark_initcap.xml", "product_code":"dli", - "code":"154", + "code":"155", "des":"This function is used to convert the first letter of each word of a string to upper case and all other letters to lower case.The return value is of the STRING type. In th", "doc_type":"sqlreference", "kw":"initcap,String Functions,SQL Syntax Reference", @@ -2768,7 +3094,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"initcap", @@ -2778,7 +3106,7 @@ "uri":"dli_spark_keyvalue.html", "node_id":"dli_spark_keyvalue.xml", "product_code":"dli", - "code":"155", + "code":"156", "des":"This function is used to split str by split1, convert each group into a key-value pair by split2, and return the value corresponding to the key.The return value is of the", "doc_type":"sqlreference", "kw":"keyvalue,String Functions,SQL Syntax Reference", @@ -2786,7 +3114,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"keyvalue", @@ -2796,7 +3126,7 @@ "uri":"dli_spark_length.html", "node_id":"dli_spark_length.xml", "product_code":"dli", - "code":"156", + "code":"157", "des":"This function is used to return the length of a string.Similar function: lengthb. The lengthb function is used to return the length of string str in bytes and return a va", "doc_type":"sqlreference", "kw":"length,String Functions,SQL Syntax Reference", @@ -2804,7 +3134,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"length", @@ -2814,7 +3146,7 @@ "uri":"dli_spark_lengthb.html", "node_id":"dli_spark_lengthb.xml", "product_code":"dli", - "code":"157", + "code":"158", "des":"This function is used to return the length of a specified string in bytes.Similar function: length. The length function is used to return the length of a string and retur", "doc_type":"sqlreference", "kw":"lengthb,String Functions,SQL Syntax Reference", @@ -2822,7 +3154,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"lengthb", @@ -2832,7 +3166,7 @@ "uri":"dli_spark_levenshtein.html", "node_id":"dli_spark_levenshtein.xml", "product_code":"dli", - "code":"158", + "code":"159", "des":"This function is used to returns the Levenshtein distance between two strings, for example, levenshtein('kitten','sitting') = 3.Levenshtein distance is a type of edit dis", "doc_type":"sqlreference", "kw":"levenshtein,String Functions,SQL Syntax Reference", @@ -2840,7 +3174,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"levenshtein", @@ -2850,7 +3186,7 @@ "uri":"dli_spark_locate.html", "node_id":"dli_spark_locate.xml", "product_code":"dli", - "code":"159", + "code":"160", "des":"This function is used to return the position of substr in str. You can specify the starting position of your search using \"start_pos,\" which starts from 1.The return valu", "doc_type":"sqlreference", "kw":"locate,String Functions,SQL Syntax Reference", @@ -2858,7 +3194,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"locate", @@ -2868,7 +3206,7 @@ "uri":"dli_spark_lower_lcase.html", "node_id":"dli_spark_lower_lcase.xml", "product_code":"dli", - "code":"160", + "code":"161", "des":"This function is used to convert all characters of a string to the lower case.The return value is of the STRING type.If the value of the input parameter is not of the STR", "doc_type":"sqlreference", "kw":"lower/lcase,String Functions,SQL Syntax Reference", @@ -2876,7 +3214,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"lower/lcase", @@ -2886,7 +3226,7 @@ "uri":"dli_spark_lpad.html", "node_id":"dli_spark_lpad.xml", "product_code":"dli", - "code":"161", + "code":"162", "des":"This function is used to return a string of a specified length. If the length of the given string (str1) is shorter than the specified length (length), the given string i", "doc_type":"sqlreference", "kw":"lpad,String Functions,SQL Syntax Reference", @@ -2894,7 +3234,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"lpad", @@ -2904,7 +3246,7 @@ "uri":"dli_spark_ltrim.html", "node_id":"dli_spark_ltrim.xml", "product_code":"dli", - "code":"162", + "code":"163", "des":"This function is used to remove characters from the left of str.If trimChars is not specified, spaces are removed by default.If trimChars is specified, the function remov", "doc_type":"sqlreference", "kw":"ltrim,String Functions,SQL Syntax Reference", @@ -2912,7 +3254,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"ltrim", @@ -2922,7 +3266,7 @@ "uri":"dli_spark_parse_url.html", "node_id":"dli_spark_parse_url.xml", "product_code":"dli", - "code":"163", + "code":"164", "des":"This character is used to return the specified part of a given URL. Valid values of partToExtract include HOST, PATH, QUERY, REF, PROTOCOL, AUTHORITY, FILE, and USERINFO.", "doc_type":"sqlreference", "kw":"parse_url,String Functions,SQL Syntax Reference", @@ -2930,7 +3274,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"parse_url", @@ -2940,7 +3286,7 @@ "uri":"dli_spark_printf.html", "node_id":"dli_spark_printf.xml", "product_code":"dli", - "code":"164", + "code":"165", "des":"This function is used to print the input in a specific format.The return value is of the STRING type.The value is returned after the parameters that filled in Obj are spe", "doc_type":"sqlreference", "kw":"printf,String Functions,SQL Syntax Reference", @@ -2948,7 +3294,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"printf", @@ -2958,7 +3306,7 @@ "uri":"dli_spark_regexp_count.html", "node_id":"dli_spark_regexp_count.xml", "product_code":"dli", - "code":"165", + "code":"166", "des":"This function is used to return the number of substrings that match a specified pattern in the source, starting from the start_position position.The return value is of th", "doc_type":"sqlreference", "kw":"regexp_count,String Functions,SQL Syntax Reference", @@ -2966,7 +3314,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"regexp_count", @@ -2976,7 +3326,7 @@ "uri":"dli_spark_regexp_extract.html", "node_id":"dli_spark_regexp_extract.xml", "product_code":"dli", - "code":"166", + "code":"167", "des":"This function is used to match the string source based on the pattern grouping rule and return the string content that matches groupid.regexp_extract(string , str", "doc_type":"sqlreference", "kw":"regexp_extract,String Functions,SQL Syntax Reference", @@ -2984,7 +3334,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"regexp_extract", @@ -2994,7 +3346,7 @@ "uri":"dli_spark_replace.html", "node_id":"dli_spark_replace.xml", "product_code":"dli", - "code":"167", + "code":"168", "des":"This function is used to replace the part in a specified string that is the same as the string old with the string new and return the result.If the string has no same cha", "doc_type":"sqlreference", "kw":"replace,String Functions,SQL Syntax Reference", @@ -3002,7 +3354,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"replace", @@ -3012,7 +3366,7 @@ "uri":"dli_spark_regexp_replace.html", "node_id":"dli_spark_regexp_replace.xml", "product_code":"dli", - "code":"168", + "code":"169", "des":"This function has slight variations in its functionality depending on the version of Spark being used.For Spark 2.4.5 or earlier: Replaces the substring that matches patt", "doc_type":"sqlreference", "kw":"regexp_replace,String Functions,SQL Syntax Reference", @@ -3020,7 +3374,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"regexp_replace", @@ -3030,7 +3386,7 @@ "uri":"dli_spark_regexp_replace1.html", "node_id":"dli_spark_regexp_replace1.xml", "product_code":"dli", - "code":"169", + "code":"170", "des":"This function is used to replace the substring that matches pattern for the occurrence time in the source string with the specified string replace_string and return the r", "doc_type":"sqlreference", "kw":"regexp_replace1,String Functions,SQL Syntax Reference", @@ -3038,7 +3394,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"regexp_replace1", @@ -3048,7 +3406,7 @@ "uri":"dli_spark_regexp_instr.html", "node_id":"dli_spark_regexp_instr.xml", "product_code":"dli", - "code":"170", + "code":"171", "des":"This function is used to return the start or end position of the substring that matches a specified pattern for the occurrence time, starting from start_position in the s", "doc_type":"sqlreference", "kw":"regexp_instr,String Functions,SQL Syntax Reference", @@ -3056,7 +3414,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"regexp_instr", @@ -3066,7 +3426,7 @@ "uri":"dli_spark_regexp_substr.html", "node_id":"dli_spark_regexp_substr.xml", "product_code":"dli", - "code":"171", + "code":"172", "des":"This function is used to return the substring that matches a specified pattern for the occurrence time, starting from start_position in the string source.The return value", "doc_type":"sqlreference", "kw":"regexp_substr,String Functions,SQL Syntax Reference", @@ -3074,7 +3434,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"regexp_substr", @@ -3084,7 +3446,7 @@ "uri":"dli_spark_repeat.html", "node_id":"dli_spark_repeat.xml", "product_code":"dli", - "code":"172", + "code":"173", "des":"This function is used to return the string after str is repeated for n times.The return value is of the STRING type.If the value of str is not of the STRING, BIGINT, DOUB", "doc_type":"sqlreference", "kw":"repeat,String Functions,SQL Syntax Reference", @@ -3092,7 +3454,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"repeat", @@ -3102,7 +3466,7 @@ "uri":"dli_spark_reverse.html", "node_id":"dli_spark_reverse.xml", "product_code":"dli", - "code":"173", + "code":"174", "des":"This function is used to return a string in reverse order.The return value is of the STRING type.If the value of str is not of the STRING, BIGINT, DOUBLE, DECIMAL, or DAT", "doc_type":"sqlreference", "kw":"reverse,String Functions,SQL Syntax Reference", @@ -3110,7 +3474,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"reverse", @@ -3120,7 +3486,7 @@ "uri":"dli_spark_rpad.html", "node_id":"dli_spark_rpad.xml", "product_code":"dli", - "code":"174", + "code":"175", "des":"This function is used to right pad str1 with str2 to the specified length.The return value is of the STRING type.If the value of length is smaller than the number of digi", "doc_type":"sqlreference", "kw":"rpad,String Functions,SQL Syntax Reference", @@ -3128,7 +3494,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"rpad", @@ -3138,7 +3506,7 @@ "uri":"dli_spark_rtrim.html", "node_id":"dli_spark_rtrim.xml", "product_code":"dli", - "code":"175", + "code":"176", "des":"This function is used to remove characters from the right of str.If trimChars is not specified, spaces are removed by default.If trimChars is specified, the function remo", "doc_type":"sqlreference", "kw":"rtrim,String Functions,SQL Syntax Reference", @@ -3146,7 +3514,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"rtrim", @@ -3156,7 +3526,7 @@ "uri":"dli_spark_soundex.html", "node_id":"dli_spark_soundex.xml", "product_code":"dli", - "code":"176", + "code":"177", "des":"This function is used to return the soundex string from str, for example, soundex('Miller') = M460.The return value is of the STRING type.If the value of str is NULL, NUL", "doc_type":"sqlreference", "kw":"soundex,String Functions,SQL Syntax Reference", @@ -3164,7 +3534,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"soundex", @@ -3174,7 +3546,7 @@ "uri":"dli_spark_space.html", "node_id":"dli_spark_space.xml", "product_code":"dli", - "code":"177", + "code":"178", "des":"This function is used to return a specified number of spaces.The return value is of the STRING type.If the value of n is empty, an error is reported.If the value of n is ", "doc_type":"sqlreference", "kw":"space,String Functions,SQL Syntax Reference", @@ -3182,7 +3554,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"space", @@ -3192,7 +3566,7 @@ "uri":"dli_spark_substr_substring.html", "node_id":"dli_spark_substr_substring.xml", "product_code":"dli", - "code":"178", + "code":"179", "des":"This function is used to return the substring of str, starting from start_position and with a length of length.orThe return value is of the STRING type.If the value of st", "doc_type":"sqlreference", "kw":"substr/substring,String Functions,SQL Syntax Reference", @@ -3200,7 +3574,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"substr/substring", @@ -3210,7 +3586,7 @@ "uri":"dli_spark_substring_index.html", "node_id":"dli_spark_substring_index.xml", "product_code":"dli", - "code":"179", + "code":"180", "des":"This function is used to truncate the string before the count separator of str. If the value of count is positive, the string is truncated from the left. If the value of ", "doc_type":"sqlreference", "kw":"substring_index,String Functions,SQL Syntax Reference", @@ -3218,7 +3594,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"substring_index", @@ -3228,7 +3606,7 @@ "uri":"dli_spark_split_part.html", "node_id":"dli_spark_split_part.xml", "product_code":"dli", - "code":"180", + "code":"181", "des":"This function is used to split a specified string based on a specified separator and return a substring from the start to end position.The return value is of the STRING t", "doc_type":"sqlreference", "kw":"split_part,String Functions,SQL Syntax Reference", @@ -3236,7 +3614,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"split_part", @@ -3246,7 +3626,7 @@ "uri":"dli_spark_translate.html", "node_id":"dli_spark_translate.xml", "product_code":"dli", - "code":"181", + "code":"182", "des":"This function is used to translate the input string by replacing the characters or string specified by from with the characters or string specified by to.For example, it ", "doc_type":"sqlreference", "kw":"translate,String Functions,SQL Syntax Reference", @@ -3254,7 +3634,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"translate", @@ -3264,7 +3646,7 @@ "uri":"dli_spark_trim.html", "node_id":"dli_spark_trim.xml", "product_code":"dli", - "code":"182", + "code":"183", "des":"This function is used to remove characters from the left and right of str.If trimChars is not specified, spaces are removed by default.If trimChars is specified, the func", "doc_type":"sqlreference", "kw":"trim,String Functions,SQL Syntax Reference", @@ -3272,7 +3654,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"trim", @@ -3282,7 +3666,7 @@ "uri":"dli_spark_upper_ucase.html", "node_id":"dli_spark_upper_ucase.xml", "product_code":"dli", - "code":"183", + "code":"184", "des":"This function is used to convert all characters of a string to the upper case.orThe return value is of the STRING type.If the value of the input parameter is not of the S", "doc_type":"sqlreference", "kw":"upper/ucase,String Functions,SQL Syntax Reference", @@ -3290,7 +3674,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"upper/ucase", @@ -3300,7 +3686,7 @@ "uri":"dli_08_0473.html", "node_id":"dli_08_0473.xml", "product_code":"dli", - "code":"184", + "code":"185", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Mathematical Functions", @@ -3308,7 +3694,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Mathematical Functions", @@ -3318,7 +3706,7 @@ "uri":"dli_08_0065.html", "node_id":"dli_08_0065.xml", "product_code":"dli", - "code":"185", + "code":"186", "des":"Table 1 lists the mathematical functions supported by DLI.", "doc_type":"sqlreference", "kw":"Overview,Mathematical Functions,SQL Syntax Reference", @@ -3326,7 +3714,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Overview", @@ -3336,7 +3726,7 @@ "uri":"dli_spark_abs.html", "node_id":"dli_spark_abs.xml", "product_code":"dli", - "code":"186", + "code":"187", "des":"This function is used to calculate the absolute value of an input parameter.The return value is of the DOUBLE or INT type.If the value of a is NULL, NULL is returned.The ", "doc_type":"sqlreference", "kw":"abs,Mathematical Functions,SQL Syntax Reference", @@ -3344,7 +3734,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"abs", @@ -3354,7 +3746,7 @@ "uri":"dli_spark_aocs.html", "node_id":"dli_spark_aocs.xml", "product_code":"dli", - "code":"187", + "code":"188", "des":"This function is used to return the arc cosine value of a given angle a.The return value is of the DOUBLE type. The value ranges from 0 to π.If the value of a is not with", "doc_type":"sqlreference", "kw":"acos,Mathematical Functions,SQL Syntax Reference", @@ -3362,7 +3754,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"acos", @@ -3372,7 +3766,7 @@ "uri":"dli_spark_asin.html", "node_id":"dli_spark_asin.xml", "product_code":"dli", - "code":"188", + "code":"189", "des":"This function is used to return the arc sine value of a given angle a.The return value is of the DOUBLE type. The value ranges from -π/2 to π/2.If the value of a is not w", "doc_type":"sqlreference", "kw":"asin,Mathematical Functions,SQL Syntax Reference", @@ -3380,7 +3774,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"asin", @@ -3390,7 +3786,7 @@ "uri":"dli_spark_atan.html", "node_id":"dli_spark_atan.xml", "product_code":"dli", - "code":"189", + "code":"190", "des":"This function is used to return the arc tangent value of a given angle a.The return value is of the DOUBLE type. The value ranges from -π/2 to π/2.If the value of a is no", "doc_type":"sqlreference", "kw":"atan,Mathematical Functions,SQL Syntax Reference", @@ -3398,7 +3794,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"atan", @@ -3408,7 +3806,7 @@ "uri":"dli_spark_bin.html", "node_id":"dli_spark_bin.xml", "product_code":"dli", - "code":"190", + "code":"191", "des":"This function is used to return the binary format of a.The return value is of the STRING type.If the value of a is NULL, NULL is returned.The value 1 is returned.The valu", "doc_type":"sqlreference", "kw":"bin,Mathematical Functions,SQL Syntax Reference", @@ -3416,7 +3814,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"bin", @@ -3426,7 +3826,7 @@ "uri":"dli_spark_bround.html", "node_id":"dli_spark_bround.xml", "product_code":"dli", - "code":"191", + "code":"192", "des":"This function is used to return a value that is rounded off to d decimal places.The return value is of the DOUBLE type.If the value of a or d is NULL, NULL is returned.Th", "doc_type":"sqlreference", "kw":"bround,Mathematical Functions,SQL Syntax Reference", @@ -3434,7 +3834,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"bround", @@ -3444,7 +3846,7 @@ "uri":"dli_spark_cbrt.html", "node_id":"dli_spark_cbrt.xml", "product_code":"dli", - "code":"192", + "code":"193", "des":"This function is used to return the cube root of a.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 3 is returned.select cbrt(", "doc_type":"sqlreference", "kw":"cbrt,Mathematical Functions,SQL Syntax Reference", @@ -3452,7 +3854,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"cbrt", @@ -3462,7 +3866,7 @@ "uri":"dli_spark_ceil.html", "node_id":"dli_spark_ceil.xml", "product_code":"dli", - "code":"193", + "code":"194", "des":"This function is used to round up a to the nearest integer.The return value is of the DECIMAL type.If the value of a is NULL, NULL is returned.The value 2 is returned.The", "doc_type":"sqlreference", "kw":"ceil,Mathematical Functions,SQL Syntax Reference", @@ -3470,7 +3874,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"ceil", @@ -3480,7 +3886,7 @@ "uri":"dli_spark_conv.html", "node_id":"dli_spark_conv.xml", "product_code":"dli", - "code":"194", + "code":"195", "des":"This function is used to convert a number from from_base to to_base.The return value is of the STRING type.If the value of num, from_base, or to_base is NULL, NULL is ret", "doc_type":"sqlreference", "kw":"conv,Mathematical Functions,SQL Syntax Reference", @@ -3488,7 +3894,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"conv", @@ -3498,7 +3906,7 @@ "uri":"dli_spark_cos.html", "node_id":"dli_spark_cos.xml", "product_code":"dli", - "code":"195", + "code":"196", "des":"This function is used to calculate the cosine value of a, with input in radians.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The val", "doc_type":"sqlreference", "kw":"cos,Mathematical Functions,SQL Syntax Reference", @@ -3506,7 +3914,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"cos", @@ -3516,7 +3926,7 @@ "uri":"dli_spark_cot1.html", "node_id":"dli_spark_cot1.xml", "product_code":"dli", - "code":"196", + "code":"197", "des":"This function is used to calculate the cotangent value of a, with input in radians.The return value is of the DOUBLE or DECIMAL type.If the value of a is NULL, NULL is re", "doc_type":"sqlreference", "kw":"cot1,Mathematical Functions,SQL Syntax Reference", @@ -3524,7 +3934,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"cot1", @@ -3534,7 +3946,7 @@ "uri":"dli_spark_degress.html", "node_id":"dli_spark_degress.xml", "product_code":"dli", - "code":"197", + "code":"198", "des":"This function is used to calculate the angle corresponding to the returned radian.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The v", "doc_type":"sqlreference", "kw":"degrees,Mathematical Functions,SQL Syntax Reference", @@ -3542,7 +3954,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"degrees", @@ -3552,7 +3966,7 @@ "uri":"dli_spark_e.html", "node_id":"dli_spark_e.xml", "product_code":"dli", - "code":"198", + "code":"199", "des":"This function is used to return the value of e.The return value is of the DOUBLE type.The value 2.718281828459045 is returned.select e();", "doc_type":"sqlreference", "kw":"e,Mathematical Functions,SQL Syntax Reference", @@ -3560,7 +3974,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"e", @@ -3570,7 +3986,7 @@ "uri":"dli_spark_exp.html", "node_id":"dli_spark_exp.xml", "product_code":"dli", - "code":"199", + "code":"200", "des":"This function is used to return the value of e raised to the power of a.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 7.389", "doc_type":"sqlreference", "kw":"exp,Mathematical Functions,SQL Syntax Reference", @@ -3578,7 +3994,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"exp", @@ -3588,7 +4006,7 @@ "uri":"dli_spark_factorial.html", "node_id":"dli_spark_factorial.xml", "product_code":"dli", - "code":"200", + "code":"201", "des":"This function is used to return the factorial of a.The return value is of the BIGINT type.If the value of a is 0, 1 is returned.If the value of a is NULL or outside the r", "doc_type":"sqlreference", "kw":"factorial,Mathematical Functions,SQL Syntax Reference", @@ -3596,7 +4014,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"factorial", @@ -3606,7 +4026,7 @@ "uri":"dli_spark_floor.html", "node_id":"dli_spark_floor.xml", "product_code":"dli", - "code":"201", + "code":"202", "des":"This function is used to round down a to the nearest integer.The return value is of the BIGINT type.If the value of a is NULL, NULL is returned.The value 1 is returned.Th", "doc_type":"sqlreference", "kw":"floor,Mathematical Functions,SQL Syntax Reference", @@ -3614,7 +4034,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"floor", @@ -3624,7 +4046,7 @@ "uri":"dli_spark_greatest.html", "node_id":"dli_spark_greatest.xml", "product_code":"dli", - "code":"202", + "code":"203", "des":"This function is used to return the greatest value in a list of values.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 4.0 is", "doc_type":"sqlreference", "kw":"greatest,Mathematical Functions,SQL Syntax Reference", @@ -3632,7 +4054,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"greatest", @@ -3642,7 +4066,7 @@ "uri":"dli_spark_hex.html", "node_id":"dli_spark_hex.xml", "product_code":"dli", - "code":"203", + "code":"204", "des":"This function is used to convert an integer or character into its hexadecimal representation.The return value is of the STRING type.If the value of a is 0, 0 is returned.", "doc_type":"sqlreference", "kw":"hex,Mathematical Functions,SQL Syntax Reference", @@ -3650,7 +4074,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"hex", @@ -3660,7 +4086,7 @@ "uri":"dli_spark_least.html", "node_id":"dli_spark_least.xml", "product_code":"dli", - "code":"204", + "code":"205", "des":"This function is used to return the smallest value in a list of values.The return value is of the DOUBLE type.If the value of v1 or v2 is of the STRING type, an error is ", "doc_type":"sqlreference", "kw":"least,Mathematical Functions,SQL Syntax Reference", @@ -3668,7 +4094,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"least", @@ -3678,7 +4106,7 @@ "uri":"dli_spark_ln.html", "node_id":"dli_spark_ln.xml", "product_code":"dli", - "code":"205", + "code":"206", "des":"This function is used to return the natural logarithm of a given value.The return value is of the DOUBLE type.If the value of a is negative or 0, NULL is returned.If the ", "doc_type":"sqlreference", "kw":"ln,Mathematical Functions,SQL Syntax Reference", @@ -3686,7 +4114,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"ln", @@ -3696,7 +4126,7 @@ "uri":"dli_spark_log.html", "node_id":"dli_spark_log.xml", "product_code":"dli", - "code":"206", + "code":"207", "des":"This function is used to return the natural logarithm of a given base and exponent.The return value is of the DOUBLE type.If the value of base or a is NULL, NULL is retur", "doc_type":"sqlreference", "kw":"log,Mathematical Functions,SQL Syntax Reference", @@ -3704,7 +4134,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"log", @@ -3714,7 +4146,7 @@ "uri":"dli_spark_log10.html", "node_id":"dli_spark_log10.xml", "product_code":"dli", - "code":"207", + "code":"208", "des":"This function is used to return the natural logarithm of a given value with a base of 10.The return value is of the DOUBLE type.If the value of a is negative, 0, or NULL,", "doc_type":"sqlreference", "kw":"log10,Mathematical Functions,SQL Syntax Reference", @@ -3722,7 +4154,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"log10", @@ -3732,7 +4166,7 @@ "uri":"dli_spark_log2.html", "node_id":"dli_spark_log2.xml", "product_code":"dli", - "code":"208", + "code":"209", "des":"This function is used to return the natural logarithm of a given value with a base of 2.The return value is of the DOUBLE type.If the value of a is negative, 0, or NULL, ", "doc_type":"sqlreference", "kw":"log2,Mathematical Functions,SQL Syntax Reference", @@ -3740,7 +4174,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"log2", @@ -3750,7 +4186,7 @@ "uri":"dli_spark_median.html", "node_id":"dli_spark_median.xml", "product_code":"dli", - "code":"209", + "code":"210", "des":"This function is used to calculate the median of input parameters.The return value is of the DOUBLE or DECIMAL type.If the column name does not exist, an error is reporte", "doc_type":"sqlreference", "kw":"median,Mathematical Functions,SQL Syntax Reference", @@ -3758,7 +4194,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"median", @@ -3768,7 +4206,7 @@ "uri":"dli_spark_negative.html", "node_id":"dli_spark_negative.xml", "product_code":"dli", - "code":"210", + "code":"211", "des":"This function is used to return the additive inverse of a.The return value is of the DECIMAL or INT type.If the value of a is NULL, NULL is returned.The value -1 is retur", "doc_type":"sqlreference", "kw":"negative,Mathematical Functions,SQL Syntax Reference", @@ -3776,7 +4214,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"negative", @@ -3786,7 +4226,7 @@ "uri":"dli_spark_percentlie.html", "node_id":"dli_spark_percentlie.xml", "product_code":"dli", - "code":"211", + "code":"212", "des":"This function is used to return the exact percentile, which is applicable to a small amount of data. It sorts a specified column in ascending order, and then obtains the ", "doc_type":"sqlreference", "kw":"percentlie,Mathematical Functions,SQL Syntax Reference", @@ -3794,7 +4234,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"percentlie", @@ -3804,7 +4246,7 @@ "uri":"dli_spark_percentlie_approx.html", "node_id":"dli_spark_percentlie_approx.xml", "product_code":"dli", - "code":"212", + "code":"213", "des":"This function is used to return the approximate percentile, which is applicable to a large amount of data. It sorts a specified column in ascending order, and then obtain", "doc_type":"sqlreference", "kw":"percentlie_approx,Mathematical Functions,SQL Syntax Reference", @@ -3812,7 +4254,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"percentlie_approx", @@ -3822,7 +4266,7 @@ "uri":"dli_spark_pi.html", "node_id":"dli_spark_pi.xml", "product_code":"dli", - "code":"213", + "code":"214", "des":"This function is used to return the value of π.The return value is of the DOUBLE type.The value 3.141592653589793 is returned.", "doc_type":"sqlreference", "kw":"pi,Mathematical Functions,SQL Syntax Reference", @@ -3830,7 +4274,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"pi", @@ -3840,7 +4286,7 @@ "uri":"dli_spark_pmod.html", "node_id":"dli_spark_pmod.xml", "product_code":"dli", - "code":"214", + "code":"215", "des":"This function is used to return the positive value of the remainder after division of x by y.pmod(INT a, INT b)The return value is of the DECIMAL or INT type.If the value", "doc_type":"sqlreference", "kw":"pmod,Mathematical Functions,SQL Syntax Reference", @@ -3848,7 +4294,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"pmod", @@ -3858,7 +4306,7 @@ "uri":"dli_spark_positive.html", "node_id":"dli_spark_positive.xml", "product_code":"dli", - "code":"215", + "code":"216", "des":"This function is used to return the value of a.The return value is of the DECIMAL, DOUBLE, or INT type.If the value of a is NULL, NULL is returned.The value 3 is returned", "doc_type":"sqlreference", "kw":"positive,Mathematical Functions,SQL Syntax Reference", @@ -3866,7 +4314,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"positive", @@ -3876,7 +4326,7 @@ "uri":"dli_spark_pow.html", "node_id":"dli_spark_pow.xml", "product_code":"dli", - "code":"216", + "code":"217", "des":"This function is used to calculate and return the pth power of a.The return value is of the DOUBLE type.If the value of a or p is NULL, NULL is returned.The value 16 retu", "doc_type":"sqlreference", "kw":"pow,Mathematical Functions,SQL Syntax Reference", @@ -3884,7 +4334,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"pow", @@ -3894,7 +4346,7 @@ "uri":"dli_spark_radians.html", "node_id":"dli_spark_radians.xml", "product_code":"dli", - "code":"217", + "code":"218", "des":"This function is used to return the radian corresponding to an angle.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 1.047197", "doc_type":"sqlreference", "kw":"radians,Mathematical Functions,SQL Syntax Reference", @@ -3902,7 +4354,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"radians", @@ -3912,7 +4366,7 @@ "uri":"dli_spark_rand.html", "node_id":"dli_spark_rand.xml", "product_code":"dli", - "code":"218", + "code":"219", "des":"This function is used to return an evenly distributed random number that is greater than or equal to 0 and less than 1.The return value is of the DOUBLE type.The value 0.", "doc_type":"sqlreference", "kw":"rand,Mathematical Functions,SQL Syntax Reference", @@ -3920,7 +4374,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"rand", @@ -3930,7 +4386,7 @@ "uri":"dli_spark_round.html", "node_id":"dli_spark_round.xml", "product_code":"dli", - "code":"219", + "code":"220", "des":"This function is used to calculate the rounded value of a up to d decimal places.The return value is of the DOUBLE type.If the value of d is negative, an error is reporte", "doc_type":"sqlreference", "kw":"round,Mathematical Functions,SQL Syntax Reference", @@ -3938,7 +4394,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"round", @@ -3948,7 +4406,7 @@ "uri":"dli_spark_shiftleft.html", "node_id":"dli_spark_shiftleft.xml", "product_code":"dli", - "code":"220", + "code":"221", "des":"This function is used to perform a signed bitwise left shift. It takes the binary number a and shifts it b positions to the left.shiftleft(BIGINT a, BIGINT b)The return v", "doc_type":"sqlreference", "kw":"shiftleft,Mathematical Functions,SQL Syntax Reference", @@ -3956,7 +4414,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"shiftleft", @@ -3966,7 +4426,7 @@ "uri":"dli_spark_shiftright.html", "node_id":"dli_spark_shiftright.xml", "product_code":"dli", - "code":"221", + "code":"222", "des":"This function is used to perform a signed bitwise right shift. It takes the binary number a and shifts it b positions to the right.The return value is of the INT type.If ", "doc_type":"sqlreference", "kw":"shiftright,Mathematical Functions,SQL Syntax Reference", @@ -3974,7 +4434,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"shiftright", @@ -3984,7 +4446,7 @@ "uri":"dli_spark_shiftrightunsigned.html", "node_id":"dli_spark_shiftrightunsigned.xml", "product_code":"dli", - "code":"222", + "code":"223", "des":"This function is used to perform an unsigned bitwise right shift. It takes the binary number a and shifts it b positions to the right.The return value is of the INT type.", "doc_type":"sqlreference", "kw":"shiftrightunsigned,Mathematical Functions,SQL Syntax Reference", @@ -3992,7 +4454,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"shiftrightunsigned", @@ -4002,7 +4466,7 @@ "uri":"dli_spark_sign.html", "node_id":"dli_spark_sign.xml", "product_code":"dli", - "code":"223", + "code":"224", "des":"This function is used to return the positive and negative signs corresponding to a.The return value is of the DOUBLE type.If the value of a is a positive number, 1 is ret", "doc_type":"sqlreference", "kw":"sign,Mathematical Functions,SQL Syntax Reference", @@ -4010,7 +4474,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"sign", @@ -4020,7 +4486,7 @@ "uri":"dli_spark_sin.html", "node_id":"dli_spark_sin.xml", "product_code":"dli", - "code":"224", + "code":"225", "des":"This function is used to return the sine value of a, with input in radians.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 1 ", "doc_type":"sqlreference", "kw":"sin,Mathematical Functions,SQL Syntax Reference", @@ -4028,7 +4494,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"sin", @@ -4038,7 +4506,7 @@ "uri":"dli_spark_sqrt.html", "node_id":"dli_spark_sqrt.xml", "product_code":"dli", - "code":"225", + "code":"226", "des":"This function is used to return the square root of a value.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 2.8284271247461903", "doc_type":"sqlreference", "kw":"sqrt,Mathematical Functions,SQL Syntax Reference", @@ -4046,7 +4514,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"sqrt", @@ -4056,7 +4526,7 @@ "uri":"dli_spark_tan.html", "node_id":"dli_spark_tan.xml", "product_code":"dli", - "code":"226", + "code":"227", "des":"This function is used to return the tangent value of a, with input in radians.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value", "doc_type":"sqlreference", "kw":"tan,Mathematical Functions,SQL Syntax Reference", @@ -4064,7 +4534,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"tan", @@ -4074,7 +4546,7 @@ "uri":"dli_08_0474.html", "node_id":"dli_08_0474.xml", "product_code":"dli", - "code":"227", + "code":"228", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Aggregate Functions", @@ -4082,7 +4554,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Aggregate Functions", @@ -4092,7 +4566,7 @@ "uri":"dli_08_0068.html", "node_id":"dli_08_0068.xml", "product_code":"dli", - "code":"228", + "code":"229", "des":"Table 1 lists the aggregate functions supported by DLI.", "doc_type":"sqlreference", "kw":"Overview,Aggregate Functions,SQL Syntax Reference", @@ -4100,7 +4574,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Overview", @@ -4110,7 +4586,7 @@ "uri":"dli_spark_avg.html", "node_id":"dli_spark_avg.xml", "product_code":"dli", - "code":"229", + "code":"230", "des":"This function is used to return the average value.The return value is of the DOUBLE type.If the value of col is NULL, the column is not involved in calculation.Calculates", "doc_type":"sqlreference", "kw":"avg,Aggregate Functions,SQL Syntax Reference", @@ -4118,7 +4594,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"avg", @@ -4128,7 +4606,7 @@ "uri":"dli_spark_corr.html", "node_id":"dli_spark_corr.xml", "product_code":"dli", - "code":"230", + "code":"231", "des":"This function is used to return the correlation coefficient between two columns of numerical values.The return value is of the DOUBLE type.Calculates the correlation coef", "doc_type":"sqlreference", "kw":"corr,Aggregate Functions,SQL Syntax Reference", @@ -4136,7 +4614,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"corr", @@ -4146,7 +4626,7 @@ "uri":"dli_spark_count.html", "node_id":"dli_spark_count.xml", "product_code":"dli", - "code":"231", + "code":"232", "des":"This function is used to return the number of records.The return value is of the BIGINT type.If the value of colname is NULL, the row is not involved in calculation.Calcu", "doc_type":"sqlreference", "kw":"count,Aggregate Functions,SQL Syntax Reference", @@ -4154,7 +4634,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"count", @@ -4164,7 +4646,7 @@ "uri":"dli_spark_covar_pop.html", "node_id":"dli_spark_covar_pop.xml", "product_code":"dli", - "code":"232", + "code":"233", "des":"This function is used to return the covariance between two columns of numerical values.The return value is of the DOUBLE type.Calculates the covariance between the invent", "doc_type":"sqlreference", "kw":"covar_pop,Aggregate Functions,SQL Syntax Reference", @@ -4172,7 +4654,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"covar_pop", @@ -4182,7 +4666,7 @@ "uri":"dli_spark_covar_samp.html", "node_id":"dli_spark_covar_samp.xml", "product_code":"dli", - "code":"233", + "code":"234", "des":"This function is used to return the sample covariance between two columns of numerical values.The return value is of the DOUBLE type.Calculates the sample covariance betw", "doc_type":"sqlreference", "kw":"covar_samp,Aggregate Functions,SQL Syntax Reference", @@ -4190,7 +4674,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"covar_samp", @@ -4200,7 +4686,7 @@ "uri":"dli_spark_max.html", "node_id":"dli_spark_max.xml", "product_code":"dli", - "code":"234", + "code":"235", "des":"This function is used to return the maximum value.The return value is of the DOUBLE type.The return type is the same as the type of col. The return rules are as follows:I", "doc_type":"sqlreference", "kw":"max,Aggregate Functions,SQL Syntax Reference", @@ -4208,7 +4694,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"max", @@ -4218,7 +4706,7 @@ "uri":"dli_spark_min.html", "node_id":"dli_spark_min.xml", "product_code":"dli", - "code":"235", + "code":"236", "des":"This function is used to return the minimum value.The return value is of the DOUBLE type.The return type is the same as the type of col. The return rules are as follows:I", "doc_type":"sqlreference", "kw":"min,Aggregate Functions,SQL Syntax Reference", @@ -4226,7 +4714,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"min", @@ -4236,7 +4726,7 @@ "uri":"dli_spark_percentile.html", "node_id":"dli_spark_percentile.xml", "product_code":"dli", - "code":"236", + "code":"237", "des":"This function is used to return the numerical value at a certain percentage point within a range of values.The return value is of the DOUBLE type.The value should be betw", "doc_type":"sqlreference", "kw":"percentile,Aggregate Functions,SQL Syntax Reference", @@ -4244,7 +4734,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"percentile", @@ -4254,7 +4746,7 @@ "uri":"dli_spark_percentile_approx.html", "node_id":"dli_spark_percentile_approx.xml", "product_code":"dli", - "code":"237", + "code":"238", "des":"This function is used to approximate the pth percentile (including floating-point numbers) of a numeric column within a group.The return value is of the DOUBLE type.Calcu", "doc_type":"sqlreference", "kw":"percentile_approx,Aggregate Functions,SQL Syntax Reference", @@ -4262,7 +4754,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"percentile_approx", @@ -4272,7 +4766,7 @@ "uri":"dli_spark_stddev_pop.html", "node_id":"dli_spark_stddev_pop.xml", "product_code":"dli", - "code":"238", + "code":"239", "des":"This function is used to return the deviation of a specified column.The return value is of the DOUBLE type.Calculates the deviation of all offering inventories (items). A", "doc_type":"sqlreference", "kw":"stddev_pop,Aggregate Functions,SQL Syntax Reference", @@ -4280,7 +4774,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"stddev_pop", @@ -4290,15 +4786,17 @@ "uri":"dli_spark_stddev_samp.html", "node_id":"dli_spark_stddev_samp.xml", "product_code":"dli", - "code":"239", - "des":"This function is used to return the sample deviation of a specified column.The return value is of the DOUBLE type.Calculates the sample covariance between the inventory (", + "code":"240", + "des":"This function is used to return the sample deviation of a specified column.The return value is of the DOUBLE type.Calculates the sample deviation of all offering inventor", "doc_type":"sqlreference", "kw":"stddev_samp,Aggregate Functions,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"stddev_samp", @@ -4308,7 +4806,7 @@ "uri":"dli_spark_sum.html", "node_id":"dli_spark_sum.xml", "product_code":"dli", - "code":"240", + "code":"241", "des":"This function is used to calculate the total sum.The return value is of the DOUBLE type.If the value of col is NULL, the row is not involved in calculation.Calculates the", "doc_type":"sqlreference", "kw":"sum,Aggregate Functions,SQL Syntax Reference", @@ -4316,7 +4814,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"sum", @@ -4326,7 +4826,7 @@ "uri":"dli_spark_variance_var_pop.html", "node_id":"dli_spark_variance_var_pop.xml", "product_code":"dli", - "code":"241", + "code":"242", "des":"This function is used to return the variance of a column.The return value is of the DOUBLE type.Calculates the variance of all offering inventories (items). An example co", "doc_type":"sqlreference", "kw":"variance/var_pop,Aggregate Functions,SQL Syntax Reference", @@ -4334,7 +4834,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"variance/var_pop", @@ -4344,7 +4846,7 @@ "uri":"dli_spark_war_samp.html", "node_id":"dli_spark_war_samp.xml", "product_code":"dli", - "code":"242", + "code":"243", "des":"This function is used to return the sample variance of a specified column.The return value is of the DOUBLE type.Calculates the sample variance of all offering inventorie", "doc_type":"sqlreference", "kw":"var_samp,Aggregate Functions,SQL Syntax Reference", @@ -4352,7 +4854,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"var_samp", @@ -4362,7 +4866,7 @@ "uri":"dli_08_0475.html", "node_id":"dli_08_0475.xml", "product_code":"dli", - "code":"243", + "code":"244", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Window Functions", @@ -4370,7 +4874,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Window Functions", @@ -4380,7 +4886,7 @@ "uri":"dli_08_0069.html", "node_id":"dli_08_0069.xml", "product_code":"dli", - "code":"244", + "code":"245", "des":"Table 1 lists the window functions supported by DLI.", "doc_type":"sqlreference", "kw":"Overview,Window Functions,SQL Syntax Reference", @@ -4388,7 +4894,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Overview", @@ -4398,7 +4906,7 @@ "uri":"dli_spark_cume_dist.html", "node_id":"dli_spark_cume_dist.xml", "product_code":"dli", - "code":"245", + "code":"246", "des":"This function is used to return the cumulative distribution, which is equivalent to calculating the proportion of data in the partition that is greater than or equal to, ", "doc_type":"sqlreference", "kw":"cume_dist,Window Functions,SQL Syntax Reference", @@ -4406,7 +4914,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"cume_dist", @@ -4416,7 +4926,7 @@ "uri":"dli_spark_first_value.html", "node_id":"dli_spark_first_value.xml", "product_code":"dli", - "code":"246", + "code":"247", "des":"This function is used to obtain the value of the first data record in the window corresponding to the current row.The restrictions on using window functions are as follow", "doc_type":"sqlreference", "kw":"first_value,Window Functions,SQL Syntax Reference", @@ -4424,7 +4934,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"first_value", @@ -4434,7 +4946,7 @@ "uri":"dli_spark_last_value.html", "node_id":"dli_spark_last_value.xml", "product_code":"dli", - "code":"247", + "code":"248", "des":"This function is used to obtain the value of the last data record in the window corresponding to the current row.The restrictions on using window functions are as follows", "doc_type":"sqlreference", "kw":"last_value,Window Functions,SQL Syntax Reference", @@ -4442,7 +4954,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"last_value", @@ -4452,7 +4966,7 @@ "uri":"dli_spark_lag.html", "node_id":"dli_spark_lag.xml", "product_code":"dli", - "code":"248", + "code":"249", "des":"This function is used to return the value of the nth row upwards within a specified window.The restrictions on using window functions are as follows:Window functions can ", "doc_type":"sqlreference", "kw":"lag,Window Functions,SQL Syntax Reference", @@ -4460,7 +4974,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"lag", @@ -4470,7 +4986,7 @@ "uri":"dli_spark_lead.html", "node_id":"dli_spark_lead.xml", "product_code":"dli", - "code":"249", + "code":"250", "des":"This function is used to return the value of the nth row downwards within a specified window.The restrictions on using window functions are as follows:Window functions ca", "doc_type":"sqlreference", "kw":"lead,Window Functions,SQL Syntax Reference", @@ -4478,7 +4994,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"lead", @@ -4488,7 +5006,7 @@ "uri":"dli_spark_percent_rank.html", "node_id":"dli_spark_percent_rank.xml", "product_code":"dli", - "code":"250", + "code":"251", "des":"This function is used to return the value of the column specified in the ORDER BY clause of a window, expressed as a decimal between 0 and 1. It is calculated as (the ran", "doc_type":"sqlreference", "kw":"percent_rank,Window Functions,SQL Syntax Reference", @@ -4496,7 +5014,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"percent_rank", @@ -4506,7 +5026,7 @@ "uri":"dli_spark_rank.html", "node_id":"dli_spark_rank.xml", "product_code":"dli", - "code":"251", + "code":"252", "des":"This function is used to return the rank of a value in a set of values. When multiple values share the same rank, the next rank in the sequence is not consecutive.The res", "doc_type":"sqlreference", "kw":"rank,Window Functions,SQL Syntax Reference", @@ -4514,7 +5034,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"rank", @@ -4524,7 +5046,7 @@ "uri":"dli_spark_row_number.html", "node_id":"dli_spark_row_number.xml", "product_code":"dli", - "code":"252", + "code":"253", "des":"This function is used to return the row number, starting from 1 and increasing incrementally.The restrictions on using window functions are as follows:Window functions ca", "doc_type":"sqlreference", "kw":"row_number,Window Functions,SQL Syntax Reference", @@ -4532,7 +5054,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"row_number", @@ -4542,7 +5066,7 @@ "uri":"dli_08_0476.html", "node_id":"dli_08_0476.xml", "product_code":"dli", - "code":"253", + "code":"254", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Other Functions", @@ -4550,7 +5074,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Other Functions", @@ -4560,7 +5086,7 @@ "uri":"dli_08_0469.html", "node_id":"dli_08_0469.xml", "product_code":"dli", - "code":"254", + "code":"255", "des":"The following table lists the functions provided by DLI, such as decode1, javahash, and max_pt.", "doc_type":"sqlreference", "kw":"Overview,Other Functions,SQL Syntax Reference", @@ -4568,7 +5094,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Overview", @@ -4578,7 +5106,7 @@ "uri":"dli_spark_decode1.html", "node_id":"dli_spark_decode1.xml", "product_code":"dli", - "code":"255", + "code":"256", "des":"This function is used to implement if-then-else branch selection.result and default are return values. These values can be of any data type.If they match, the value of re", "doc_type":"sqlreference", "kw":"decode1,Other Functions,SQL Syntax Reference", @@ -4586,7 +5114,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"decode1", @@ -4596,7 +5126,7 @@ "uri":"dli_spark_javahash.html", "node_id":"dli_spark_javahash.xml", "product_code":"dli", - "code":"256", + "code":"257", "des":"This function is used to return the hash value of a.The return value is of the STRING type.The hash value is returned. If the value of a is null, an error is reported.The", "doc_type":"sqlreference", "kw":"javahash,Other Functions,SQL Syntax Reference", @@ -4604,7 +5134,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"javahash", @@ -4614,7 +5146,7 @@ "uri":"dli_spark_max_pt.html", "node_id":"dli_spark_max_pt.xml", "product_code":"dli", - "code":"257", + "code":"258", "des":"This function is used to return the name of the largest level-1 partition that contains data in a partitioned table and read the data of this partition.The return value i", "doc_type":"sqlreference", "kw":"max_pt,Other Functions,SQL Syntax Reference", @@ -4622,7 +5154,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"max_pt", @@ -4632,7 +5166,7 @@ "uri":"dli_spark_ordinal.html", "node_id":"dli_spark_ordinal.xml", "product_code":"dli", - "code":"258", + "code":"259", "des":"This function is used to sort input variables in ascending order and return the value at the position specified by nth.The return value is of the DOUBLE or DECIMAL type.V", "doc_type":"sqlreference", "kw":"ordinal,Other Functions,SQL Syntax Reference", @@ -4640,7 +5174,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"ordinal", @@ -4650,7 +5186,7 @@ "uri":"dli_spark_trans_array.html", "node_id":"dli_spark_trans_array.xml", "product_code":"dli", - "code":"259", + "code":"260", "des":"This function is used to convert an array split by a fixed separator in a column into multiple rows.All columns used as keys must be placed before the columns to be trans", "doc_type":"sqlreference", "kw":"trans_array,Other Functions,SQL Syntax Reference", @@ -4658,7 +5194,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"trans_array", @@ -4668,7 +5206,7 @@ "uri":"dli_spark_trunc_numeric.html", "node_id":"dli_spark_trunc_numeric.xml", "product_code":"dli", - "code":"260", + "code":"261", "des":"This function is used to truncate the number value to a specified decimal place.The return value is of the DOUBLE or DECIMAL type.The return rules are as follows:If the n", "doc_type":"sqlreference", "kw":"trunc_numeric,Other Functions,SQL Syntax Reference", @@ -4676,7 +5214,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"trunc_numeric", @@ -4686,7 +5226,7 @@ "uri":"dli_spark_url_decode.html", "node_id":"dli_spark_url_decode.xml", "product_code":"dli", - "code":"261", + "code":"262", "des":"This function is used to convert a string from the application/x-www-form-urlencoded MIME format to regular characters.The return value is of the STRING type.UTF-8-encode", "doc_type":"sqlreference", "kw":"url_decode,Other Functions,SQL Syntax Reference", @@ -4694,7 +5234,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"url_decode", @@ -4704,7 +5246,7 @@ "uri":"dli_spark_url_encode.html", "node_id":"dli_spark_url_encode.xml", "product_code":"dli", - "code":"262", + "code":"263", "des":"This function is used to encode a string in the application/x-www-form-urlencoded MIME format.url_encode(string [, string ])The return value is of the ST", "doc_type":"sqlreference", "kw":"url_encode,Other Functions,SQL Syntax Reference", @@ -4712,7 +5254,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"url_encode", @@ -4722,15 +5266,17 @@ "uri":"dli_08_0150.html", "node_id":"dli_08_0150.xml", "product_code":"dli", - "code":"263", - "des":"This statement is a basic query statement and is used to return the query results.The table to be queried must exist. Otherwise, an error is reported.To filter the record", + "code":"264", + "des":"This statement is a basic query statement and is used to return the query results.The table to be queried must already exist, or an error message will be displayed.When s", "doc_type":"sqlreference", "kw":"Basic SELECT Statements,Spark SQL Syntax Reference,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Basic SELECT Statements", @@ -4740,7 +5286,7 @@ "uri":"dli_08_0151.html", "node_id":"dli_08_0151.xml", "product_code":"dli", - "code":"264", + "code":"265", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Filtering", @@ -4748,7 +5294,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Filtering", @@ -4758,7 +5306,7 @@ "uri":"dli_08_0152.html", "node_id":"dli_08_0152.xml", "product_code":"dli", - "code":"265", + "code":"266", "des":"This statement is used to filter the query results using the WHERE clause.All is used to return repeated rows. By default, all repeated rows are returned. It is followed ", "doc_type":"sqlreference", "kw":"WHERE Filtering Clause,Filtering,SQL Syntax Reference", @@ -4766,7 +5314,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"WHERE Filtering Clause", @@ -4776,7 +5326,7 @@ "uri":"dli_08_0153.html", "node_id":"dli_08_0153.xml", "product_code":"dli", - "code":"266", + "code":"267", "des":"This statement is used to filter the query results using the HAVING clause.All is used to return repeated rows. By default, all repeated rows are returned. It is followed", "doc_type":"sqlreference", "kw":"HAVING Filtering Clause,Filtering,SQL Syntax Reference", @@ -4784,7 +5334,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"HAVING Filtering Clause", @@ -4794,7 +5346,7 @@ "uri":"dli_08_0154.html", "node_id":"dli_08_0154.xml", "product_code":"dli", - "code":"267", + "code":"268", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Sorting", @@ -4802,7 +5354,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Sorting", @@ -4812,7 +5366,7 @@ "uri":"dli_08_0155.html", "node_id":"dli_08_0155.xml", "product_code":"dli", - "code":"268", + "code":"269", "des":"This statement is used to order the result set of a query by the specified field.ASC/DESC: ASC sorts from the lowest value to the highest value. DESC sorts from the highe", "doc_type":"sqlreference", "kw":"ORDER BY,Sorting,SQL Syntax Reference", @@ -4820,7 +5374,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"ORDER BY", @@ -4830,7 +5386,7 @@ "uri":"dli_08_0156.html", "node_id":"dli_08_0156.xml", "product_code":"dli", - "code":"269", + "code":"270", "des":"This statement is used to achieve the partial sorting of tables according to fields.ASC/DESC: ASC sorts from the lowest value to the highest value. DESC sorts from the hi", "doc_type":"sqlreference", "kw":"SORT BY,Sorting,SQL Syntax Reference", @@ -4838,7 +5394,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"SORT BY", @@ -4848,7 +5406,7 @@ "uri":"dli_08_0157.html", "node_id":"dli_08_0157.xml", "product_code":"dli", - "code":"270", + "code":"271", "des":"This statement is used to bucket a table and sort the table within buckets.CLUSTER BY: Buckets are created based on specified fields. Single fields and multiple fields ar", "doc_type":"sqlreference", "kw":"CLUSTER BY,Sorting,SQL Syntax Reference", @@ -4856,7 +5414,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"CLUSTER BY", @@ -4866,7 +5426,7 @@ "uri":"dli_08_0158.html", "node_id":"dli_08_0158.xml", "product_code":"dli", - "code":"271", + "code":"272", "des":"This statement is used to bucket a table according to the field.DISTRIBUTE BY: Buckets are created based on specified fields. A single field or multiple fields are suppor", "doc_type":"sqlreference", "kw":"DISTRIBUTE BY,Sorting,SQL Syntax Reference", @@ -4874,7 +5434,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"DISTRIBUTE BY", @@ -4884,7 +5446,7 @@ "uri":"dli_08_0159.html", "node_id":"dli_08_0159.xml", "product_code":"dli", - "code":"272", + "code":"273", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Grouping", @@ -4892,7 +5454,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Grouping", @@ -4902,7 +5466,7 @@ "uri":"dli_08_0160.html", "node_id":"dli_08_0160.xml", "product_code":"dli", - "code":"273", + "code":"274", "des":"This statement is used to group a table based on columns.Column-based GROUP BY can be categorized into single-column GROUP BY and multi-column GROUP BY.Single-column GROU", "doc_type":"sqlreference", "kw":"Column-Based GROUP BY,Grouping,SQL Syntax Reference", @@ -4910,7 +5474,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Column-Based GROUP BY", @@ -4920,7 +5486,7 @@ "uri":"dli_08_0161.html", "node_id":"dli_08_0161.xml", "product_code":"dli", - "code":"274", + "code":"275", "des":"This statement is used to group a table according to expressions.The groupby_expression can contain a single field or multiple fields, and also can call aggregate functio", "doc_type":"sqlreference", "kw":"Expression-Based GROUP BY,Grouping,SQL Syntax Reference", @@ -4928,7 +5494,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Expression-Based GROUP BY", @@ -4938,7 +5506,7 @@ "uri":"dli_08_0162.html", "node_id":"dli_08_0162.xml", "product_code":"dli", - "code":"275", + "code":"276", "des":"This statement filters a table after grouping it using the HAVING clause.The groupby_expression can contain a single field or multiple fields, and can also call aggregate", "doc_type":"sqlreference", "kw":"GROUP BY Using HAVING,Grouping,SQL Syntax Reference", @@ -4946,7 +5514,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"GROUP BY Using HAVING", @@ -4956,7 +5526,7 @@ "uri":"dli_08_0163.html", "node_id":"dli_08_0163.xml", "product_code":"dli", - "code":"276", + "code":"277", "des":"This statement is used to generate the aggregate row, super-aggregate row, and the total row. The statement can achieve multi-layer statistics from right to left and disp", "doc_type":"sqlreference", "kw":"ROLLUP,Grouping,SQL Syntax Reference", @@ -4964,7 +5534,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"ROLLUP", @@ -4974,7 +5546,7 @@ "uri":"dli_08_0164.html", "node_id":"dli_08_0164.xml", "product_code":"dli", - "code":"277", + "code":"278", "des":"This statement is used to generate the cross-table row and achieve the cross-statistics of the GROUP BY field.GROUPING SETS is the expansion of GROUP BY. For example:SELE", "doc_type":"sqlreference", "kw":"GROUPING SETS,Grouping,SQL Syntax Reference", @@ -4982,7 +5554,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"GROUPING SETS", @@ -4992,7 +5566,7 @@ "uri":"dli_08_0165.html", "node_id":"dli_08_0165.xml", "product_code":"dli", - "code":"278", + "code":"279", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"JOIN", @@ -5000,7 +5574,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"JOIN", @@ -5010,7 +5586,7 @@ "uri":"dli_08_0166.html", "node_id":"dli_08_0166.xml", "product_code":"dli", - "code":"279", + "code":"280", "des":"This statement is used to join and return the rows that meet the JOIN conditions from two tables as the result set.JOIN/INNER JOIN: Only the records that meet the JOIN co", "doc_type":"sqlreference", "kw":"INNER JOIN,JOIN,SQL Syntax Reference", @@ -5018,7 +5594,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"INNER JOIN", @@ -5028,7 +5606,7 @@ "uri":"dli_08_0167.html", "node_id":"dli_08_0167.xml", "product_code":"dli", - "code":"280", + "code":"281", "des":"Join the left table with the right table and return all joined records of the left table. If no joined record is found, NULL will be returned.LEFT OUTER JOIN: Returns all", "doc_type":"sqlreference", "kw":"LEFT OUTER JOIN,JOIN,SQL Syntax Reference", @@ -5036,7 +5614,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"LEFT OUTER JOIN", @@ -5046,7 +5626,7 @@ "uri":"dli_08_0168.html", "node_id":"dli_08_0168.xml", "product_code":"dli", - "code":"281", + "code":"282", "des":"Match the right table with the left table and return all matched records of the right table. If no matched record is found, NULL will be returned.RIGHT OUTER JOIN: Return", "doc_type":"sqlreference", "kw":"RIGHT OUTER JOIN,JOIN,SQL Syntax Reference", @@ -5054,7 +5634,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"RIGHT OUTER JOIN", @@ -5064,7 +5646,7 @@ "uri":"dli_08_0169.html", "node_id":"dli_08_0169.xml", "product_code":"dli", - "code":"282", + "code":"283", "des":"Join all records from the right table and the left table and return all joined records. If no joined record is found, NULL will be returned.FULL OUTER JOIN: Matches all r", "doc_type":"sqlreference", "kw":"FULL OUTER JOIN,JOIN,SQL Syntax Reference", @@ -5072,7 +5654,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"FULL OUTER JOIN", @@ -5082,7 +5666,7 @@ "uri":"dli_08_0170.html", "node_id":"dli_08_0170.xml", "product_code":"dli", - "code":"283", + "code":"284", "des":"This statement has the same function as INNER JOIN, that is, the result set that meet the WHERE condition is returned. However, IMPLICIT JOIN does not use the condition s", "doc_type":"sqlreference", "kw":"IMPLICIT JOIN,JOIN,SQL Syntax Reference", @@ -5090,7 +5674,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"IMPLICIT JOIN", @@ -5100,7 +5686,7 @@ "uri":"dli_08_0171.html", "node_id":"dli_08_0171.xml", "product_code":"dli", - "code":"284", + "code":"285", "des":"Cartesian JOIN joins each record of table A with all records in table B. For example, if there are m records in table A and n records in table B, m x n records will be ge", "doc_type":"sqlreference", "kw":"Cartesian JOIN,JOIN,SQL Syntax Reference", @@ -5108,7 +5694,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Cartesian JOIN", @@ -5118,7 +5706,7 @@ "uri":"dli_08_0172.html", "node_id":"dli_08_0172.xml", "product_code":"dli", - "code":"285", + "code":"286", "des":"This statement is used to query the records that meet the JOIN condition from the left table.LEFT SEMI JOIN: Indicates to only return the records from the left table. LEF", "doc_type":"sqlreference", "kw":"LEFT SEMI JOIN,JOIN,SQL Syntax Reference", @@ -5126,7 +5714,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"LEFT SEMI JOIN", @@ -5136,7 +5726,7 @@ "uri":"dli_08_0173.html", "node_id":"dli_08_0173.xml", "product_code":"dli", - "code":"286", + "code":"287", "des":"This statement is used to join multiple tables using unequal values and return the result set that meet the condition.The non_equi_join_condition is similar to join_condi", "doc_type":"sqlreference", "kw":"NON-EQUIJOIN,JOIN,SQL Syntax Reference", @@ -5144,7 +5734,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"NON-EQUIJOIN", @@ -5154,7 +5746,7 @@ "uri":"dli_08_0174.html", "node_id":"dli_08_0174.xml", "product_code":"dli", - "code":"287", + "code":"288", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Subquery", @@ -5162,7 +5754,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Subquery", @@ -5172,7 +5766,7 @@ "uri":"dli_08_0175.html", "node_id":"dli_08_0175.xml", "product_code":"dli", - "code":"288", + "code":"289", "des":"Subqueries are nested in the WHERE clause, and the subquery result is used as the filtering condition.All is used to return repeated rows. By default, all repeated rows a", "doc_type":"sqlreference", "kw":"Subquery Nested by WHERE,Subquery,SQL Syntax Reference", @@ -5180,7 +5774,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Subquery Nested by WHERE", @@ -5190,7 +5786,7 @@ "uri":"dli_08_0176.html", "node_id":"dli_08_0176.xml", "product_code":"dli", - "code":"289", + "code":"290", "des":"This statement is used to nest subquery by FROM and use the subquery results as the data source of the external SELECT statement.All is used to return repeated rows. By d", "doc_type":"sqlreference", "kw":"Subquery Nested by FROM,Subquery,SQL Syntax Reference", @@ -5198,7 +5794,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Subquery Nested by FROM", @@ -5208,7 +5806,7 @@ "uri":"dli_08_0177.html", "node_id":"dli_08_0177.xml", "product_code":"dli", - "code":"290", + "code":"291", "des":"This statement is used to embed a subquery in the HAVING clause. The subquery result is used as a part of the HAVING clause.All is used to return repeated rows. By defaul", "doc_type":"sqlreference", "kw":"Subquery Nested by HAVING,Subquery,SQL Syntax Reference", @@ -5216,7 +5814,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Subquery Nested by HAVING", @@ -5226,7 +5826,7 @@ "uri":"dli_08_0178.html", "node_id":"dli_08_0178.xml", "product_code":"dli", - "code":"291", + "code":"292", "des":"This statement is used to nest queries in the subquery.All is used to return repeated rows. By default, all repeated rows are returned. It is followed by asterisks (*) on", "doc_type":"sqlreference", "kw":"Multi-Layer Nested Subquery,Subquery,SQL Syntax Reference", @@ -5234,7 +5834,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Multi-Layer Nested Subquery", @@ -5244,7 +5846,7 @@ "uri":"dli_08_0179.html", "node_id":"dli_08_0179.xml", "product_code":"dli", - "code":"292", + "code":"293", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Alias", @@ -5252,7 +5854,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Alias", @@ -5262,7 +5866,7 @@ "uri":"dli_08_0180.html", "node_id":"dli_08_0180.xml", "product_code":"dli", - "code":"293", + "code":"294", "des":"This statement is used to specify an alias for a table or the subquery result.table_reference: Can be a table, view, or subquery.As: Is used to connect to table_reference", "doc_type":"sqlreference", "kw":"AS for Table,Alias,SQL Syntax Reference", @@ -5270,7 +5874,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"AS for Table", @@ -5280,7 +5886,7 @@ "uri":"dli_08_0181.html", "node_id":"dli_08_0181.xml", "product_code":"dli", - "code":"294", + "code":"295", "des":"This statement is used to specify an alias for a column.alias: gives an alias for the attr_expr field.AS: Whether to add AS does not affect the result.The to-be-queried t", "doc_type":"sqlreference", "kw":"AS for Column,Alias,SQL Syntax Reference", @@ -5288,7 +5894,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"AS for Column", @@ -5298,7 +5906,7 @@ "uri":"dli_08_0182.html", "node_id":"dli_08_0182.xml", "product_code":"dli", - "code":"295", + "code":"296", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Set Operations", @@ -5306,7 +5914,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Set Operations", @@ -5316,7 +5926,7 @@ "uri":"dli_08_0183.html", "node_id":"dli_08_0183.xml", "product_code":"dli", - "code":"296", + "code":"297", "des":"This statement is used to return the union set of multiple query results.UNION: The set operation is used to join the head and tail of a table based on certain conditions", "doc_type":"sqlreference", "kw":"UNION,Set Operations,SQL Syntax Reference", @@ -5324,7 +5934,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"UNION", @@ -5334,7 +5946,7 @@ "uri":"dli_08_0184.html", "node_id":"dli_08_0184.xml", "product_code":"dli", - "code":"297", + "code":"298", "des":"This statement is used to return the intersection set of multiple query results.INTERSECT returns the intersection of multiple query results. The number of columns return", "doc_type":"sqlreference", "kw":"INTERSECT,Set Operations,SQL Syntax Reference", @@ -5342,7 +5954,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"INTERSECT", @@ -5352,7 +5966,7 @@ "uri":"dli_08_0185.html", "node_id":"dli_08_0185.xml", "product_code":"dli", - "code":"298", + "code":"299", "des":"This statement is used to return the difference set of two query results.EXCEPT minus the sets. A EXCEPT B indicates to remove the records that exist in both A and B from", "doc_type":"sqlreference", "kw":"EXCEPT,Set Operations,SQL Syntax Reference", @@ -5360,7 +5974,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"EXCEPT", @@ -5370,7 +5986,7 @@ "uri":"dli_08_0186.html", "node_id":"dli_08_0186.xml", "product_code":"dli", - "code":"299", + "code":"300", "des":"This statement is used to define the common table expression (CTE) using WITH...AS to simplify the query and make the result easier to read and maintain.cte_name: Name of", "doc_type":"sqlreference", "kw":"WITH...AS,Spark SQL Syntax Reference,SQL Syntax Reference", @@ -5378,7 +5994,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"WITH...AS", @@ -5388,7 +6006,7 @@ "uri":"dli_08_0187.html", "node_id":"dli_08_0187.xml", "product_code":"dli", - "code":"300", + "code":"301", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"CASE...WHEN", @@ -5396,7 +6014,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"CASE...WHEN", @@ -5406,7 +6026,7 @@ "uri":"dli_08_0188.html", "node_id":"dli_08_0188.xml", "product_code":"dli", - "code":"301", + "code":"302", "des":"This statement is used to display result_expression according to the joined results of input_expression and when_expression.CASE: Subquery is supported in basic CASE stat", "doc_type":"sqlreference", "kw":"Basic CASE Statement,CASE...WHEN,SQL Syntax Reference", @@ -5414,7 +6034,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Basic CASE Statement", @@ -5424,7 +6046,7 @@ "uri":"dli_08_0189.html", "node_id":"dli_08_0189.xml", "product_code":"dli", - "code":"302", + "code":"303", "des":"This statement is used to obtain the value of boolean_expression for each WHEN statement in a specified order. Then return the first result_expression with the value TRUE", "doc_type":"sqlreference", "kw":"CASE Query Statement,CASE...WHEN,SQL Syntax Reference", @@ -5432,7 +6054,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"CASE Query Statement", @@ -5442,7 +6066,7 @@ "uri":"dli_08_0190.html", "node_id":"dli_08_0190.xml", "product_code":"dli", - "code":"303", + "code":"304", "des":"This statement is used together with the window function. The OVER statement is used to group data and sort the data within the group. The window function is used to gene", "doc_type":"sqlreference", "kw":"OVER Clause,Spark SQL Syntax Reference,SQL Syntax Reference", @@ -5450,35 +6074,39 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"OVER Clause", "githuburl":"" }, { - "uri":"dli_08_0370.html", - "node_id":"dli_08_0370.xml", + "uri":"en-us_topic_0000002172127506.html", + "node_id":"en-us_topic_0000002172127506.xml", "product_code":"dli", - "code":"304", + "code":"305", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", - "kw":"Flink OpenSource SQL 1.12 Syntax Reference", + "kw":"Flink OpenSource SQL 1.15 Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], - "title":"Flink OpenSource SQL 1.12 Syntax Reference", + "title":"Flink OpenSource SQL 1.15 Syntax Reference", "githuburl":"" }, { - "uri":"dli_08_0371.html", - "node_id":"dli_08_0371.xml", + "uri":"dli_08_15001.html", + "node_id":"dli_08_15001.xml", "product_code":"dli", - "code":"305", + "code":"306", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Constraints and Definitions", @@ -5493,11 +6121,11 @@ "githuburl":"" }, { - "uri":"dli_08_0372.html", - "node_id":"dli_08_0372.xml", + "uri":"dli_08_15002.html", + "node_id":"dli_08_15002.xml", "product_code":"dli", - "code":"306", - "des":"STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP WITH LOCAL TIME ZONE, INTERVAL, ARRAY, MULTISET, MAP,", + "code":"307", + "des":"DLI supports the following data types:CHAR, VARCHAR, STRING, BOOLEAN, BINARY, VARBINARY, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TI", "doc_type":"sqlreference", "kw":"Supported Data Types,Constraints and Definitions,SQL Syntax Reference", "search_title":"", @@ -5511,13 +6139,13 @@ "githuburl":"" }, { - "uri":"dli_08_0373.html", - "node_id":"dli_08_0373.xml", + "uri":"dli_08_15003.html", + "node_id":"dli_08_15003.xml", "product_code":"dli", - "code":"307", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "code":"308", + "des":"Certain combinations of strings have been reserved as keywords for future use.If you use any of the following strings as field names, enclose them in backticks when using", "doc_type":"sqlreference", - "kw":"Syntax", + "kw":"Reserved Keywords,Constraints and Definitions,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -5525,14 +6153,14 @@ "documenttype":"sqlreference" } ], - "title":"Syntax", + "title":"Reserved Keywords", "githuburl":"" }, { - "uri":"dli_08_0374.html", - "node_id":"dli_08_0374.xml", + "uri":"dli_08_15005.html", + "node_id":"dli_08_15005.xml", "product_code":"dli", - "code":"308", + "code":"309", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Data Definition Language (DDL)", @@ -5547,11 +6175,11 @@ "githuburl":"" }, { - "uri":"dli_08_0375.html", - "node_id":"dli_08_0375.xml", + "uri":"dli_08_15006.html", + "node_id":"dli_08_15006.xml", "product_code":"dli", - "code":"309", - "des":"Create a table with a specified name.COMPUTED COLUMNA computed column is a virtual column generated using column_name AS computed_column_expression. A computed column eva", + "code":"310", + "des":"This statement creates a table using the specified table name. However, if a table with the same name already exists in the catalog, the registration process will fail.CO", "doc_type":"sqlreference", "kw":"CREATE TABLE,Data Definition Language (DDL),SQL Syntax Reference", "search_title":"", @@ -5565,11 +6193,47 @@ "githuburl":"" }, { - "uri":"dli_08_0376.html", - "node_id":"dli_08_0376.xml", + "uri":"dli_08_15007.html", + "node_id":"dli_08_15007.xml", "product_code":"dli", - "code":"310", - "des":"Create a view with multiple layers nested in it to simplify the development process.IF NOT EXISTSIf the view already exists, nothing happens.Create a view named viewName.", + "code":"311", + "des":"This statement creates a catalog using specified attributes. If a catalog with the same name already exists, an exception is thrown.WITH OPTIONSCatalog attributes typical", + "doc_type":"sqlreference", + "kw":"CREATE CATALOG,Data Definition Language (DDL),SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"CREATE CATALOG", + "githuburl":"" + }, + { + "uri":"dli_08_15008.html", + "node_id":"dli_08_15008.xml", + "product_code":"dli", + "code":"312", + "des":"This statement creates a database using specified table attributes. If a table with the same name already exists in the database, an exception is thrown.IF NOT EXISTSIf t", + "doc_type":"sqlreference", + "kw":"CREATE DATABASE,Data Definition Language (DDL),SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"CREATE DATABASE", + "githuburl":"" + }, + { + "uri":"dli_08_15009.html", + "node_id":"dli_08_15009.xml", + "product_code":"dli", + "code":"313", + "des":"This statement creates a view based on the given query statement. If a view with the same name already exists in the database, an exception is thrown.TEMPORARYCreate a te", "doc_type":"sqlreference", "kw":"CREATE VIEW,Data Definition Language (DDL),SQL Syntax Reference", "search_title":"", @@ -5583,11 +6247,11 @@ "githuburl":"" }, { - "uri":"dli_08_0377.html", - "node_id":"dli_08_0377.xml", + "uri":"dli_08_15010.html", + "node_id":"dli_08_15010.xml", "product_code":"dli", - "code":"311", - "des":"Create a user-defined function.For details about how to create a user-defined function, see User-Defined Functions (UDFs).IF NOT EXISTSIf the function already exists, not", + "code":"314", + "des":"To create a catalog function with a catalog and a database namespace, you will need to specify an identifier. You can specify a language tag. You cannot register the func", "doc_type":"sqlreference", "kw":"CREATE FUNCTION,Data Definition Language (DDL),SQL Syntax Reference", "search_title":"", @@ -5601,13 +6265,13 @@ "githuburl":"" }, { - "uri":"dli_08_0378.html", - "node_id":"dli_08_0378.xml", + "uri":"dli_08_15011.html", + "node_id":"dli_08_15011.xml", "product_code":"dli", - "code":"312", - "des":"SyntaxPrecautionsFlink SQL uses a lexical policy for identifier (table, attribute, function names) similar to Java:The case of identifiers is preserved whether they are q", + "code":"315", + "des":"Flink SQL uses a lexical policy for identifier (table, attribute, function names) similar to Java:The case of identifiers is preserved whether they are quoted.Identifiers", "doc_type":"sqlreference", - "kw":"Data Manipulation Language (DML),Syntax,SQL Syntax Reference", + "kw":"Data Manipulation Language (DML),Constraints and Definitions,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -5619,13 +6283,13 @@ "githuburl":"" }, { - "uri":"dli_08_0379.html", - "node_id":"dli_08_0379.xml", + "uri":"dli_08_15012.html", + "node_id":"dli_08_15012.xml", "product_code":"dli", - "code":"313", - "des":"This section describes the Flink open source SQL 1.12 syntax supported by DLI. For details about the parameters and examples, see the syntax description.", + "code":"316", + "des":"This section describes the Flink OpenSource SQL 1.15 syntax supported by DLI. For details about the parameters and examples, see the syntax description.", "doc_type":"sqlreference", - "kw":"Overview,Flink OpenSource SQL 1.12 Syntax Reference,SQL Syntax Reference", + "kw":"Overview,Flink OpenSource SQL 1.15 Syntax Reference,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -5637,67 +6301,13 @@ "githuburl":"" }, { - "uri":"dli_08_0380.html", - "node_id":"dli_08_0380.xml", - "product_code":"dli", - "code":"314", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"sqlreference", - "kw":"DDL Syntax", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"DDL Syntax", - "githuburl":"" - }, - { - "uri":"dli_08_0381.html", - "node_id":"dli_08_0381.xml", - "product_code":"dli", - "code":"315", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"sqlreference", - "kw":"Creating Source Tables", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Creating Source Tables", - "githuburl":"" - }, - { - "uri":"dli_08_0382.html", - "node_id":"dli_08_0382.xml", - "product_code":"dli", - "code":"316", - "des":"DataGen is used to generate random data for debugging and testing.NoneWhen you create a DataGen table, the table field type cannot be Array, Map, or Row. You can use COMP", - "doc_type":"sqlreference", - "kw":"DataGen Source Table,Creating Source Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"DataGen Source Table", - "githuburl":"" - }, - { - "uri":"dli_08_0383.html", - "node_id":"dli_08_0383.xml", + "uri":"dli_08_15108.html", + "node_id":"dli_08_15108.xml", "product_code":"dli", "code":"317", - "des":"DLI reads data of Flink jobs from GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex types an", + "des":"When switching from Flink 1.12 to Flink 1.15 for job execution, keep in mind the following considerations when utilizing Flink OpenSource SQL 1.15:Flink SQL utilizes a SQ", "doc_type":"sqlreference", - "kw":"GaussDB(DWS) Source Table,Creating Source Tables,SQL Syntax Reference", + "kw":"Flink OpenSource SQL 1.15 Usage,Flink OpenSource SQL 1.15 Syntax Reference,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -5705,17 +6315,17 @@ "documenttype":"sqlreference" } ], - "title":"GaussDB(DWS) Source Table", + "title":"Flink OpenSource SQL 1.15 Usage", "githuburl":"" }, { - "uri":"dli_08_0384.html", - "node_id":"dli_08_0384.xml", + "uri":"dli_08_15014.html", + "node_id":"dli_08_15014.xml", "product_code":"dli", "code":"318", - "des":"Create a source stream to obtain data from HBase as input for jobs. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excell", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", - "kw":"HBase Source Table,Creating Source Tables,SQL Syntax Reference", + "kw":"Formats", "search_title":"", "metedata":[ { @@ -5723,17 +6333,17 @@ "documenttype":"sqlreference" } ], - "title":"HBase Source Table", + "title":"Formats", "githuburl":"" }, { - "uri":"dli_08_0385.html", - "node_id":"dli_08_0385.xml", + "uri":"dli_08_15015.html", + "node_id":"dli_08_15015.xml", "product_code":"dli", "code":"319", - "des":"The JDBC connector is a Flink's built-in connector to read data from a database.An enhanced datasource connection with the instances has been established, so that you can", + "des":"Flink provides a set of table formats that can be used with table connectors.A table format is a storage format that defines how to map binary data onto table columns.", "doc_type":"sqlreference", - "kw":"JDBC Source Table,Creating Source Tables,SQL Syntax Reference", + "kw":"Overview,Formats,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -5741,431 +6351,17 @@ "documenttype":"sqlreference" } ], - "title":"JDBC Source Table", + "title":"Overview", "githuburl":"" }, { - "uri":"dli_08_0386.html", - "node_id":"dli_08_0386.xml", + "uri":"dli_08_15016.html", + "node_id":"dli_08_15016.xml", "product_code":"dli", "code":"320", - "des":"Create a source stream to obtain data from Kafka as input data for jobs.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscripti", + "des":"Apache Avro is supported for you to read and write Avro data based on an Avro schema with Flink. The Avro schema is derived from the table schema.For details, see Avro Fo", "doc_type":"sqlreference", - "kw":"Kafka Source Table,Creating Source Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Kafka Source Table", - "githuburl":"" - }, - { - "uri":"dli_08_0387.html", - "node_id":"dli_08_0387.xml", - "product_code":"dli", - "code":"321", - "des":"The MySQL CDC source table, that is, the MySQL streaming source table, reads all historical data in the database first and then smoothly switches data read to the Binlog ", - "doc_type":"sqlreference", - "kw":"MySQL CDC Source Table,Creating Source Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"MySQL CDC Source Table", - "githuburl":"" - }, - { - "uri":"dli_08_0388.html", - "node_id":"dli_08_0388.xml", - "product_code":"dli", - "code":"322", - "des":"The Postgres CDC source table, that is, Postgres streaming source table, is used to read the full snapshot data and changed data of the PostgreSQL database in sequence. T", - "doc_type":"sqlreference", - "kw":"Postgres CDC Source Table,Creating Source Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Postgres CDC Source Table", - "githuburl":"" - }, - { - "uri":"dli_08_0389.html", - "node_id":"dli_08_0389.xml", - "product_code":"dli", - "code":"323", - "des":"Create a source stream to obtain data from Redis as input for jobs.An enhanced datasource connection has been created for DLI to connect to the Redis database, so that yo", - "doc_type":"sqlreference", - "kw":"Redis Source Table,Creating Source Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Redis Source Table", - "githuburl":"" - }, - { - "uri":"dli_08_0390.html", - "node_id":"dli_08_0390.xml", - "product_code":"dli", - "code":"324", - "des":"Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It delivers high throughput and built-in partitions and provi", - "doc_type":"sqlreference", - "kw":"Upsert Kafka Source Table,Creating Source Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Upsert Kafka Source Table", - "githuburl":"" - }, - { - "uri":"dli_08_0391.html", - "node_id":"dli_08_0391.xml", - "product_code":"dli", - "code":"325", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"sqlreference", - "kw":"Creating Result Tables", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Creating Result Tables", - "githuburl":"" - }, - { - "uri":"dli_08_0392.html", - "node_id":"dli_08_0392.xml", - "product_code":"dli", - "code":"326", - "des":"The BlackHole connector allows for swallowing all input records. It is designed for high-performance testing and UDF output. It is not a substantive sink. The BlackHole r", - "doc_type":"sqlreference", - "kw":"BlackHole Result Table,Creating Result Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"BlackHole Result Table", - "githuburl":"" - }, - { - "uri":"dli_08_0393.html", - "node_id":"dli_08_0393.xml", - "product_code":"dli", - "code":"327", - "des":"DLI can output Flink job data to the ClickHouse database. ClickHouse is a column-based database oriented to online analysis and processing. It supports SQL query and prov", - "doc_type":"sqlreference", - "kw":"ClickHouse Result Table,Creating Result Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"ClickHouse Result Table", - "githuburl":"" - }, - { - "uri":"dli_08_0394.html", - "node_id":"dli_08_0394.xml", - "product_code":"dli", - "code":"328", - "des":"DLI outputs the Flink job output data to GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex t", - "doc_type":"sqlreference", - "kw":"GaussDB(DWS) Result Table,Creating Result Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"GaussDB(DWS) Result Table", - "githuburl":"" - }, - { - "uri":"dli_08_0395.html", - "node_id":"dli_08_0395.xml", - "product_code":"dli", - "code":"329", - "des":"DLI outputs Flink job output data to Elasticsearch of Cloud Search Service (CSS). Elasticsearch is a popular enterprise-class Lucene-powered search server and provides th", - "doc_type":"sqlreference", - "kw":"Elasticsearch Result Table,Creating Result Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Elasticsearch Result Table", - "githuburl":"" - }, - { - "uri":"dli_08_0396.html", - "node_id":"dli_08_0396.xml", - "product_code":"dli", - "code":"330", - "des":"DLI outputs the job data to HBase. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excellent performance, and elastic scal", - "doc_type":"sqlreference", - "kw":"HBase Result Table,Creating Result Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"HBase Result Table", - "githuburl":"" - }, - { - "uri":"dli_08_0397.html", - "node_id":"dli_08_0397.xml", - "product_code":"dli", - "code":"331", - "des":"DLI outputs the Flink job output data to RDS through the JDBC result table.An enhanced datasource connection with the instances has been established, so that you can conf", - "doc_type":"sqlreference", - "kw":"JDBC Result Table,Creating Result Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"JDBC Result Table", - "githuburl":"" - }, - { - "uri":"dli_08_0398.html", - "node_id":"dli_08_0398.xml", - "product_code":"dli", - "code":"332", - "des":"DLI outputs the Flink job output data to Kafka through the Kafka result table.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subs", - "doc_type":"sqlreference", - "kw":"Kafka Result Table,Creating Result Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Kafka Result Table", - "githuburl":"" - }, - { - "uri":"dli_08_0399.html", - "node_id":"dli_08_0399.xml", - "product_code":"dli", - "code":"333", - "des":"The Print connector is used to print output data to the error file or TaskManager file, making it easier for you to view the result in code debugging.NoneThe Print result", - "doc_type":"sqlreference", - "kw":"Print Result Table,Creating Result Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Print Result Table", - "githuburl":"" - }, - { - "uri":"dli_08_0400.html", - "node_id":"dli_08_0400.xml", - "product_code":"dli", - "code":"334", - "des":"DLI outputs the Flink job output data to Redis. Redis is a key-value storage system that supports multiple types of data structures. It can be used in scenarios such as c", - "doc_type":"sqlreference", - "kw":"Redis Result Table,Creating Result Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Redis Result Table", - "githuburl":"" - }, - { - "uri":"dli_08_0401.html", - "node_id":"dli_08_0401.xml", - "product_code":"dli", - "code":"335", - "des":"Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It delivers high throughput and built-in partitions and provi", - "doc_type":"sqlreference", - "kw":"Upsert Kafka Result Table,Creating Result Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Upsert Kafka Result Table", - "githuburl":"" - }, - { - "uri":"dli_08_0439.html", - "node_id":"dli_08_0439.xml", - "product_code":"dli", - "code":"336", - "des":"The FileSystem result (sink) table is used to export data to the HDFS or OBS file system. It is applicable to scenarios such as data dumping, big data analysis, data back", - "doc_type":"sqlreference", - "kw":"FileSystem Result Table,Creating Result Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"FileSystem Result Table", - "githuburl":"" - }, - { - "uri":"dli_08_0402.html", - "node_id":"dli_08_0402.xml", - "product_code":"dli", - "code":"337", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"sqlreference", - "kw":"Creating Dimension Tables", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Creating Dimension Tables", - "githuburl":"" - }, - { - "uri":"dli_08_0403.html", - "node_id":"dli_08_0403.xml", - "product_code":"dli", - "code":"338", - "des":"Create a GaussDB(DWS) table to connect to source streams for wide table generation.Ensure that you have created a GaussDB(DWS) cluster using your account.A DWS database t", - "doc_type":"sqlreference", - "kw":"GaussDB(DWS) Dimension Table,Creating Dimension Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"GaussDB(DWS) Dimension Table", - "githuburl":"" - }, - { - "uri":"dli_08_0404.html", - "node_id":"dli_08_0404.xml", - "product_code":"dli", - "code":"339", - "des":"Create a Hbase dimension table to connect to the source streams for wide table generation.An enhanced datasource connection has been created for DLI to connect to HBase, ", - "doc_type":"sqlreference", - "kw":"HBase Dimension Table,Creating Dimension Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"HBase Dimension Table", - "githuburl":"" - }, - { - "uri":"dli_08_0405.html", - "node_id":"dli_08_0405.xml", - "product_code":"dli", - "code":"340", - "des":"Create a JDBC dimension table to connect to the source stream.You have created a JDBC instance for your account.When you create a Flink OpenSource SQL job, set Flink Vers", - "doc_type":"sqlreference", - "kw":"JDBC Dimension Table,Creating Dimension Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"JDBC Dimension Table", - "githuburl":"" - }, - { - "uri":"dli_08_0406.html", - "node_id":"dli_08_0406.xml", - "product_code":"dli", - "code":"341", - "des":"Create a Redis table to connect to source streams for wide table generation.An enhanced datasource connection with Redis has been established, so that you can configure s", - "doc_type":"sqlreference", - "kw":"Redis Dimension Table,Creating Dimension Tables,SQL Syntax Reference", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Redis Dimension Table", - "githuburl":"" - }, - { - "uri":"dli_08_0407.html", - "node_id":"dli_08_0407.xml", - "product_code":"dli", - "code":"342", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"sqlreference", - "kw":"Format", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"sqlreference" - } - ], - "title":"Format", - "githuburl":"" - }, - { - "uri":"dli_08_0408.html", - "node_id":"dli_08_0408.xml", - "product_code":"dli", - "code":"343", - "des":"Apache Avro is supported for you to read and write Avro data based on an Avro schema with Flink. The Avro schema is derived from the table schema.KafkaUpsert KafkaCurrent", - "doc_type":"sqlreference", - "kw":"Avro,Format,SQL Syntax Reference", + "kw":"Avro,Formats,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -6177,13 +6373,13 @@ "githuburl":"" }, { - "uri":"dli_08_0409.html", - "node_id":"dli_08_0409.xml", + "uri":"dli_08_15017.html", + "node_id":"dli_08_15017.xml", "product_code":"dli", - "code":"344", + "code":"321", "des":"Canal is a Changelog Data Capture (CDC) tool that can stream changes in real-time from MySQL into other systems. Canal provides a unified format schema for changelog and ", "doc_type":"sqlreference", - "kw":"Canal,Format,SQL Syntax Reference", + "kw":"Canal,Formats,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -6195,13 +6391,13 @@ "githuburl":"" }, { - "uri":"dli_08_0410.html", - "node_id":"dli_08_0410.xml", + "uri":"dli_08_15018.html", + "node_id":"dli_08_15018.xml", "product_code":"dli", - "code":"345", + "code":"322", "des":"The Avro Schema Registry (avro-confluent) format allows you to read records that were serialized by the io.confluent.kafka.serializers.KafkaAvroSerializer and to write re", "doc_type":"sqlreference", - "kw":"Confluent Avro,Format,SQL Syntax Reference", + "kw":"Confluent Avro,Formats,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -6213,13 +6409,13 @@ "githuburl":"" }, { - "uri":"dli_08_0411.html", - "node_id":"dli_08_0411.xml", + "uri":"dli_08_15019.html", + "node_id":"dli_08_15019.xml", "product_code":"dli", - "code":"346", - "des":"The CSV format allows you to read and write CSV data based on a CSV schema. Currently, the CSV schema is derived from table schema.KafkaUpsert KafkaUse Kafka to send data", + "code":"323", + "des":"The CSV format allows you to read and write CSV data based on a CSV schema. Currently, the CSV schema is derived from table schema. For details, see CSV Format.KafkaUpser", "doc_type":"sqlreference", - "kw":"CSV,Format,SQL Syntax Reference", + "kw":"CSV,Formats,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -6231,13 +6427,13 @@ "githuburl":"" }, { - "uri":"dli_08_0412.html", - "node_id":"dli_08_0412.xml", + "uri":"dli_08_15020.html", + "node_id":"dli_08_15020.xml", "product_code":"dli", - "code":"347", - "des":"Debezium is a Changelog Data Capture (CDC) tool that can stream changes in real-time from other databases into Kafka. Debezium provides a unified format schema for change", + "code":"324", + "des":"Debezium is a Changelog Data Capture (CDC) tool that can stream changes in real-time from MySQL, PostgreSQL, Oracle, Microsoft SQL Server and many other databases into Ka", "doc_type":"sqlreference", - "kw":"Debezium,Format,SQL Syntax Reference", + "kw":"Debezium,Formats,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -6249,13 +6445,13 @@ "githuburl":"" }, { - "uri":"dli_08_0413.html", - "node_id":"dli_08_0413.xml", + "uri":"dli_08_15021.html", + "node_id":"dli_08_15021.xml", "product_code":"dli", - "code":"348", - "des":"The JSON format allows you to read and write JSON data based on a JSON schema. Currently, the JSON schema is derived from table schema.KafkaUpsert KafkaElasticsearchIn th", + "code":"325", + "des":"The JSON format allows you to read and write JSON data based on a JSON schema. Currently, the JSON schema is derived from table schema. For details, see JSON Format.Kafka", "doc_type":"sqlreference", - "kw":"JSON,Format,SQL Syntax Reference", + "kw":"JSON,Formats,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -6267,13 +6463,13 @@ "githuburl":"" }, { - "uri":"dli_08_0414.html", - "node_id":"dli_08_0414.xml", + "uri":"dli_08_15022.html", + "node_id":"dli_08_15022.xml", "product_code":"dli", - "code":"349", - "des":"Flink supports to interpret Maxwell JSON messages as INSERT/UPDATE/DELETE messages into Flink SQL system. This is useful in many cases to leverage this feature,such as:Sy", + "code":"326", + "des":"Maxwell is a Changelog Data Capture (CDC) tool that can stream changes in real-time from MySQL into Kafka and other streaming connectors. Maxwell provides a unified forma", "doc_type":"sqlreference", - "kw":"Maxwell,Format,SQL Syntax Reference", + "kw":"Maxwell,Formats,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -6285,13 +6481,67 @@ "githuburl":"" }, { - "uri":"dli_08_0415.html", - "node_id":"dli_08_0415.xml", + "uri":"dli_08_15023.html", + "node_id":"dli_08_15023.xml", "product_code":"dli", - "code":"350", - "des":"The raw format allows you to read and write raw (byte based) values as a single column.Note: This format encodes null values as null of the byte[] type. This may have lim", + "code":"327", + "des":"Oracle GoldenGate (a.k.a ogg) is a comprehensive software package for real-time data capture and replication in heterogeneous IT environments. The product set enables hig", "doc_type":"sqlreference", - "kw":"Raw,Format,SQL Syntax Reference", + "kw":"Ogg,Formats,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Ogg", + "githuburl":"" + }, + { + "uri":"dli_08_15024.html", + "node_id":"dli_08_15024.xml", + "product_code":"dli", + "code":"328", + "des":"The Apache ORC format allows to read and write ORC data. For details, see ORC Format.FileSystemORC format also supports table properties from Table properties. For exampl", + "doc_type":"sqlreference", + "kw":"ORC,Formats,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"ORC", + "githuburl":"" + }, + { + "uri":"dli_08_15025.html", + "node_id":"dli_08_15025.xml", + "product_code":"dli", + "code":"329", + "des":"The Apache Parquet format allows to read and write Parquet data. For details, see Parquet Format.FileSystemCurrently, Parquet format type mapping is compatible with Apach", + "doc_type":"sqlreference", + "kw":"Parquet,Formats,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Parquet", + "githuburl":"" + }, + { + "uri":"dli_08_15026.html", + "node_id":"dli_08_15026.xml", + "product_code":"dli", + "code":"330", + "des":"The Raw format allows to read and write raw (byte based) values as a single column.This format encodes null values as null of byte[] type. This may have limitation when u", + "doc_type":"sqlreference", + "kw":"Raw,Formats,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -6303,10 +6553,730 @@ "githuburl":"" }, { - "uri":"dli_08_0416.html", - "node_id":"dli_08_0416.xml", + "uri":"dli_08_15027.html", + "node_id":"dli_08_15027.xml", + "product_code":"dli", + "code":"331", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Connectors", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Connectors", + "githuburl":"" + }, + { + "uri":"dli_08_15028.html", + "node_id":"dli_08_15028.xml", + "product_code":"dli", + "code":"332", + "des":"Source table: A source table is the data input table for Flink jobs, such as real-time streaming data input from Kafka, etc.Dimension Table: An auxiliary table for the da", + "doc_type":"sqlreference", + "kw":"Overview,Connectors,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Overview", + "githuburl":"" + }, + { + "uri":"dli_08_15029.html", + "node_id":"dli_08_15029.xml", + "product_code":"dli", + "code":"333", + "des":"The BlackHole connector allows for swallowing all input records. It is designed for high-performance testing and UDF output. It is not a substantive sink. The BlackHole r", + "doc_type":"sqlreference", + "kw":"BlackHole,Connectors,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"BlackHole", + "githuburl":"" + }, + { + "uri":"dli_08_15030.html", + "node_id":"dli_08_15030.xml", + "product_code":"dli", + "code":"334", + "des":"DLI has the capability to export data from Flink jobs to the ClickHouse database. However, it only supports exporting data to result tables.ClickHouse is a column-based d", + "doc_type":"sqlreference", + "kw":"ClickHouse,Connectors,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"ClickHouse", + "githuburl":"" + }, + { + "uri":"dli_08_15031.html", + "node_id":"dli_08_15031.xml", + "product_code":"dli", + "code":"335", + "des":"DataGen is used to generate random data for debugging and testing.When you create a DataGen table, the table field type cannot be Array, Map, or Row. You can use COMPUTED", + "doc_type":"sqlreference", + "kw":"DataGen,Connectors,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"DataGen", + "githuburl":"" + }, + { + "uri":"dli_08_15032.html", + "node_id":"dli_08_15032.xml", + "product_code":"dli", + "code":"336", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Doris", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Doris", + "githuburl":"" + }, + { + "uri":"dli_08_15033.html", + "node_id":"dli_08_15033.xml", + "product_code":"dli", + "code":"337", + "des":"The Flink Doris Connector can support operations (read, insert, modify, delete) data stored in Doris through Flink.Only tables in the Unique Key model can be modified or ", + "doc_type":"sqlreference", + "kw":"Overview,Doris,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Overview", + "githuburl":"" + }, + { + "uri":"dli_08_15034.html", + "node_id":"dli_08_15034.xml", + "product_code":"dli", + "code":"338", + "des":"Flink SQL jobs read from the Doris source table.An enhanced datasource connection has been created for DLI to connect to Doris, so that jobs can run on the dedicated queu", + "doc_type":"sqlreference", + "kw":"Source Table,Doris,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_15035.html", + "node_id":"dli_08_15035.xml", + "product_code":"dli", + "code":"339", + "des":"Flink SQL jobs write to the Doris result table.An enhanced datasource connection has been created for DLI to connect to Doris, so that jobs can run on the dedicated queue", + "doc_type":"sqlreference", + "kw":"Result Table,Doris,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_15036.html", + "node_id":"dli_08_15036.xml", + "product_code":"dli", + "code":"340", + "des":"Create a Doris dimension table to connect to the source streams for wide table generation.An enhanced datasource connection has been created for DLI to connect to HBase, ", + "doc_type":"sqlreference", + "kw":"Dimension Table,Doris,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Dimension Table", + "githuburl":"" + }, + { + "uri":"dli_08_15103.html", + "node_id":"dli_08_15103.xml", + "product_code":"dli", + "code":"341", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"GaussDB(DWS)", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"GaussDB(DWS)", + "githuburl":"" + }, + { + "uri":"dli_08_15037.html", + "node_id":"dli_08_15037.xml", + "product_code":"dli", + "code":"342", + "des":"GaussDB(DWS) is an online data processing database based on the cloud infrastructure and platform and helps you mine and analyze massive sets of data. DLI reads data of F", + "doc_type":"sqlreference", + "kw":"Overview,GaussDB(DWS),SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Overview", + "githuburl":"" + }, + { + "uri":"dli_08_15104.html", + "node_id":"dli_08_15104.xml", + "product_code":"dli", + "code":"343", + "des":"DLI reads data of Flink jobs from GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex types an", + "doc_type":"sqlreference", + "kw":"GaussDB(DWS) Source Table (Not Recommended),GaussDB(DWS),SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"GaussDB(DWS) Source Table (Not Recommended)", + "githuburl":"" + }, + { + "uri":"dli_08_15105.html", + "node_id":"dli_08_15105.xml", + "product_code":"dli", + "code":"344", + "des":"DLI outputs the Flink job output data to GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex t", + "doc_type":"sqlreference", + "kw":"GaussDB(DWS) Result Table (Not Recommended),GaussDB(DWS),SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"GaussDB(DWS) Result Table (Not Recommended)", + "githuburl":"" + }, + { + "uri":"dli_08_15106.html", + "node_id":"dli_08_15106.xml", + "product_code":"dli", + "code":"345", + "des":"Create a GaussDB(DWS) table to connect to source streams for wide table generation.You are advised to use GaussDB(DWS) self-developed GaussDB(DWS) connector.Ensure that y", + "doc_type":"sqlreference", + "kw":"GaussDB(DWS) Dimension Table (Not Recommended),GaussDB(DWS),SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"GaussDB(DWS) Dimension Table (Not Recommended)", + "githuburl":"" + }, + { + "uri":"dli_08_15038.html", + "node_id":"dli_08_15038.xml", + "product_code":"dli", + "code":"346", + "des":"DLI outputs the output data of the Flink job to an index in the Elasticsearch engine of the Cloud Search Service (CSS).Elasticsearch is a popular enterprise-class Lucene-", + "doc_type":"sqlreference", + "kw":"Elasticsearch,Connectors,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Elasticsearch", + "githuburl":"" + }, + { + "uri":"dli_08_15039.html", + "node_id":"dli_08_15039.xml", + "product_code":"dli", + "code":"347", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"OBS", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"OBS", + "githuburl":"" + }, + { + "uri":"dli_08_15040.html", + "node_id":"dli_08_15040.xml", + "product_code":"dli", + "code":"348", + "des":"The file system connector can be used to read single files or entire directories into a single table.When using a directory as the source path, there is no defined order ", + "doc_type":"sqlreference", + "kw":"OBS Source Table,OBS,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"OBS Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_15041.html", + "node_id":"dli_08_15041.xml", + "product_code":"dli", + "code":"349", + "des":"The FileSystem result (sink) table is used to export data to the HDFS or OBS file system. It is applicable to scenarios such as data dumping, big data analysis, data back", + "doc_type":"sqlreference", + "kw":"OBS Result Table,OBS,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"OBS Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_15042.html", + "node_id":"dli_08_15042.xml", + "product_code":"dli", + "code":"350", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"HBase", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"HBase", + "githuburl":"" + }, + { + "uri":"dli_08_15043.html", + "node_id":"dli_08_15043.xml", "product_code":"dli", "code":"351", + "des":"Create a source stream to obtain data from HBase as input for jobs. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excell", + "doc_type":"sqlreference", + "kw":"Source Table,HBase,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_15044.html", + "node_id":"dli_08_15044.xml", + "product_code":"dli", + "code":"352", + "des":"DLI outputs the job data to HBase. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excellent performance, and elastic scal", + "doc_type":"sqlreference", + "kw":"HBase Result Table,HBase,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"HBase Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_15045.html", + "node_id":"dli_08_15045.xml", + "product_code":"dli", + "code":"353", + "des":"Create an HBase dimension table to connect to the source streams for wide table generation.An enhanced datasource connection has been created for DLI to connect to HBase,", + "doc_type":"sqlreference", + "kw":"Dimension Table,HBase,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Dimension Table", + "githuburl":"" + }, + { + "uri":"dli_08_15046.html", + "node_id":"dli_08_15046.xml", + "product_code":"dli", + "code":"354", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Hive", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Hive", + "githuburl":"" + }, + { + "uri":"dli_08_15047.html", + "node_id":"dli_08_15047.xml", + "product_code":"dli", + "code":"355", + "des":"Catalogs provide metadata, such as databases, tables, partitions, views, and functions and information needed to access data stored in a database or other external system", + "doc_type":"sqlreference", + "kw":"Creating a Hive Catalog,Hive,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Creating a Hive Catalog", + "githuburl":"" + }, + { + "uri":"dli_08_15048.html", + "node_id":"dli_08_15048.xml", + "product_code":"dli", + "code":"356", + "des":"Starting from 1.11.0, Flink allows users to write SQL statements in Hive syntax when Hive dialect is used. By providing compatibility with Hive syntax, we aim to improve ", + "doc_type":"sqlreference", + "kw":"Hive Dialect,Hive,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Hive Dialect", + "githuburl":"" + }, + { + "uri":"dli_08_15049.html", + "node_id":"dli_08_15049.xml", + "product_code":"dli", + "code":"357", + "des":"Apache Hive has established itself as a focal point of the data warehousing ecosystem. It serves as not only a SQL engine for big data analytics and ETL, but also a data ", + "doc_type":"sqlreference", + "kw":"Hive Source Table,Hive,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Hive Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_15050.html", + "node_id":"dli_08_15050.xml", + "product_code":"dli", + "code":"358", + "des":"This section describes how to use Flink to write Hive tables, the definition of the Hive result table, parameters used for creating the result table, and sample code. For", + "doc_type":"sqlreference", + "kw":"Result Table,Hive,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_15051.html", + "node_id":"dli_08_15051.xml", + "product_code":"dli", + "code":"359", + "des":"You can use Hive tables as temporal tables and associate them through temporal joins. For more information on temporal joins, refer to temporal join.Flink supports proces", + "doc_type":"sqlreference", + "kw":"Hive Dimension Table,Hive,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Hive Dimension Table", + "githuburl":"" + }, + { + "uri":"dli_08_15052.html", + "node_id":"dli_08_15052.xml", + "product_code":"dli", + "code":"360", + "des":"For partitioned tables that change over time, we can read them as unbounded streams. If each partition contains a complete set of data for a certain version, the partitio", + "doc_type":"sqlreference", + "kw":"Using Temporal Join to Associate the Latest Partition of a Dimension Table,Hive,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Using Temporal Join to Associate the Latest Partition of a Dimension Table", + "githuburl":"" + }, + { + "uri":"dli_08_15053.html", + "node_id":"dli_08_15053.xml", + "product_code":"dli", + "code":"361", + "des":"For Hive tables, we can read them as bounded streams. In this case, the Hive table can only track its latest version when queried. The latest version of the table retains", + "doc_type":"sqlreference", + "kw":"Using Temporal Join to Associate the Latest Version of a Dimension Table,Hive,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Using Temporal Join to Associate the Latest Version of a Dimension Table", + "githuburl":"" + }, + { + "uri":"dli_08_15057.html", + "node_id":"dli_08_15057.xml", + "product_code":"dli", + "code":"362", + "des":"The JDBC connector is provided by Apache Flink and can be used to read data from and write data to common databases, such as MySQL and PostgreSQL. Source tables, result t", + "doc_type":"sqlreference", + "kw":"JDBC,Connectors,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"JDBC", + "githuburl":"" + }, + { + "uri":"dli_08_15058.html", + "node_id":"dli_08_15058.xml", + "product_code":"dli", + "code":"363", + "des":"The Kafka connector allows for reading data from and writing data into Kafka topics.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing an", + "doc_type":"sqlreference", + "kw":"Kafka,Connectors,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Kafka", + "githuburl":"" + }, + { + "uri":"dli_08_15059.html", + "node_id":"dli_08_15059.xml", + "product_code":"dli", + "code":"364", + "des":"The MySQL CDC source table, that is, the MySQL streaming source table, reads all historical data in the database first and then smoothly switches data read to the Binlog ", + "doc_type":"sqlreference", + "kw":"MySQL CDC,Connectors,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"MySQL CDC", + "githuburl":"" + }, + { + "uri":"dli_08_15060.html", + "node_id":"dli_08_15060.xml", + "product_code":"dli", + "code":"365", + "des":"The Print connector is used to print output data to the error file or out file in the TaskManager, making it easier for you to view the result in code debugging.NoneThe P", + "doc_type":"sqlreference", + "kw":"Print,Connectors,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Print", + "githuburl":"" + }, + { + "uri":"dli_08_15061.html", + "node_id":"dli_08_15061.xml", + "product_code":"dli", + "code":"366", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Redis", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Redis", + "githuburl":"" + }, + { + "uri":"dli_08_15062.html", + "node_id":"dli_08_15062.xml", + "product_code":"dli", + "code":"367", + "des":"Create a source stream to obtain data from Redis as input for jobs.An enhanced datasource connection has been created for DLI to connect to the Redis database, so that yo", + "doc_type":"sqlreference", + "kw":"Source Table,Redis,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_15063.html", + "node_id":"dli_08_15063.xml", + "product_code":"dli", + "code":"368", + "des":"DLI outputs the Flink job output data to Redis. Redis is a key-value storage system that supports multiple types of data structures. It can be used in scenarios such as c", + "doc_type":"sqlreference", + "kw":"Result Table,Redis,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_15064.html", + "node_id":"dli_08_15064.xml", + "product_code":"dli", + "code":"369", + "des":"Create a Redis table to connect to source streams for wide table generation.An enhanced datasource connection with Redis has been established, so that you can configure s", + "doc_type":"sqlreference", + "kw":"Dimension Table,Redis,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Dimension Table", + "githuburl":"" + }, + { + "uri":"dli_08_15065.html", + "node_id":"dli_08_15065.xml", + "product_code":"dli", + "code":"370", + "des":"Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It delivers high throughput and built-in partitions and provi", + "doc_type":"sqlreference", + "kw":"Upsert Kafka,Connectors,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Upsert Kafka", + "githuburl":"" + }, + { + "uri":"dli_08_15066.html", + "node_id":"dli_08_15066.xml", + "product_code":"dli", + "code":"371", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"DML Snytax", @@ -6321,10 +7291,10 @@ "githuburl":"" }, { - "uri":"dli_08_0417.html", - "node_id":"dli_08_0417.xml", + "uri":"dli_08_15067.html", + "node_id":"dli_08_15067.xml", "product_code":"dli", - "code":"352", + "code":"372", "des":"SyntaxDescriptionSELECT is used to select data from a table.ALL indicates that all results are returned.DISTINCT indicates that the duplicated results are removed.Precaut", "doc_type":"sqlreference", "kw":"SELECT,DML Snytax,SQL Syntax Reference", @@ -6339,10 +7309,28 @@ "githuburl":"" }, { - "uri":"dli_08_0418.html", - "node_id":"dli_08_0418.xml", + "uri":"dli_08_15107.html", + "node_id":"dli_08_15107.xml", "product_code":"dli", - "code":"353", + "code":"373", + "des":"This section describes how to use the INSERT INTO statement to write job results to a sink table.SyntaxINSERT INTO your_sink\n SELECT ... FROM your_source WHERE ...Exampl", + "doc_type":"sqlreference", + "kw":"INSERT INTO,DML Snytax,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"INSERT INTO", + "githuburl":"" + }, + { + "uri":"dli_08_15068.html", + "node_id":"dli_08_15068.xml", + "product_code":"dli", + "code":"374", "des":"SyntaxDescriptionUNION is used to return the union set of multiple query results.INTERSECT is used to return the intersection of multiple query results.EXCEPT is used to ", "doc_type":"sqlreference", "kw":"Set Operations,DML Snytax,SQL Syntax Reference", @@ -6357,13 +7345,13 @@ "githuburl":"" }, { - "uri":"dli_08_0419.html", - "node_id":"dli_08_0419.xml", + "uri":"dli_08_15069.html", + "node_id":"dli_08_15069.xml", "product_code":"dli", - "code":"354", - "des":"DescriptionGroup Window is defined in GROUP BY. One record is generated from each group. Group Window involves the following functions:Array functionsArray functionsGroup", + "code":"375", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", - "kw":"Window,DML Snytax,SQL Syntax Reference", + "kw":"Window", "search_title":"", "metedata":[ { @@ -6375,10 +7363,136 @@ "githuburl":"" }, { - "uri":"dli_08_0420.html", - "node_id":"dli_08_0420.xml", + "uri":"dli_08_15070.html", + "node_id":"dli_08_15070.xml", "product_code":"dli", - "code":"355", + "code":"376", + "des":"Windows are at the heart of processing infinite streams. Windows split the stream into \"buckets\" of finite size, over which we can apply computations.Apache Flink provide", + "doc_type":"sqlreference", + "kw":"Window Functions,Window,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Window Functions", + "githuburl":"" + }, + { + "uri":"dli_08_15071.html", + "node_id":"dli_08_15071.xml", + "product_code":"dli", + "code":"377", + "des":"Window aggregations are defined in the GROUP BY clause contains \"window_start\" and \"window_end\" columns of the relation applied Windowing TVF. Just like queries with regu", + "doc_type":"sqlreference", + "kw":"Window Aggregation,Window,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Window Aggregation", + "githuburl":"" + }, + { + "uri":"dli_08_15072.html", + "node_id":"dli_08_15072.xml", + "product_code":"dli", + "code":"378", + "des":"Window Top-N is a special Top-N which returns the N smallest or largest values for each window and other partitioned keys.Unlike regular Top-N on continuous tables, windo", + "doc_type":"sqlreference", + "kw":"Window Top-N,Window,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Window Top-N", + "githuburl":"" + }, + { + "uri":"dli_08_15073.html", + "node_id":"dli_08_15073.xml", + "product_code":"dli", + "code":"379", + "des":"Window Deduplication is a special Deduplication which removes rows that duplicate over a set of columns, keeping the first one or the last one for each window and partiti", + "doc_type":"sqlreference", + "kw":"Window Deduplication,Window,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Window Deduplication", + "githuburl":"" + }, + { + "uri":"dli_08_15074.html", + "node_id":"dli_08_15074.xml", + "product_code":"dli", + "code":"380", + "des":"A window join adds the dimension of time into the join criteria themselves. By doing so, the window join joins the elements of two streams that share a common key and are", + "doc_type":"sqlreference", + "kw":"Window Join,Window,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Window Join", + "githuburl":"" + }, + { + "uri":"dli_08_15075.html", + "node_id":"dli_08_15075.xml", + "product_code":"dli", + "code":"381", + "des":"An aggregate function computes a single result from multiple input rows. For example, there are aggregates to compute the COUNT, SUM, AVG (average), MAX (maximum) and MIN", + "doc_type":"sqlreference", + "kw":"Group Aggregation,DML Snytax,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Group Aggregation", + "githuburl":"" + }, + { + "uri":"dli_08_15076.html", + "node_id":"dli_08_15076.xml", + "product_code":"dli", + "code":"382", + "des":"OVER aggregates compute an aggregated value for every input row over a range of ordered rows. In contrast to GROUP BY aggregates, OVER aggregates do not reduce the number", + "doc_type":"sqlreference", + "kw":"Over Aggregation,DML Snytax,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Over Aggregation", + "githuburl":"" + }, + { + "uri":"dli_08_15077.html", + "node_id":"dli_08_15077.xml", + "product_code":"dli", + "code":"383", "des":"SyntaxPrecautionsCurrently, only equi-joins are supported, for example, joins that have at least one conjunctive condition with an equality predicate. Arbitrary cross or ", "doc_type":"sqlreference", "kw":"JOIN,DML Snytax,SQL Syntax Reference", @@ -6393,10 +7507,10 @@ "githuburl":"" }, { - "uri":"dli_08_0421.html", - "node_id":"dli_08_0421.xml", + "uri":"dli_08_15078.html", + "node_id":"dli_08_15078.xml", "product_code":"dli", - "code":"356", + "code":"384", "des":"FunctionThis clause is used to sort data in ascending order on a time attribute.PrecautionsCurrently, only sorting by time attribute is supported.ExampleSort data in asce", "doc_type":"sqlreference", "kw":"OrderBy & Limit,DML Snytax,SQL Syntax Reference", @@ -6411,10 +7525,10 @@ "githuburl":"" }, { - "uri":"dli_08_0422.html", - "node_id":"dli_08_0422.xml", + "uri":"dli_08_15079.html", + "node_id":"dli_08_15079.xml", "product_code":"dli", - "code":"357", + "code":"385", "des":"Top-N queries ask for the N smallest or largest values ordered by columns. Both smallest and largest values sets are considered Top-N queries. Top-N queries are useful in", "doc_type":"sqlreference", "kw":"Top-N,DML Snytax,SQL Syntax Reference", @@ -6429,10 +7543,10 @@ "githuburl":"" }, { - "uri":"dli_08_0423.html", - "node_id":"dli_08_0423.xml", + "uri":"dli_08_15080.html", + "node_id":"dli_08_15080.xml", "product_code":"dli", - "code":"358", + "code":"386", "des":"Deduplication removes rows that duplicate over a set of columns, keeping only the first one or the last one.ROW_NUMBER(): Assigns a unique, sequential number to each row,", "doc_type":"sqlreference", "kw":"Deduplication,DML Snytax,SQL Syntax Reference", @@ -6447,10 +7561,10 @@ "githuburl":"" }, { - "uri":"dli_08_0424.html", - "node_id":"dli_08_0424.xml", + "uri":"dli_08_15081.html", + "node_id":"dli_08_15081.xml", "product_code":"dli", - "code":"359", + "code":"387", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Functions", @@ -6465,13 +7579,13 @@ "githuburl":"" }, { - "uri":"dli_08_0425.html", - "node_id":"dli_08_0425.xml", + "uri":"dli_08_15082.html", + "node_id":"dli_08_15082.xml", "product_code":"dli", - "code":"360", + "code":"388", "des":"DLI supports the following three types of user-defined functions (UDFs):Regular UDF: takes in one or more input parameters and returns a single result.User-defined table-", "doc_type":"sqlreference", - "kw":"User-Defined Functions (UDFs),Functions,SQL Syntax Reference", + "kw":"UDFs,Functions,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -6479,14 +7593,50 @@ "documenttype":"sqlreference" } ], - "title":"User-Defined Functions (UDFs)", + "title":"UDFs", "githuburl":"" }, { - "uri":"dli_08_0426.html", - "node_id":"dli_08_0426.xml", + "uri":"dli_08_15083.html", + "node_id":"dli_08_15083.xml", "product_code":"dli", - "code":"361", + "code":"389", + "des":"Type inference summarizes the logic for validating input arguments and deriving data types for both the parameters and the result of a function. From a logical perspectiv", + "doc_type":"sqlreference", + "kw":"Type Inference,Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Type Inference", + "githuburl":"" + }, + { + "uri":"dli_08_15084.html", + "node_id":"dli_08_15084.xml", + "product_code":"dli", + "code":"390", + "des":"A UDF can be used in many jobs, and some parameter values vary with jobs. To easily modify the parameter values, you can set pipeline.global-job-parameters in the Runtime", + "doc_type":"sqlreference", + "kw":"Parameter Transfer,Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Parameter Transfer", + "githuburl":"" + }, + { + "uri":"dli_08_15085.html", + "node_id":"dli_08_15085.xml", + "product_code":"dli", + "code":"391", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Built-In Functions", @@ -6501,13 +7651,13 @@ "githuburl":"" }, { - "uri":"dli_08_0427.html", - "node_id":"dli_08_0427.xml", + "uri":"dli_08_15086.html", + "node_id":"dli_08_15086.xml", "product_code":"dli", - "code":"362", - "des":"All data types can be compared by using relational operators and the result is returned as a BOOLEAN value.Relationship operators are binary operators. Two compared data ", + "code":"392", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", - "kw":"Mathematical Operation Functions,Built-In Functions,SQL Syntax Reference", + "kw":"Comparison Functions,Built-In Functions,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -6515,15 +7665,51 @@ "documenttype":"sqlreference" } ], - "title":"Mathematical Operation Functions", + "title":"Comparison Functions", "githuburl":"" }, { - "uri":"dli_08_0428.html", - "node_id":"dli_08_0428.xml", + "uri":"dli_08_15087.html", + "node_id":"dli_08_15087.xml", "product_code":"dli", - "code":"363", - "des":"SyntaxExampleTest input data.Test the data source kafka. The message content is as follows:{name:James,age:24,sex:male,grade:{math:95,science:[80,85],english:100}}\n{name:", + "code":"393", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Logical Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Logical Functions", + "githuburl":"" + }, + { + "uri":"dli_08_15088.html", + "node_id":"dli_08_15088.xml", + "product_code":"dli", + "code":"394", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Arithmetic Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Arithmetic Functions", + "githuburl":"" + }, + { + "uri":"dli_08_15089.html", + "node_id":"dli_08_15089.xml", + "product_code":"dli", + "code":"395", + "des":"DLI offers a wide range of string functions for processing and transforming string data. These functions include concatenation, case conversion, substring extraction, rep", "doc_type":"sqlreference", "kw":"String Functions,Built-In Functions,SQL Syntax Reference", "search_title":"", @@ -6537,10 +7723,10 @@ "githuburl":"" }, { - "uri":"dli_08_0429.html", - "node_id":"dli_08_0429.xml", + "uri":"dli_08_15090.html", + "node_id":"dli_08_15090.xml", "product_code":"dli", - "code":"364", + "code":"396", "des":"Table 1 lists the time functions supported by Flink OpenSource SQL.FunctionReturns a SQL date parsed from string in form of yyyy-MM-dd.Returns a SQL date parsed from stri", "doc_type":"sqlreference", "kw":"Temporal Functions,Built-In Functions,SQL Syntax Reference", @@ -6555,10 +7741,10 @@ "githuburl":"" }, { - "uri":"dli_08_0430.html", - "node_id":"dli_08_0430.xml", + "uri":"dli_08_15091.html", + "node_id":"dli_08_15091.xml", "product_code":"dli", - "code":"365", + "code":"397", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Conditional Functions,Built-In Functions,SQL Syntax Reference", @@ -6573,11 +7759,11 @@ "githuburl":"" }, { - "uri":"dli_08_0431.html", - "node_id":"dli_08_0431.xml", + "uri":"dli_08_15092.html", + "node_id":"dli_08_15092.xml", "product_code":"dli", - "code":"366", - "des":"This function is used to forcibly convert types.If the input is NULL, NULL is returned.The cast function does not support converting a string to the JSON format.The follo", + "code":"398", + "des":"This syntax is used to forcibly convert types.If the input is NULL, NULL is returned.The following example converts the amount value to an integer.Flink jobs do not suppo", "doc_type":"sqlreference", "kw":"Type Conversion Functions,Built-In Functions,SQL Syntax Reference", "search_title":"", @@ -6591,10 +7777,10 @@ "githuburl":"" }, { - "uri":"dli_08_0432.html", - "node_id":"dli_08_0432.xml", + "uri":"dli_08_15093.html", + "node_id":"dli_08_15093.xml", "product_code":"dli", - "code":"367", + "code":"399", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Collection Functions,Built-In Functions,SQL Syntax Reference", @@ -6609,10 +7795,28 @@ "githuburl":"" }, { - "uri":"dli_08_0433.html", - "node_id":"dli_08_0433.xml", + "uri":"dli_08_15094.html", + "node_id":"dli_08_15094.xml", "product_code":"dli", - "code":"368", + "code":"400", + "des":"JSON functions use JSON path expressions described in the SQL standard ISO/IEC TR 19075-6. Their syntax is inspired by ECMAScript and adopts many of its features, but is ", + "doc_type":"sqlreference", + "kw":"JSON Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"JSON Functions", + "githuburl":"" + }, + { + "uri":"dli_08_15095.html", + "node_id":"dli_08_15095.xml", + "product_code":"dli", + "code":"401", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Value Construction Functions,Built-In Functions,SQL Syntax Reference", @@ -6627,13 +7831,13 @@ "githuburl":"" }, { - "uri":"dli_08_0434.html", - "node_id":"dli_08_0434.xml", + "uri":"dli_08_15096.html", + "node_id":"dli_08_15096.xml", "product_code":"dli", - "code":"369", + "code":"402", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", - "kw":"Value Access Functions,Built-In Functions,SQL Syntax Reference", + "kw":"Value Retrieval Functions,Built-In Functions,SQL Syntax Reference", "search_title":"", "metedata":[ { @@ -6641,14 +7845,32 @@ "documenttype":"sqlreference" } ], - "title":"Value Access Functions", + "title":"Value Retrieval Functions", "githuburl":"" }, { - "uri":"dli_08_0435.html", - "node_id":"dli_08_0435.xml", + "uri":"dli_08_15097.html", + "node_id":"dli_08_15097.xml", "product_code":"dli", - "code":"370", + "code":"403", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Grouping Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference" + } + ], + "title":"Grouping Functions", + "githuburl":"" + }, + { + "uri":"dli_08_15098.html", + "node_id":"dli_08_15098.xml", + "product_code":"dli", + "code":"404", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Hash Functions,Built-In Functions,SQL Syntax Reference", @@ -6663,11 +7885,11 @@ "githuburl":"" }, { - "uri":"dli_08_0436.html", - "node_id":"dli_08_0436.xml", + "uri":"dli_08_15099.html", + "node_id":"dli_08_15099.xml", "product_code":"dli", - "code":"371", - "des":"An aggregate function performs a calculation operation on a set of input values and returns a value. For example, the COUNT function counts the number of rows retrieved b", + "code":"405", + "des":"Aggregate functions process all rows as input and produce a single aggregate value as the output.", "doc_type":"sqlreference", "kw":"Aggregate Functions,Built-In Functions,SQL Syntax Reference", "search_title":"", @@ -6681,10 +7903,10 @@ "githuburl":"" }, { - "uri":"dli_08_0437.html", - "node_id":"dli_08_0437.xml", + "uri":"dli_08_15101.html", + "node_id":"dli_08_15101.xml", "product_code":"dli", - "code":"372", + "code":"406", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Table-Valued Functions", @@ -6699,10 +7921,10 @@ "githuburl":"" }, { - "uri":"dli_08_0438.html", - "node_id":"dli_08_0438.xml", + "uri":"dli_08_15102.html", + "node_id":"dli_08_15102.xml", "product_code":"dli", - "code":"373", + "code":"407", "des":"The string_split function splits a target string into substrings based on the specified separator and returns a substring list.Create a Flink OpenSource SQL job by referr", "doc_type":"sqlreference", "kw":"string_split,Table-Valued Functions,SQL Syntax Reference", @@ -6716,11 +7938,1411 @@ "title":"string_split", "githuburl":"" }, + { + "uri":"dli_08_0370.html", + "node_id":"dli_08_0370.xml", + "product_code":"dli", + "code":"408", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Flink OpenSource SQL 1.12 Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Flink OpenSource SQL 1.12 Syntax Reference", + "githuburl":"" + }, + { + "uri":"dli_08_0371.html", + "node_id":"dli_08_0371.xml", + "product_code":"dli", + "code":"409", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Constraints and Definitions", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Constraints and Definitions", + "githuburl":"" + }, + { + "uri":"dli_08_0372.html", + "node_id":"dli_08_0372.xml", + "product_code":"dli", + "code":"410", + "des":"The DLI SQL syntax supports the following data types:STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP ", + "doc_type":"sqlreference", + "kw":"Supported Data Types,Constraints and Definitions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Supported Data Types", + "githuburl":"" + }, + { + "uri":"dli_08_0373.html", + "node_id":"dli_08_0373.xml", + "product_code":"dli", + "code":"411", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Syntax", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Syntax", + "githuburl":"" + }, + { + "uri":"dli_08_0374.html", + "node_id":"dli_08_0374.xml", + "product_code":"dli", + "code":"412", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Data Definition Language (DDL)", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Data Definition Language (DDL)", + "githuburl":"" + }, + { + "uri":"dli_08_0375.html", + "node_id":"dli_08_0375.xml", + "product_code":"dli", + "code":"413", + "des":"Create a table with a specified name.COMPUTED COLUMNA computed column is a virtual column generated using column_name AS computed_column_expression. A computed column eva", + "doc_type":"sqlreference", + "kw":"CREATE TABLE,Data Definition Language (DDL),SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"CREATE TABLE", + "githuburl":"" + }, + { + "uri":"dli_08_0376.html", + "node_id":"dli_08_0376.xml", + "product_code":"dli", + "code":"414", + "des":"Create a view with multiple layers nested in it to simplify the development process.IF NOT EXISTSIf the view already exists, nothing happens.Create a view named viewName.", + "doc_type":"sqlreference", + "kw":"CREATE VIEW,Data Definition Language (DDL),SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"CREATE VIEW", + "githuburl":"" + }, + { + "uri":"dli_08_0377.html", + "node_id":"dli_08_0377.xml", + "product_code":"dli", + "code":"415", + "des":"Create a user-defined function.For details about how to create a user-defined function, see User-Defined Functions (UDFs).IF NOT EXISTSIf the function already exists, not", + "doc_type":"sqlreference", + "kw":"CREATE FUNCTION,Data Definition Language (DDL),SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"CREATE FUNCTION", + "githuburl":"" + }, + { + "uri":"dli_08_0378.html", + "node_id":"dli_08_0378.xml", + "product_code":"dli", + "code":"416", + "des":"SyntaxPrecautionsFlink SQL uses a lexical policy for identifier (table, attribute, function names) similar to Java:The case of identifiers is preserved whether they are q", + "doc_type":"sqlreference", + "kw":"Data Manipulation Language (DML),Syntax,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Data Manipulation Language (DML)", + "githuburl":"" + }, + { + "uri":"dli_08_0379.html", + "node_id":"dli_08_0379.xml", + "product_code":"dli", + "code":"417", + "des":"This section describes the Flink open source SQL 1.12 syntax supported by DLI. For details about the parameters and examples, see the syntax description.", + "doc_type":"sqlreference", + "kw":"Overview,Flink OpenSource SQL 1.12 Syntax Reference,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Overview", + "githuburl":"" + }, + { + "uri":"dli_08_0380.html", + "node_id":"dli_08_0380.xml", + "product_code":"dli", + "code":"418", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"DDL Syntax", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"DDL Syntax", + "githuburl":"" + }, + { + "uri":"dli_08_0381.html", + "node_id":"dli_08_0381.xml", + "product_code":"dli", + "code":"419", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Creating Source Tables", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Creating Source Tables", + "githuburl":"" + }, + { + "uri":"dli_08_0382.html", + "node_id":"dli_08_0382.xml", + "product_code":"dli", + "code":"420", + "des":"DataGen is used to generate random data for debugging and testing.NoneWhen you create a DataGen table, the table field type cannot be Array, Map, or Row. You can use COMP", + "doc_type":"sqlreference", + "kw":"DataGen Source Table,Creating Source Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"DataGen Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_0383.html", + "node_id":"dli_08_0383.xml", + "product_code":"dli", + "code":"421", + "des":"DLI reads data of Flink jobs from GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex types an", + "doc_type":"sqlreference", + "kw":"GaussDB(DWS) Source Table,Creating Source Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"GaussDB(DWS) Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_0384.html", + "node_id":"dli_08_0384.xml", + "product_code":"dli", + "code":"422", + "des":"Create a source stream to obtain data from HBase as input for jobs. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excell", + "doc_type":"sqlreference", + "kw":"HBase Source Table,Creating Source Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"HBase Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_0385.html", + "node_id":"dli_08_0385.xml", + "product_code":"dli", + "code":"423", + "des":"The JDBC connector is a Flink's built-in connector to read data from a database.An enhanced datasource connection with the instances has been established, so that you can", + "doc_type":"sqlreference", + "kw":"JDBC Source Table,Creating Source Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"JDBC Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_0386.html", + "node_id":"dli_08_0386.xml", + "product_code":"dli", + "code":"424", + "des":"Create a source stream to obtain data from Kafka as input data for jobs.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscripti", + "doc_type":"sqlreference", + "kw":"Kafka Source Table,Creating Source Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Kafka Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_0387.html", + "node_id":"dli_08_0387.xml", + "product_code":"dli", + "code":"425", + "des":"The MySQL CDC source table, that is, the MySQL streaming source table, reads all historical data in the database first and then smoothly switches data read to the Binlog ", + "doc_type":"sqlreference", + "kw":"MySQL CDC Source Table,Creating Source Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"MySQL CDC Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_0388.html", + "node_id":"dli_08_0388.xml", + "product_code":"dli", + "code":"426", + "des":"The Postgres CDC source table, that is, Postgres streaming source table, is used to read the full snapshot data and changed data of the PostgreSQL database in sequence. T", + "doc_type":"sqlreference", + "kw":"Postgres CDC Source Table,Creating Source Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Postgres CDC Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_0389.html", + "node_id":"dli_08_0389.xml", + "product_code":"dli", + "code":"427", + "des":"Create a source stream to obtain data from Redis as input for jobs.An enhanced datasource connection has been created for DLI to connect to the Redis database, so that yo", + "doc_type":"sqlreference", + "kw":"Redis Source Table,Creating Source Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Redis Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_0390.html", + "node_id":"dli_08_0390.xml", + "product_code":"dli", + "code":"428", + "des":"Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It delivers high throughput and built-in partitions and provi", + "doc_type":"sqlreference", + "kw":"Upsert Kafka Source Table,Creating Source Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Upsert Kafka Source Table", + "githuburl":"" + }, + { + "uri":"dli_08_0391.html", + "node_id":"dli_08_0391.xml", + "product_code":"dli", + "code":"429", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Creating Result Tables", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Creating Result Tables", + "githuburl":"" + }, + { + "uri":"dli_08_0392.html", + "node_id":"dli_08_0392.xml", + "product_code":"dli", + "code":"430", + "des":"The BlackHole connector allows for swallowing all input records. It is designed for high-performance testing and UDF output. It is not a substantive sink. The BlackHole r", + "doc_type":"sqlreference", + "kw":"BlackHole Result Table,Creating Result Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"BlackHole Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_0393.html", + "node_id":"dli_08_0393.xml", + "product_code":"dli", + "code":"431", + "des":"DLI can output Flink job data to the ClickHouse database. ClickHouse is a column-based database oriented to online analysis and processing. It supports SQL query and prov", + "doc_type":"sqlreference", + "kw":"ClickHouse Result Table,Creating Result Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"ClickHouse Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_0394.html", + "node_id":"dli_08_0394.xml", + "product_code":"dli", + "code":"432", + "des":"DLI outputs the Flink job output data to GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex t", + "doc_type":"sqlreference", + "kw":"GaussDB(DWS) Result Table,Creating Result Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"GaussDB(DWS) Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_0395.html", + "node_id":"dli_08_0395.xml", + "product_code":"dli", + "code":"433", + "des":"DLI outputs Flink job output data to Elasticsearch of Cloud Search Service (CSS). Elasticsearch is a popular enterprise-class Lucene-powered search server and provides th", + "doc_type":"sqlreference", + "kw":"Elasticsearch Result Table,Creating Result Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Elasticsearch Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_0396.html", + "node_id":"dli_08_0396.xml", + "product_code":"dli", + "code":"434", + "des":"DLI outputs the job data to HBase. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excellent performance, and elastic scal", + "doc_type":"sqlreference", + "kw":"HBase Result Table,Creating Result Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"HBase Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_0397.html", + "node_id":"dli_08_0397.xml", + "product_code":"dli", + "code":"435", + "des":"DLI outputs the Flink job output data to RDS through the JDBC result table.An enhanced datasource connection with the instances has been established, so that you can conf", + "doc_type":"sqlreference", + "kw":"JDBC Result Table,Creating Result Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"JDBC Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_0398.html", + "node_id":"dli_08_0398.xml", + "product_code":"dli", + "code":"436", + "des":"DLI outputs the Flink job output data to Kafka through the Kafka result table.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subs", + "doc_type":"sqlreference", + "kw":"Kafka Result Table,Creating Result Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Kafka Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_0399.html", + "node_id":"dli_08_0399.xml", + "product_code":"dli", + "code":"437", + "des":"The Print connector is used to print output data to the error file or TaskManager file, making it easier for you to view the result in code debugging.NoneThe Print result", + "doc_type":"sqlreference", + "kw":"Print Result Table,Creating Result Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Print Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_0400.html", + "node_id":"dli_08_0400.xml", + "product_code":"dli", + "code":"438", + "des":"DLI outputs the Flink job output data to Redis. Redis is a key-value storage system that supports multiple types of data structures. It can be used in scenarios such as c", + "doc_type":"sqlreference", + "kw":"Redis Result Table,Creating Result Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Redis Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_0401.html", + "node_id":"dli_08_0401.xml", + "product_code":"dli", + "code":"439", + "des":"Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It delivers high throughput and built-in partitions and provi", + "doc_type":"sqlreference", + "kw":"Upsert Kafka Result Table,Creating Result Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Upsert Kafka Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_0439.html", + "node_id":"dli_08_0439.xml", + "product_code":"dli", + "code":"440", + "des":"The FileSystem result (sink) table is used to export data to the HDFS or OBS file system. It is applicable to scenarios such as data dumping, big data analysis, data back", + "doc_type":"sqlreference", + "kw":"FileSystem Result Table,Creating Result Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"FileSystem Result Table", + "githuburl":"" + }, + { + "uri":"dli_08_0402.html", + "node_id":"dli_08_0402.xml", + "product_code":"dli", + "code":"441", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Creating Dimension Tables", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Creating Dimension Tables", + "githuburl":"" + }, + { + "uri":"dli_08_0403.html", + "node_id":"dli_08_0403.xml", + "product_code":"dli", + "code":"442", + "des":"Create a GaussDB(DWS) table to connect to source streams for wide table generation.Ensure that you have created a GaussDB(DWS) cluster using your account.A DWS database t", + "doc_type":"sqlreference", + "kw":"GaussDB(DWS) Dimension Table,Creating Dimension Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"GaussDB(DWS) Dimension Table", + "githuburl":"" + }, + { + "uri":"dli_08_0404.html", + "node_id":"dli_08_0404.xml", + "product_code":"dli", + "code":"443", + "des":"Create a Hbase dimension table to connect to the source streams for wide table generation.An enhanced datasource connection has been created for DLI to connect to HBase, ", + "doc_type":"sqlreference", + "kw":"HBase Dimension Table,Creating Dimension Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"HBase Dimension Table", + "githuburl":"" + }, + { + "uri":"dli_08_0405.html", + "node_id":"dli_08_0405.xml", + "product_code":"dli", + "code":"444", + "des":"Create a JDBC dimension table to connect to the source stream.You have created a JDBC instance for your account.When you create a Flink OpenSource SQL job, set Flink Vers", + "doc_type":"sqlreference", + "kw":"JDBC Dimension Table,Creating Dimension Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"JDBC Dimension Table", + "githuburl":"" + }, + { + "uri":"dli_08_0406.html", + "node_id":"dli_08_0406.xml", + "product_code":"dli", + "code":"445", + "des":"Create a Redis table to connect to source streams for wide table generation.An enhanced datasource connection with Redis has been established, so that you can configure s", + "doc_type":"sqlreference", + "kw":"Redis Dimension Table,Creating Dimension Tables,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Redis Dimension Table", + "githuburl":"" + }, + { + "uri":"dli_08_0407.html", + "node_id":"dli_08_0407.xml", + "product_code":"dli", + "code":"446", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Format", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Format", + "githuburl":"" + }, + { + "uri":"dli_08_0408.html", + "node_id":"dli_08_0408.xml", + "product_code":"dli", + "code":"447", + "des":"Apache Avro is supported for you to read and write Avro data based on an Avro schema with Flink. The Avro schema is derived from the table schema.KafkaUpsert KafkaCurrent", + "doc_type":"sqlreference", + "kw":"Avro,Format,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Avro", + "githuburl":"" + }, + { + "uri":"dli_08_0409.html", + "node_id":"dli_08_0409.xml", + "product_code":"dli", + "code":"448", + "des":"Canal is a Changelog Data Capture (CDC) tool that can stream changes in real-time from MySQL into other systems. Canal provides a unified format schema for changelog and ", + "doc_type":"sqlreference", + "kw":"Canal,Format,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Canal", + "githuburl":"" + }, + { + "uri":"dli_08_0410.html", + "node_id":"dli_08_0410.xml", + "product_code":"dli", + "code":"449", + "des":"The Avro Schema Registry (avro-confluent) format allows you to read records that were serialized by the io.confluent.kafka.serializers.KafkaAvroSerializer and to write re", + "doc_type":"sqlreference", + "kw":"Confluent Avro,Format,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Confluent Avro", + "githuburl":"" + }, + { + "uri":"dli_08_0411.html", + "node_id":"dli_08_0411.xml", + "product_code":"dli", + "code":"450", + "des":"The CSV format allows you to read and write CSV data based on a CSV schema. Currently, the CSV schema is derived from table schema.KafkaUpsert KafkaUse Kafka to send data", + "doc_type":"sqlreference", + "kw":"CSV,Format,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"CSV", + "githuburl":"" + }, + { + "uri":"dli_08_0412.html", + "node_id":"dli_08_0412.xml", + "product_code":"dli", + "code":"451", + "des":"Debezium is a Changelog Data Capture (CDC) tool that can stream changes in real-time from other databases into Kafka. Debezium provides a unified format schema for change", + "doc_type":"sqlreference", + "kw":"Debezium,Format,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Debezium", + "githuburl":"" + }, + { + "uri":"dli_08_0413.html", + "node_id":"dli_08_0413.xml", + "product_code":"dli", + "code":"452", + "des":"The JSON format allows you to read and write JSON data based on a JSON schema. Currently, the JSON schema is derived from table schema.KafkaUpsert KafkaElasticsearchIn th", + "doc_type":"sqlreference", + "kw":"JSON,Format,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"JSON", + "githuburl":"" + }, + { + "uri":"dli_08_0414.html", + "node_id":"dli_08_0414.xml", + "product_code":"dli", + "code":"453", + "des":"Flink supports to interpret Maxwell JSON messages as INSERT/UPDATE/DELETE messages into Flink SQL system. This is useful in many cases to leverage this feature,such as:Sy", + "doc_type":"sqlreference", + "kw":"Maxwell,Format,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Maxwell", + "githuburl":"" + }, + { + "uri":"dli_08_0415.html", + "node_id":"dli_08_0415.xml", + "product_code":"dli", + "code":"454", + "des":"The raw format allows you to read and write raw (byte based) values as a single column.Note: This format encodes null values as null of the byte[] type. This may have lim", + "doc_type":"sqlreference", + "kw":"Raw,Format,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Raw", + "githuburl":"" + }, + { + "uri":"dli_08_0416.html", + "node_id":"dli_08_0416.xml", + "product_code":"dli", + "code":"455", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"DML Snytax", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"DML Snytax", + "githuburl":"" + }, + { + "uri":"dli_08_0417.html", + "node_id":"dli_08_0417.xml", + "product_code":"dli", + "code":"456", + "des":"SyntaxDescriptionSELECT is used to select data from a table.ALL indicates that all results are returned.DISTINCT indicates that the duplicated results are removed.Precaut", + "doc_type":"sqlreference", + "kw":"SELECT,DML Snytax,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"SELECT", + "githuburl":"" + }, + { + "uri":"dli_08_0418.html", + "node_id":"dli_08_0418.xml", + "product_code":"dli", + "code":"457", + "des":"SyntaxDescriptionUNION is used to return the union set of multiple query results.INTERSECT is used to return the intersection of multiple query results.EXCEPT is used to ", + "doc_type":"sqlreference", + "kw":"Set Operations,DML Snytax,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Set Operations", + "githuburl":"" + }, + { + "uri":"dli_08_0419.html", + "node_id":"dli_08_0419.xml", + "product_code":"dli", + "code":"458", + "des":"DescriptionGroup Window is defined in GROUP BY. One record is generated from each group. Group Window involves the following functions:Array functionsArray functionsGroup", + "doc_type":"sqlreference", + "kw":"Window,DML Snytax,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Window", + "githuburl":"" + }, + { + "uri":"dli_08_0420.html", + "node_id":"dli_08_0420.xml", + "product_code":"dli", + "code":"459", + "des":"SyntaxPrecautionsCurrently, only equi-joins are supported, for example, joins that have at least one conjunctive condition with an equality predicate. Arbitrary cross or ", + "doc_type":"sqlreference", + "kw":"JOIN,DML Snytax,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"JOIN", + "githuburl":"" + }, + { + "uri":"dli_08_0421.html", + "node_id":"dli_08_0421.xml", + "product_code":"dli", + "code":"460", + "des":"FunctionThis clause is used to sort data in ascending order on a time attribute.PrecautionsCurrently, only sorting by time attribute is supported.ExampleSort data in asce", + "doc_type":"sqlreference", + "kw":"OrderBy & Limit,DML Snytax,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"OrderBy & Limit", + "githuburl":"" + }, + { + "uri":"dli_08_0422.html", + "node_id":"dli_08_0422.xml", + "product_code":"dli", + "code":"461", + "des":"Top-N queries ask for the N smallest or largest values ordered by columns. Both smallest and largest values sets are considered Top-N queries. Top-N queries are useful in", + "doc_type":"sqlreference", + "kw":"Top-N,DML Snytax,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Top-N", + "githuburl":"" + }, + { + "uri":"dli_08_0423.html", + "node_id":"dli_08_0423.xml", + "product_code":"dli", + "code":"462", + "des":"Deduplication removes rows that duplicate over a set of columns, keeping only the first one or the last one.ROW_NUMBER(): Assigns a unique, sequential number to each row,", + "doc_type":"sqlreference", + "kw":"Deduplication,DML Snytax,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Deduplication", + "githuburl":"" + }, + { + "uri":"dli_08_0424.html", + "node_id":"dli_08_0424.xml", + "product_code":"dli", + "code":"463", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Functions", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0425.html", + "node_id":"dli_08_0425.xml", + "product_code":"dli", + "code":"464", + "des":"DLI supports the following three types of user-defined functions (UDFs):Regular UDF: takes in one or more input parameters and returns a single result.User-defined table-", + "doc_type":"sqlreference", + "kw":"User-Defined Functions (UDFs),Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"User-Defined Functions (UDFs)", + "githuburl":"" + }, + { + "uri":"dli_08_0426.html", + "node_id":"dli_08_0426.xml", + "product_code":"dli", + "code":"465", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Built-In Functions", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Built-In Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0427.html", + "node_id":"dli_08_0427.xml", + "product_code":"dli", + "code":"466", + "des":"All data types can be compared by using relational operators and the result is returned as a BOOLEAN value.Relationship operators are binary operators. Two compared data ", + "doc_type":"sqlreference", + "kw":"Mathematical Operation Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Mathematical Operation Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0428.html", + "node_id":"dli_08_0428.xml", + "product_code":"dli", + "code":"467", + "des":"DLI offers a wide range of string functions for processing and transforming string data. These functions include concatenation, case conversion, substring extraction, rep", + "doc_type":"sqlreference", + "kw":"String Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"String Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0429.html", + "node_id":"dli_08_0429.xml", + "product_code":"dli", + "code":"468", + "des":"Table 1 lists the time functions supported by Flink OpenSource SQL.FunctionReturns a SQL date parsed from string in form of yyyy-MM-dd.Returns a SQL date parsed from stri", + "doc_type":"sqlreference", + "kw":"Temporal Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Temporal Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0430.html", + "node_id":"dli_08_0430.xml", + "product_code":"dli", + "code":"469", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Conditional Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Conditional Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0431.html", + "node_id":"dli_08_0431.xml", + "product_code":"dli", + "code":"470", + "des":"This function is used to forcibly convert types.If the input is NULL, NULL is returned.The cast function does not support converting a string to the JSON format.The follo", + "doc_type":"sqlreference", + "kw":"Type Conversion Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Type Conversion Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0432.html", + "node_id":"dli_08_0432.xml", + "product_code":"dli", + "code":"471", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Collection Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Collection Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0433.html", + "node_id":"dli_08_0433.xml", + "product_code":"dli", + "code":"472", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Value Construction Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Value Construction Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0434.html", + "node_id":"dli_08_0434.xml", + "product_code":"dli", + "code":"473", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Value Access Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Value Access Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0435.html", + "node_id":"dli_08_0435.xml", + "product_code":"dli", + "code":"474", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Hash Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Hash Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0436.html", + "node_id":"dli_08_0436.xml", + "product_code":"dli", + "code":"475", + "des":"An aggregate function performs a calculation operation on a set of input values and returns a value. For example, the COUNT function counts the number of rows retrieved b", + "doc_type":"sqlreference", + "kw":"Aggregate Functions,Built-In Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Aggregate Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0437.html", + "node_id":"dli_08_0437.xml", + "product_code":"dli", + "code":"476", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"sqlreference", + "kw":"Table-Valued Functions", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"Table-Valued Functions", + "githuburl":"" + }, + { + "uri":"dli_08_0438.html", + "node_id":"dli_08_0438.xml", + "product_code":"dli", + "code":"477", + "des":"The string_split function splits a target string into substrings based on the specified separator and returns a substring list.Create a Flink OpenSource SQL job by referr", + "doc_type":"sqlreference", + "kw":"string_split,Table-Valued Functions,SQL Syntax Reference", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" + } + ], + "title":"string_split", + "githuburl":"" + }, { "uri":"dli_08_0289.html", "node_id":"dli_08_0289.xml", "product_code":"dli", - "code":"374", + "code":"478", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Flink Opensource SQL 1.10 Syntax Reference", @@ -6728,7 +9350,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Flink Opensource SQL 1.10 Syntax Reference", @@ -6738,7 +9362,7 @@ "uri":"dli_08_0290.html", "node_id":"dli_08_0290.xml", "product_code":"dli", - "code":"375", + "code":"479", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Constraints and Definitions", @@ -6746,7 +9370,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Constraints and Definitions", @@ -6756,15 +9382,17 @@ "uri":"dli_08_0291.html", "node_id":"dli_08_0291.xml", "product_code":"dli", - "code":"376", - "des":"STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP WITH LOCAL TIME ZONE, INTERVAL, ARRAY, MULTISET, MAP,", + "code":"480", + "des":"The DLI SQL syntax supports the following data types:STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP ", "doc_type":"sqlreference", "kw":"Supported Data Types,Constraints and Definitions,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Supported Data Types", @@ -6774,7 +9402,7 @@ "uri":"dli_08_0292.html", "node_id":"dli_08_0292.xml", "product_code":"dli", - "code":"377", + "code":"481", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Syntax Definition", @@ -6782,7 +9410,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Syntax Definition", @@ -6792,7 +9422,7 @@ "uri":"dli_08_0293.html", "node_id":"dli_08_0293.xml", "product_code":"dli", - "code":"378", + "code":"482", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Data Definition Language (DDL)", @@ -6800,7 +9430,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Data Definition Language (DDL)", @@ -6810,7 +9442,7 @@ "uri":"dli_08_0294.html", "node_id":"dli_08_0294.xml", "product_code":"dli", - "code":"379", + "code":"483", "des":"This clause is used to create a table with a specified name.COMPUTED COLUMNA computed column is a virtual column generated using column_name AS computed_column_expression", "doc_type":"sqlreference", "kw":"CREATE TABLE,Data Definition Language (DDL),SQL Syntax Reference", @@ -6818,7 +9450,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"CREATE TABLE", @@ -6828,7 +9462,7 @@ "uri":"dli_08_0295.html", "node_id":"dli_08_0295.xml", "product_code":"dli", - "code":"380", + "code":"484", "des":"Create a view with multiple layers nested in it to simplify the development process.IF NOT EXISTSIf the view already exists, nothing happens.Create a view named viewName.", "doc_type":"sqlreference", "kw":"CREATE VIEW,Data Definition Language (DDL),SQL Syntax Reference", @@ -6836,7 +9470,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"CREATE VIEW", @@ -6846,7 +9482,7 @@ "uri":"dli_08_0296.html", "node_id":"dli_08_0296.xml", "product_code":"dli", - "code":"381", + "code":"485", "des":"Create a user-defined function.IF NOT EXISTSIf the function already exists, nothing happens.LANGUAGE JAVA|SCALALanguage tag is used to instruct Flink runtime how to execu", "doc_type":"sqlreference", "kw":"CREATE FUNCTION,Data Definition Language (DDL),SQL Syntax Reference", @@ -6854,7 +9490,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"CREATE FUNCTION", @@ -6864,7 +9502,7 @@ "uri":"dli_08_0297.html", "node_id":"dli_08_0297.xml", "product_code":"dli", - "code":"382", + "code":"486", "des":"SyntaxPrecautionsFlink SQL uses a lexical policy for identifier (table, attribute, function names) similar to Java:The case of identifiers is preserved whether they are q", "doc_type":"sqlreference", "kw":"Data Manipulation Language (DML),Syntax Definition,SQL Syntax Reference", @@ -6872,7 +9510,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Data Manipulation Language (DML)", @@ -6882,7 +9522,7 @@ "uri":"dli_08_0298.html", "node_id":"dli_08_0298.xml", "product_code":"dli", - "code":"383", + "code":"487", "des":"This section describes the Flink OpenSource SQL syntax supported by DLI. For details about the parameters and examples, see the syntax description.", "doc_type":"sqlreference", "kw":"Flink OpenSource SQL 1.10 Syntax,Flink Opensource SQL 1.10 Syntax Reference,SQL Syntax Reference", @@ -6890,7 +9530,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Flink OpenSource SQL 1.10 Syntax", @@ -6900,7 +9542,7 @@ "uri":"dli_08_0299.html", "node_id":"dli_08_0299.xml", "product_code":"dli", - "code":"384", + "code":"488", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Data Definition Language (DDL)", @@ -6908,7 +9550,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Data Definition Language (DDL)", @@ -6918,7 +9562,7 @@ "uri":"dli_08_0300.html", "node_id":"dli_08_0300.xml", "product_code":"dli", - "code":"385", + "code":"489", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Source Table", @@ -6926,7 +9570,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Source Table", @@ -6936,7 +9582,7 @@ "uri":"dli_08_0301.html", "node_id":"dli_08_0301.xml", "product_code":"dli", - "code":"386", + "code":"490", "des":"Create a source stream to obtain data from Kafka as input data for jobs.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscripti", "doc_type":"sqlreference", "kw":"Kafka Source Table,Creating a Source Table,SQL Syntax Reference", @@ -6944,7 +9590,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Kafka Source Table", @@ -6954,7 +9602,7 @@ "uri":"dli_08_0302.html", "node_id":"dli_08_0302.xml", "product_code":"dli", - "code":"387", + "code":"491", "des":"Create a source stream to read data from DIS. DIS accesses user data and Flink job reads data from the DIS stream as input data for jobs. Flink jobs can quickly remove da", "doc_type":"sqlreference", "kw":"DIS Source Table,Creating a Source Table,SQL Syntax Reference", @@ -6962,7 +9610,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"DIS Source Table", @@ -6972,7 +9622,7 @@ "uri":"dli_08_0303.html", "node_id":"dli_08_0303.xml", "product_code":"dli", - "code":"388", + "code":"492", "des":"The JDBC connector is a Flink's built-in connector to read data from a database.An enhanced datasource connection with the database has been established, so that you can ", "doc_type":"sqlreference", "kw":"JDBC Source Table,Creating a Source Table,SQL Syntax Reference", @@ -6980,7 +9630,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"JDBC Source Table", @@ -6990,7 +9642,7 @@ "uri":"dli_08_0304.html", "node_id":"dli_08_0304.xml", "product_code":"dli", - "code":"389", + "code":"493", "des":"DLI reads data of Flink jobs from GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex types an", "doc_type":"sqlreference", "kw":"GaussDB(DWS) Source Table,Creating a Source Table,SQL Syntax Reference", @@ -6998,7 +9650,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"GaussDB(DWS) Source Table", @@ -7008,7 +9662,7 @@ "uri":"dli_08_0305.html", "node_id":"dli_08_0305.xml", "product_code":"dli", - "code":"390", + "code":"494", "des":"Create a source stream to obtain data from Redis as input for jobs.An enhanced datasource connection with Redis has been established, so that you can configure security g", "doc_type":"sqlreference", "kw":"Redis Source Table,Creating a Source Table,SQL Syntax Reference", @@ -7016,7 +9670,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Redis Source Table", @@ -7026,7 +9682,7 @@ "uri":"dli_08_0306.html", "node_id":"dli_08_0306.xml", "product_code":"dli", - "code":"391", + "code":"495", "des":"Create a source stream to obtain data from HBase as input for jobs. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excell", "doc_type":"sqlreference", "kw":"HBase Source Table,Creating a Source Table,SQL Syntax Reference", @@ -7034,7 +9690,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"HBase Source Table", @@ -7044,7 +9702,7 @@ "uri":"dli_08_0358.html", "node_id":"dli_08_0358.xml", "product_code":"dli", - "code":"392", + "code":"496", "des":"You can call APIs to obtain data from the cloud ecosystem or an open source ecosystem and use the obtained data as input of Flink jobs.The customized source class needs t", "doc_type":"sqlreference", "kw":"userDefined Source Table,Creating a Source Table,SQL Syntax Reference", @@ -7052,7 +9710,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"userDefined Source Table", @@ -7062,7 +9722,7 @@ "uri":"dli_08_0307.html", "node_id":"dli_08_0307.xml", "product_code":"dli", - "code":"393", + "code":"497", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Result Table", @@ -7070,7 +9730,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Result Table", @@ -7080,7 +9742,7 @@ "uri":"dli_08_0344.html", "node_id":"dli_08_0344.xml", "product_code":"dli", - "code":"394", + "code":"498", "des":"DLI exports Flink job data to ClickHouse result tables.ClickHouse is a column-based database oriented to online analysis and processing. It supports SQL query and provide", "doc_type":"sqlreference", "kw":"ClickHouse Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7088,7 +9750,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"ClickHouse Result Table", @@ -7098,7 +9762,7 @@ "uri":"dli_08_0308.html", "node_id":"dli_08_0308.xml", "product_code":"dli", - "code":"395", + "code":"499", "des":"DLI exports the output data of the Flink job to Kafka.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It deli", "doc_type":"sqlreference", "kw":"Kafka Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7106,7 +9770,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Kafka Result Table", @@ -7116,7 +9782,7 @@ "uri":"dli_08_0309.html", "node_id":"dli_08_0309.xml", "product_code":"dli", - "code":"396", + "code":"500", "des":"DLI exports the output data of the Flink job to Kafka in upsert mode.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription ", "doc_type":"sqlreference", "kw":"Upsert Kafka Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7124,7 +9790,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Upsert Kafka Result Table", @@ -7134,7 +9802,7 @@ "uri":"dli_08_0310.html", "node_id":"dli_08_0310.xml", "product_code":"dli", - "code":"397", + "code":"501", "des":"DLI writes the Flink job output data into DIS. The data is filtered and imported to the DIS stream for future processing.DIS addresses the challenge of transmitting data ", "doc_type":"sqlreference", "kw":"DIS Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7142,7 +9810,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"DIS Result Table", @@ -7152,7 +9822,7 @@ "uri":"dli_08_0311.html", "node_id":"dli_08_0311.xml", "product_code":"dli", - "code":"398", + "code":"502", "des":"DLI exports the output data of the Flink job to RDS.An enhanced datasource connection with the database has been established, so that you can configure security group rul", "doc_type":"sqlreference", "kw":"JDBC Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7160,7 +9830,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"JDBC Result Table", @@ -7170,7 +9842,7 @@ "uri":"dli_08_0312.html", "node_id":"dli_08_0312.xml", "product_code":"dli", - "code":"399", + "code":"503", "des":"DLI outputs the Flink job output data to GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex t", "doc_type":"sqlreference", "kw":"GaussDB(DWS) Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7178,7 +9850,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"GaussDB(DWS) Result Table", @@ -7188,7 +9862,7 @@ "uri":"dli_08_0313.html", "node_id":"dli_08_0313.xml", "product_code":"dli", - "code":"400", + "code":"504", "des":"DLI exports the output data of the Flink job to Redis. Redis is a storage system that supports multiple types of data structures such as key-value. It can be used in scen", "doc_type":"sqlreference", "kw":"Redis Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7196,7 +9870,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Redis Result Table", @@ -7206,7 +9882,7 @@ "uri":"dli_08_0314.html", "node_id":"dli_08_0314.xml", "product_code":"dli", - "code":"401", + "code":"505", "des":"DLI exports Flink job output data to SMN.SMN provides reliable and flexible large-scale message notification services to DLI. It significantly simplifies system coupling ", "doc_type":"sqlreference", "kw":"SMN Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7214,7 +9890,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"SMN Result Table", @@ -7224,7 +9902,7 @@ "uri":"dli_08_0315.html", "node_id":"dli_08_0315.xml", "product_code":"dli", - "code":"402", + "code":"506", "des":"DLI outputs the job data to HBase. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excellent performance, and elastic scal", "doc_type":"sqlreference", "kw":"HBase Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7232,7 +9910,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"HBase Result Table", @@ -7242,7 +9922,7 @@ "uri":"dli_08_0316.html", "node_id":"dli_08_0316.xml", "product_code":"dli", - "code":"403", + "code":"507", "des":"DLI exports Flink job output data to Elasticsearch of Cloud Search Service (CSS). Elasticsearch is a popular enterprise-class Lucene-powered search server and provides th", "doc_type":"sqlreference", "kw":"Elasticsearch Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7250,7 +9930,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Elasticsearch Result Table", @@ -7260,7 +9942,7 @@ "uri":"dli_08_0348.html", "node_id":"dli_08_0348.xml", "product_code":"dli", - "code":"404", + "code":"508", "des":"OpenTSDB is a distributed, scalable time series database based on HBase. OpenTSDB is designed to collect monitoring information of a large-scale cluster and query data in", "doc_type":"sqlreference", "kw":"OpenTSDB Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7268,7 +9950,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"OpenTSDB Result Table", @@ -7278,7 +9962,7 @@ "uri":"dli_08_0347.html", "node_id":"dli_08_0347.xml", "product_code":"dli", - "code":"405", + "code":"509", "des":"Write your Java code to insert the processed data into a specified database supported by your cloud service.Implement the custom sink class :The custom sink class is inhe", "doc_type":"sqlreference", "kw":"User-defined Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7286,7 +9970,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"User-defined Result Table", @@ -7296,7 +9982,7 @@ "uri":"dli_08_0345.html", "node_id":"dli_08_0345.xml", "product_code":"dli", - "code":"406", + "code":"510", "des":"The print connector exports your data output to the error file or the out file of TaskManager. It is mainly used for code debugging and output viewing.Read data from Kafk", "doc_type":"sqlreference", "kw":"Print Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7304,7 +9990,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Print Result Table", @@ -7314,7 +10002,7 @@ "uri":"dli_08_0346.html", "node_id":"dli_08_0346.xml", "product_code":"dli", - "code":"407", + "code":"511", "des":"You can create a file system result table to export data to a file system such as HDFS or OBS. After the data is generated, a non-DLI table can be created directly accord", "doc_type":"sqlreference", "kw":"File System Result Table,Creating a Result Table,SQL Syntax Reference", @@ -7322,7 +10010,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"File System Result Table", @@ -7332,7 +10022,7 @@ "uri":"dli_08_0317.html", "node_id":"dli_08_0317.xml", "product_code":"dli", - "code":"408", + "code":"512", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Dimension Table", @@ -7340,7 +10030,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Dimension Table", @@ -7350,7 +10042,7 @@ "uri":"dli_08_0318.html", "node_id":"dli_08_0318.xml", "product_code":"dli", - "code":"409", + "code":"513", "des":"Create a JDBC dimension table to connect to the source stream.You have created a JDBC instance for your account.The RDS table is used to connect to the source stream.CREA", "doc_type":"sqlreference", "kw":"JDBC Dimension Table,Creating a Dimension Table,SQL Syntax Reference", @@ -7358,7 +10050,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"JDBC Dimension Table", @@ -7368,7 +10062,7 @@ "uri":"dli_08_0319.html", "node_id":"dli_08_0319.xml", "product_code":"dli", - "code":"410", + "code":"514", "des":"Create a GaussDB(DWS) dimension table to connect to the input stream.You have created a GaussDB(DWS) instance for your account.Use an RDS table to connect to the source s", "doc_type":"sqlreference", "kw":"GaussDB(DWS) Dimension Table,Creating a Dimension Table,SQL Syntax Reference", @@ -7376,7 +10070,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"GaussDB(DWS) Dimension Table", @@ -7386,7 +10082,7 @@ "uri":"dli_08_0320.html", "node_id":"dli_08_0320.xml", "product_code":"dli", - "code":"411", + "code":"515", "des":"Create a Hbase dimension table to connect to the source stream.An enhanced datasource connection has been created for DLI to connect to HBase, so that jobs can run on the", "doc_type":"sqlreference", "kw":"HBase Dimension Table,Creating a Dimension Table,SQL Syntax Reference", @@ -7394,7 +10090,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"HBase Dimension Table", @@ -7404,7 +10102,7 @@ "uri":"dli_08_0321.html", "node_id":"dli_08_0321.xml", "product_code":"dli", - "code":"412", + "code":"516", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Data Manipulation Language (DML)", @@ -7412,7 +10110,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Data Manipulation Language (DML)", @@ -7422,7 +10122,7 @@ "uri":"dli_08_0322.html", "node_id":"dli_08_0322.xml", "product_code":"dli", - "code":"413", + "code":"517", "des":"SyntaxDescriptionThis clause is used to select data from a table.ALL indicates that all results are returned.DISTINCT indicates that the duplicated results are removed.Pr", "doc_type":"sqlreference", "kw":"SELECT,Data Manipulation Language (DML),SQL Syntax Reference", @@ -7430,7 +10130,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"SELECT", @@ -7440,7 +10142,7 @@ "uri":"dli_08_0323.html", "node_id":"dli_08_0323.xml", "product_code":"dli", - "code":"414", + "code":"518", "des":"SyntaxDescriptionUNION is used to return the union set of multiple query results.INTERSECT is used to return the intersection of multiple query results.EXCEPT is used to ", "doc_type":"sqlreference", "kw":"Set Operations,Data Manipulation Language (DML),SQL Syntax Reference", @@ -7448,7 +10150,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Set Operations", @@ -7458,15 +10162,17 @@ "uri":"dli_08_0324.html", "node_id":"dli_08_0324.xml", "product_code":"dli", - "code":"415", - "des":"DescriptionGroup Window is defined in GROUP BY. One record is generated from each group. Group Window involves the following functions:Array functionsArray functionsGroup", + "code":"519", + "des":"DescriptionGroup Window is defined in GROUP BY. One record is generated from each group. Group Window involves the following functions:Array functionsFor SQL queries in s", "doc_type":"sqlreference", "kw":"Window,Data Manipulation Language (DML),SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Window", @@ -7476,7 +10182,7 @@ "uri":"dli_08_0325.html", "node_id":"dli_08_0325.xml", "product_code":"dli", - "code":"416", + "code":"520", "des":"SyntaxPrecautionsCurrently, only equi-joins are supported, for example, joins that have at least one conjunctive condition with an equality predicate. Arbitrary cross or ", "doc_type":"sqlreference", "kw":"JOIN,Data Manipulation Language (DML),SQL Syntax Reference", @@ -7484,7 +10190,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"JOIN", @@ -7494,7 +10202,7 @@ "uri":"dli_08_0326.html", "node_id":"dli_08_0326.xml", "product_code":"dli", - "code":"417", + "code":"521", "des":"FunctionThis clause is used to sort data in ascending order on a time attribute.PrecautionsCurrently, only sorting by time attribute is supported.ExampleSort data in asce", "doc_type":"sqlreference", "kw":"OrderBy & Limit,Data Manipulation Language (DML),SQL Syntax Reference", @@ -7502,7 +10210,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"OrderBy & Limit", @@ -7512,7 +10222,7 @@ "uri":"dli_08_0327.html", "node_id":"dli_08_0327.xml", "product_code":"dli", - "code":"418", + "code":"522", "des":"Top-N queries ask for the N smallest or largest values ordered by columns. Both smallest and largest values sets are considered Top-N queries. Top-N queries are useful in", "doc_type":"sqlreference", "kw":"Top-N,Data Manipulation Language (DML),SQL Syntax Reference", @@ -7520,7 +10230,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Top-N", @@ -7530,7 +10242,7 @@ "uri":"dli_08_0328.html", "node_id":"dli_08_0328.xml", "product_code":"dli", - "code":"419", + "code":"523", "des":"Deduplication removes rows that duplicate over a set of columns, keeping only the first one or the last one.ROW_NUMBER(): Assigns a unique, sequential number to each row,", "doc_type":"sqlreference", "kw":"Deduplication,Data Manipulation Language (DML),SQL Syntax Reference", @@ -7538,7 +10250,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Deduplication", @@ -7548,7 +10262,7 @@ "uri":"dli_08_0329.html", "node_id":"dli_08_0329.xml", "product_code":"dli", - "code":"420", + "code":"524", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Functions", @@ -7556,7 +10270,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Functions", @@ -7566,7 +10282,7 @@ "uri":"dli_08_0330.html", "node_id":"dli_08_0330.xml", "product_code":"dli", - "code":"421", + "code":"525", "des":"DLI supports the following three types of user-defined functions (UDFs):Regular UDF: takes in one or more input parameters and returns a single result.User-defined table-", "doc_type":"sqlreference", "kw":"User-Defined Functions,Functions,SQL Syntax Reference", @@ -7574,7 +10290,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"User-Defined Functions", @@ -7584,7 +10302,7 @@ "uri":"dli_08_0331.html", "node_id":"dli_08_0331.xml", "product_code":"dli", - "code":"422", + "code":"526", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Built-In Functions", @@ -7592,7 +10310,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Built-In Functions", @@ -7602,7 +10322,7 @@ "uri":"dli_08_0332.html", "node_id":"dli_08_0332.xml", "product_code":"dli", - "code":"423", + "code":"527", "des":"All data types can be compared by using relational operators and the result is returned as a BOOLEAN value.Relationship operators are binary operators. Two compared data ", "doc_type":"sqlreference", "kw":"Mathematical Operation Functions,Built-In Functions,SQL Syntax Reference", @@ -7610,7 +10330,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Mathematical Operation Functions", @@ -7620,15 +10342,17 @@ "uri":"dli_08_0333.html", "node_id":"dli_08_0333.xml", "product_code":"dli", - "code":"424", - "des":"SyntaxExampleTest input data.Test the data source kafka. The message content is as follows:\"{name:James,age:24,sex:male,grade:{math:95,science:[80,85],english:100}}\"\n\"{na", + "code":"528", + "des":"SyntaxExampleTest input data.Test the data source kafka. The message content is as follows:\"{name:James,age:24,gender:male,grade:{math:95,science:[80,85],english:100}}\"\n\"", "doc_type":"sqlreference", "kw":"String Functions,Built-In Functions,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"String Functions", @@ -7638,7 +10362,7 @@ "uri":"dli_08_0334.html", "node_id":"dli_08_0334.xml", "product_code":"dli", - "code":"425", + "code":"529", "des":"Table 1 lists the temporal functions supported by Flink OpenSource SQL.FunctionReturns a date parsed from string in form of yyyy-MM-dd.Returns a date parsed from string i", "doc_type":"sqlreference", "kw":"Temporal Functions,Built-In Functions,SQL Syntax Reference", @@ -7646,7 +10370,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Temporal Functions", @@ -7656,7 +10382,7 @@ "uri":"dli_08_0335.html", "node_id":"dli_08_0335.xml", "product_code":"dli", - "code":"426", + "code":"530", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Conditional Functions,Built-In Functions,SQL Syntax Reference", @@ -7664,7 +10390,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Conditional Functions", @@ -7674,7 +10402,7 @@ "uri":"dli_08_0336.html", "node_id":"dli_08_0336.xml", "product_code":"dli", - "code":"427", + "code":"531", "des":"This function is used to forcibly convert types.If the input is NULL, NULL is returned.The following example converts the amount value to an integer.Flink jobs do not sup", "doc_type":"sqlreference", "kw":"Type Conversion Function,Built-In Functions,SQL Syntax Reference", @@ -7682,7 +10410,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Type Conversion Function", @@ -7692,7 +10422,7 @@ "uri":"dli_08_0337.html", "node_id":"dli_08_0337.xml", "product_code":"dli", - "code":"428", + "code":"532", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Collection Functions,Built-In Functions,SQL Syntax Reference", @@ -7700,7 +10430,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Collection Functions", @@ -7710,7 +10442,7 @@ "uri":"dli_08_0338.html", "node_id":"dli_08_0338.xml", "product_code":"dli", - "code":"429", + "code":"533", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Value Construction Functions,Built-In Functions,SQL Syntax Reference", @@ -7718,7 +10450,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Value Construction Functions", @@ -7728,7 +10462,7 @@ "uri":"dli_08_0339.html", "node_id":"dli_08_0339.xml", "product_code":"dli", - "code":"430", + "code":"534", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Value Access Functions,Built-In Functions,SQL Syntax Reference", @@ -7736,7 +10470,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Value Access Functions", @@ -7746,7 +10482,7 @@ "uri":"dli_08_0340.html", "node_id":"dli_08_0340.xml", "product_code":"dli", - "code":"431", + "code":"535", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Hash Functions,Built-In Functions,SQL Syntax Reference", @@ -7754,7 +10490,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Hash Functions", @@ -7764,7 +10502,7 @@ "uri":"dli_08_0341.html", "node_id":"dli_08_0341.xml", "product_code":"dli", - "code":"432", + "code":"536", "des":"An aggregate function performs a calculation operation on a set of input values and returns a value. For example, the COUNT function counts the number of rows retrieved b", "doc_type":"sqlreference", "kw":"Aggregate Function,Built-In Functions,SQL Syntax Reference", @@ -7772,7 +10510,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Aggregate Function", @@ -7782,7 +10522,7 @@ "uri":"dli_08_0342.html", "node_id":"dli_08_0342.xml", "product_code":"dli", - "code":"433", + "code":"537", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Table-Valued Functions", @@ -7790,7 +10530,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Table-Valued Functions", @@ -7800,7 +10542,7 @@ "uri":"dli_08_0357.html", "node_id":"dli_08_0357.xml", "product_code":"dli", - "code":"434", + "code":"538", "des":"The split_cursor function can convert one row of records into multiple rows or convert one column of records into multiple columns. Table-valued functions can only be use", "doc_type":"sqlreference", "kw":"split_cursor,Table-Valued Functions,SQL Syntax Reference", @@ -7808,7 +10550,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"split_cursor", @@ -7818,7 +10562,7 @@ "uri":"dli_08_0356.html", "node_id":"dli_08_0356.xml", "product_code":"dli", - "code":"435", + "code":"539", "des":"The string_split function splits a target string into substrings based on the specified separator and returns a substring list.Prepare test input data.Source table disSou", "doc_type":"sqlreference", "kw":"string_split,Table-Valued Functions,SQL Syntax Reference", @@ -7826,7 +10570,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"string_split", @@ -7836,7 +10582,7 @@ "uri":"dli_08_0450.html", "node_id":"dli_08_0450.xml", "product_code":"dli", - "code":"436", + "code":"540", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Historical Versions", @@ -7844,7 +10590,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Historical Versions", @@ -7854,7 +10602,7 @@ "uri":"dli_08_0233.html", "node_id":"dli_08_0233.xml", "product_code":"dli", - "code":"437", + "code":"541", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Flink SQL Syntax", @@ -7862,7 +10610,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Flink SQL Syntax", @@ -7872,7 +10622,7 @@ "uri":"dli_08_0075.html", "node_id":"dli_08_0075.xml", "product_code":"dli", - "code":"438", + "code":"542", "des":"Currently, Flink SQL only supports the following operations: SELECT, FROM, WHERE, UNION, aggregation, window, JOIN between stream and table data, and JOIN between streams", "doc_type":"sqlreference", "kw":"SQL Syntax Constraints and Definitions,Flink SQL Syntax,SQL Syntax Reference", @@ -7880,7 +10630,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"SQL Syntax Constraints and Definitions", @@ -7890,7 +10642,7 @@ "uri":"dli_08_0275.html", "node_id":"dli_08_0275.xml", "product_code":"dli", - "code":"439", + "code":"543", "des":"This section describes the Flink SQL syntax list provided by DLI. For details about the parameters and examples, see the syntax description.", "doc_type":"sqlreference", "kw":"SQL Syntax Overview of Stream Jobs,Flink SQL Syntax,SQL Syntax Reference", @@ -7898,7 +10650,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"SQL Syntax Overview of Stream Jobs", @@ -7908,7 +10662,7 @@ "uri":"dli_08_0234.html", "node_id":"dli_08_0234.xml", "product_code":"dli", - "code":"440", + "code":"544", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Source Stream", @@ -7916,7 +10670,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Source Stream", @@ -7926,7 +10682,7 @@ "uri":"dli_08_0237.html", "node_id":"dli_08_0237.xml", "product_code":"dli", - "code":"441", + "code":"545", "des":"Create a source stream to obtain data from HBase of CloudTable as input data of the job. HBase is a column-oriented distributed cloud storage system that features enhance", "doc_type":"sqlreference", "kw":"CloudTable HBase Source Stream,Creating a Source Stream,SQL Syntax Reference", @@ -7934,7 +10690,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"CloudTable HBase Source Stream", @@ -7944,7 +10702,7 @@ "uri":"dli_08_0235.html", "node_id":"dli_08_0235.xml", "product_code":"dli", - "code":"442", + "code":"546", "des":"Create a source stream to read data from DIS. DIS accesses user data and Flink job reads data from the DIS stream as input data for jobs. Flink jobs can quickly remove da", "doc_type":"sqlreference", "kw":"DIS Source Stream,Creating a Source Stream,SQL Syntax Reference", @@ -7952,7 +10710,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"DIS Source Stream", @@ -7962,7 +10722,7 @@ "uri":"dli_08_0270.html", "node_id":"dli_08_0270.xml", "product_code":"dli", - "code":"443", + "code":"547", "des":"DMS (Distributed Message Service) is a message middleware service based on distributed, high-availability clustering technology. It provides reliable, scalable, fully man", "doc_type":"sqlreference", "kw":"DMS Source Stream,Creating a Source Stream,SQL Syntax Reference", @@ -7970,7 +10730,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"DMS Source Stream", @@ -7980,7 +10742,7 @@ "uri":"dli_08_0238.html", "node_id":"dli_08_0238.xml", "product_code":"dli", - "code":"444", + "code":"548", "des":"Create a source stream to obtain data from Kafka as input data for jobs.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscripti", "doc_type":"sqlreference", "kw":"MRS Kafka Source Stream,Creating a Source Stream,SQL Syntax Reference", @@ -7988,7 +10750,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"MRS Kafka Source Stream", @@ -7998,7 +10762,7 @@ "uri":"dli_08_0239.html", "node_id":"dli_08_0239.xml", "product_code":"dli", - "code":"445", + "code":"549", "des":"Create a source stream to obtain data from Kafka as input data for jobs.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscripti", "doc_type":"sqlreference", "kw":"Open-Source Kafka Source Stream,Creating a Source Stream,SQL Syntax Reference", @@ -8006,7 +10770,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Open-Source Kafka Source Stream", @@ -8016,7 +10782,7 @@ "uri":"dli_08_0236.html", "node_id":"dli_08_0236.xml", "product_code":"dli", - "code":"446", + "code":"550", "des":"Create a source stream to obtain data from OBS. DLI reads data stored by users in OBS as input data for jobs. OBS applies to various scenarios, such as big data analysis,", "doc_type":"sqlreference", "kw":"OBS Source Stream,Creating a Source Stream,SQL Syntax Reference", @@ -8024,7 +10790,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"OBS Source Stream", @@ -8034,7 +10802,7 @@ "uri":"dli_08_0240.html", "node_id":"dli_08_0240.xml", "product_code":"dli", - "code":"447", + "code":"551", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Sink Stream", @@ -8042,7 +10810,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Sink Stream", @@ -8052,7 +10822,7 @@ "uri":"dli_08_0243.html", "node_id":"dli_08_0243.xml", "product_code":"dli", - "code":"448", + "code":"552", "des":"DLI exports the job output data to HBase of CloudTable. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excellent performa", "doc_type":"sqlreference", "kw":"CloudTable HBase Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8060,7 +10830,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"CloudTable HBase Sink Stream", @@ -8070,7 +10842,7 @@ "uri":"dli_08_0244.html", "node_id":"dli_08_0244.xml", "product_code":"dli", - "code":"449", + "code":"553", "des":"DLI exports the job output data to OpenTSDB of CloudTable. OpenTSDB is a distributed, scalable time series database based on HBase. It stores time series data. Time serie", "doc_type":"sqlreference", "kw":"CloudTable OpenTSDB Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8078,7 +10850,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"CloudTable OpenTSDB Sink Stream", @@ -8088,7 +10862,7 @@ "uri":"dli_08_0286.html", "node_id":"dli_08_0286.xml", "product_code":"dli", - "code":"450", + "code":"554", "des":"DLI exports the output data of the Flink job to OpenTSDB of MRS.OpenTSDB has been installed in the MRS cluster.In this scenario, jobs must run on the dedicated queue of D", "doc_type":"sqlreference", "kw":"MRS OpenTSDB Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8096,7 +10870,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"MRS OpenTSDB Sink Stream", @@ -8106,7 +10882,7 @@ "uri":"dli_08_0252.html", "node_id":"dli_08_0252.xml", "product_code":"dli", - "code":"451", + "code":"555", "des":"DLI exports Flink job output data to Elasticsearch of Cloud Search Service (CSS). Elasticsearch is a popular enterprise-class Lucene-powered search server and provides th", "doc_type":"sqlreference", "kw":"CSS Elasticsearch Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8114,7 +10890,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"CSS Elasticsearch Sink Stream", @@ -8124,7 +10902,7 @@ "uri":"dli_08_0253.html", "node_id":"dli_08_0253.xml", "product_code":"dli", - "code":"452", + "code":"556", "des":"DLI exports the Flink job output data to Redis of DCS. Redis is a storage system that supports multiple types of data structures such as key-value. It can be used in scen", "doc_type":"sqlreference", "kw":"DCS Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8132,7 +10910,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"DCS Sink Stream", @@ -8142,7 +10922,7 @@ "uri":"dli_08_0249.html", "node_id":"dli_08_0249.xml", "product_code":"dli", - "code":"453", + "code":"557", "des":"DLI outputs the job output data to Document Database Service (DDS).DDS is compatible with the MongoDB protocol and is secure, highly available, reliable, scalable, and ea", "doc_type":"sqlreference", "kw":"DDS Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8150,7 +10930,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"DDS Sink Stream", @@ -8160,7 +10942,7 @@ "uri":"dli_08_0241.html", "node_id":"dli_08_0241.xml", "product_code":"dli", - "code":"454", + "code":"558", "des":"DLI writes the Flink job output data into DIS. This cloud ecosystem is applicable to scenarios where data is filtered and imported to the DIS stream for future processing", "doc_type":"sqlreference", "kw":"DIS Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8168,7 +10950,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"DIS Sink Stream", @@ -8178,15 +10962,17 @@ "uri":"dli_08_0271.html", "node_id":"dli_08_0271.xml", "product_code":"dli", - "code":"455", - "des":"DMS (Distributed Message Service) is a message middleware service based on distributed, high-availability clustering technology. It provides reliable, scalable, fully man", + "code":"559", + "des":"DMS for Kafka is a message queuing service based on Apache Kafka. This service provides Kafka premium instances.DLI can write the job output data into the Kafka instance.", "doc_type":"sqlreference", "kw":"DMS Sink Stream,Creating a Sink Stream,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"DMS Sink Stream", @@ -8196,7 +10982,7 @@ "uri":"dli_08_0247.html", "node_id":"dli_08_0247.xml", "product_code":"dli", - "code":"456", + "code":"560", "des":"DLI outputs the Flink job output data to Data Warehouse Service (DWS). DWS database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more co", "doc_type":"sqlreference", "kw":"DWS Sink Stream (JDBC Mode),Creating a Sink Stream,SQL Syntax Reference", @@ -8204,7 +10990,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"DWS Sink Stream (JDBC Mode)", @@ -8214,7 +11002,7 @@ "uri":"dli_08_0248.html", "node_id":"dli_08_0248.xml", "product_code":"dli", - "code":"457", + "code":"561", "des":"Create a sink stream to export Flink job data to DWS through OBS-based dumping, specifically, output Flink job data to OBS and then import data from OBS to DWS. For detai", "doc_type":"sqlreference", "kw":"DWS Sink Stream (OBS-based Dumping),Creating a Sink Stream,SQL Syntax Reference", @@ -8222,7 +11010,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"DWS Sink Stream (OBS-based Dumping)", @@ -8232,7 +11022,7 @@ "uri":"dli_08_0255.html", "node_id":"dli_08_0255.xml", "product_code":"dli", - "code":"458", + "code":"562", "des":"DLI exports the output data of the Flink job to HBase of MRS.An MRS cluster has been created by using your account. DLI can interconnect with HBase clusters with Kerberos", "doc_type":"sqlreference", "kw":"MRS HBase Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8240,7 +11030,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"MRS HBase Sink Stream", @@ -8250,7 +11042,7 @@ "uri":"dli_08_0254.html", "node_id":"dli_08_0254.xml", "product_code":"dli", - "code":"459", + "code":"563", "des":"DLI exports the output data of the Flink job to Kafka.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It deli", "doc_type":"sqlreference", "kw":"MRS Kafka Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8258,7 +11050,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"MRS Kafka Sink Stream", @@ -8268,7 +11062,7 @@ "uri":"dli_08_0257.html", "node_id":"dli_08_0257.xml", "product_code":"dli", - "code":"460", + "code":"564", "des":"DLI exports the output data of the Flink job to Kafka.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It deli", "doc_type":"sqlreference", "kw":"Open-Source Kafka Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8276,7 +11070,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Open-Source Kafka Sink Stream", @@ -8286,7 +11082,7 @@ "uri":"dli_08_0267.html", "node_id":"dli_08_0267.xml", "product_code":"dli", - "code":"461", + "code":"565", "des":"You can create a sink stream to export data to a file system such as HDFS or OBS. After the data is generated, a non-DLI table can be created directly according to the ge", "doc_type":"sqlreference", "kw":"File System Sink Stream (Recommended),Creating a Sink Stream,SQL Syntax Reference", @@ -8294,7 +11090,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"File System Sink Stream (Recommended)", @@ -8304,7 +11102,7 @@ "uri":"dli_08_0242.html", "node_id":"dli_08_0242.xml", "product_code":"dli", - "code":"462", + "code":"566", "des":"Create a sink stream to export DLI data to OBS. DLI can export the job analysis results to OBS. OBS applies to various scenarios, such as big data analysis, cloud-native ", "doc_type":"sqlreference", "kw":"OBS Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8312,7 +11110,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"OBS Sink Stream", @@ -8322,7 +11122,7 @@ "uri":"dli_08_0245.html", "node_id":"dli_08_0245.xml", "product_code":"dli", - "code":"463", + "code":"567", "des":"DLI outputs the Flink job output data to RDS. Currently, PostgreSQL and MySQL databases are supported. The PostgreSQL database can store data of more complex types and de", "doc_type":"sqlreference", "kw":"RDS Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8330,7 +11130,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"RDS Sink Stream", @@ -8340,7 +11142,7 @@ "uri":"dli_08_0251.html", "node_id":"dli_08_0251.xml", "product_code":"dli", - "code":"464", + "code":"568", "des":"DLI exports Flink job output data to SMN.SMN provides reliable and flexible large-scale message notification services to DLI. It significantly simplifies system coupling ", "doc_type":"sqlreference", "kw":"SMN Sink Stream,Creating a Sink Stream,SQL Syntax Reference", @@ -8348,7 +11150,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"SMN Sink Stream", @@ -8358,7 +11162,7 @@ "uri":"dli_08_0258.html", "node_id":"dli_08_0258.xml", "product_code":"dli", - "code":"465", + "code":"569", "des":"The temporary stream is used to simplify SQL logic. If complex SQL logic is followed, write SQL statements concatenated with temporary streams. The temporary stream is ju", "doc_type":"sqlreference", "kw":"Creating a Temporary Stream,Flink SQL Syntax,SQL Syntax Reference", @@ -8366,7 +11170,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Temporary Stream", @@ -8376,7 +11182,7 @@ "uri":"dli_08_0259.html", "node_id":"dli_08_0259.xml", "product_code":"dli", - "code":"466", + "code":"570", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Creating a Dimension Table", @@ -8384,7 +11190,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Dimension Table", @@ -8394,7 +11202,7 @@ "uri":"dli_08_0260.html", "node_id":"dli_08_0260.xml", "product_code":"dli", - "code":"467", + "code":"571", "des":"Create a Redis table to connect to the source stream.For details about the JOIN syntax, see JOIN Between Stream Data and Table Data.Redis clusters are not supported.Ensur", "doc_type":"sqlreference", "kw":"Creating a Redis Table,Creating a Dimension Table,SQL Syntax Reference", @@ -8402,7 +11210,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating a Redis Table", @@ -8412,7 +11222,7 @@ "uri":"dli_08_0261.html", "node_id":"dli_08_0261.xml", "product_code":"dli", - "code":"468", + "code":"572", "des":"Create an RDS/DWS table to connect to the source stream.For details about the JOIN syntax, see JOIN.Ensure that you have created a PostgreSQL or MySQL RDS instance in RDS", "doc_type":"sqlreference", "kw":"Creating an RDS Table,Creating a Dimension Table,SQL Syntax Reference", @@ -8420,7 +11230,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Creating an RDS Table", @@ -8430,7 +11242,7 @@ "uri":"dli_08_0272.html", "node_id":"dli_08_0272.xml", "product_code":"dli", - "code":"469", + "code":"573", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Custom Stream Ecosystem", @@ -8438,7 +11250,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Custom Stream Ecosystem", @@ -8448,7 +11262,7 @@ "uri":"dli_08_0273.html", "node_id":"dli_08_0273.xml", "product_code":"dli", - "code":"470", + "code":"574", "des":"Compile code to obtain data from the desired cloud ecosystem or open-source ecosystem as the input data of Flink jobs.The user-defined source class needs to inherit the R", "doc_type":"sqlreference", "kw":"Custom Source Stream,Custom Stream Ecosystem,SQL Syntax Reference", @@ -8456,7 +11270,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Custom Source Stream", @@ -8466,7 +11282,7 @@ "uri":"dli_08_0274.html", "node_id":"dli_08_0274.xml", "product_code":"dli", - "code":"471", + "code":"575", "des":"Compile code to write the data processed by DLI to a specified cloud ecosystem or open-source ecosystem.The user-defined sink class needs to inherit the RichSinkFunction ", "doc_type":"sqlreference", "kw":"Custom Sink Stream,Custom Stream Ecosystem,SQL Syntax Reference", @@ -8474,7 +11290,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Custom Sink Stream", @@ -8484,7 +11302,7 @@ "uri":"dli_08_0207.html", "node_id":"dli_08_0207.xml", "product_code":"dli", - "code":"472", + "code":"576", "des":"Data type is a basic attribute of data and used to distinguish different types of data. Different data types occupy different storage space and support different operatio", "doc_type":"sqlreference", "kw":"Data Type,Flink SQL Syntax,SQL Syntax Reference", @@ -8492,7 +11310,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Data Type", @@ -8502,7 +11322,7 @@ "uri":"dli_08_0086.html", "node_id":"dli_08_0086.xml", "product_code":"dli", - "code":"473", + "code":"577", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Built-In Functions", @@ -8510,7 +11330,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Built-In Functions", @@ -8520,7 +11342,7 @@ "uri":"dli_08_0191.html", "node_id":"dli_08_0191.xml", "product_code":"dli", - "code":"474", + "code":"578", "des":"All data types can be compared by using relational operators and the result is returned as a BOOLEAN value.Relationship operators are binary operators. Two compared data ", "doc_type":"sqlreference", "kw":"Mathematical Operation Functions,Built-In Functions,SQL Syntax Reference", @@ -8528,7 +11350,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Mathematical Operation Functions", @@ -8538,7 +11362,7 @@ "uri":"dli_08_0096.html", "node_id":"dli_08_0096.xml", "product_code":"dli", - "code":"475", + "code":"579", "des":"The common string functions of DLI are as follows:FunctionConcatenates two strings.Concatenates two strings.SyntaxVARCHAR VARCHAR a || VARCHAR bParametersa: string.b: str", "doc_type":"sqlreference", "kw":"String Functions,Built-In Functions,SQL Syntax Reference", @@ -8546,7 +11370,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"String Functions", @@ -8556,7 +11382,7 @@ "uri":"dli_08_0097.html", "node_id":"dli_08_0097.xml", "product_code":"dli", - "code":"476", + "code":"580", "des":"Table 1 lists the time functions supported by Flink SQL.None", "doc_type":"sqlreference", "kw":"Temporal Functions,Built-In Functions,SQL Syntax Reference", @@ -8564,7 +11390,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Temporal Functions", @@ -8574,7 +11402,7 @@ "uri":"dli_08_0112.html", "node_id":"dli_08_0112.xml", "product_code":"dli", - "code":"477", + "code":"581", "des":"This function is used to forcibly convert types.If the input is NULL, NULL is returned.Flink jobs do not support the conversion of bigint to timestamp using CAST. You can", "doc_type":"sqlreference", "kw":"Type Conversion Functions,Built-In Functions,SQL Syntax Reference", @@ -8582,7 +11410,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Type Conversion Functions", @@ -8592,7 +11422,7 @@ "uri":"dli_08_0104.html", "node_id":"dli_08_0104.xml", "product_code":"dli", - "code":"478", + "code":"582", "des":"An aggregate function performs a calculation operation on a set of input values and returns a value. For example, the COUNT function counts the number of rows retrieved b", "doc_type":"sqlreference", "kw":"Aggregate Functions,Built-In Functions,SQL Syntax Reference", @@ -8600,7 +11430,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Aggregate Functions", @@ -8610,7 +11442,7 @@ "uri":"dli_08_0206.html", "node_id":"dli_08_0206.xml", "product_code":"dli", - "code":"479", + "code":"583", "des":"Table-valued functions can convert one row of records into multiple rows or convert one column of records into multiple columns. Table-valued functions can only be used i", "doc_type":"sqlreference", "kw":"Table-Valued Functions,Built-In Functions,SQL Syntax Reference", @@ -8618,7 +11450,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Table-Valued Functions", @@ -8628,7 +11462,7 @@ "uri":"dli_08_0101.html", "node_id":"dli_08_0101.xml", "product_code":"dli", - "code":"480", + "code":"584", "des":"Example:The returned number of elements in the array is 3.HELLO WORLD is returned.", "doc_type":"sqlreference", "kw":"Other Functions,Built-In Functions,SQL Syntax Reference", @@ -8636,7 +11470,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Other Functions", @@ -8646,7 +11482,7 @@ "uri":"dli_08_0099.html", "node_id":"dli_08_0099.xml", "product_code":"dli", - "code":"481", + "code":"585", "des":"DLI supports the following three types of user-defined functions (UDFs):Regular UDF: takes in one or more input parameters and returns a single result.User-defined table-", "doc_type":"sqlreference", "kw":"User-Defined Functions,Flink SQL Syntax,SQL Syntax Reference", @@ -8654,7 +11490,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"User-Defined Functions", @@ -8664,7 +11502,7 @@ "uri":"dli_08_0209.html", "node_id":"dli_08_0209.xml", "product_code":"dli", - "code":"482", + "code":"586", "des":"Table 1 describes the basic geospatial geometric elements.You can build complex geospatial geometries based on basic geospatial geometric elements. Table 2 describes the ", "doc_type":"sqlreference", "kw":"Geographical Functions,Flink SQL Syntax,SQL Syntax Reference", @@ -8672,7 +11510,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Geographical Functions", @@ -8682,7 +11522,7 @@ "uri":"dli_08_0102.html", "node_id":"dli_08_0102.xml", "product_code":"dli", - "code":"483", + "code":"587", "des":"SyntaxDescriptionThe SELECT statement is used to select data from a table or insert constant data into a table.PrecautionsThe table to be queried must exist. Otherwise, a", "doc_type":"sqlreference", "kw":"SELECT,Flink SQL Syntax,SQL Syntax Reference", @@ -8690,7 +11530,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"SELECT", @@ -8700,15 +11542,17 @@ "uri":"dli_08_0103.html", "node_id":"dli_08_0103.xml", "product_code":"dli", - "code":"484", - "des":"SyntaxorDescriptionIf the value of value is value1, result1 is returned. If the value is not any of the values listed in the clause, resultZ is returned. If no else state", + "code":"588", + "des":"SyntaxorDescriptionIf the value of value is value1, result1 is returned; otherwise, resultZ is returned. If there is no else statement, null is returned.If the value of c", "doc_type":"sqlreference", "kw":"Condition Expression,Flink SQL Syntax,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Condition Expression", @@ -8718,7 +11562,7 @@ "uri":"dli_08_0218.html", "node_id":"dli_08_0218.xml", "product_code":"dli", - "code":"485", + "code":"589", "des":"DescriptionGroup Window is defined in GROUP BY. One record is generated from each group. Group Window involves the following functions:time_attr can be processing-time or", "doc_type":"sqlreference", "kw":"Window,Flink SQL Syntax,SQL Syntax Reference", @@ -8726,7 +11570,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Window", @@ -8736,7 +11582,7 @@ "uri":"dli_08_0106.html", "node_id":"dli_08_0106.xml", "product_code":"dli", - "code":"486", + "code":"590", "des":"The JOIN operation allows you to query data from a table and write the query result to the sink stream. Currently, only RDSs and DCS Redis tables are supported. The ON ke", "doc_type":"sqlreference", "kw":"JOIN Between Stream Data and Table Data,Flink SQL Syntax,SQL Syntax Reference", @@ -8744,7 +11590,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"JOIN Between Stream Data and Table Data", @@ -8754,7 +11602,7 @@ "uri":"dli_08_0107.html", "node_id":"dli_08_0107.xml", "product_code":"dli", - "code":"487", + "code":"591", "des":"Flink provides two time models: processing time and event time.DLI allows you to specify the time model during creation of the source stream and temporary stream.Processi", "doc_type":"sqlreference", "kw":"Configuring Time Models,Flink SQL Syntax,SQL Syntax Reference", @@ -8762,7 +11610,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Configuring Time Models", @@ -8772,7 +11622,7 @@ "uri":"dli_08_0108.html", "node_id":"dli_08_0108.xml", "product_code":"dli", - "code":"488", + "code":"592", "des":"Complex event processing (CEP) is used to detect complex patterns in endless data streams so as to identify and search patterns in various data rows. Pattern matching is ", "doc_type":"sqlreference", "kw":"Pattern Matching,Flink SQL Syntax,SQL Syntax Reference", @@ -8780,7 +11630,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Pattern Matching", @@ -8790,7 +11642,7 @@ "uri":"dli_08_0109.html", "node_id":"dli_08_0109.xml", "product_code":"dli", - "code":"489", + "code":"593", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"StreamingML", @@ -8798,7 +11650,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"StreamingML", @@ -8808,7 +11662,7 @@ "uri":"dli_08_0110.html", "node_id":"dli_08_0110.xml", "product_code":"dli", - "code":"490", + "code":"594", "des":"Anomaly detection applies to various scenarios, including intrusion detection, financial fraud detection, sensor data monitoring, medical diagnosis, natural data detectio", "doc_type":"sqlreference", "kw":"Anomaly Detection,StreamingML,SQL Syntax Reference", @@ -8816,7 +11670,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Anomaly Detection", @@ -8826,7 +11682,7 @@ "uri":"dli_08_0111.html", "node_id":"dli_08_0111.xml", "product_code":"dli", - "code":"491", + "code":"595", "des":"Modeling and forecasting time series is a common task in many business verticals. Modeling is used to extract meaningful statistics and other characteristics of the data.", "doc_type":"sqlreference", "kw":"Time Series Forecasting,StreamingML,SQL Syntax Reference", @@ -8834,7 +11690,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Time Series Forecasting", @@ -8844,7 +11702,7 @@ "uri":"dli_08_0216.html", "node_id":"dli_08_0216.xml", "product_code":"dli", - "code":"492", + "code":"596", "des":"Clustering algorithms belong to unsupervised algorithms. K-Means, a clustering algorithm, partitions data points into related clusters by calculating the distance between", "doc_type":"sqlreference", "kw":"Real-Time Clustering,StreamingML,SQL Syntax Reference", @@ -8852,7 +11710,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Real-Time Clustering", @@ -8862,7 +11722,7 @@ "uri":"dli_08_0088.html", "node_id":"dli_08_0088.xml", "product_code":"dli", - "code":"493", + "code":"597", "des":"Deep learning has a wide range of applications in many industries, such as image classification, image recognition, and speech recognition. DLI provides several functions", "doc_type":"sqlreference", "kw":"Deep Learning Model Prediction,StreamingML,SQL Syntax Reference", @@ -8870,7 +11730,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Deep Learning Model Prediction", @@ -8880,7 +11742,7 @@ "uri":"dli_08_0125.html", "node_id":"dli_08_0125.xml", "product_code":"dli", - "code":"494", + "code":"598", "des":"Flink SQL reserves some strings as keywords. If you want to use the following strings as field names, ensure that they are enclosed by back quotes, for example, `value` a", "doc_type":"sqlreference", "kw":"Reserved Keywords,Flink SQL Syntax,SQL Syntax Reference", @@ -8888,7 +11750,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Reserved Keywords", @@ -8898,7 +11762,7 @@ "uri":"dli_08_0001.html", "node_id":"dli_08_0001.xml", "product_code":"dli", - "code":"495", + "code":"599", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Identifiers", @@ -8906,7 +11770,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Identifiers", @@ -8916,15 +11782,17 @@ "uri":"dli_08_0002.html", "node_id":"dli_08_0002.xml", "product_code":"dli", - "code":"496", - "des":"None.Aggregate function.", + "code":"600", + "des":"NoneAggregate function.aggregate_func is typically used in database queries to perform calculations on a set of values and return a single result.", "doc_type":"sqlreference", "kw":"aggregate_func,Identifiers,SQL Syntax Reference", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"aggregate_func", @@ -8934,7 +11802,7 @@ "uri":"dli_08_0003.html", "node_id":"dli_08_0003.xml", "product_code":"dli", - "code":"497", + "code":"601", "des":"None.Alias, which must be STRING type. It can be assigned to a field, table, view, or subquery.", "doc_type":"sqlreference", "kw":"alias,Identifiers,SQL Syntax Reference", @@ -8942,7 +11810,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"alias", @@ -8952,7 +11822,7 @@ "uri":"dli_08_0004.html", "node_id":"dli_08_0004.xml", "product_code":"dli", - "code":"498", + "code":"602", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"attr_expr,Identifiers,SQL Syntax Reference", @@ -8960,7 +11830,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"attr_expr", @@ -8970,7 +11842,7 @@ "uri":"dli_08_0005.html", "node_id":"dli_08_0005.xml", "product_code":"dli", - "code":"499", + "code":"603", "des":"None.List of attr_expr, which is separated by commas (,).", "doc_type":"sqlreference", "kw":"attr_expr_list,Identifiers,SQL Syntax Reference", @@ -8978,7 +11850,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"attr_expr_list", @@ -8988,7 +11862,7 @@ "uri":"dli_08_0006.html", "node_id":"dli_08_0006.xml", "product_code":"dli", - "code":"500", + "code":"604", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"attrs_value_set_expr,Identifiers,SQL Syntax Reference", @@ -8996,7 +11870,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"attrs_value_set_expr", @@ -9006,7 +11882,7 @@ "uri":"dli_08_0007.html", "node_id":"dli_08_0007.xml", "product_code":"dli", - "code":"501", + "code":"605", "des":"None.Return a boolean expression.", "doc_type":"sqlreference", "kw":"boolean_expression,Identifiers,SQL Syntax Reference", @@ -9014,7 +11890,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"boolean_expression", @@ -9024,7 +11902,7 @@ "uri":"dli_08_0009.html", "node_id":"dli_08_0009.xml", "product_code":"dli", - "code":"502", + "code":"606", "des":"None.Formal parameter for function call. It is usually a field name, which is the same as col_name.", "doc_type":"sqlreference", "kw":"col,Identifiers,SQL Syntax Reference", @@ -9032,7 +11910,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"col", @@ -9042,7 +11922,7 @@ "uri":"dli_08_0010.html", "node_id":"dli_08_0010.xml", "product_code":"dli", - "code":"503", + "code":"607", "des":"None.Column (field) description, which must be STRING type and cannot exceed 256 bytes.", "doc_type":"sqlreference", "kw":"col_comment,Identifiers,SQL Syntax Reference", @@ -9050,7 +11930,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"col_comment", @@ -9060,7 +11942,7 @@ "uri":"dli_08_0011.html", "node_id":"dli_08_0011.xml", "product_code":"dli", - "code":"504", + "code":"608", "des":"None.Column name, which must be STRING type and cannot exceed 128 bytes.", "doc_type":"sqlreference", "kw":"col_name,Identifiers,SQL Syntax Reference", @@ -9068,7 +11950,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"col_name", @@ -9078,7 +11962,7 @@ "uri":"dli_08_0012.html", "node_id":"dli_08_0012.xml", "product_code":"dli", - "code":"505", + "code":"609", "des":"None.Field list, which consists of one col_name or more. If there is more than one col_name, separate them by using a comma (,).", "doc_type":"sqlreference", "kw":"col_name_list,Identifiers,SQL Syntax Reference", @@ -9086,7 +11970,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"col_name_list", @@ -9096,7 +11982,7 @@ "uri":"dli_08_0013.html", "node_id":"dli_08_0013.xml", "product_code":"dli", - "code":"506", + "code":"610", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"condition,Identifiers,SQL Syntax Reference", @@ -9104,7 +11990,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"condition", @@ -9114,7 +12002,7 @@ "uri":"dli_08_0014.html", "node_id":"dli_08_0014.xml", "product_code":"dli", - "code":"507", + "code":"611", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"condition_list,Identifiers,SQL Syntax Reference", @@ -9122,7 +12010,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"condition_list", @@ -9132,7 +12022,7 @@ "uri":"dli_08_0015.html", "node_id":"dli_08_0015.xml", "product_code":"dli", - "code":"508", + "code":"612", "des":"None.Common expression name.", "doc_type":"sqlreference", "kw":"cte_name,Identifiers,SQL Syntax Reference", @@ -9140,7 +12030,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"cte_name", @@ -9150,7 +12042,7 @@ "uri":"dli_08_0016.html", "node_id":"dli_08_0016.xml", "product_code":"dli", - "code":"509", + "code":"613", "des":"None.Data type. Currently, only the primitive data types are supported.", "doc_type":"sqlreference", "kw":"data_type,Identifiers,SQL Syntax Reference", @@ -9158,7 +12050,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"data_type", @@ -9168,7 +12062,7 @@ "uri":"dli_08_0017.html", "node_id":"dli_08_0017.xml", "product_code":"dli", - "code":"510", + "code":"614", "des":"None.Database description, which must be STRING type and cannot exceed 256 characters.", "doc_type":"sqlreference", "kw":"db_comment,Identifiers,SQL Syntax Reference", @@ -9176,7 +12070,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"db_comment", @@ -9186,7 +12082,7 @@ "uri":"dli_08_0018.html", "node_id":"dli_08_0018.xml", "product_code":"dli", - "code":"511", + "code":"615", "des":"None.Database name, which must be STRING type and cannot exceed 128 bytes.", "doc_type":"sqlreference", "kw":"db_name,Identifiers,SQL Syntax Reference", @@ -9194,7 +12090,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"db_name", @@ -9204,7 +12102,7 @@ "uri":"dli_08_0019.html", "node_id":"dli_08_0019.xml", "product_code":"dli", - "code":"512", + "code":"616", "des":"None.Returned result for the ELSE clause of the CASE WHEN statement.", "doc_type":"sqlreference", "kw":"else_result_expression,Identifiers,SQL Syntax Reference", @@ -9212,7 +12110,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"else_result_expression", @@ -9222,7 +12122,7 @@ "uri":"dli_08_0020.html", "node_id":"dli_08_0020.xml", "product_code":"dli", - "code":"513", + "code":"617", "des":"| AVRO| CSV| JSON| ORC| PARQUETCurrently, the preceding formats are supported.Both USING and STORED AS can be used for specifying the data format. You can specify the pre", "doc_type":"sqlreference", "kw":"file_format,Identifiers,SQL Syntax Reference", @@ -9230,7 +12130,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"file_format", @@ -9240,7 +12142,7 @@ "uri":"dli_08_0021.html", "node_id":"dli_08_0021.xml", "product_code":"dli", - "code":"514", + "code":"618", "des":"None.File path, which is the OBS path", "doc_type":"sqlreference", "kw":"file_path,Identifiers,SQL Syntax Reference", @@ -9248,7 +12150,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"file_path", @@ -9258,7 +12162,7 @@ "uri":"dli_08_0022.html", "node_id":"dli_08_0022.xml", "product_code":"dli", - "code":"515", + "code":"619", "des":"None.Function name, which must be STRING type.", "doc_type":"sqlreference", "kw":"function_name,Identifiers,SQL Syntax Reference", @@ -9266,7 +12170,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"function_name", @@ -9276,7 +12182,7 @@ "uri":"dli_08_0023.html", "node_id":"dli_08_0023.xml", "product_code":"dli", - "code":"516", + "code":"620", "des":"None.Expression that includes GROUP BY.", "doc_type":"sqlreference", "kw":"groupby_expression,Identifiers,SQL Syntax Reference", @@ -9284,7 +12190,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"groupby_expression", @@ -9294,7 +12202,7 @@ "uri":"dli_08_0024.html", "node_id":"dli_08_0024.xml", "product_code":"dli", - "code":"517", + "code":"621", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"having_condition,Identifiers,SQL Syntax Reference", @@ -9302,7 +12210,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"having_condition", @@ -9312,7 +12222,7 @@ "uri":"dli_08_0026.html", "node_id":"dli_08_0026.xml", "product_code":"dli", - "code":"518", + "code":"622", "des":"None.Input expression of the CASE WHEN statement.", "doc_type":"sqlreference", "kw":"input_expression,Identifiers,SQL Syntax Reference", @@ -9320,7 +12230,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"input_expression", @@ -9330,7 +12242,7 @@ "uri":"dli_08_0029.html", "node_id":"dli_08_0029.xml", "product_code":"dli", - "code":"519", + "code":"623", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"join_condition,Identifiers,SQL Syntax Reference", @@ -9338,7 +12250,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"join_condition", @@ -9348,7 +12262,7 @@ "uri":"dli_08_0030.html", "node_id":"dli_08_0030.xml", "product_code":"dli", - "code":"520", + "code":"624", "des":"None.The condition of an inequality join.", "doc_type":"sqlreference", "kw":"non_equi_join_condition,Identifiers,SQL Syntax Reference", @@ -9356,7 +12270,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"non_equi_join_condition", @@ -9366,7 +12282,7 @@ "uri":"dli_08_0031.html", "node_id":"dli_08_0031.xml", "product_code":"dli", - "code":"521", + "code":"625", "des":"None.Maximum number of output lines specified by LIMIT. Which must be INT type.", "doc_type":"sqlreference", "kw":"number,Identifiers,SQL Syntax Reference", @@ -9374,7 +12290,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"number", @@ -9384,7 +12302,7 @@ "uri":"dli_08_0034.html", "node_id":"dli_08_0034.xml", "product_code":"dli", - "code":"522", + "code":"626", "des":"None.Partition column name, that is, partition field name, which must be STRING type.", "doc_type":"sqlreference", "kw":"partition_col_name,Identifiers,SQL Syntax Reference", @@ -9392,7 +12310,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"partition_col_name", @@ -9402,7 +12322,7 @@ "uri":"dli_08_0035.html", "node_id":"dli_08_0035.xml", "product_code":"dli", - "code":"523", + "code":"627", "des":"None.Partition column value, that is, partition field value.", "doc_type":"sqlreference", "kw":"partition_col_value,Identifiers,SQL Syntax Reference", @@ -9410,7 +12330,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"partition_col_value", @@ -9420,7 +12342,7 @@ "uri":"dli_08_0036.html", "node_id":"dli_08_0036.xml", "product_code":"dli", - "code":"524", + "code":"628", "des":"partition_specs : (partition_col_name = partition_col_value, partition_col_name = partition_col_value, ...);Table partition list, which is expressed by using key=value pa", "doc_type":"sqlreference", "kw":"partition_specs,Identifiers,SQL Syntax Reference", @@ -9428,7 +12350,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"partition_specs", @@ -9438,7 +12362,7 @@ "uri":"dli_08_0037.html", "node_id":"dli_08_0037.xml", "product_code":"dli", - "code":"525", + "code":"629", "des":"None.Property name, which must be STRING type.", "doc_type":"sqlreference", "kw":"property_name,Identifiers,SQL Syntax Reference", @@ -9446,7 +12370,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"property_name", @@ -9456,7 +12382,7 @@ "uri":"dli_08_0038.html", "node_id":"dli_08_0038.xml", "product_code":"dli", - "code":"526", + "code":"630", "des":"None.Property value, which must be STRING type.", "doc_type":"sqlreference", "kw":"property_value,Identifiers,SQL Syntax Reference", @@ -9464,7 +12390,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"property_value", @@ -9474,7 +12402,7 @@ "uri":"dli_08_0039.html", "node_id":"dli_08_0039.xml", "product_code":"dli", - "code":"527", + "code":"631", "des":"None.Pattern matching string, which supports wildcard matching.", "doc_type":"sqlreference", "kw":"regex_expression,Identifiers,SQL Syntax Reference", @@ -9482,7 +12410,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"regex_expression", @@ -9492,7 +12422,7 @@ "uri":"dli_08_0040.html", "node_id":"dli_08_0040.xml", "product_code":"dli", - "code":"528", + "code":"632", "des":"None.Returned result for the THEN clause of the CASE WHEN statement.", "doc_type":"sqlreference", "kw":"result_expression,Identifiers,SQL Syntax Reference", @@ -9500,7 +12430,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"result_expression", @@ -9510,7 +12442,7 @@ "uri":"dli_08_0042.html", "node_id":"dli_08_0042.xml", "product_code":"dli", - "code":"529", + "code":"633", "des":"None.Query clause for the basic SELECT statement.", "doc_type":"sqlreference", "kw":"select_statement,Identifiers,SQL Syntax Reference", @@ -9518,7 +12450,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"select_statement", @@ -9528,7 +12462,7 @@ "uri":"dli_08_0043.html", "node_id":"dli_08_0043.xml", "product_code":"dli", - "code":"530", + "code":"634", "des":"None.Separator, which can be customized by users, for example, comma (,), semicolon (;), and colon (:). Which must be CHAR type.", "doc_type":"sqlreference", "kw":"separator,Identifiers,SQL Syntax Reference", @@ -9536,7 +12470,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"separator", @@ -9546,7 +12482,7 @@ "uri":"dli_08_0045.html", "node_id":"dli_08_0045.xml", "product_code":"dli", - "code":"531", + "code":"635", "des":"None.SQL statement containing the common expression defined by cte_name.", "doc_type":"sqlreference", "kw":"sql_containing_cte_name,Identifiers,SQL Syntax Reference", @@ -9554,7 +12490,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"sql_containing_cte_name", @@ -9564,7 +12502,7 @@ "uri":"dli_08_0046.html", "node_id":"dli_08_0046.xml", "product_code":"dli", - "code":"532", + "code":"636", "des":"None.Subquery.", "doc_type":"sqlreference", "kw":"sub_query,Identifiers,SQL Syntax Reference", @@ -9572,7 +12510,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"sub_query", @@ -9582,7 +12522,7 @@ "uri":"dli_08_0047.html", "node_id":"dli_08_0047.xml", "product_code":"dli", - "code":"533", + "code":"637", "des":"None.Table description, which must be STRING type and cannot exceed 256 bytes.", "doc_type":"sqlreference", "kw":"table_comment,Identifiers,SQL Syntax Reference", @@ -9590,7 +12530,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"table_comment", @@ -9600,7 +12542,7 @@ "uri":"dli_08_0048.html", "node_id":"dli_08_0048.xml", "product_code":"dli", - "code":"534", + "code":"638", "des":"NoneTable name, which cannot exceed 128 bytes. The string type and \"$\" symbol are supported.", "doc_type":"sqlreference", "kw":"table_name,Identifiers,SQL Syntax Reference", @@ -9608,7 +12550,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"table_name", @@ -9618,7 +12562,7 @@ "uri":"dli_08_0049.html", "node_id":"dli_08_0049.xml", "product_code":"dli", - "code":"535", + "code":"639", "des":"None.Table property list, which is expressed by using key=value pairs. key represents property_name, and value represents property_value. If there is more than one key=va", "doc_type":"sqlreference", "kw":"table_properties,Identifiers,SQL Syntax Reference", @@ -9626,7 +12570,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"table_properties", @@ -9636,7 +12582,7 @@ "uri":"dli_08_0050.html", "node_id":"dli_08_0050.xml", "product_code":"dli", - "code":"536", + "code":"640", "des":"None.Table or view name, which must be STRING type. It can also be a subquery. If it is subquery, an alias must also be provided.", "doc_type":"sqlreference", "kw":"table_reference,Identifiers,SQL Syntax Reference", @@ -9644,7 +12590,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"table_reference", @@ -9654,7 +12602,7 @@ "uri":"dli_08_0053.html", "node_id":"dli_08_0053.xml", "product_code":"dli", - "code":"537", + "code":"641", "des":"None.When expression of the CASE WHEN statement. It is used for matching with the input expression.", "doc_type":"sqlreference", "kw":"when_expression,Identifiers,SQL Syntax Reference", @@ -9662,7 +12610,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"when_expression", @@ -9672,7 +12622,7 @@ "uri":"dli_08_0054.html", "node_id":"dli_08_0054.xml", "product_code":"dli", - "code":"538", + "code":"642", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"where_condition,Identifiers,SQL Syntax Reference", @@ -9680,7 +12630,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"where_condition", @@ -9690,7 +12642,7 @@ "uri":"dli_08_0055.html", "node_id":"dli_08_0055.xml", "product_code":"dli", - "code":"539", + "code":"643", "des":"NoneAnalysis window function.", "doc_type":"sqlreference", "kw":"window_function,Identifiers,SQL Syntax Reference", @@ -9698,7 +12650,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"window_function", @@ -9708,7 +12662,7 @@ "uri":"dli_08_0060.html", "node_id":"dli_08_0060.xml", "product_code":"dli", - "code":"540", + "code":"644", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Operators", @@ -9716,7 +12670,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Operators", @@ -9726,7 +12682,7 @@ "uri":"dli_08_0061.html", "node_id":"dli_08_0061.xml", "product_code":"dli", - "code":"541", + "code":"645", "des":"All data types can be compared by using relational operators and the result is returned as a BOOLEAN value.Relationship operators are binary operators. Two compared data ", "doc_type":"sqlreference", "kw":"Relational Operators,Operators,SQL Syntax Reference", @@ -9734,7 +12690,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Relational Operators", @@ -9744,7 +12702,7 @@ "uri":"dli_08_0062.html", "node_id":"dli_08_0062.xml", "product_code":"dli", - "code":"542", + "code":"646", "des":"Arithmetic operators include binary operators and unary operators. For both types of operators, the returned results are numbers. Table 1 lists the arithmetic operators s", "doc_type":"sqlreference", "kw":"Arithmetic Operators,Operators,SQL Syntax Reference", @@ -9752,7 +12710,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Arithmetic Operators", @@ -9762,7 +12722,7 @@ "uri":"dli_08_0063.html", "node_id":"dli_08_0063.xml", "product_code":"dli", - "code":"543", + "code":"647", "des":"Common logical operators include AND, OR, and NOT. The operation result can be TRUE, FALSE, or NULL (which means unknown). The priorities of the operators are as follows:", "doc_type":"sqlreference", "kw":"Logical Operators,Operators,SQL Syntax Reference", @@ -9770,7 +12730,9 @@ "metedata":[ { "prodname":"dli", - "documenttype":"sqlreference" + "documenttype":"sqlreference", + "opensource":"true", + "IsBot":"Yes" } ], "title":"Logical Operators", @@ -9780,7 +12742,7 @@ "uri":"dli_08_00005.html", "node_id":"dli_08_00005.xml", "product_code":"dli", - "code":"544", + "code":"648", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"sqlreference", "kw":"Change History,SQL Syntax Reference", @@ -9789,8 +12751,8 @@ { "prodname":"dli", "documenttype":"sqlreference", - "IsBot":"Yes", - "opensource":"true" + "opensource":"true", + "IsBot":"Yes" } ], "title":"Change History", diff --git a/docs/dli/sqlreference/CLASS.TXT.json b/docs/dli/sqlreference/CLASS.TXT.json index 4116898a3..734d2c19b 100644 --- a/docs/dli/sqlreference/CLASS.TXT.json +++ b/docs/dli/sqlreference/CLASS.TXT.json @@ -65,7 +65,7 @@ { "desc":"This syntax is used to view the information about a specified database, including the database name and database description.EXTENDED: Displays the database properties.If", "product_code":"dli", - "title":"Viewing a Specified Database", + "title":"Checking a Specified Database", "uri":"dli_08_0073.html", "doc_type":"sqlreference", "p_code":"5", @@ -74,7 +74,7 @@ { "desc":"This syntax is used to query all current databases.NoneKeyword DATABASES is equivalent to SCHEMAS. You can use either of them in this statement.View all the current datab", "product_code":"dli", - "title":"Viewing All Databases", + "title":"Checking All Databases", "uri":"dli_08_0074.html", "doc_type":"sqlreference", "p_code":"5", @@ -146,25 +146,25 @@ { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Viewing Tables", + "title":"Checking Tables", "uri":"dli_08_0089.html", "doc_type":"sqlreference", "p_code":"1", "code":"17" }, { - "desc":"This statement is used to view all tables and views in the current database.FROM/IN: followed by the name of a database whose tables and views will be displayed.NoneCreat", + "desc":"This statement is used to check all tables and views in the current database.FROM/IN: followed by the name of a database whose tables and views will be displayed.NoneCrea", "product_code":"dli", - "title":"Viewing All Tables", + "title":"Checking All Tables", "uri":"dli_08_0090.html", "doc_type":"sqlreference", "p_code":"17", "code":"18" }, { - "desc":"This statement is used to show the statements for creating a table.CREATE TABLE: statement for creating a tableThe table specified in this statement must exist. Otherwise", + "desc":"This statement is used to show the statements for creating a table.Use the following syntax to view table creation statements when using Spark 3.3.1 (this applies only to", "product_code":"dli", - "title":"Viewing Table Creation Statements", + "title":"Checking Table Creation Statements", "uri":"dli_08_0091.html", "doc_type":"sqlreference", "p_code":"17", @@ -173,7 +173,7 @@ { "desc":"Check the properties of a table.TBLPROPERTIES: This statement allows you to add a key/value property to a table.property_name is case sensitive. You cannot specify multip", "product_code":"dli", - "title":"Viewing Table Properties", + "title":"Checking Table Properties", "uri":"dli_08_0092.html", "doc_type":"sqlreference", "p_code":"17", @@ -182,25 +182,25 @@ { "desc":"This statement is used to query all columns in a specified table.COLUMNS: columns in the current tableFROM/IN: followed by the name of a database whose tables and views w", "product_code":"dli", - "title":"Viewing All Columns in a Specified Table", + "title":"Checking All Columns in a Specified Table", "uri":"dli_08_0093.html", "doc_type":"sqlreference", "p_code":"17", "code":"21" }, { - "desc":"This statement is used to view all partitions in a specified table.PARTITIONS: partitions in a specified tablePARTITION: a specified partitionThe table specified in this ", + "desc":"This statement is used to check all partitions in a specified table.PARTITIONS: partitions in a specified tablePARTITION: a specified partitionThe table specified in this", "product_code":"dli", - "title":"Viewing All Partitions in a Specified Table", + "title":"Checking All Partitions in a Specified Table", "uri":"dli_08_0094.html", "doc_type":"sqlreference", "p_code":"17", "code":"22" }, { - "desc":"This statement is used to view the table statistics. The names and data types of all columns in a specified table will be returned.EXTENDED: displays all metadata of the ", + "desc":"This statement is used to check table statistics. The names and data types of all columns in a specified table will be returned.EXTENDED: displays all metadata of the spe", "product_code":"dli", - "title":"Viewing Table Statistics", + "title":"Checking Table Statistics", "uri":"dli_08_0105.html", "doc_type":"sqlreference", "p_code":"17", @@ -324,7 +324,7 @@ "code":"36" }, { - "desc":"This statement is used to insert the SELECT query result or a certain data record into a table.Insert the SELECT query result into a table.INSERT INTO [TABLE] [db_name.]t", + "desc":"This statement is used to insert the SELECT query result or a certain data record into a table.The INSERT OVERWRITE syntax is not suitable for \"read-write\" scenarios, whe", "product_code":"dli", "title":"Inserting Data", "uri":"dli_08_0095.html", @@ -332,6 +332,15 @@ "p_code":"1", "code":"37" }, + { + "desc":"To improve query performance, caching the results of a subquery and reusing them in different parts of the query can be done to avoid redundant calculations of the same s", + "product_code":"dli", + "title":"Reusing Results of Subqueries", + "uri":"en-us_topic_0000001873107668.html", + "doc_type":"sqlreference", + "p_code":"1", + "code":"38" + }, { "desc":"This statement is used to delete data from the DLI or OBS table.Only data in the DLI or OBS table can be deleted.", "product_code":"dli", @@ -339,7 +348,7 @@ "uri":"dli_08_0217.html", "doc_type":"sqlreference", "p_code":"1", - "code":"38" + "code":"39" }, { "desc":"This statement is used to directly write query results to a specified directory. The query results can be stored in CSV, Parquet, ORC, JSON, or Avro format.USING: Specifi", @@ -348,7 +357,7 @@ "uri":"dli_08_0205.html", "doc_type":"sqlreference", "p_code":"1", - "code":"39" + "code":"40" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -357,7 +366,7 @@ "uri":"dli_08_0349.html", "doc_type":"sqlreference", "p_code":"1", - "code":"40" + "code":"41" }, { "desc":"After multiversion is enabled, backup data is retained for seven days by default. You can change the retention period by setting system parameterdli.multi.version.retenti", @@ -365,17 +374,17 @@ "title":"Setting the Retention Period for Multiversion Backup Data", "uri":"dli_08_0350.html", "doc_type":"sqlreference", - "p_code":"40", - "code":"41" + "p_code":"41", + "code":"42" }, { "desc":"After the multiversion function is enabled, you can run the SHOW HISTORY command to view the backup data of a table. For details about the syntax for enabling or disablin", "product_code":"dli", - "title":"Viewing Multiversion Backup Data", + "title":"Checking Multiversion Backup Data", "uri":"dli_08_0351.html", "doc_type":"sqlreference", - "p_code":"40", - "code":"42" + "p_code":"41", + "code":"43" }, { "desc":"After the multiversion function is enabled, you can run the RESTORE TABLE statement to restore a table or partition of a specified version. For details about the syntax f", @@ -383,8 +392,8 @@ "title":"Restoring Multiversion Backup Data", "uri":"dli_08_0352.html", "doc_type":"sqlreference", - "p_code":"40", - "code":"43" + "p_code":"41", + "code":"44" }, { "desc":"After the multiversion function is enabled, expired backup data will be directly deleted by the system when theinsert overwrite or truncate statement is executed. You can", @@ -392,8 +401,8 @@ "title":"Configuring the Trash Bin for Expired Multiversion Data", "uri":"dli_08_0353.html", "doc_type":"sqlreference", - "p_code":"40", - "code":"44" + "p_code":"41", + "code":"45" }, { "desc":"The retention period of multiversion backup data takes effect each time the insert overwrite or truncate statement is executed. If neither statement is executed for the t", @@ -401,44 +410,44 @@ "title":"Deleting Multiversion Backup Data", "uri":"dli_08_0355.html", "doc_type":"sqlreference", - "p_code":"40", - "code":"45" + "p_code":"41", + "code":"46" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", "title":"Table Lifecycle Management", - "uri":"en-us_topic_0000001571023676.html", + "uri":"dli_08_0465.html", "doc_type":"sqlreference", "p_code":"1", - "code":"46" + "code":"47" }, { "desc":"DLI provides table lifecycle management to allow you to specify the lifecycle of a table when creating the table. DLI determines whether to reclaim a table based on the t", "product_code":"dli", "title":"Specifying the Lifecycle of a Table When Creating the Table", - "uri":"en-us_topic_0000001621263317.html", + "uri":"dli_08_0466.html", "doc_type":"sqlreference", - "p_code":"46", - "code":"47" + "p_code":"47", + "code":"48" }, { "desc":"This section describes how to modify the lifecycle of an existing partitioned or non-partitioned table.When the lifecycle function is enabled for the first time, the syst", "product_code":"dli", "title":"Modifying the Lifecycle of a Table", - "uri":"en-us_topic_0000001621382957.html", + "uri":"dli_08_0467.html", "doc_type":"sqlreference", - "p_code":"46", - "code":"48" + "p_code":"47", + "code":"49" }, { "desc":"This section describes how to disable or restore the lifecycle of a specified table or partition.You can disable or restore the lifecycle of a table in either of the foll", "product_code":"dli", "title":"Disabling or Restoring the Lifecycle of a Table", - "uri":"en-us_topic_0000001621542965.html", + "uri":"dli_08_0468.html", "doc_type":"sqlreference", - "p_code":"46", - "code":"49" + "p_code":"47", + "code":"50" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -447,7 +456,7 @@ "uri":"dli_08_0118.html", "doc_type":"sqlreference", "p_code":"1", - "code":"50" + "code":"51" }, { "desc":"This statement is used to create a DLI table and associate it with an existing HBase table.In Spark cross-source development scenarios, there is a risk of password leakag", @@ -455,8 +464,8 @@ "title":"Creating a DLI Table and Associating It with HBase", "uri":"dli_08_0119.html", "doc_type":"sqlreference", - "p_code":"50", - "code":"51" + "p_code":"51", + "code":"52" }, { "desc":"This statement is used to insert data in a DLI table to the associated HBase table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field", @@ -464,8 +473,8 @@ "title":"Inserting Data to an HBase Table", "uri":"dli_08_0120.html", "doc_type":"sqlreference", - "p_code":"50", - "code":"52" + "p_code":"51", + "code":"53" }, { "desc":"This statement is used to query data in an HBase table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.The table to be querie", @@ -473,8 +482,8 @@ "title":"Querying an HBase Table", "uri":"dli_08_0121.html", "doc_type":"sqlreference", - "p_code":"50", - "code":"53" + "p_code":"51", + "code":"54" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -483,7 +492,7 @@ "uri":"dli_08_0220.html", "doc_type":"sqlreference", "p_code":"1", - "code":"54" + "code":"55" }, { "desc":"Run the CREATE TABLE statement to create the DLI table and associate it with the existing metric in OpenTSDB. This syntax supports the OpenTSDB of CloudTable and MRS.Befo", @@ -491,8 +500,8 @@ "title":"Creating a DLI Table and Associating It with OpenTSDB", "uri":"dli_08_0122.html", "doc_type":"sqlreference", - "p_code":"54", - "code":"55" + "p_code":"55", + "code":"56" }, { "desc":"Run the INSERT INTO statement to insert the data in the DLI table to the associated OpenTSDB metric.If no metric exists on the OpenTSDB, a new metric is automatically cre", @@ -500,8 +509,8 @@ "title":"Inserting Data to the OpenTSDB Table", "uri":"dli_08_0123.html", "doc_type":"sqlreference", - "p_code":"54", - "code":"56" + "p_code":"55", + "code":"57" }, { "desc":"This SELECT command is used to query data in an OpenTSDB table.If no metric exists in OpenTSDB, an error will be reported when the corresponding DLI table is queried.If t", @@ -509,8 +518,8 @@ "title":"Querying an OpenTSDB Table", "uri":"dli_08_0124.html", "doc_type":"sqlreference", - "p_code":"54", - "code":"57" + "p_code":"55", + "code":"58" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -519,16 +528,16 @@ "uri":"dli_08_0192.html", "doc_type":"sqlreference", "p_code":"1", - "code":"58" + "code":"59" }, { - "desc":"This statement is used to create a DLI table and associate it with an existing DWS table.In Spark cross-source development scenarios, there is a risk of password leakage ", + "desc":"This statement is used to create a DLI table and associate it with an existing GaussDB(DWS) table.In Spark cross-source development scenarios, there is a risk of password", "product_code":"dli", - "title":"Creating a DLI Table and Associating It with DWS", + "title":"Creating a DLI Table and Associating It with GaussDB(DWS)", "uri":"dli_08_0193.html", "doc_type":"sqlreference", - "p_code":"58", - "code":"59" + "p_code":"59", + "code":"60" }, { "desc":"This statement is used to insert data in a DLI table to the associated DWS table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2.", @@ -536,8 +545,8 @@ "title":"Inserting Data to the DWS Table", "uri":"dli_08_0194.html", "doc_type":"sqlreference", - "p_code":"58", - "code":"60" + "p_code":"59", + "code":"61" }, { "desc":"This statement is used to query data in a DWS table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.The table to be queried m", @@ -545,8 +554,8 @@ "title":"Querying the DWS Table", "uri":"dli_08_0195.html", "doc_type":"sqlreference", - "p_code":"58", - "code":"61" + "p_code":"59", + "code":"62" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -555,7 +564,7 @@ "uri":"dli_08_0196.html", "doc_type":"sqlreference", "p_code":"1", - "code":"62" + "code":"63" }, { "desc":"This statement is used to create a DLI table and associate it with an existing RDS table. This function supports access to the MySQL and PostgreSQL clusters of RDS.In Spa", @@ -563,8 +572,8 @@ "title":"Creating a DLI Table and Associating It with RDS", "uri":"dli_08_0197.html", "doc_type":"sqlreference", - "p_code":"62", - "code":"63" + "p_code":"63", + "code":"64" }, { "desc":"This statement is used to insert data in a DLI table to the associated RDS table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2.", @@ -572,8 +581,8 @@ "title":"Inserting Data to the RDS Table", "uri":"dli_08_0198.html", "doc_type":"sqlreference", - "p_code":"62", - "code":"64" + "p_code":"63", + "code":"65" }, { "desc":"This statement is used to query data in an RDS table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.The table to be queried ", @@ -581,8 +590,8 @@ "title":"Querying the RDS Table", "uri":"dli_08_0199.html", "doc_type":"sqlreference", - "p_code":"62", - "code":"65" + "p_code":"63", + "code":"66" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -591,7 +600,7 @@ "uri":"dli_08_0200.html", "doc_type":"sqlreference", "p_code":"1", - "code":"66" + "code":"67" }, { "desc":"This statement is used to create a DLI table and associate it with an existing CSS table.In Spark cross-source development scenarios, there is a risk of password leakage ", @@ -599,8 +608,8 @@ "title":"Creating a DLI Table and Associating It with CSS", "uri":"dli_08_0201.html", "doc_type":"sqlreference", - "p_code":"66", - "code":"67" + "p_code":"67", + "code":"68" }, { "desc":"This statement is used to insert data in a DLI table to the associated CSS table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2.", @@ -608,8 +617,8 @@ "title":"Inserting Data to the CSS Table", "uri":"dli_08_0202.html", "doc_type":"sqlreference", - "p_code":"66", - "code":"68" + "p_code":"67", + "code":"69" }, { "desc":"This statement is used to query data in a CSS table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.The table to be queried m", @@ -617,8 +626,8 @@ "title":"Querying the CSS Table", "uri":"dli_08_0203.html", "doc_type":"sqlreference", - "p_code":"66", - "code":"69" + "p_code":"67", + "code":"70" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -627,7 +636,7 @@ "uri":"dli_08_0225.html", "doc_type":"sqlreference", "p_code":"1", - "code":"70" + "code":"71" }, { "desc":"This statement is used to create a DLI table and associate it with an existing DCS key.In Spark cross-source development scenarios, there is a risk of password leakage if", @@ -635,8 +644,8 @@ "title":"Creating a DLI Table and Associating It with DCS", "uri":"dli_08_0226.html", "doc_type":"sqlreference", - "p_code":"70", - "code":"71" + "p_code":"71", + "code":"72" }, { "desc":"This statement is used to insert data in a DLI table to the DCS key.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2...\n [FROM DL", @@ -644,8 +653,8 @@ "title":"Inserting Data to a DCS Table", "uri":"dli_08_0227.html", "doc_type":"sqlreference", - "p_code":"70", - "code":"72" + "p_code":"71", + "code":"73" }, { "desc":"This statement is used to query data in a DCS table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.Query data in the test_re", @@ -653,8 +662,8 @@ "title":"Querying the DCS Table", "uri":"dli_08_0228.html", "doc_type":"sqlreference", - "p_code":"70", - "code":"73" + "p_code":"71", + "code":"74" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -663,7 +672,7 @@ "uri":"dli_08_0229.html", "doc_type":"sqlreference", "p_code":"1", - "code":"74" + "code":"75" }, { "desc":"This statement is used to create a DLI table and associate it with an existing DDS collection.In Spark cross-source development scenarios, there is a risk of password lea", @@ -671,8 +680,8 @@ "title":"Creating a DLI Table and Associating It with DDS", "uri":"dli_08_0230.html", "doc_type":"sqlreference", - "p_code":"74", - "code":"75" + "p_code":"75", + "code":"76" }, { "desc":"This statement is used to insert data in a DLI table to the associated DDS table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2.", @@ -680,8 +689,8 @@ "title":"Inserting Data to the DDS Table", "uri":"dli_08_0231.html", "doc_type":"sqlreference", - "p_code":"74", - "code":"76" + "p_code":"75", + "code":"77" }, { "desc":"This statement is used to query data in a DDS table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.If schema information is ", @@ -689,8 +698,8 @@ "title":"Querying the DDS Table", "uri":"dli_08_0232.html", "doc_type":"sqlreference", - "p_code":"74", - "code":"77" + "p_code":"75", + "code":"78" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -699,7 +708,7 @@ "uri":"dli_08_0460.html", "doc_type":"sqlreference", "p_code":"1", - "code":"78" + "code":"79" }, { "desc":"This statement is used to create a DLI table and associate it with an existing Oracle table.Before creating a DLI table and associating it with Oracle, you need to create", @@ -707,8 +716,8 @@ "title":"Creating a DLI Table and Associating It with Oracle", "uri":"dli_08_0461.html", "doc_type":"sqlreference", - "p_code":"78", - "code":"79" + "p_code":"79", + "code":"80" }, { "desc":"This statement is used to insert data into an associated Oracle table.Insert the SELECT query result into a table.INSERT INTO DLI_TABLE\n SELECT field1,field2...\n [FROM ", @@ -716,8 +725,8 @@ "title":"Inserting Data to an Oracle Table", "uri":"dli_08_0462.html", "doc_type":"sqlreference", - "p_code":"78", - "code":"80" + "p_code":"79", + "code":"81" }, { "desc":"This statement is used to query data in an Oracle table.LIMIT is used to limit the query results. Only INT type is supported by the number parameter.If schema information", @@ -725,8 +734,8 @@ "title":"Querying an Oracle Table", "uri":"dli_08_0463.html", "doc_type":"sqlreference", - "p_code":"78", - "code":"81" + "p_code":"79", + "code":"82" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -735,7 +744,7 @@ "uri":"dli_08_0129.html", "doc_type":"sqlreference", "p_code":"1", - "code":"82" + "code":"83" }, { "desc":"This statement is used to create views.CREATE VIEW: creates views based on the given select statement. The result of the select statement will not be written into the dis", @@ -743,8 +752,8 @@ "title":"Creating a View", "uri":"dli_08_0130.html", "doc_type":"sqlreference", - "p_code":"82", - "code":"83" + "p_code":"83", + "code":"84" }, { "desc":"This statement is used to delete views.DROP: Deletes the metadata of a specified view. Although views and tables have many common points, the DROP TABLE statement cannot ", @@ -752,17 +761,17 @@ "title":"Deleting a View", "uri":"dli_08_0131.html", "doc_type":"sqlreference", - "p_code":"82", - "code":"84" + "p_code":"83", + "code":"85" }, { "desc":"This statement returns the logical plan and physical execution plan for the SQL statement.EXTENDED: After this keyword is specified, the logical and physical plans are ou", "product_code":"dli", - "title":"Viewing the Execution Plan", + "title":"Checking the Execution Plan", "uri":"dli_08_0138.html", "doc_type":"sqlreference", "p_code":"1", - "code":"85" + "code":"86" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -771,7 +780,7 @@ "uri":"dli_08_0139.html", "doc_type":"sqlreference", "p_code":"1", - "code":"86" + "code":"87" }, { "desc":"Table 1 describes the SQL statement permission matrix in DLI in terms of permissions on databases, tables, and roles.For privilege granting or revocation on databases and", @@ -779,8 +788,8 @@ "title":"Data Permissions List", "uri":"dli_08_0140.html", "doc_type":"sqlreference", - "p_code":"86", - "code":"87" + "p_code":"87", + "code":"88" }, { "desc":"This statement is used to create a role in the current database or a specified database.Only users with the CREATE_ROLE permission on the database can create roles. For e", @@ -788,8 +797,8 @@ "title":"Creating a Role", "uri":"dli_08_0141.html", "doc_type":"sqlreference", - "p_code":"86", - "code":"88" + "p_code":"87", + "code":"89" }, { "desc":"This statement is used to delete a role in the current database or a specified database.NoneThe role_name to be deleted must exist in the current database or the specifie", @@ -797,8 +806,8 @@ "title":"Deleting a Role", "uri":"dli_08_0148.html", "doc_type":"sqlreference", - "p_code":"86", - "code":"89" + "p_code":"87", + "code":"90" }, { "desc":"This statement is used to bind a user with a role.NoneThe role_name and username must exist. Otherwise, an error will be reported.", @@ -806,8 +815,8 @@ "title":"Binding a Role", "uri":"dli_08_0142.html", "doc_type":"sqlreference", - "p_code":"86", - "code":"90" + "p_code":"87", + "code":"91" }, { "desc":"This statement is used to unbind the user with the role.Nonerole_name and user_name must exist and user_name has been bound to role_name.To unbind the user_name1 from rol", @@ -815,8 +824,8 @@ "title":"Unbinding a Role", "uri":"dli_08_0147.html", "doc_type":"sqlreference", - "p_code":"86", - "code":"91" + "p_code":"87", + "code":"92" }, { "desc":"This statement is used to display all roles or roles bound to the user_name in the current database.ALL: Displays all roles.Keywords ALL and user_name cannot coexist.To d", @@ -824,8 +833,8 @@ "title":"Displaying a Role", "uri":"dli_08_0143.html", "doc_type":"sqlreference", - "p_code":"86", - "code":"92" + "p_code":"87", + "code":"93" }, { "desc":"This statement is used to grant permissions to a user or role.ROLE: The subsequent role_name must be a role.USER: The subsequent user_name must be a user.The privilege mu", @@ -833,8 +842,8 @@ "title":"Granting a Permission", "uri":"dli_08_0144.html", "doc_type":"sqlreference", - "p_code":"86", - "code":"93" + "p_code":"87", + "code":"94" }, { "desc":"This statement is used to revoke permissions granted to a user or role.ROLE: The subsequent role_name must be a role.USER: The subsequent user_name must be a user.The pri", @@ -842,8 +851,8 @@ "title":"Revoking a Permission", "uri":"dli_08_0146.html", "doc_type":"sqlreference", - "p_code":"86", - "code":"94" + "p_code":"87", + "code":"95" }, { "desc":"This statement is used to show the permissions granted to a user on a resource.USER: The subsequent user_name must be a user.The resource can be a queue, database, table,", @@ -851,8 +860,8 @@ "title":"Showing Granted Permissions", "uri":"dli_08_0145.html", "doc_type":"sqlreference", - "p_code":"86", - "code":"95" + "p_code":"87", + "code":"96" }, { "desc":"This statement is used to display the binding relationship between roles and a user in the current database.NoneThe ROLE variable must exist.", @@ -860,8 +869,8 @@ "title":"Displaying the Binding Relationship Between All Roles and Users", "uri":"dli_08_0149.html", "doc_type":"sqlreference", - "p_code":"86", - "code":"96" + "p_code":"87", + "code":"97" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -870,7 +879,7 @@ "uri":"dli_08_0056.html", "doc_type":"sqlreference", "p_code":"1", - "code":"97" + "code":"98" }, { "desc":"Data type is a basic attribute of data. It is used to distinguish different types of data. Different data types occupy different storage space and support different opera", @@ -878,8 +887,8 @@ "title":"Overview", "uri":"dli_08_0057.html", "doc_type":"sqlreference", - "p_code":"97", - "code":"98" + "p_code":"98", + "code":"99" }, { "desc":"Table 1 lists the primitive data types supported by DLI.VARCHAR and CHAR data is stored in STRING type on DLI. Therefore, the string that exceeds the specified length wil", @@ -887,8 +896,8 @@ "title":"Primitive Data Types", "uri":"dli_08_0058.html", "doc_type":"sqlreference", - "p_code":"97", - "code":"99" + "p_code":"98", + "code":"100" }, { "desc":"Spark SQL supports complex data types, as shown in Table 1.When a table containing fields of the complex data type is created, the storage format of this table cannot be ", @@ -896,8 +905,8 @@ "title":"Complex Data Types", "uri":"dli_08_0059.html", "doc_type":"sqlreference", - "p_code":"97", - "code":"100" + "p_code":"98", + "code":"101" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -906,7 +915,7 @@ "uri":"dli_08_0282.html", "doc_type":"sqlreference", "p_code":"1", - "code":"101" + "code":"102" }, { "desc":"DLI allows you to create and use user-defined functions (UDF) and user-defined table functions (UDTF) in Spark jobs.OrIf a function with the same name exists in the datab", @@ -914,8 +923,8 @@ "title":"Creating a Function", "uri":"dli_08_0283.html", "doc_type":"sqlreference", - "p_code":"101", - "code":"102" + "p_code":"102", + "code":"103" }, { "desc":"This statement is used to delete functions.TEMPORARY: Indicates whether the function to be deleted is a temporary function.IF EXISTS: Used when the function to be deleted", @@ -923,8 +932,8 @@ "title":"Deleting a Function", "uri":"dli_08_0284.html", "doc_type":"sqlreference", - "p_code":"101", - "code":"103" + "p_code":"102", + "code":"104" }, { "desc":"Displays information about a specified function.EXTENDED: displays extended usage information.The metadata (implementation class and usage) of an existing function is ret", @@ -932,8 +941,8 @@ "title":"Displaying Function Details", "uri":"dli_08_0281.html", "doc_type":"sqlreference", - "p_code":"101", - "code":"104" + "p_code":"102", + "code":"105" }, { "desc":"View all functions in the current project.In the preceding statement, regex is a regular expression. For details about its parameters, see Table 1.For details about other", @@ -941,8 +950,8 @@ "title":"Displaying All Functions", "uri":"dli_08_0285.html", "doc_type":"sqlreference", - "p_code":"101", - "code":"105" + "p_code":"102", + "code":"106" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -951,7 +960,7 @@ "uri":"dli_08_0064.html", "doc_type":"sqlreference", "p_code":"1", - "code":"106" + "code":"107" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -959,8 +968,8 @@ "title":"Date Functions", "uri":"dli_08_0471.html", "doc_type":"sqlreference", - "p_code":"106", - "code":"107" + "p_code":"107", + "code":"108" }, { "desc":"Table 1 lists the date functions supported by DLI.", @@ -968,8 +977,8 @@ "title":"Overview", "uri":"dli_08_0066.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"108" + "p_code":"108", + "code":"109" }, { "desc":"This function is used to calculate the date after a date value is increased by a specified number of months. That is, it calculates the data that is num_months after star", @@ -977,8 +986,8 @@ "title":"add_months", "uri":"dli_spark_add_months.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"109" + "p_code":"108", + "code":"110" }, { "desc":"This function is used to return the current date, in the yyyy-mm-dd format.Similar function: getdate. The getdate function is used to return the current system time, in t", @@ -986,8 +995,8 @@ "title":"current_date", "uri":"dli_spark_current_date.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"110" + "p_code":"108", + "code":"111" }, { "desc":"This function is used to return the current timestamp.NoneThe return value is of the TIMESTAMP type.The value 1692002816300 is returned.", @@ -995,8 +1004,8 @@ "title":"current_timestamp", "uri":"dli_spark_current_timestamp.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"111" + "p_code":"108", + "code":"112" }, { "desc":"This function is used to calculate the number of days in which start_date is increased by days.To obtain the date with a specified change range based on the current date,", @@ -1004,8 +1013,8 @@ "title":"date_add", "uri":"dli_spark_date_add.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"112" + "p_code":"108", + "code":"113" }, { "desc":"This function is used to change a date based on datepart and delta.To obtain the date with a specified change range based on the current date, use this function together ", @@ -1013,8 +1022,8 @@ "title":"dateadd", "uri":"dli_spark_dateadd.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"113" + "p_code":"108", + "code":"114" }, { "desc":"This function is used to calculate the number of days in which start_date is subtracted by days.To obtain the date with a specified change range based on the current date", @@ -1022,8 +1031,8 @@ "title":"date_sub", "uri":"dli_spark_date_sub.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"114" + "p_code":"108", + "code":"115" }, { "desc":"This function is used to convert a date into a string based on the format specified by format.The return value is of the STRING type.If the value of date is not of the DA", @@ -1031,8 +1040,8 @@ "title":"date_format", "uri":"dli_spark_date_format.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"115" + "p_code":"108", + "code":"116" }, { "desc":"This function is used to calculate the difference between date1 and date2.Similar function: datediff1. The datediff1 function is used to calculate the difference between ", @@ -1040,8 +1049,8 @@ "title":"datediff", "uri":"dli_spark_datediff.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"116" + "p_code":"108", + "code":"117" }, { "desc":"This function is used to calculate the difference between date1 and date2 and return the difference in a specified datepart.Similar function: datediff. The datediff funct", @@ -1049,8 +1058,8 @@ "title":"datediff1", "uri":"dli_spark_datediff1.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"117" + "p_code":"108", + "code":"118" }, { "desc":"This function is used to calculate the value that meets the specified datepart in date.The return value is of the BIGINT type.If the value of date is not of the DATE or S", @@ -1058,8 +1067,8 @@ "title":"datepart", "uri":"dli_spark_datepart.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"118" + "p_code":"108", + "code":"119" }, { "desc":"This function is used to calculate the date otained through the truncation of a specified date based on a specified datepart.It truncates the date before the specified da", @@ -1067,8 +1076,8 @@ "title":"datetrunc", "uri":"dli_spark_datetrunc.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"119" + "p_code":"108", + "code":"120" }, { "desc":"This function is used to return the day of a specified date.The return value is of the INT type.If the value of date is not of the DATE or STRING type, the error message ", @@ -1076,8 +1085,8 @@ "title":"day/dayofmonth", "uri":"dli_spark_daydayofmonth.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"120" + "p_code":"108", + "code":"121" }, { "desc":"This function is used to convert a timestamp represented by a numeric UNIX value to a date value.The return value is of the STRING type, in the yyyy-mm-dd hh:mi:ss format", @@ -1085,8 +1094,8 @@ "title":"from_unixtime", "uri":"dli_spark_from_unixtime.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"121" + "p_code":"108", + "code":"122" }, { "desc":"This function is used to convert a UTC timestamp to a UNIX timestamp in a given time zone.The return value is of the TIMESTAMP type.If the value of timestamp is not of th", @@ -1094,8 +1103,8 @@ "title":"from_utc_timestamp", "uri":"dli_spark_from_utc_timestamp.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"122" + "p_code":"108", + "code":"123" }, { "desc":"This function is used to return the current system time, in the yyyy-mm-dd hh:mi:ss format.Similar function: current_date. The current_date function is used to return the", @@ -1103,8 +1112,8 @@ "title":"getdate", "uri":"dli_spark_getdate.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"123" + "p_code":"108", + "code":"124" }, { "desc":"This function is used to return the hour (from 0 to 23) of a specified time.The return value is of the INT type.If the value of date is not of the DATE or STRING type, th", @@ -1112,8 +1121,8 @@ "title":"hour", "uri":"dli_spark_hour.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"124" + "p_code":"108", + "code":"125" }, { "desc":"This function is used to determine whether a date string can be converted into a date value based on a specified format.The return value is of the BOOLEAN type.If the val", @@ -1121,8 +1130,8 @@ "title":"isdate", "uri":"dli_spark_isdate.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"125" + "p_code":"108", + "code":"126" }, { "desc":"This function is used to return the last day of the month a date belongs to.Similar function: lastday. The lastday function is used to return the last day of the month a ", @@ -1130,8 +1139,8 @@ "title":"last_day", "uri":"dli_spark_last_day.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"126" + "p_code":"108", + "code":"127" }, { "desc":"This function is used to return the last day of the month a date belongs to. The hour, minute, and second part is 00:00:00.Similar function: last_day. The last_day functi", @@ -1139,8 +1148,8 @@ "title":"lastday", "uri":"dli_spark_lastday.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"127" + "p_code":"108", + "code":"128" }, { "desc":"This function is used to return the minute (from 0 to 59) of a specified time.The return value is of the INT type.If the value of date is not of the DATE or STRING type, ", @@ -1148,8 +1157,8 @@ "title":"minute", "uri":"dli_spark_minute.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"128" + "p_code":"108", + "code":"129" }, { "desc":"This function is used to return the month (from January to December) of a specified time.The return value is of the INT type.If the value of date is not of the DATE or ST", @@ -1157,8 +1166,8 @@ "title":"month", "uri":"dli_spark_month.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"129" + "p_code":"108", + "code":"130" }, { "desc":"This function returns the month difference between date1 and date2.The return value is of the DOUBLE type.If the values of date1 and date2 are not of the DATE or STRING t", @@ -1166,8 +1175,8 @@ "title":"months_between", "uri":"dli_spark_months_between.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"130" + "p_code":"108", + "code":"131" }, { "desc":"This function is used to return the date closest to day_of_week after start_date.The return value is of the DATE type, in the yyyy-mm-dd format.If the value of start_date", @@ -1175,8 +1184,8 @@ "title":"next_day", "uri":"dli_spark_next_day.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"131" + "p_code":"108", + "code":"132" }, { "desc":"This function is used to return the quarter of a date. The value ranges from 1 to 4.The return value is of the INT type.If the value of date is not of the DATE or STRING ", @@ -1184,8 +1193,8 @@ "title":"quarter", "uri":"dli_spark_quarter.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"132" + "p_code":"108", + "code":"133" }, { "desc":"This function is used to return the second (from 0 to 59) of a specified time.The return value is of the INT type.If the value of date is not of the DATE or STRING type, ", @@ -1193,8 +1202,8 @@ "title":"second", "uri":"dli_spark_second.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"133" + "p_code":"108", + "code":"134" }, { "desc":"This function is used to convert a date into a string in a specified format.The return value is of the STRING type.If the value of date is not of the DATE or STRING type,", @@ -1202,8 +1211,8 @@ "title":"to_char", "uri":"dli_spark_to_char.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"134" + "p_code":"108", + "code":"135" }, { "desc":"This function is used to return the year, month, and day in a time.Similar function: to_date1. The to_date1 function is used to convert a string in a specified format to ", @@ -1211,8 +1220,8 @@ "title":"to_date", "uri":"dli_spark_to_date.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"135" + "p_code":"108", + "code":"136" }, { "desc":"This function is used to convert a string in a specified format to a date value.Similar function: to_date. The to_date function is used to return the year, month, and day", @@ -1220,8 +1229,8 @@ "title":"to_date1", "uri":"dli_spark_to_date1.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"136" + "p_code":"108", + "code":"137" }, { "desc":"This function is used to convert a timestamp in a given time zone to a UTC timestamp.The return value is of the BIGINT type.If the value of timestamp is not of the DATE o", @@ -1229,8 +1238,8 @@ "title":"to_utc_timestamp", "uri":"dli_spark_to_utc_timestamp.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"137" + "p_code":"108", + "code":"138" }, { "desc":"This function is used to reset a date to a specific format.Resetting means returning to default values, where the default values for year, month, and day are 01, and the ", @@ -1238,8 +1247,8 @@ "title":"trunc", "uri":"dli_spark_trunc.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"138" + "p_code":"108", + "code":"139" }, { "desc":"This function is used to convert a date value to a numeric date value in UNIX format.The function returns the first ten digits of the timestamp in normal UNIX format.The ", @@ -1247,8 +1256,8 @@ "title":"unix_timestamp", "uri":"dli_spark_unix_timestamp.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"139" + "p_code":"108", + "code":"140" }, { "desc":"This function is used to return the day of the current week.The return value is of the INT type.If Monday is used as the first day of a week, the value 0 is returned. For", @@ -1256,8 +1265,8 @@ "title":"weekday", "uri":"dli_spark_weekday.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"140" + "p_code":"108", + "code":"141" }, { "desc":"This function is used to return the week number (from 0 to 53) of a specified date.The return value is of the INT type.If the value of date is not of the DATE or STRING t", @@ -1265,8 +1274,8 @@ "title":"weekofyear", "uri":"dli_spark_weekofyear.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"141" + "p_code":"108", + "code":"142" }, { "desc":"This function is used to return the year of a specified date.The return value is of the INT type.If the value of date is not of the DATE or STRING type, the error message", @@ -1274,8 +1283,8 @@ "title":"year", "uri":"dli_spark_year.html", "doc_type":"sqlreference", - "p_code":"107", - "code":"142" + "p_code":"108", + "code":"143" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1283,8 +1292,8 @@ "title":"String Functions", "uri":"dli_08_0472.html", "doc_type":"sqlreference", - "p_code":"106", - "code":"143" + "p_code":"107", + "code":"144" }, { "desc":"Table 1 lists the string functions supported by DLI.", @@ -1292,8 +1301,8 @@ "title":"Overview", "uri":"dli_08_0067.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"144" + "p_code":"144", + "code":"145" }, { "desc":"This function is used to return the ASCII code of the first character in str.The return value is of the BIGINT type.If the value of str is not of the STRING, BIGINT, DOUB", @@ -1301,8 +1310,8 @@ "title":"ascii", "uri":"dli_spark_ascii.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"145" + "p_code":"144", + "code":"146" }, { "desc":"This function is used to concatenate arrays or strings.If multiple arrays are used as the input, all elements in the arrays are connected to generate a new array.If multi", @@ -1310,8 +1319,8 @@ "title":"concat", "uri":"dli_spark_concat.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"146" + "p_code":"144", + "code":"147" }, { "desc":"This function is used to return a string concatenated from multiple input strings that are separated by specified separators.orReturns the result of joining all the strin", @@ -1319,8 +1328,8 @@ "title":"concat_ws", "uri":"dli_spark_concat_ws.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"147" + "p_code":"144", + "code":"148" }, { "desc":"This parameter is used to return the number of characters in str1 that appear in str2.The return value is of the BIGINT type.If the value of str1 or str2 is NULL, NULL is", @@ -1328,8 +1337,8 @@ "title":"char_matchcount", "uri":"dli_spark_char_matchcount.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"148" + "p_code":"144", + "code":"149" }, { "desc":"This function is used to encode str in charset format.encode(string , string )The return value is of the BINARY type.If the value of str or charset is NULL,", @@ -1337,8 +1346,8 @@ "title":"encode", "uri":"dli_spark_encode.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"149" + "p_code":"144", + "code":"150" }, { "desc":"This function is used to return the position (stating from 1) of str1 in str2 separated by commas (,).The return value is of the BIGINT type.If str1 cannot be matched in ", @@ -1346,8 +1355,8 @@ "title":"find_in_set", "uri":"dli_spark_find_in_set.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"150" + "p_code":"144", + "code":"151" }, { "desc":"This function is used to parse the JSON object in a specified JSON path. The function will return NULL if the JSON object is invalid.The return value is of the STRING typ", @@ -1355,8 +1364,8 @@ "title":"get_json_object", "uri":"dli_spark_get_json_object.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"151" + "p_code":"144", + "code":"152" }, { "desc":"This function is used to return the index of substr that appears earliest in str.It returns NULL if either of the arguments are NULL and returns 0 if substr does not exis", @@ -1364,8 +1373,8 @@ "title":"instr", "uri":"dli_spark_instr.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"152" + "p_code":"144", + "code":"153" }, { "desc":"This function is used to return the position of substring str2 in string str1.Similar function: instr. The instr function is used to return the index of substr that appea", @@ -1373,8 +1382,8 @@ "title":"instr1", "uri":"dli_spark_instr1.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"153" + "p_code":"144", + "code":"154" }, { "desc":"This function is used to convert the first letter of each word of a string to upper case and all other letters to lower case.The return value is of the STRING type. In th", @@ -1382,8 +1391,8 @@ "title":"initcap", "uri":"dli_spark_initcap.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"154" + "p_code":"144", + "code":"155" }, { "desc":"This function is used to split str by split1, convert each group into a key-value pair by split2, and return the value corresponding to the key.The return value is of the", @@ -1391,8 +1400,8 @@ "title":"keyvalue", "uri":"dli_spark_keyvalue.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"155" + "p_code":"144", + "code":"156" }, { "desc":"This function is used to return the length of a string.Similar function: lengthb. The lengthb function is used to return the length of string str in bytes and return a va", @@ -1400,8 +1409,8 @@ "title":"length", "uri":"dli_spark_length.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"156" + "p_code":"144", + "code":"157" }, { "desc":"This function is used to return the length of a specified string in bytes.Similar function: length. The length function is used to return the length of a string and retur", @@ -1409,8 +1418,8 @@ "title":"lengthb", "uri":"dli_spark_lengthb.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"157" + "p_code":"144", + "code":"158" }, { "desc":"This function is used to returns the Levenshtein distance between two strings, for example, levenshtein('kitten','sitting') = 3.Levenshtein distance is a type of edit dis", @@ -1418,8 +1427,8 @@ "title":"levenshtein", "uri":"dli_spark_levenshtein.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"158" + "p_code":"144", + "code":"159" }, { "desc":"This function is used to return the position of substr in str. You can specify the starting position of your search using \"start_pos,\" which starts from 1.The return valu", @@ -1427,8 +1436,8 @@ "title":"locate", "uri":"dli_spark_locate.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"159" + "p_code":"144", + "code":"160" }, { "desc":"This function is used to convert all characters of a string to the lower case.The return value is of the STRING type.If the value of the input parameter is not of the STR", @@ -1436,8 +1445,8 @@ "title":"lower/lcase", "uri":"dli_spark_lower_lcase.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"160" + "p_code":"144", + "code":"161" }, { "desc":"This function is used to return a string of a specified length. If the length of the given string (str1) is shorter than the specified length (length), the given string i", @@ -1445,8 +1454,8 @@ "title":"lpad", "uri":"dli_spark_lpad.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"161" + "p_code":"144", + "code":"162" }, { "desc":"This function is used to remove characters from the left of str.If trimChars is not specified, spaces are removed by default.If trimChars is specified, the function remov", @@ -1454,8 +1463,8 @@ "title":"ltrim", "uri":"dli_spark_ltrim.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"162" + "p_code":"144", + "code":"163" }, { "desc":"This character is used to return the specified part of a given URL. Valid values of partToExtract include HOST, PATH, QUERY, REF, PROTOCOL, AUTHORITY, FILE, and USERINFO.", @@ -1463,8 +1472,8 @@ "title":"parse_url", "uri":"dli_spark_parse_url.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"163" + "p_code":"144", + "code":"164" }, { "desc":"This function is used to print the input in a specific format.The return value is of the STRING type.The value is returned after the parameters that filled in Obj are spe", @@ -1472,8 +1481,8 @@ "title":"printf", "uri":"dli_spark_printf.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"164" + "p_code":"144", + "code":"165" }, { "desc":"This function is used to return the number of substrings that match a specified pattern in the source, starting from the start_position position.The return value is of th", @@ -1481,8 +1490,8 @@ "title":"regexp_count", "uri":"dli_spark_regexp_count.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"165" + "p_code":"144", + "code":"166" }, { "desc":"This function is used to match the string source based on the pattern grouping rule and return the string content that matches groupid.regexp_extract(string , str", @@ -1490,8 +1499,8 @@ "title":"regexp_extract", "uri":"dli_spark_regexp_extract.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"166" + "p_code":"144", + "code":"167" }, { "desc":"This function is used to replace the part in a specified string that is the same as the string old with the string new and return the result.If the string has no same cha", @@ -1499,8 +1508,8 @@ "title":"replace", "uri":"dli_spark_replace.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"167" + "p_code":"144", + "code":"168" }, { "desc":"This function has slight variations in its functionality depending on the version of Spark being used.For Spark 2.4.5 or earlier: Replaces the substring that matches patt", @@ -1508,8 +1517,8 @@ "title":"regexp_replace", "uri":"dli_spark_regexp_replace.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"168" + "p_code":"144", + "code":"169" }, { "desc":"This function is used to replace the substring that matches pattern for the occurrence time in the source string with the specified string replace_string and return the r", @@ -1517,8 +1526,8 @@ "title":"regexp_replace1", "uri":"dli_spark_regexp_replace1.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"169" + "p_code":"144", + "code":"170" }, { "desc":"This function is used to return the start or end position of the substring that matches a specified pattern for the occurrence time, starting from start_position in the s", @@ -1526,8 +1535,8 @@ "title":"regexp_instr", "uri":"dli_spark_regexp_instr.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"170" + "p_code":"144", + "code":"171" }, { "desc":"This function is used to return the substring that matches a specified pattern for the occurrence time, starting from start_position in the string source.The return value", @@ -1535,8 +1544,8 @@ "title":"regexp_substr", "uri":"dli_spark_regexp_substr.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"171" + "p_code":"144", + "code":"172" }, { "desc":"This function is used to return the string after str is repeated for n times.The return value is of the STRING type.If the value of str is not of the STRING, BIGINT, DOUB", @@ -1544,8 +1553,8 @@ "title":"repeat", "uri":"dli_spark_repeat.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"172" + "p_code":"144", + "code":"173" }, { "desc":"This function is used to return a string in reverse order.The return value is of the STRING type.If the value of str is not of the STRING, BIGINT, DOUBLE, DECIMAL, or DAT", @@ -1553,8 +1562,8 @@ "title":"reverse", "uri":"dli_spark_reverse.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"173" + "p_code":"144", + "code":"174" }, { "desc":"This function is used to right pad str1 with str2 to the specified length.The return value is of the STRING type.If the value of length is smaller than the number of digi", @@ -1562,8 +1571,8 @@ "title":"rpad", "uri":"dli_spark_rpad.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"174" + "p_code":"144", + "code":"175" }, { "desc":"This function is used to remove characters from the right of str.If trimChars is not specified, spaces are removed by default.If trimChars is specified, the function remo", @@ -1571,8 +1580,8 @@ "title":"rtrim", "uri":"dli_spark_rtrim.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"175" + "p_code":"144", + "code":"176" }, { "desc":"This function is used to return the soundex string from str, for example, soundex('Miller') = M460.The return value is of the STRING type.If the value of str is NULL, NUL", @@ -1580,8 +1589,8 @@ "title":"soundex", "uri":"dli_spark_soundex.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"176" + "p_code":"144", + "code":"177" }, { "desc":"This function is used to return a specified number of spaces.The return value is of the STRING type.If the value of n is empty, an error is reported.If the value of n is ", @@ -1589,8 +1598,8 @@ "title":"space", "uri":"dli_spark_space.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"177" + "p_code":"144", + "code":"178" }, { "desc":"This function is used to return the substring of str, starting from start_position and with a length of length.orThe return value is of the STRING type.If the value of st", @@ -1598,8 +1607,8 @@ "title":"substr/substring", "uri":"dli_spark_substr_substring.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"178" + "p_code":"144", + "code":"179" }, { "desc":"This function is used to truncate the string before the count separator of str. If the value of count is positive, the string is truncated from the left. If the value of ", @@ -1607,8 +1616,8 @@ "title":"substring_index", "uri":"dli_spark_substring_index.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"179" + "p_code":"144", + "code":"180" }, { "desc":"This function is used to split a specified string based on a specified separator and return a substring from the start to end position.The return value is of the STRING t", @@ -1616,8 +1625,8 @@ "title":"split_part", "uri":"dli_spark_split_part.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"180" + "p_code":"144", + "code":"181" }, { "desc":"This function is used to translate the input string by replacing the characters or string specified by from with the characters or string specified by to.For example, it ", @@ -1625,8 +1634,8 @@ "title":"translate", "uri":"dli_spark_translate.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"181" + "p_code":"144", + "code":"182" }, { "desc":"This function is used to remove characters from the left and right of str.If trimChars is not specified, spaces are removed by default.If trimChars is specified, the func", @@ -1634,8 +1643,8 @@ "title":"trim", "uri":"dli_spark_trim.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"182" + "p_code":"144", + "code":"183" }, { "desc":"This function is used to convert all characters of a string to the upper case.orThe return value is of the STRING type.If the value of the input parameter is not of the S", @@ -1643,8 +1652,8 @@ "title":"upper/ucase", "uri":"dli_spark_upper_ucase.html", "doc_type":"sqlreference", - "p_code":"143", - "code":"183" + "p_code":"144", + "code":"184" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1652,8 +1661,8 @@ "title":"Mathematical Functions", "uri":"dli_08_0473.html", "doc_type":"sqlreference", - "p_code":"106", - "code":"184" + "p_code":"107", + "code":"185" }, { "desc":"Table 1 lists the mathematical functions supported by DLI.", @@ -1661,8 +1670,8 @@ "title":"Overview", "uri":"dli_08_0065.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"185" + "p_code":"185", + "code":"186" }, { "desc":"This function is used to calculate the absolute value of an input parameter.The return value is of the DOUBLE or INT type.If the value of a is NULL, NULL is returned.The ", @@ -1670,8 +1679,8 @@ "title":"abs", "uri":"dli_spark_abs.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"186" + "p_code":"185", + "code":"187" }, { "desc":"This function is used to return the arc cosine value of a given angle a.The return value is of the DOUBLE type. The value ranges from 0 to π.If the value of a is not with", @@ -1679,8 +1688,8 @@ "title":"acos", "uri":"dli_spark_aocs.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"187" + "p_code":"185", + "code":"188" }, { "desc":"This function is used to return the arc sine value of a given angle a.The return value is of the DOUBLE type. The value ranges from -π/2 to π/2.If the value of a is not w", @@ -1688,8 +1697,8 @@ "title":"asin", "uri":"dli_spark_asin.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"188" + "p_code":"185", + "code":"189" }, { "desc":"This function is used to return the arc tangent value of a given angle a.The return value is of the DOUBLE type. The value ranges from -π/2 to π/2.If the value of a is no", @@ -1697,8 +1706,8 @@ "title":"atan", "uri":"dli_spark_atan.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"189" + "p_code":"185", + "code":"190" }, { "desc":"This function is used to return the binary format of a.The return value is of the STRING type.If the value of a is NULL, NULL is returned.The value 1 is returned.The valu", @@ -1706,8 +1715,8 @@ "title":"bin", "uri":"dli_spark_bin.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"190" + "p_code":"185", + "code":"191" }, { "desc":"This function is used to return a value that is rounded off to d decimal places.The return value is of the DOUBLE type.If the value of a or d is NULL, NULL is returned.Th", @@ -1715,8 +1724,8 @@ "title":"bround", "uri":"dli_spark_bround.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"191" + "p_code":"185", + "code":"192" }, { "desc":"This function is used to return the cube root of a.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 3 is returned.select cbrt(", @@ -1724,8 +1733,8 @@ "title":"cbrt", "uri":"dli_spark_cbrt.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"192" + "p_code":"185", + "code":"193" }, { "desc":"This function is used to round up a to the nearest integer.The return value is of the DECIMAL type.If the value of a is NULL, NULL is returned.The value 2 is returned.The", @@ -1733,8 +1742,8 @@ "title":"ceil", "uri":"dli_spark_ceil.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"193" + "p_code":"185", + "code":"194" }, { "desc":"This function is used to convert a number from from_base to to_base.The return value is of the STRING type.If the value of num, from_base, or to_base is NULL, NULL is ret", @@ -1742,8 +1751,8 @@ "title":"conv", "uri":"dli_spark_conv.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"194" + "p_code":"185", + "code":"195" }, { "desc":"This function is used to calculate the cosine value of a, with input in radians.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The val", @@ -1751,8 +1760,8 @@ "title":"cos", "uri":"dli_spark_cos.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"195" + "p_code":"185", + "code":"196" }, { "desc":"This function is used to calculate the cotangent value of a, with input in radians.The return value is of the DOUBLE or DECIMAL type.If the value of a is NULL, NULL is re", @@ -1760,8 +1769,8 @@ "title":"cot1", "uri":"dli_spark_cot1.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"196" + "p_code":"185", + "code":"197" }, { "desc":"This function is used to calculate the angle corresponding to the returned radian.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The v", @@ -1769,8 +1778,8 @@ "title":"degrees", "uri":"dli_spark_degress.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"197" + "p_code":"185", + "code":"198" }, { "desc":"This function is used to return the value of e.The return value is of the DOUBLE type.The value 2.718281828459045 is returned.select e();", @@ -1778,8 +1787,8 @@ "title":"e", "uri":"dli_spark_e.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"198" + "p_code":"185", + "code":"199" }, { "desc":"This function is used to return the value of e raised to the power of a.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 7.389", @@ -1787,8 +1796,8 @@ "title":"exp", "uri":"dli_spark_exp.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"199" + "p_code":"185", + "code":"200" }, { "desc":"This function is used to return the factorial of a.The return value is of the BIGINT type.If the value of a is 0, 1 is returned.If the value of a is NULL or outside the r", @@ -1796,8 +1805,8 @@ "title":"factorial", "uri":"dli_spark_factorial.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"200" + "p_code":"185", + "code":"201" }, { "desc":"This function is used to round down a to the nearest integer.The return value is of the BIGINT type.If the value of a is NULL, NULL is returned.The value 1 is returned.Th", @@ -1805,8 +1814,8 @@ "title":"floor", "uri":"dli_spark_floor.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"201" + "p_code":"185", + "code":"202" }, { "desc":"This function is used to return the greatest value in a list of values.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 4.0 is", @@ -1814,8 +1823,8 @@ "title":"greatest", "uri":"dli_spark_greatest.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"202" + "p_code":"185", + "code":"203" }, { "desc":"This function is used to convert an integer or character into its hexadecimal representation.The return value is of the STRING type.If the value of a is 0, 0 is returned.", @@ -1823,8 +1832,8 @@ "title":"hex", "uri":"dli_spark_hex.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"203" + "p_code":"185", + "code":"204" }, { "desc":"This function is used to return the smallest value in a list of values.The return value is of the DOUBLE type.If the value of v1 or v2 is of the STRING type, an error is ", @@ -1832,8 +1841,8 @@ "title":"least", "uri":"dli_spark_least.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"204" + "p_code":"185", + "code":"205" }, { "desc":"This function is used to return the natural logarithm of a given value.The return value is of the DOUBLE type.If the value of a is negative or 0, NULL is returned.If the ", @@ -1841,8 +1850,8 @@ "title":"ln", "uri":"dli_spark_ln.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"205" + "p_code":"185", + "code":"206" }, { "desc":"This function is used to return the natural logarithm of a given base and exponent.The return value is of the DOUBLE type.If the value of base or a is NULL, NULL is retur", @@ -1850,8 +1859,8 @@ "title":"log", "uri":"dli_spark_log.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"206" + "p_code":"185", + "code":"207" }, { "desc":"This function is used to return the natural logarithm of a given value with a base of 10.The return value is of the DOUBLE type.If the value of a is negative, 0, or NULL,", @@ -1859,8 +1868,8 @@ "title":"log10", "uri":"dli_spark_log10.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"207" + "p_code":"185", + "code":"208" }, { "desc":"This function is used to return the natural logarithm of a given value with a base of 2.The return value is of the DOUBLE type.If the value of a is negative, 0, or NULL, ", @@ -1868,8 +1877,8 @@ "title":"log2", "uri":"dli_spark_log2.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"208" + "p_code":"185", + "code":"209" }, { "desc":"This function is used to calculate the median of input parameters.The return value is of the DOUBLE or DECIMAL type.If the column name does not exist, an error is reporte", @@ -1877,8 +1886,8 @@ "title":"median", "uri":"dli_spark_median.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"209" + "p_code":"185", + "code":"210" }, { "desc":"This function is used to return the additive inverse of a.The return value is of the DECIMAL or INT type.If the value of a is NULL, NULL is returned.The value -1 is retur", @@ -1886,8 +1895,8 @@ "title":"negative", "uri":"dli_spark_negative.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"210" + "p_code":"185", + "code":"211" }, { "desc":"This function is used to return the exact percentile, which is applicable to a small amount of data. It sorts a specified column in ascending order, and then obtains the ", @@ -1895,8 +1904,8 @@ "title":"percentlie", "uri":"dli_spark_percentlie.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"211" + "p_code":"185", + "code":"212" }, { "desc":"This function is used to return the approximate percentile, which is applicable to a large amount of data. It sorts a specified column in ascending order, and then obtain", @@ -1904,8 +1913,8 @@ "title":"percentlie_approx", "uri":"dli_spark_percentlie_approx.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"212" + "p_code":"185", + "code":"213" }, { "desc":"This function is used to return the value of π.The return value is of the DOUBLE type.The value 3.141592653589793 is returned.", @@ -1913,8 +1922,8 @@ "title":"pi", "uri":"dli_spark_pi.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"213" + "p_code":"185", + "code":"214" }, { "desc":"This function is used to return the positive value of the remainder after division of x by y.pmod(INT a, INT b)The return value is of the DECIMAL or INT type.If the value", @@ -1922,8 +1931,8 @@ "title":"pmod", "uri":"dli_spark_pmod.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"214" + "p_code":"185", + "code":"215" }, { "desc":"This function is used to return the value of a.The return value is of the DECIMAL, DOUBLE, or INT type.If the value of a is NULL, NULL is returned.The value 3 is returned", @@ -1931,8 +1940,8 @@ "title":"positive", "uri":"dli_spark_positive.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"215" + "p_code":"185", + "code":"216" }, { "desc":"This function is used to calculate and return the pth power of a.The return value is of the DOUBLE type.If the value of a or p is NULL, NULL is returned.The value 16 retu", @@ -1940,8 +1949,8 @@ "title":"pow", "uri":"dli_spark_pow.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"216" + "p_code":"185", + "code":"217" }, { "desc":"This function is used to return the radian corresponding to an angle.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 1.047197", @@ -1949,8 +1958,8 @@ "title":"radians", "uri":"dli_spark_radians.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"217" + "p_code":"185", + "code":"218" }, { "desc":"This function is used to return an evenly distributed random number that is greater than or equal to 0 and less than 1.The return value is of the DOUBLE type.The value 0.", @@ -1958,8 +1967,8 @@ "title":"rand", "uri":"dli_spark_rand.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"218" + "p_code":"185", + "code":"219" }, { "desc":"This function is used to calculate the rounded value of a up to d decimal places.The return value is of the DOUBLE type.If the value of d is negative, an error is reporte", @@ -1967,8 +1976,8 @@ "title":"round", "uri":"dli_spark_round.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"219" + "p_code":"185", + "code":"220" }, { "desc":"This function is used to perform a signed bitwise left shift. It takes the binary number a and shifts it b positions to the left.shiftleft(BIGINT a, BIGINT b)The return v", @@ -1976,8 +1985,8 @@ "title":"shiftleft", "uri":"dli_spark_shiftleft.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"220" + "p_code":"185", + "code":"221" }, { "desc":"This function is used to perform a signed bitwise right shift. It takes the binary number a and shifts it b positions to the right.The return value is of the INT type.If ", @@ -1985,8 +1994,8 @@ "title":"shiftright", "uri":"dli_spark_shiftright.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"221" + "p_code":"185", + "code":"222" }, { "desc":"This function is used to perform an unsigned bitwise right shift. It takes the binary number a and shifts it b positions to the right.The return value is of the INT type.", @@ -1994,8 +2003,8 @@ "title":"shiftrightunsigned", "uri":"dli_spark_shiftrightunsigned.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"222" + "p_code":"185", + "code":"223" }, { "desc":"This function is used to return the positive and negative signs corresponding to a.The return value is of the DOUBLE type.If the value of a is a positive number, 1 is ret", @@ -2003,8 +2012,8 @@ "title":"sign", "uri":"dli_spark_sign.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"223" + "p_code":"185", + "code":"224" }, { "desc":"This function is used to return the sine value of a, with input in radians.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 1 ", @@ -2012,8 +2021,8 @@ "title":"sin", "uri":"dli_spark_sin.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"224" + "p_code":"185", + "code":"225" }, { "desc":"This function is used to return the square root of a value.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value 2.8284271247461903", @@ -2021,8 +2030,8 @@ "title":"sqrt", "uri":"dli_spark_sqrt.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"225" + "p_code":"185", + "code":"226" }, { "desc":"This function is used to return the tangent value of a, with input in radians.The return value is of the DOUBLE type.If the value of a is NULL, NULL is returned.The value", @@ -2030,8 +2039,8 @@ "title":"tan", "uri":"dli_spark_tan.html", "doc_type":"sqlreference", - "p_code":"184", - "code":"226" + "p_code":"185", + "code":"227" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2039,8 +2048,8 @@ "title":"Aggregate Functions", "uri":"dli_08_0474.html", "doc_type":"sqlreference", - "p_code":"106", - "code":"227" + "p_code":"107", + "code":"228" }, { "desc":"Table 1 lists the aggregate functions supported by DLI.", @@ -2048,8 +2057,8 @@ "title":"Overview", "uri":"dli_08_0068.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"228" + "p_code":"228", + "code":"229" }, { "desc":"This function is used to return the average value.The return value is of the DOUBLE type.If the value of col is NULL, the column is not involved in calculation.Calculates", @@ -2057,8 +2066,8 @@ "title":"avg", "uri":"dli_spark_avg.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"229" + "p_code":"228", + "code":"230" }, { "desc":"This function is used to return the correlation coefficient between two columns of numerical values.The return value is of the DOUBLE type.Calculates the correlation coef", @@ -2066,8 +2075,8 @@ "title":"corr", "uri":"dli_spark_corr.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"230" + "p_code":"228", + "code":"231" }, { "desc":"This function is used to return the number of records.The return value is of the BIGINT type.If the value of colname is NULL, the row is not involved in calculation.Calcu", @@ -2075,8 +2084,8 @@ "title":"count", "uri":"dli_spark_count.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"231" + "p_code":"228", + "code":"232" }, { "desc":"This function is used to return the covariance between two columns of numerical values.The return value is of the DOUBLE type.Calculates the covariance between the invent", @@ -2084,8 +2093,8 @@ "title":"covar_pop", "uri":"dli_spark_covar_pop.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"232" + "p_code":"228", + "code":"233" }, { "desc":"This function is used to return the sample covariance between two columns of numerical values.The return value is of the DOUBLE type.Calculates the sample covariance betw", @@ -2093,8 +2102,8 @@ "title":"covar_samp", "uri":"dli_spark_covar_samp.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"233" + "p_code":"228", + "code":"234" }, { "desc":"This function is used to return the maximum value.The return value is of the DOUBLE type.The return type is the same as the type of col. The return rules are as follows:I", @@ -2102,8 +2111,8 @@ "title":"max", "uri":"dli_spark_max.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"234" + "p_code":"228", + "code":"235" }, { "desc":"This function is used to return the minimum value.The return value is of the DOUBLE type.The return type is the same as the type of col. The return rules are as follows:I", @@ -2111,8 +2120,8 @@ "title":"min", "uri":"dli_spark_min.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"235" + "p_code":"228", + "code":"236" }, { "desc":"This function is used to return the numerical value at a certain percentage point within a range of values.The return value is of the DOUBLE type.The value should be betw", @@ -2120,8 +2129,8 @@ "title":"percentile", "uri":"dli_spark_percentile.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"236" + "p_code":"228", + "code":"237" }, { "desc":"This function is used to approximate the pth percentile (including floating-point numbers) of a numeric column within a group.The return value is of the DOUBLE type.Calcu", @@ -2129,8 +2138,8 @@ "title":"percentile_approx", "uri":"dli_spark_percentile_approx.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"237" + "p_code":"228", + "code":"238" }, { "desc":"This function is used to return the deviation of a specified column.The return value is of the DOUBLE type.Calculates the deviation of all offering inventories (items). A", @@ -2138,17 +2147,17 @@ "title":"stddev_pop", "uri":"dli_spark_stddev_pop.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"238" + "p_code":"228", + "code":"239" }, { - "desc":"This function is used to return the sample deviation of a specified column.The return value is of the DOUBLE type.Calculates the sample covariance between the inventory (", + "desc":"This function is used to return the sample deviation of a specified column.The return value is of the DOUBLE type.Calculates the sample deviation of all offering inventor", "product_code":"dli", "title":"stddev_samp", "uri":"dli_spark_stddev_samp.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"239" + "p_code":"228", + "code":"240" }, { "desc":"This function is used to calculate the total sum.The return value is of the DOUBLE type.If the value of col is NULL, the row is not involved in calculation.Calculates the", @@ -2156,8 +2165,8 @@ "title":"sum", "uri":"dli_spark_sum.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"240" + "p_code":"228", + "code":"241" }, { "desc":"This function is used to return the variance of a column.The return value is of the DOUBLE type.Calculates the variance of all offering inventories (items). An example co", @@ -2165,8 +2174,8 @@ "title":"variance/var_pop", "uri":"dli_spark_variance_var_pop.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"241" + "p_code":"228", + "code":"242" }, { "desc":"This function is used to return the sample variance of a specified column.The return value is of the DOUBLE type.Calculates the sample variance of all offering inventorie", @@ -2174,8 +2183,8 @@ "title":"var_samp", "uri":"dli_spark_war_samp.html", "doc_type":"sqlreference", - "p_code":"227", - "code":"242" + "p_code":"228", + "code":"243" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2183,8 +2192,8 @@ "title":"Window Functions", "uri":"dli_08_0475.html", "doc_type":"sqlreference", - "p_code":"106", - "code":"243" + "p_code":"107", + "code":"244" }, { "desc":"Table 1 lists the window functions supported by DLI.", @@ -2192,8 +2201,8 @@ "title":"Overview", "uri":"dli_08_0069.html", "doc_type":"sqlreference", - "p_code":"243", - "code":"244" + "p_code":"244", + "code":"245" }, { "desc":"This function is used to return the cumulative distribution, which is equivalent to calculating the proportion of data in the partition that is greater than or equal to, ", @@ -2201,8 +2210,8 @@ "title":"cume_dist", "uri":"dli_spark_cume_dist.html", "doc_type":"sqlreference", - "p_code":"243", - "code":"245" + "p_code":"244", + "code":"246" }, { "desc":"This function is used to obtain the value of the first data record in the window corresponding to the current row.The restrictions on using window functions are as follow", @@ -2210,8 +2219,8 @@ "title":"first_value", "uri":"dli_spark_first_value.html", "doc_type":"sqlreference", - "p_code":"243", - "code":"246" + "p_code":"244", + "code":"247" }, { "desc":"This function is used to obtain the value of the last data record in the window corresponding to the current row.The restrictions on using window functions are as follows", @@ -2219,8 +2228,8 @@ "title":"last_value", "uri":"dli_spark_last_value.html", "doc_type":"sqlreference", - "p_code":"243", - "code":"247" + "p_code":"244", + "code":"248" }, { "desc":"This function is used to return the value of the nth row upwards within a specified window.The restrictions on using window functions are as follows:Window functions can ", @@ -2228,8 +2237,8 @@ "title":"lag", "uri":"dli_spark_lag.html", "doc_type":"sqlreference", - "p_code":"243", - "code":"248" + "p_code":"244", + "code":"249" }, { "desc":"This function is used to return the value of the nth row downwards within a specified window.The restrictions on using window functions are as follows:Window functions ca", @@ -2237,8 +2246,8 @@ "title":"lead", "uri":"dli_spark_lead.html", "doc_type":"sqlreference", - "p_code":"243", - "code":"249" + "p_code":"244", + "code":"250" }, { "desc":"This function is used to return the value of the column specified in the ORDER BY clause of a window, expressed as a decimal between 0 and 1. It is calculated as (the ran", @@ -2246,8 +2255,8 @@ "title":"percent_rank", "uri":"dli_spark_percent_rank.html", "doc_type":"sqlreference", - "p_code":"243", - "code":"250" + "p_code":"244", + "code":"251" }, { "desc":"This function is used to return the rank of a value in a set of values. When multiple values share the same rank, the next rank in the sequence is not consecutive.The res", @@ -2255,8 +2264,8 @@ "title":"rank", "uri":"dli_spark_rank.html", "doc_type":"sqlreference", - "p_code":"243", - "code":"251" + "p_code":"244", + "code":"252" }, { "desc":"This function is used to return the row number, starting from 1 and increasing incrementally.The restrictions on using window functions are as follows:Window functions ca", @@ -2264,8 +2273,8 @@ "title":"row_number", "uri":"dli_spark_row_number.html", "doc_type":"sqlreference", - "p_code":"243", - "code":"252" + "p_code":"244", + "code":"253" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2273,8 +2282,8 @@ "title":"Other Functions", "uri":"dli_08_0476.html", "doc_type":"sqlreference", - "p_code":"106", - "code":"253" + "p_code":"107", + "code":"254" }, { "desc":"The following table lists the functions provided by DLI, such as decode1, javahash, and max_pt.", @@ -2282,8 +2291,8 @@ "title":"Overview", "uri":"dli_08_0469.html", "doc_type":"sqlreference", - "p_code":"253", - "code":"254" + "p_code":"254", + "code":"255" }, { "desc":"This function is used to implement if-then-else branch selection.result and default are return values. These values can be of any data type.If they match, the value of re", @@ -2291,8 +2300,8 @@ "title":"decode1", "uri":"dli_spark_decode1.html", "doc_type":"sqlreference", - "p_code":"253", - "code":"255" + "p_code":"254", + "code":"256" }, { "desc":"This function is used to return the hash value of a.The return value is of the STRING type.The hash value is returned. If the value of a is null, an error is reported.The", @@ -2300,8 +2309,8 @@ "title":"javahash", "uri":"dli_spark_javahash.html", "doc_type":"sqlreference", - "p_code":"253", - "code":"256" + "p_code":"254", + "code":"257" }, { "desc":"This function is used to return the name of the largest level-1 partition that contains data in a partitioned table and read the data of this partition.The return value i", @@ -2309,8 +2318,8 @@ "title":"max_pt", "uri":"dli_spark_max_pt.html", "doc_type":"sqlreference", - "p_code":"253", - "code":"257" + "p_code":"254", + "code":"258" }, { "desc":"This function is used to sort input variables in ascending order and return the value at the position specified by nth.The return value is of the DOUBLE or DECIMAL type.V", @@ -2318,8 +2327,8 @@ "title":"ordinal", "uri":"dli_spark_ordinal.html", "doc_type":"sqlreference", - "p_code":"253", - "code":"258" + "p_code":"254", + "code":"259" }, { "desc":"This function is used to convert an array split by a fixed separator in a column into multiple rows.All columns used as keys must be placed before the columns to be trans", @@ -2327,8 +2336,8 @@ "title":"trans_array", "uri":"dli_spark_trans_array.html", "doc_type":"sqlreference", - "p_code":"253", - "code":"259" + "p_code":"254", + "code":"260" }, { "desc":"This function is used to truncate the number value to a specified decimal place.The return value is of the DOUBLE or DECIMAL type.The return rules are as follows:If the n", @@ -2336,8 +2345,8 @@ "title":"trunc_numeric", "uri":"dli_spark_trunc_numeric.html", "doc_type":"sqlreference", - "p_code":"253", - "code":"260" + "p_code":"254", + "code":"261" }, { "desc":"This function is used to convert a string from the application/x-www-form-urlencoded MIME format to regular characters.The return value is of the STRING type.UTF-8-encode", @@ -2345,8 +2354,8 @@ "title":"url_decode", "uri":"dli_spark_url_decode.html", "doc_type":"sqlreference", - "p_code":"253", - "code":"261" + "p_code":"254", + "code":"262" }, { "desc":"This function is used to encode a string in the application/x-www-form-urlencoded MIME format.url_encode(string [, string ])The return value is of the ST", @@ -2354,17 +2363,17 @@ "title":"url_encode", "uri":"dli_spark_url_encode.html", "doc_type":"sqlreference", - "p_code":"253", - "code":"262" + "p_code":"254", + "code":"263" }, { - "desc":"This statement is a basic query statement and is used to return the query results.The table to be queried must exist. Otherwise, an error is reported.To filter the record", + "desc":"This statement is a basic query statement and is used to return the query results.The table to be queried must already exist, or an error message will be displayed.When s", "product_code":"dli", "title":"Basic SELECT Statements", "uri":"dli_08_0150.html", "doc_type":"sqlreference", "p_code":"1", - "code":"263" + "code":"264" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2373,7 +2382,7 @@ "uri":"dli_08_0151.html", "doc_type":"sqlreference", "p_code":"1", - "code":"264" + "code":"265" }, { "desc":"This statement is used to filter the query results using the WHERE clause.All is used to return repeated rows. By default, all repeated rows are returned. It is followed ", @@ -2381,8 +2390,8 @@ "title":"WHERE Filtering Clause", "uri":"dli_08_0152.html", "doc_type":"sqlreference", - "p_code":"264", - "code":"265" + "p_code":"265", + "code":"266" }, { "desc":"This statement is used to filter the query results using the HAVING clause.All is used to return repeated rows. By default, all repeated rows are returned. It is followed", @@ -2390,8 +2399,8 @@ "title":"HAVING Filtering Clause", "uri":"dli_08_0153.html", "doc_type":"sqlreference", - "p_code":"264", - "code":"266" + "p_code":"265", + "code":"267" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2400,7 +2409,7 @@ "uri":"dli_08_0154.html", "doc_type":"sqlreference", "p_code":"1", - "code":"267" + "code":"268" }, { "desc":"This statement is used to order the result set of a query by the specified field.ASC/DESC: ASC sorts from the lowest value to the highest value. DESC sorts from the highe", @@ -2408,8 +2417,8 @@ "title":"ORDER BY", "uri":"dli_08_0155.html", "doc_type":"sqlreference", - "p_code":"267", - "code":"268" + "p_code":"268", + "code":"269" }, { "desc":"This statement is used to achieve the partial sorting of tables according to fields.ASC/DESC: ASC sorts from the lowest value to the highest value. DESC sorts from the hi", @@ -2417,8 +2426,8 @@ "title":"SORT BY", "uri":"dli_08_0156.html", "doc_type":"sqlreference", - "p_code":"267", - "code":"269" + "p_code":"268", + "code":"270" }, { "desc":"This statement is used to bucket a table and sort the table within buckets.CLUSTER BY: Buckets are created based on specified fields. Single fields and multiple fields ar", @@ -2426,8 +2435,8 @@ "title":"CLUSTER BY", "uri":"dli_08_0157.html", "doc_type":"sqlreference", - "p_code":"267", - "code":"270" + "p_code":"268", + "code":"271" }, { "desc":"This statement is used to bucket a table according to the field.DISTRIBUTE BY: Buckets are created based on specified fields. A single field or multiple fields are suppor", @@ -2435,8 +2444,8 @@ "title":"DISTRIBUTE BY", "uri":"dli_08_0158.html", "doc_type":"sqlreference", - "p_code":"267", - "code":"271" + "p_code":"268", + "code":"272" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2445,7 +2454,7 @@ "uri":"dli_08_0159.html", "doc_type":"sqlreference", "p_code":"1", - "code":"272" + "code":"273" }, { "desc":"This statement is used to group a table based on columns.Column-based GROUP BY can be categorized into single-column GROUP BY and multi-column GROUP BY.Single-column GROU", @@ -2453,8 +2462,8 @@ "title":"Column-Based GROUP BY", "uri":"dli_08_0160.html", "doc_type":"sqlreference", - "p_code":"272", - "code":"273" + "p_code":"273", + "code":"274" }, { "desc":"This statement is used to group a table according to expressions.The groupby_expression can contain a single field or multiple fields, and also can call aggregate functio", @@ -2462,8 +2471,8 @@ "title":"Expression-Based GROUP BY", "uri":"dli_08_0161.html", "doc_type":"sqlreference", - "p_code":"272", - "code":"274" + "p_code":"273", + "code":"275" }, { "desc":"This statement filters a table after grouping it using the HAVING clause.The groupby_expression can contain a single field or multiple fields, and can also call aggregate", @@ -2471,8 +2480,8 @@ "title":"GROUP BY Using HAVING", "uri":"dli_08_0162.html", "doc_type":"sqlreference", - "p_code":"272", - "code":"275" + "p_code":"273", + "code":"276" }, { "desc":"This statement is used to generate the aggregate row, super-aggregate row, and the total row. The statement can achieve multi-layer statistics from right to left and disp", @@ -2480,8 +2489,8 @@ "title":"ROLLUP", "uri":"dli_08_0163.html", "doc_type":"sqlreference", - "p_code":"272", - "code":"276" + "p_code":"273", + "code":"277" }, { "desc":"This statement is used to generate the cross-table row and achieve the cross-statistics of the GROUP BY field.GROUPING SETS is the expansion of GROUP BY. For example:SELE", @@ -2489,8 +2498,8 @@ "title":"GROUPING SETS", "uri":"dli_08_0164.html", "doc_type":"sqlreference", - "p_code":"272", - "code":"277" + "p_code":"273", + "code":"278" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2499,7 +2508,7 @@ "uri":"dli_08_0165.html", "doc_type":"sqlreference", "p_code":"1", - "code":"278" + "code":"279" }, { "desc":"This statement is used to join and return the rows that meet the JOIN conditions from two tables as the result set.JOIN/INNER JOIN: Only the records that meet the JOIN co", @@ -2507,8 +2516,8 @@ "title":"INNER JOIN", "uri":"dli_08_0166.html", "doc_type":"sqlreference", - "p_code":"278", - "code":"279" + "p_code":"279", + "code":"280" }, { "desc":"Join the left table with the right table and return all joined records of the left table. If no joined record is found, NULL will be returned.LEFT OUTER JOIN: Returns all", @@ -2516,8 +2525,8 @@ "title":"LEFT OUTER JOIN", "uri":"dli_08_0167.html", "doc_type":"sqlreference", - "p_code":"278", - "code":"280" + "p_code":"279", + "code":"281" }, { "desc":"Match the right table with the left table and return all matched records of the right table. If no matched record is found, NULL will be returned.RIGHT OUTER JOIN: Return", @@ -2525,8 +2534,8 @@ "title":"RIGHT OUTER JOIN", "uri":"dli_08_0168.html", "doc_type":"sqlreference", - "p_code":"278", - "code":"281" + "p_code":"279", + "code":"282" }, { "desc":"Join all records from the right table and the left table and return all joined records. If no joined record is found, NULL will be returned.FULL OUTER JOIN: Matches all r", @@ -2534,8 +2543,8 @@ "title":"FULL OUTER JOIN", "uri":"dli_08_0169.html", "doc_type":"sqlreference", - "p_code":"278", - "code":"282" + "p_code":"279", + "code":"283" }, { "desc":"This statement has the same function as INNER JOIN, that is, the result set that meet the WHERE condition is returned. However, IMPLICIT JOIN does not use the condition s", @@ -2543,8 +2552,8 @@ "title":"IMPLICIT JOIN", "uri":"dli_08_0170.html", "doc_type":"sqlreference", - "p_code":"278", - "code":"283" + "p_code":"279", + "code":"284" }, { "desc":"Cartesian JOIN joins each record of table A with all records in table B. For example, if there are m records in table A and n records in table B, m x n records will be ge", @@ -2552,8 +2561,8 @@ "title":"Cartesian JOIN", "uri":"dli_08_0171.html", "doc_type":"sqlreference", - "p_code":"278", - "code":"284" + "p_code":"279", + "code":"285" }, { "desc":"This statement is used to query the records that meet the JOIN condition from the left table.LEFT SEMI JOIN: Indicates to only return the records from the left table. LEF", @@ -2561,8 +2570,8 @@ "title":"LEFT SEMI JOIN", "uri":"dli_08_0172.html", "doc_type":"sqlreference", - "p_code":"278", - "code":"285" + "p_code":"279", + "code":"286" }, { "desc":"This statement is used to join multiple tables using unequal values and return the result set that meet the condition.The non_equi_join_condition is similar to join_condi", @@ -2570,8 +2579,8 @@ "title":"NON-EQUIJOIN", "uri":"dli_08_0173.html", "doc_type":"sqlreference", - "p_code":"278", - "code":"286" + "p_code":"279", + "code":"287" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2580,7 +2589,7 @@ "uri":"dli_08_0174.html", "doc_type":"sqlreference", "p_code":"1", - "code":"287" + "code":"288" }, { "desc":"Subqueries are nested in the WHERE clause, and the subquery result is used as the filtering condition.All is used to return repeated rows. By default, all repeated rows a", @@ -2588,8 +2597,8 @@ "title":"Subquery Nested by WHERE", "uri":"dli_08_0175.html", "doc_type":"sqlreference", - "p_code":"287", - "code":"288" + "p_code":"288", + "code":"289" }, { "desc":"This statement is used to nest subquery by FROM and use the subquery results as the data source of the external SELECT statement.All is used to return repeated rows. By d", @@ -2597,8 +2606,8 @@ "title":"Subquery Nested by FROM", "uri":"dli_08_0176.html", "doc_type":"sqlreference", - "p_code":"287", - "code":"289" + "p_code":"288", + "code":"290" }, { "desc":"This statement is used to embed a subquery in the HAVING clause. The subquery result is used as a part of the HAVING clause.All is used to return repeated rows. By defaul", @@ -2606,8 +2615,8 @@ "title":"Subquery Nested by HAVING", "uri":"dli_08_0177.html", "doc_type":"sqlreference", - "p_code":"287", - "code":"290" + "p_code":"288", + "code":"291" }, { "desc":"This statement is used to nest queries in the subquery.All is used to return repeated rows. By default, all repeated rows are returned. It is followed by asterisks (*) on", @@ -2615,8 +2624,8 @@ "title":"Multi-Layer Nested Subquery", "uri":"dli_08_0178.html", "doc_type":"sqlreference", - "p_code":"287", - "code":"291" + "p_code":"288", + "code":"292" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2625,7 +2634,7 @@ "uri":"dli_08_0179.html", "doc_type":"sqlreference", "p_code":"1", - "code":"292" + "code":"293" }, { "desc":"This statement is used to specify an alias for a table or the subquery result.table_reference: Can be a table, view, or subquery.As: Is used to connect to table_reference", @@ -2633,8 +2642,8 @@ "title":"AS for Table", "uri":"dli_08_0180.html", "doc_type":"sqlreference", - "p_code":"292", - "code":"293" + "p_code":"293", + "code":"294" }, { "desc":"This statement is used to specify an alias for a column.alias: gives an alias for the attr_expr field.AS: Whether to add AS does not affect the result.The to-be-queried t", @@ -2642,8 +2651,8 @@ "title":"AS for Column", "uri":"dli_08_0181.html", "doc_type":"sqlreference", - "p_code":"292", - "code":"294" + "p_code":"293", + "code":"295" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2652,7 +2661,7 @@ "uri":"dli_08_0182.html", "doc_type":"sqlreference", "p_code":"1", - "code":"295" + "code":"296" }, { "desc":"This statement is used to return the union set of multiple query results.UNION: The set operation is used to join the head and tail of a table based on certain conditions", @@ -2660,8 +2669,8 @@ "title":"UNION", "uri":"dli_08_0183.html", "doc_type":"sqlreference", - "p_code":"295", - "code":"296" + "p_code":"296", + "code":"297" }, { "desc":"This statement is used to return the intersection set of multiple query results.INTERSECT returns the intersection of multiple query results. The number of columns return", @@ -2669,8 +2678,8 @@ "title":"INTERSECT", "uri":"dli_08_0184.html", "doc_type":"sqlreference", - "p_code":"295", - "code":"297" + "p_code":"296", + "code":"298" }, { "desc":"This statement is used to return the difference set of two query results.EXCEPT minus the sets. A EXCEPT B indicates to remove the records that exist in both A and B from", @@ -2678,8 +2687,8 @@ "title":"EXCEPT", "uri":"dli_08_0185.html", "doc_type":"sqlreference", - "p_code":"295", - "code":"298" + "p_code":"296", + "code":"299" }, { "desc":"This statement is used to define the common table expression (CTE) using WITH...AS to simplify the query and make the result easier to read and maintain.cte_name: Name of", @@ -2688,7 +2697,7 @@ "uri":"dli_08_0186.html", "doc_type":"sqlreference", "p_code":"1", - "code":"299" + "code":"300" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2697,7 +2706,7 @@ "uri":"dli_08_0187.html", "doc_type":"sqlreference", "p_code":"1", - "code":"300" + "code":"301" }, { "desc":"This statement is used to display result_expression according to the joined results of input_expression and when_expression.CASE: Subquery is supported in basic CASE stat", @@ -2705,8 +2714,8 @@ "title":"Basic CASE Statement", "uri":"dli_08_0188.html", "doc_type":"sqlreference", - "p_code":"300", - "code":"301" + "p_code":"301", + "code":"302" }, { "desc":"This statement is used to obtain the value of boolean_expression for each WHEN statement in a specified order. Then return the first result_expression with the value TRUE", @@ -2714,8 +2723,8 @@ "title":"CASE Query Statement", "uri":"dli_08_0189.html", "doc_type":"sqlreference", - "p_code":"300", - "code":"302" + "p_code":"301", + "code":"303" }, { "desc":"This statement is used together with the window function. The OVER statement is used to group data and sort the data within the group. The window function is used to gene", @@ -2724,7 +2733,934 @@ "uri":"dli_08_0190.html", "doc_type":"sqlreference", "p_code":"1", - "code":"303" + "code":"304" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Flink OpenSource SQL 1.15 Syntax Reference", + "uri":"en-us_topic_0000002172127506.html", + "doc_type":"sqlreference", + "p_code":"", + "code":"305" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Constraints and Definitions", + "uri":"dli_08_15001.html", + "doc_type":"sqlreference", + "p_code":"305", + "code":"306" + }, + { + "desc":"DLI supports the following data types:CHAR, VARCHAR, STRING, BOOLEAN, BINARY, VARBINARY, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TI", + "product_code":"dli", + "title":"Supported Data Types", + "uri":"dli_08_15002.html", + "doc_type":"sqlreference", + "p_code":"306", + "code":"307" + }, + { + "desc":"Certain combinations of strings have been reserved as keywords for future use.If you use any of the following strings as field names, enclose them in backticks when using", + "product_code":"dli", + "title":"Reserved Keywords", + "uri":"dli_08_15003.html", + "doc_type":"sqlreference", + "p_code":"306", + "code":"308" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Data Definition Language (DDL)", + "uri":"dli_08_15005.html", + "doc_type":"sqlreference", + "p_code":"306", + "code":"309" + }, + { + "desc":"This statement creates a table using the specified table name. However, if a table with the same name already exists in the catalog, the registration process will fail.CO", + "product_code":"dli", + "title":"CREATE TABLE", + "uri":"dli_08_15006.html", + "doc_type":"sqlreference", + "p_code":"309", + "code":"310" + }, + { + "desc":"This statement creates a catalog using specified attributes. If a catalog with the same name already exists, an exception is thrown.WITH OPTIONSCatalog attributes typical", + "product_code":"dli", + "title":"CREATE CATALOG", + "uri":"dli_08_15007.html", + "doc_type":"sqlreference", + "p_code":"309", + "code":"311" + }, + { + "desc":"This statement creates a database using specified table attributes. If a table with the same name already exists in the database, an exception is thrown.IF NOT EXISTSIf t", + "product_code":"dli", + "title":"CREATE DATABASE", + "uri":"dli_08_15008.html", + "doc_type":"sqlreference", + "p_code":"309", + "code":"312" + }, + { + "desc":"This statement creates a view based on the given query statement. If a view with the same name already exists in the database, an exception is thrown.TEMPORARYCreate a te", + "product_code":"dli", + "title":"CREATE VIEW", + "uri":"dli_08_15009.html", + "doc_type":"sqlreference", + "p_code":"309", + "code":"313" + }, + { + "desc":"To create a catalog function with a catalog and a database namespace, you will need to specify an identifier. You can specify a language tag. You cannot register the func", + "product_code":"dli", + "title":"CREATE FUNCTION", + "uri":"dli_08_15010.html", + "doc_type":"sqlreference", + "p_code":"309", + "code":"314" + }, + { + "desc":"Flink SQL uses a lexical policy for identifier (table, attribute, function names) similar to Java:The case of identifiers is preserved whether they are quoted.Identifiers", + "product_code":"dli", + "title":"Data Manipulation Language (DML)", + "uri":"dli_08_15011.html", + "doc_type":"sqlreference", + "p_code":"306", + "code":"315" + }, + { + "desc":"This section describes the Flink OpenSource SQL 1.15 syntax supported by DLI. For details about the parameters and examples, see the syntax description.", + "product_code":"dli", + "title":"Overview", + "uri":"dli_08_15012.html", + "doc_type":"sqlreference", + "p_code":"305", + "code":"316" + }, + { + "desc":"When switching from Flink 1.12 to Flink 1.15 for job execution, keep in mind the following considerations when utilizing Flink OpenSource SQL 1.15:Flink SQL utilizes a SQ", + "product_code":"dli", + "title":"Flink OpenSource SQL 1.15 Usage", + "uri":"dli_08_15108.html", + "doc_type":"sqlreference", + "p_code":"305", + "code":"317" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Formats", + "uri":"dli_08_15014.html", + "doc_type":"sqlreference", + "p_code":"305", + "code":"318" + }, + { + "desc":"Flink provides a set of table formats that can be used with table connectors.A table format is a storage format that defines how to map binary data onto table columns.", + "product_code":"dli", + "title":"Overview", + "uri":"dli_08_15015.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"319" + }, + { + "desc":"Apache Avro is supported for you to read and write Avro data based on an Avro schema with Flink. The Avro schema is derived from the table schema.For details, see Avro Fo", + "product_code":"dli", + "title":"Avro", + "uri":"dli_08_15016.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"320" + }, + { + "desc":"Canal is a Changelog Data Capture (CDC) tool that can stream changes in real-time from MySQL into other systems. Canal provides a unified format schema for changelog and ", + "product_code":"dli", + "title":"Canal", + "uri":"dli_08_15017.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"321" + }, + { + "desc":"The Avro Schema Registry (avro-confluent) format allows you to read records that were serialized by the io.confluent.kafka.serializers.KafkaAvroSerializer and to write re", + "product_code":"dli", + "title":"Confluent Avro", + "uri":"dli_08_15018.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"322" + }, + { + "desc":"The CSV format allows you to read and write CSV data based on a CSV schema. Currently, the CSV schema is derived from table schema. For details, see CSV Format.KafkaUpser", + "product_code":"dli", + "title":"CSV", + "uri":"dli_08_15019.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"323" + }, + { + "desc":"Debezium is a Changelog Data Capture (CDC) tool that can stream changes in real-time from MySQL, PostgreSQL, Oracle, Microsoft SQL Server and many other databases into Ka", + "product_code":"dli", + "title":"Debezium", + "uri":"dli_08_15020.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"324" + }, + { + "desc":"The JSON format allows you to read and write JSON data based on a JSON schema. Currently, the JSON schema is derived from table schema. For details, see JSON Format.Kafka", + "product_code":"dli", + "title":"JSON", + "uri":"dli_08_15021.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"325" + }, + { + "desc":"Maxwell is a Changelog Data Capture (CDC) tool that can stream changes in real-time from MySQL into Kafka and other streaming connectors. Maxwell provides a unified forma", + "product_code":"dli", + "title":"Maxwell", + "uri":"dli_08_15022.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"326" + }, + { + "desc":"Oracle GoldenGate (a.k.a ogg) is a comprehensive software package for real-time data capture and replication in heterogeneous IT environments. The product set enables hig", + "product_code":"dli", + "title":"Ogg", + "uri":"dli_08_15023.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"327" + }, + { + "desc":"The Apache ORC format allows to read and write ORC data. For details, see ORC Format.FileSystemORC format also supports table properties from Table properties. For exampl", + "product_code":"dli", + "title":"ORC", + "uri":"dli_08_15024.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"328" + }, + { + "desc":"The Apache Parquet format allows to read and write Parquet data. For details, see Parquet Format.FileSystemCurrently, Parquet format type mapping is compatible with Apach", + "product_code":"dli", + "title":"Parquet", + "uri":"dli_08_15025.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"329" + }, + { + "desc":"The Raw format allows to read and write raw (byte based) values as a single column.This format encodes null values as null of byte[] type. This may have limitation when u", + "product_code":"dli", + "title":"Raw", + "uri":"dli_08_15026.html", + "doc_type":"sqlreference", + "p_code":"318", + "code":"330" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Connectors", + "uri":"dli_08_15027.html", + "doc_type":"sqlreference", + "p_code":"305", + "code":"331" + }, + { + "desc":"Source table: A source table is the data input table for Flink jobs, such as real-time streaming data input from Kafka, etc.Dimension Table: An auxiliary table for the da", + "product_code":"dli", + "title":"Overview", + "uri":"dli_08_15028.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"332" + }, + { + "desc":"The BlackHole connector allows for swallowing all input records. It is designed for high-performance testing and UDF output. It is not a substantive sink. The BlackHole r", + "product_code":"dli", + "title":"BlackHole", + "uri":"dli_08_15029.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"333" + }, + { + "desc":"DLI has the capability to export data from Flink jobs to the ClickHouse database. However, it only supports exporting data to result tables.ClickHouse is a column-based d", + "product_code":"dli", + "title":"ClickHouse", + "uri":"dli_08_15030.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"334" + }, + { + "desc":"DataGen is used to generate random data for debugging and testing.When you create a DataGen table, the table field type cannot be Array, Map, or Row. You can use COMPUTED", + "product_code":"dli", + "title":"DataGen", + "uri":"dli_08_15031.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"335" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Doris", + "uri":"dli_08_15032.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"336" + }, + { + "desc":"The Flink Doris Connector can support operations (read, insert, modify, delete) data stored in Doris through Flink.Only tables in the Unique Key model can be modified or ", + "product_code":"dli", + "title":"Overview", + "uri":"dli_08_15033.html", + "doc_type":"sqlreference", + "p_code":"336", + "code":"337" + }, + { + "desc":"Flink SQL jobs read from the Doris source table.An enhanced datasource connection has been created for DLI to connect to Doris, so that jobs can run on the dedicated queu", + "product_code":"dli", + "title":"Source Table", + "uri":"dli_08_15034.html", + "doc_type":"sqlreference", + "p_code":"336", + "code":"338" + }, + { + "desc":"Flink SQL jobs write to the Doris result table.An enhanced datasource connection has been created for DLI to connect to Doris, so that jobs can run on the dedicated queue", + "product_code":"dli", + "title":"Result Table", + "uri":"dli_08_15035.html", + "doc_type":"sqlreference", + "p_code":"336", + "code":"339" + }, + { + "desc":"Create a Doris dimension table to connect to the source streams for wide table generation.An enhanced datasource connection has been created for DLI to connect to HBase, ", + "product_code":"dli", + "title":"Dimension Table", + "uri":"dli_08_15036.html", + "doc_type":"sqlreference", + "p_code":"336", + "code":"340" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"GaussDB(DWS)", + "uri":"dli_08_15103.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"341" + }, + { + "desc":"GaussDB(DWS) is an online data processing database based on the cloud infrastructure and platform and helps you mine and analyze massive sets of data. DLI reads data of F", + "product_code":"dli", + "title":"Overview", + "uri":"dli_08_15037.html", + "doc_type":"sqlreference", + "p_code":"341", + "code":"342" + }, + { + "desc":"DLI reads data of Flink jobs from GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex types an", + "product_code":"dli", + "title":"GaussDB(DWS) Source Table (Not Recommended)", + "uri":"dli_08_15104.html", + "doc_type":"sqlreference", + "p_code":"341", + "code":"343" + }, + { + "desc":"DLI outputs the Flink job output data to GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex t", + "product_code":"dli", + "title":"GaussDB(DWS) Result Table (Not Recommended)", + "uri":"dli_08_15105.html", + "doc_type":"sqlreference", + "p_code":"341", + "code":"344" + }, + { + "desc":"Create a GaussDB(DWS) table to connect to source streams for wide table generation.You are advised to use GaussDB(DWS) self-developed GaussDB(DWS) connector.Ensure that y", + "product_code":"dli", + "title":"GaussDB(DWS) Dimension Table (Not Recommended)", + "uri":"dli_08_15106.html", + "doc_type":"sqlreference", + "p_code":"341", + "code":"345" + }, + { + "desc":"DLI outputs the output data of the Flink job to an index in the Elasticsearch engine of the Cloud Search Service (CSS).Elasticsearch is a popular enterprise-class Lucene-", + "product_code":"dli", + "title":"Elasticsearch", + "uri":"dli_08_15038.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"346" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"OBS", + "uri":"dli_08_15039.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"347" + }, + { + "desc":"The file system connector can be used to read single files or entire directories into a single table.When using a directory as the source path, there is no defined order ", + "product_code":"dli", + "title":"OBS Source Table", + "uri":"dli_08_15040.html", + "doc_type":"sqlreference", + "p_code":"347", + "code":"348" + }, + { + "desc":"The FileSystem result (sink) table is used to export data to the HDFS or OBS file system. It is applicable to scenarios such as data dumping, big data analysis, data back", + "product_code":"dli", + "title":"OBS Result Table", + "uri":"dli_08_15041.html", + "doc_type":"sqlreference", + "p_code":"347", + "code":"349" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"HBase", + "uri":"dli_08_15042.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"350" + }, + { + "desc":"Create a source stream to obtain data from HBase as input for jobs. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excell", + "product_code":"dli", + "title":"Source Table", + "uri":"dli_08_15043.html", + "doc_type":"sqlreference", + "p_code":"350", + "code":"351" + }, + { + "desc":"DLI outputs the job data to HBase. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excellent performance, and elastic scal", + "product_code":"dli", + "title":"HBase Result Table", + "uri":"dli_08_15044.html", + "doc_type":"sqlreference", + "p_code":"350", + "code":"352" + }, + { + "desc":"Create an HBase dimension table to connect to the source streams for wide table generation.An enhanced datasource connection has been created for DLI to connect to HBase,", + "product_code":"dli", + "title":"Dimension Table", + "uri":"dli_08_15045.html", + "doc_type":"sqlreference", + "p_code":"350", + "code":"353" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Hive", + "uri":"dli_08_15046.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"354" + }, + { + "desc":"Catalogs provide metadata, such as databases, tables, partitions, views, and functions and information needed to access data stored in a database or other external system", + "product_code":"dli", + "title":"Creating a Hive Catalog", + "uri":"dli_08_15047.html", + "doc_type":"sqlreference", + "p_code":"354", + "code":"355" + }, + { + "desc":"Starting from 1.11.0, Flink allows users to write SQL statements in Hive syntax when Hive dialect is used. By providing compatibility with Hive syntax, we aim to improve ", + "product_code":"dli", + "title":"Hive Dialect", + "uri":"dli_08_15048.html", + "doc_type":"sqlreference", + "p_code":"354", + "code":"356" + }, + { + "desc":"Apache Hive has established itself as a focal point of the data warehousing ecosystem. It serves as not only a SQL engine for big data analytics and ETL, but also a data ", + "product_code":"dli", + "title":"Hive Source Table", + "uri":"dli_08_15049.html", + "doc_type":"sqlreference", + "p_code":"354", + "code":"357" + }, + { + "desc":"This section describes how to use Flink to write Hive tables, the definition of the Hive result table, parameters used for creating the result table, and sample code. For", + "product_code":"dli", + "title":"Result Table", + "uri":"dli_08_15050.html", + "doc_type":"sqlreference", + "p_code":"354", + "code":"358" + }, + { + "desc":"You can use Hive tables as temporal tables and associate them through temporal joins. For more information on temporal joins, refer to temporal join.Flink supports proces", + "product_code":"dli", + "title":"Hive Dimension Table", + "uri":"dli_08_15051.html", + "doc_type":"sqlreference", + "p_code":"354", + "code":"359" + }, + { + "desc":"For partitioned tables that change over time, we can read them as unbounded streams. If each partition contains a complete set of data for a certain version, the partitio", + "product_code":"dli", + "title":"Using Temporal Join to Associate the Latest Partition of a Dimension Table", + "uri":"dli_08_15052.html", + "doc_type":"sqlreference", + "p_code":"354", + "code":"360" + }, + { + "desc":"For Hive tables, we can read them as bounded streams. In this case, the Hive table can only track its latest version when queried. The latest version of the table retains", + "product_code":"dli", + "title":"Using Temporal Join to Associate the Latest Version of a Dimension Table", + "uri":"dli_08_15053.html", + "doc_type":"sqlreference", + "p_code":"354", + "code":"361" + }, + { + "desc":"The JDBC connector is provided by Apache Flink and can be used to read data from and write data to common databases, such as MySQL and PostgreSQL. Source tables, result t", + "product_code":"dli", + "title":"JDBC", + "uri":"dli_08_15057.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"362" + }, + { + "desc":"The Kafka connector allows for reading data from and writing data into Kafka topics.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing an", + "product_code":"dli", + "title":"Kafka", + "uri":"dli_08_15058.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"363" + }, + { + "desc":"The MySQL CDC source table, that is, the MySQL streaming source table, reads all historical data in the database first and then smoothly switches data read to the Binlog ", + "product_code":"dli", + "title":"MySQL CDC", + "uri":"dli_08_15059.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"364" + }, + { + "desc":"The Print connector is used to print output data to the error file or out file in the TaskManager, making it easier for you to view the result in code debugging.NoneThe P", + "product_code":"dli", + "title":"Print", + "uri":"dli_08_15060.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"365" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Redis", + "uri":"dli_08_15061.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"366" + }, + { + "desc":"Create a source stream to obtain data from Redis as input for jobs.An enhanced datasource connection has been created for DLI to connect to the Redis database, so that yo", + "product_code":"dli", + "title":"Source Table", + "uri":"dli_08_15062.html", + "doc_type":"sqlreference", + "p_code":"366", + "code":"367" + }, + { + "desc":"DLI outputs the Flink job output data to Redis. Redis is a key-value storage system that supports multiple types of data structures. It can be used in scenarios such as c", + "product_code":"dli", + "title":"Result Table", + "uri":"dli_08_15063.html", + "doc_type":"sqlreference", + "p_code":"366", + "code":"368" + }, + { + "desc":"Create a Redis table to connect to source streams for wide table generation.An enhanced datasource connection with Redis has been established, so that you can configure s", + "product_code":"dli", + "title":"Dimension Table", + "uri":"dli_08_15064.html", + "doc_type":"sqlreference", + "p_code":"366", + "code":"369" + }, + { + "desc":"Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It delivers high throughput and built-in partitions and provi", + "product_code":"dli", + "title":"Upsert Kafka", + "uri":"dli_08_15065.html", + "doc_type":"sqlreference", + "p_code":"331", + "code":"370" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"DML Snytax", + "uri":"dli_08_15066.html", + "doc_type":"sqlreference", + "p_code":"305", + "code":"371" + }, + { + "desc":"SyntaxDescriptionSELECT is used to select data from a table.ALL indicates that all results are returned.DISTINCT indicates that the duplicated results are removed.Precaut", + "product_code":"dli", + "title":"SELECT", + "uri":"dli_08_15067.html", + "doc_type":"sqlreference", + "p_code":"371", + "code":"372" + }, + { + "desc":"This section describes how to use the INSERT INTO statement to write job results to a sink table.SyntaxINSERT INTO your_sink\n SELECT ... FROM your_source WHERE ...Exampl", + "product_code":"dli", + "title":"INSERT INTO", + "uri":"dli_08_15107.html", + "doc_type":"sqlreference", + "p_code":"371", + "code":"373" + }, + { + "desc":"SyntaxDescriptionUNION is used to return the union set of multiple query results.INTERSECT is used to return the intersection of multiple query results.EXCEPT is used to ", + "product_code":"dli", + "title":"Set Operations", + "uri":"dli_08_15068.html", + "doc_type":"sqlreference", + "p_code":"371", + "code":"374" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Window", + "uri":"dli_08_15069.html", + "doc_type":"sqlreference", + "p_code":"371", + "code":"375" + }, + { + "desc":"Windows are at the heart of processing infinite streams. Windows split the stream into \"buckets\" of finite size, over which we can apply computations.Apache Flink provide", + "product_code":"dli", + "title":"Window Functions", + "uri":"dli_08_15070.html", + "doc_type":"sqlreference", + "p_code":"375", + "code":"376" + }, + { + "desc":"Window aggregations are defined in the GROUP BY clause contains \"window_start\" and \"window_end\" columns of the relation applied Windowing TVF. Just like queries with regu", + "product_code":"dli", + "title":"Window Aggregation", + "uri":"dli_08_15071.html", + "doc_type":"sqlreference", + "p_code":"375", + "code":"377" + }, + { + "desc":"Window Top-N is a special Top-N which returns the N smallest or largest values for each window and other partitioned keys.Unlike regular Top-N on continuous tables, windo", + "product_code":"dli", + "title":"Window Top-N", + "uri":"dli_08_15072.html", + "doc_type":"sqlreference", + "p_code":"375", + "code":"378" + }, + { + "desc":"Window Deduplication is a special Deduplication which removes rows that duplicate over a set of columns, keeping the first one or the last one for each window and partiti", + "product_code":"dli", + "title":"Window Deduplication", + "uri":"dli_08_15073.html", + "doc_type":"sqlreference", + "p_code":"375", + "code":"379" + }, + { + "desc":"A window join adds the dimension of time into the join criteria themselves. By doing so, the window join joins the elements of two streams that share a common key and are", + "product_code":"dli", + "title":"Window Join", + "uri":"dli_08_15074.html", + "doc_type":"sqlreference", + "p_code":"375", + "code":"380" + }, + { + "desc":"An aggregate function computes a single result from multiple input rows. For example, there are aggregates to compute the COUNT, SUM, AVG (average), MAX (maximum) and MIN", + "product_code":"dli", + "title":"Group Aggregation", + "uri":"dli_08_15075.html", + "doc_type":"sqlreference", + "p_code":"371", + "code":"381" + }, + { + "desc":"OVER aggregates compute an aggregated value for every input row over a range of ordered rows. In contrast to GROUP BY aggregates, OVER aggregates do not reduce the number", + "product_code":"dli", + "title":"Over Aggregation", + "uri":"dli_08_15076.html", + "doc_type":"sqlreference", + "p_code":"371", + "code":"382" + }, + { + "desc":"SyntaxPrecautionsCurrently, only equi-joins are supported, for example, joins that have at least one conjunctive condition with an equality predicate. Arbitrary cross or ", + "product_code":"dli", + "title":"JOIN", + "uri":"dli_08_15077.html", + "doc_type":"sqlreference", + "p_code":"371", + "code":"383" + }, + { + "desc":"FunctionThis clause is used to sort data in ascending order on a time attribute.PrecautionsCurrently, only sorting by time attribute is supported.ExampleSort data in asce", + "product_code":"dli", + "title":"OrderBy & Limit", + "uri":"dli_08_15078.html", + "doc_type":"sqlreference", + "p_code":"371", + "code":"384" + }, + { + "desc":"Top-N queries ask for the N smallest or largest values ordered by columns. Both smallest and largest values sets are considered Top-N queries. Top-N queries are useful in", + "product_code":"dli", + "title":"Top-N", + "uri":"dli_08_15079.html", + "doc_type":"sqlreference", + "p_code":"371", + "code":"385" + }, + { + "desc":"Deduplication removes rows that duplicate over a set of columns, keeping only the first one or the last one.ROW_NUMBER(): Assigns a unique, sequential number to each row,", + "product_code":"dli", + "title":"Deduplication", + "uri":"dli_08_15080.html", + "doc_type":"sqlreference", + "p_code":"371", + "code":"386" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Functions", + "uri":"dli_08_15081.html", + "doc_type":"sqlreference", + "p_code":"305", + "code":"387" + }, + { + "desc":"DLI supports the following three types of user-defined functions (UDFs):Regular UDF: takes in one or more input parameters and returns a single result.User-defined table-", + "product_code":"dli", + "title":"UDFs", + "uri":"dli_08_15082.html", + "doc_type":"sqlreference", + "p_code":"387", + "code":"388" + }, + { + "desc":"Type inference summarizes the logic for validating input arguments and deriving data types for both the parameters and the result of a function. From a logical perspectiv", + "product_code":"dli", + "title":"Type Inference", + "uri":"dli_08_15083.html", + "doc_type":"sqlreference", + "p_code":"387", + "code":"389" + }, + { + "desc":"A UDF can be used in many jobs, and some parameter values vary with jobs. To easily modify the parameter values, you can set pipeline.global-job-parameters in the Runtime", + "product_code":"dli", + "title":"Parameter Transfer", + "uri":"dli_08_15084.html", + "doc_type":"sqlreference", + "p_code":"387", + "code":"390" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Built-In Functions", + "uri":"dli_08_15085.html", + "doc_type":"sqlreference", + "p_code":"387", + "code":"391" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Comparison Functions", + "uri":"dli_08_15086.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"392" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Logical Functions", + "uri":"dli_08_15087.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"393" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Arithmetic Functions", + "uri":"dli_08_15088.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"394" + }, + { + "desc":"DLI offers a wide range of string functions for processing and transforming string data. These functions include concatenation, case conversion, substring extraction, rep", + "product_code":"dli", + "title":"String Functions", + "uri":"dli_08_15089.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"395" + }, + { + "desc":"Table 1 lists the time functions supported by Flink OpenSource SQL.FunctionReturns a SQL date parsed from string in form of yyyy-MM-dd.Returns a SQL date parsed from stri", + "product_code":"dli", + "title":"Temporal Functions", + "uri":"dli_08_15090.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"396" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Conditional Functions", + "uri":"dli_08_15091.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"397" + }, + { + "desc":"This syntax is used to forcibly convert types.If the input is NULL, NULL is returned.The following example converts the amount value to an integer.Flink jobs do not suppo", + "product_code":"dli", + "title":"Type Conversion Functions", + "uri":"dli_08_15092.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"398" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Collection Functions", + "uri":"dli_08_15093.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"399" + }, + { + "desc":"JSON functions use JSON path expressions described in the SQL standard ISO/IEC TR 19075-6. Their syntax is inspired by ECMAScript and adopts many of its features, but is ", + "product_code":"dli", + "title":"JSON Functions", + "uri":"dli_08_15094.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"400" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Value Construction Functions", + "uri":"dli_08_15095.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"401" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Value Retrieval Functions", + "uri":"dli_08_15096.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"402" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Grouping Functions", + "uri":"dli_08_15097.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"403" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Hash Functions", + "uri":"dli_08_15098.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"404" + }, + { + "desc":"Aggregate functions process all rows as input and produce a single aggregate value as the output.", + "product_code":"dli", + "title":"Aggregate Functions", + "uri":"dli_08_15099.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"405" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Table-Valued Functions", + "uri":"dli_08_15101.html", + "doc_type":"sqlreference", + "p_code":"391", + "code":"406" + }, + { + "desc":"The string_split function splits a target string into substrings based on the specified separator and returns a substring list.Create a Flink OpenSource SQL job by referr", + "product_code":"dli", + "title":"string_split", + "uri":"dli_08_15102.html", + "doc_type":"sqlreference", + "p_code":"406", + "code":"407" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2733,7 +3669,7 @@ "uri":"dli_08_0370.html", "doc_type":"sqlreference", "p_code":"", - "code":"304" + "code":"408" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2741,17 +3677,17 @@ "title":"Constraints and Definitions", "uri":"dli_08_0371.html", "doc_type":"sqlreference", - "p_code":"304", - "code":"305" + "p_code":"408", + "code":"409" }, { - "desc":"STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP WITH LOCAL TIME ZONE, INTERVAL, ARRAY, MULTISET, MAP,", + "desc":"The DLI SQL syntax supports the following data types:STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP ", "product_code":"dli", "title":"Supported Data Types", "uri":"dli_08_0372.html", "doc_type":"sqlreference", - "p_code":"305", - "code":"306" + "p_code":"409", + "code":"410" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2759,8 +3695,8 @@ "title":"Syntax", "uri":"dli_08_0373.html", "doc_type":"sqlreference", - "p_code":"305", - "code":"307" + "p_code":"409", + "code":"411" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2768,8 +3704,8 @@ "title":"Data Definition Language (DDL)", "uri":"dli_08_0374.html", "doc_type":"sqlreference", - "p_code":"307", - "code":"308" + "p_code":"411", + "code":"412" }, { "desc":"Create a table with a specified name.COMPUTED COLUMNA computed column is a virtual column generated using column_name AS computed_column_expression. A computed column eva", @@ -2777,8 +3713,8 @@ "title":"CREATE TABLE", "uri":"dli_08_0375.html", "doc_type":"sqlreference", - "p_code":"308", - "code":"309" + "p_code":"412", + "code":"413" }, { "desc":"Create a view with multiple layers nested in it to simplify the development process.IF NOT EXISTSIf the view already exists, nothing happens.Create a view named viewName.", @@ -2786,8 +3722,8 @@ "title":"CREATE VIEW", "uri":"dli_08_0376.html", "doc_type":"sqlreference", - "p_code":"308", - "code":"310" + "p_code":"412", + "code":"414" }, { "desc":"Create a user-defined function.For details about how to create a user-defined function, see User-Defined Functions (UDFs).IF NOT EXISTSIf the function already exists, not", @@ -2795,8 +3731,8 @@ "title":"CREATE FUNCTION", "uri":"dli_08_0377.html", "doc_type":"sqlreference", - "p_code":"308", - "code":"311" + "p_code":"412", + "code":"415" }, { "desc":"SyntaxPrecautionsFlink SQL uses a lexical policy for identifier (table, attribute, function names) similar to Java:The case of identifiers is preserved whether they are q", @@ -2804,8 +3740,8 @@ "title":"Data Manipulation Language (DML)", "uri":"dli_08_0378.html", "doc_type":"sqlreference", - "p_code":"307", - "code":"312" + "p_code":"411", + "code":"416" }, { "desc":"This section describes the Flink open source SQL 1.12 syntax supported by DLI. For details about the parameters and examples, see the syntax description.", @@ -2813,8 +3749,8 @@ "title":"Overview", "uri":"dli_08_0379.html", "doc_type":"sqlreference", - "p_code":"304", - "code":"313" + "p_code":"408", + "code":"417" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2822,8 +3758,8 @@ "title":"DDL Syntax", "uri":"dli_08_0380.html", "doc_type":"sqlreference", - "p_code":"304", - "code":"314" + "p_code":"408", + "code":"418" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2831,8 +3767,8 @@ "title":"Creating Source Tables", "uri":"dli_08_0381.html", "doc_type":"sqlreference", - "p_code":"314", - "code":"315" + "p_code":"418", + "code":"419" }, { "desc":"DataGen is used to generate random data for debugging and testing.NoneWhen you create a DataGen table, the table field type cannot be Array, Map, or Row. You can use COMP", @@ -2840,8 +3776,8 @@ "title":"DataGen Source Table", "uri":"dli_08_0382.html", "doc_type":"sqlreference", - "p_code":"315", - "code":"316" + "p_code":"419", + "code":"420" }, { "desc":"DLI reads data of Flink jobs from GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex types an", @@ -2849,8 +3785,8 @@ "title":"GaussDB(DWS) Source Table", "uri":"dli_08_0383.html", "doc_type":"sqlreference", - "p_code":"315", - "code":"317" + "p_code":"419", + "code":"421" }, { "desc":"Create a source stream to obtain data from HBase as input for jobs. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excell", @@ -2858,8 +3794,8 @@ "title":"HBase Source Table", "uri":"dli_08_0384.html", "doc_type":"sqlreference", - "p_code":"315", - "code":"318" + "p_code":"419", + "code":"422" }, { "desc":"The JDBC connector is a Flink's built-in connector to read data from a database.An enhanced datasource connection with the instances has been established, so that you can", @@ -2867,8 +3803,8 @@ "title":"JDBC Source Table", "uri":"dli_08_0385.html", "doc_type":"sqlreference", - "p_code":"315", - "code":"319" + "p_code":"419", + "code":"423" }, { "desc":"Create a source stream to obtain data from Kafka as input data for jobs.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscripti", @@ -2876,8 +3812,8 @@ "title":"Kafka Source Table", "uri":"dli_08_0386.html", "doc_type":"sqlreference", - "p_code":"315", - "code":"320" + "p_code":"419", + "code":"424" }, { "desc":"The MySQL CDC source table, that is, the MySQL streaming source table, reads all historical data in the database first and then smoothly switches data read to the Binlog ", @@ -2885,8 +3821,8 @@ "title":"MySQL CDC Source Table", "uri":"dli_08_0387.html", "doc_type":"sqlreference", - "p_code":"315", - "code":"321" + "p_code":"419", + "code":"425" }, { "desc":"The Postgres CDC source table, that is, Postgres streaming source table, is used to read the full snapshot data and changed data of the PostgreSQL database in sequence. T", @@ -2894,8 +3830,8 @@ "title":"Postgres CDC Source Table", "uri":"dli_08_0388.html", "doc_type":"sqlreference", - "p_code":"315", - "code":"322" + "p_code":"419", + "code":"426" }, { "desc":"Create a source stream to obtain data from Redis as input for jobs.An enhanced datasource connection has been created for DLI to connect to the Redis database, so that yo", @@ -2903,8 +3839,8 @@ "title":"Redis Source Table", "uri":"dli_08_0389.html", "doc_type":"sqlreference", - "p_code":"315", - "code":"323" + "p_code":"419", + "code":"427" }, { "desc":"Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It delivers high throughput and built-in partitions and provi", @@ -2912,8 +3848,8 @@ "title":"Upsert Kafka Source Table", "uri":"dli_08_0390.html", "doc_type":"sqlreference", - "p_code":"315", - "code":"324" + "p_code":"419", + "code":"428" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2921,8 +3857,8 @@ "title":"Creating Result Tables", "uri":"dli_08_0391.html", "doc_type":"sqlreference", - "p_code":"314", - "code":"325" + "p_code":"418", + "code":"429" }, { "desc":"The BlackHole connector allows for swallowing all input records. It is designed for high-performance testing and UDF output. It is not a substantive sink. The BlackHole r", @@ -2930,8 +3866,8 @@ "title":"BlackHole Result Table", "uri":"dli_08_0392.html", "doc_type":"sqlreference", - "p_code":"325", - "code":"326" + "p_code":"429", + "code":"430" }, { "desc":"DLI can output Flink job data to the ClickHouse database. ClickHouse is a column-based database oriented to online analysis and processing. It supports SQL query and prov", @@ -2939,8 +3875,8 @@ "title":"ClickHouse Result Table", "uri":"dli_08_0393.html", "doc_type":"sqlreference", - "p_code":"325", - "code":"327" + "p_code":"429", + "code":"431" }, { "desc":"DLI outputs the Flink job output data to GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex t", @@ -2948,8 +3884,8 @@ "title":"GaussDB(DWS) Result Table", "uri":"dli_08_0394.html", "doc_type":"sqlreference", - "p_code":"325", - "code":"328" + "p_code":"429", + "code":"432" }, { "desc":"DLI outputs Flink job output data to Elasticsearch of Cloud Search Service (CSS). Elasticsearch is a popular enterprise-class Lucene-powered search server and provides th", @@ -2957,8 +3893,8 @@ "title":"Elasticsearch Result Table", "uri":"dli_08_0395.html", "doc_type":"sqlreference", - "p_code":"325", - "code":"329" + "p_code":"429", + "code":"433" }, { "desc":"DLI outputs the job data to HBase. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excellent performance, and elastic scal", @@ -2966,8 +3902,8 @@ "title":"HBase Result Table", "uri":"dli_08_0396.html", "doc_type":"sqlreference", - "p_code":"325", - "code":"330" + "p_code":"429", + "code":"434" }, { "desc":"DLI outputs the Flink job output data to RDS through the JDBC result table.An enhanced datasource connection with the instances has been established, so that you can conf", @@ -2975,8 +3911,8 @@ "title":"JDBC Result Table", "uri":"dli_08_0397.html", "doc_type":"sqlreference", - "p_code":"325", - "code":"331" + "p_code":"429", + "code":"435" }, { "desc":"DLI outputs the Flink job output data to Kafka through the Kafka result table.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subs", @@ -2984,8 +3920,8 @@ "title":"Kafka Result Table", "uri":"dli_08_0398.html", "doc_type":"sqlreference", - "p_code":"325", - "code":"332" + "p_code":"429", + "code":"436" }, { "desc":"The Print connector is used to print output data to the error file or TaskManager file, making it easier for you to view the result in code debugging.NoneThe Print result", @@ -2993,8 +3929,8 @@ "title":"Print Result Table", "uri":"dli_08_0399.html", "doc_type":"sqlreference", - "p_code":"325", - "code":"333" + "p_code":"429", + "code":"437" }, { "desc":"DLI outputs the Flink job output data to Redis. Redis is a key-value storage system that supports multiple types of data structures. It can be used in scenarios such as c", @@ -3002,8 +3938,8 @@ "title":"Redis Result Table", "uri":"dli_08_0400.html", "doc_type":"sqlreference", - "p_code":"325", - "code":"334" + "p_code":"429", + "code":"438" }, { "desc":"Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It delivers high throughput and built-in partitions and provi", @@ -3011,8 +3947,8 @@ "title":"Upsert Kafka Result Table", "uri":"dli_08_0401.html", "doc_type":"sqlreference", - "p_code":"325", - "code":"335" + "p_code":"429", + "code":"439" }, { "desc":"The FileSystem result (sink) table is used to export data to the HDFS or OBS file system. It is applicable to scenarios such as data dumping, big data analysis, data back", @@ -3020,8 +3956,8 @@ "title":"FileSystem Result Table", "uri":"dli_08_0439.html", "doc_type":"sqlreference", - "p_code":"325", - "code":"336" + "p_code":"429", + "code":"440" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3029,8 +3965,8 @@ "title":"Creating Dimension Tables", "uri":"dli_08_0402.html", "doc_type":"sqlreference", - "p_code":"314", - "code":"337" + "p_code":"418", + "code":"441" }, { "desc":"Create a GaussDB(DWS) table to connect to source streams for wide table generation.Ensure that you have created a GaussDB(DWS) cluster using your account.A DWS database t", @@ -3038,8 +3974,8 @@ "title":"GaussDB(DWS) Dimension Table", "uri":"dli_08_0403.html", "doc_type":"sqlreference", - "p_code":"337", - "code":"338" + "p_code":"441", + "code":"442" }, { "desc":"Create a Hbase dimension table to connect to the source streams for wide table generation.An enhanced datasource connection has been created for DLI to connect to HBase, ", @@ -3047,8 +3983,8 @@ "title":"HBase Dimension Table", "uri":"dli_08_0404.html", "doc_type":"sqlreference", - "p_code":"337", - "code":"339" + "p_code":"441", + "code":"443" }, { "desc":"Create a JDBC dimension table to connect to the source stream.You have created a JDBC instance for your account.When you create a Flink OpenSource SQL job, set Flink Vers", @@ -3056,8 +3992,8 @@ "title":"JDBC Dimension Table", "uri":"dli_08_0405.html", "doc_type":"sqlreference", - "p_code":"337", - "code":"340" + "p_code":"441", + "code":"444" }, { "desc":"Create a Redis table to connect to source streams for wide table generation.An enhanced datasource connection with Redis has been established, so that you can configure s", @@ -3065,8 +4001,8 @@ "title":"Redis Dimension Table", "uri":"dli_08_0406.html", "doc_type":"sqlreference", - "p_code":"337", - "code":"341" + "p_code":"441", + "code":"445" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3074,8 +4010,8 @@ "title":"Format", "uri":"dli_08_0407.html", "doc_type":"sqlreference", - "p_code":"314", - "code":"342" + "p_code":"418", + "code":"446" }, { "desc":"Apache Avro is supported for you to read and write Avro data based on an Avro schema with Flink. The Avro schema is derived from the table schema.KafkaUpsert KafkaCurrent", @@ -3083,8 +4019,8 @@ "title":"Avro", "uri":"dli_08_0408.html", "doc_type":"sqlreference", - "p_code":"342", - "code":"343" + "p_code":"446", + "code":"447" }, { "desc":"Canal is a Changelog Data Capture (CDC) tool that can stream changes in real-time from MySQL into other systems. Canal provides a unified format schema for changelog and ", @@ -3092,8 +4028,8 @@ "title":"Canal", "uri":"dli_08_0409.html", "doc_type":"sqlreference", - "p_code":"342", - "code":"344" + "p_code":"446", + "code":"448" }, { "desc":"The Avro Schema Registry (avro-confluent) format allows you to read records that were serialized by the io.confluent.kafka.serializers.KafkaAvroSerializer and to write re", @@ -3101,8 +4037,8 @@ "title":"Confluent Avro", "uri":"dli_08_0410.html", "doc_type":"sqlreference", - "p_code":"342", - "code":"345" + "p_code":"446", + "code":"449" }, { "desc":"The CSV format allows you to read and write CSV data based on a CSV schema. Currently, the CSV schema is derived from table schema.KafkaUpsert KafkaUse Kafka to send data", @@ -3110,8 +4046,8 @@ "title":"CSV", "uri":"dli_08_0411.html", "doc_type":"sqlreference", - "p_code":"342", - "code":"346" + "p_code":"446", + "code":"450" }, { "desc":"Debezium is a Changelog Data Capture (CDC) tool that can stream changes in real-time from other databases into Kafka. Debezium provides a unified format schema for change", @@ -3119,8 +4055,8 @@ "title":"Debezium", "uri":"dli_08_0412.html", "doc_type":"sqlreference", - "p_code":"342", - "code":"347" + "p_code":"446", + "code":"451" }, { "desc":"The JSON format allows you to read and write JSON data based on a JSON schema. Currently, the JSON schema is derived from table schema.KafkaUpsert KafkaElasticsearchIn th", @@ -3128,8 +4064,8 @@ "title":"JSON", "uri":"dli_08_0413.html", "doc_type":"sqlreference", - "p_code":"342", - "code":"348" + "p_code":"446", + "code":"452" }, { "desc":"Flink supports to interpret Maxwell JSON messages as INSERT/UPDATE/DELETE messages into Flink SQL system. This is useful in many cases to leverage this feature,such as:Sy", @@ -3137,8 +4073,8 @@ "title":"Maxwell", "uri":"dli_08_0414.html", "doc_type":"sqlreference", - "p_code":"342", - "code":"349" + "p_code":"446", + "code":"453" }, { "desc":"The raw format allows you to read and write raw (byte based) values as a single column.Note: This format encodes null values as null of the byte[] type. This may have lim", @@ -3146,8 +4082,8 @@ "title":"Raw", "uri":"dli_08_0415.html", "doc_type":"sqlreference", - "p_code":"342", - "code":"350" + "p_code":"446", + "code":"454" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3155,8 +4091,8 @@ "title":"DML Snytax", "uri":"dli_08_0416.html", "doc_type":"sqlreference", - "p_code":"304", - "code":"351" + "p_code":"408", + "code":"455" }, { "desc":"SyntaxDescriptionSELECT is used to select data from a table.ALL indicates that all results are returned.DISTINCT indicates that the duplicated results are removed.Precaut", @@ -3164,8 +4100,8 @@ "title":"SELECT", "uri":"dli_08_0417.html", "doc_type":"sqlreference", - "p_code":"351", - "code":"352" + "p_code":"455", + "code":"456" }, { "desc":"SyntaxDescriptionUNION is used to return the union set of multiple query results.INTERSECT is used to return the intersection of multiple query results.EXCEPT is used to ", @@ -3173,8 +4109,8 @@ "title":"Set Operations", "uri":"dli_08_0418.html", "doc_type":"sqlreference", - "p_code":"351", - "code":"353" + "p_code":"455", + "code":"457" }, { "desc":"DescriptionGroup Window is defined in GROUP BY. One record is generated from each group. Group Window involves the following functions:Array functionsArray functionsGroup", @@ -3182,8 +4118,8 @@ "title":"Window", "uri":"dli_08_0419.html", "doc_type":"sqlreference", - "p_code":"351", - "code":"354" + "p_code":"455", + "code":"458" }, { "desc":"SyntaxPrecautionsCurrently, only equi-joins are supported, for example, joins that have at least one conjunctive condition with an equality predicate. Arbitrary cross or ", @@ -3191,8 +4127,8 @@ "title":"JOIN", "uri":"dli_08_0420.html", "doc_type":"sqlreference", - "p_code":"351", - "code":"355" + "p_code":"455", + "code":"459" }, { "desc":"FunctionThis clause is used to sort data in ascending order on a time attribute.PrecautionsCurrently, only sorting by time attribute is supported.ExampleSort data in asce", @@ -3200,8 +4136,8 @@ "title":"OrderBy & Limit", "uri":"dli_08_0421.html", "doc_type":"sqlreference", - "p_code":"351", - "code":"356" + "p_code":"455", + "code":"460" }, { "desc":"Top-N queries ask for the N smallest or largest values ordered by columns. Both smallest and largest values sets are considered Top-N queries. Top-N queries are useful in", @@ -3209,8 +4145,8 @@ "title":"Top-N", "uri":"dli_08_0422.html", "doc_type":"sqlreference", - "p_code":"351", - "code":"357" + "p_code":"455", + "code":"461" }, { "desc":"Deduplication removes rows that duplicate over a set of columns, keeping only the first one or the last one.ROW_NUMBER(): Assigns a unique, sequential number to each row,", @@ -3218,8 +4154,8 @@ "title":"Deduplication", "uri":"dli_08_0423.html", "doc_type":"sqlreference", - "p_code":"351", - "code":"358" + "p_code":"455", + "code":"462" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3227,8 +4163,8 @@ "title":"Functions", "uri":"dli_08_0424.html", "doc_type":"sqlreference", - "p_code":"304", - "code":"359" + "p_code":"408", + "code":"463" }, { "desc":"DLI supports the following three types of user-defined functions (UDFs):Regular UDF: takes in one or more input parameters and returns a single result.User-defined table-", @@ -3236,8 +4172,8 @@ "title":"User-Defined Functions (UDFs)", "uri":"dli_08_0425.html", "doc_type":"sqlreference", - "p_code":"359", - "code":"360" + "p_code":"463", + "code":"464" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3245,8 +4181,8 @@ "title":"Built-In Functions", "uri":"dli_08_0426.html", "doc_type":"sqlreference", - "p_code":"359", - "code":"361" + "p_code":"463", + "code":"465" }, { "desc":"All data types can be compared by using relational operators and the result is returned as a BOOLEAN value.Relationship operators are binary operators. Two compared data ", @@ -3254,17 +4190,17 @@ "title":"Mathematical Operation Functions", "uri":"dli_08_0427.html", "doc_type":"sqlreference", - "p_code":"361", - "code":"362" + "p_code":"465", + "code":"466" }, { - "desc":"SyntaxExampleTest input data.Test the data source kafka. The message content is as follows:{name:James,age:24,sex:male,grade:{math:95,science:[80,85],english:100}}\n{name:", + "desc":"DLI offers a wide range of string functions for processing and transforming string data. These functions include concatenation, case conversion, substring extraction, rep", "product_code":"dli", "title":"String Functions", "uri":"dli_08_0428.html", "doc_type":"sqlreference", - "p_code":"361", - "code":"363" + "p_code":"465", + "code":"467" }, { "desc":"Table 1 lists the time functions supported by Flink OpenSource SQL.FunctionReturns a SQL date parsed from string in form of yyyy-MM-dd.Returns a SQL date parsed from stri", @@ -3272,8 +4208,8 @@ "title":"Temporal Functions", "uri":"dli_08_0429.html", "doc_type":"sqlreference", - "p_code":"361", - "code":"364" + "p_code":"465", + "code":"468" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3281,8 +4217,8 @@ "title":"Conditional Functions", "uri":"dli_08_0430.html", "doc_type":"sqlreference", - "p_code":"361", - "code":"365" + "p_code":"465", + "code":"469" }, { "desc":"This function is used to forcibly convert types.If the input is NULL, NULL is returned.The cast function does not support converting a string to the JSON format.The follo", @@ -3290,8 +4226,8 @@ "title":"Type Conversion Functions", "uri":"dli_08_0431.html", "doc_type":"sqlreference", - "p_code":"361", - "code":"366" + "p_code":"465", + "code":"470" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3299,8 +4235,8 @@ "title":"Collection Functions", "uri":"dli_08_0432.html", "doc_type":"sqlreference", - "p_code":"361", - "code":"367" + "p_code":"465", + "code":"471" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3308,8 +4244,8 @@ "title":"Value Construction Functions", "uri":"dli_08_0433.html", "doc_type":"sqlreference", - "p_code":"361", - "code":"368" + "p_code":"465", + "code":"472" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3317,8 +4253,8 @@ "title":"Value Access Functions", "uri":"dli_08_0434.html", "doc_type":"sqlreference", - "p_code":"361", - "code":"369" + "p_code":"465", + "code":"473" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3326,8 +4262,8 @@ "title":"Hash Functions", "uri":"dli_08_0435.html", "doc_type":"sqlreference", - "p_code":"361", - "code":"370" + "p_code":"465", + "code":"474" }, { "desc":"An aggregate function performs a calculation operation on a set of input values and returns a value. For example, the COUNT function counts the number of rows retrieved b", @@ -3335,8 +4271,8 @@ "title":"Aggregate Functions", "uri":"dli_08_0436.html", "doc_type":"sqlreference", - "p_code":"361", - "code":"371" + "p_code":"465", + "code":"475" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3344,8 +4280,8 @@ "title":"Table-Valued Functions", "uri":"dli_08_0437.html", "doc_type":"sqlreference", - "p_code":"361", - "code":"372" + "p_code":"465", + "code":"476" }, { "desc":"The string_split function splits a target string into substrings based on the specified separator and returns a substring list.Create a Flink OpenSource SQL job by referr", @@ -3353,8 +4289,8 @@ "title":"string_split", "uri":"dli_08_0438.html", "doc_type":"sqlreference", - "p_code":"372", - "code":"373" + "p_code":"476", + "code":"477" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3363,7 +4299,7 @@ "uri":"dli_08_0289.html", "doc_type":"sqlreference", "p_code":"", - "code":"374" + "code":"478" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3371,17 +4307,17 @@ "title":"Constraints and Definitions", "uri":"dli_08_0290.html", "doc_type":"sqlreference", - "p_code":"374", - "code":"375" + "p_code":"478", + "code":"479" }, { - "desc":"STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP WITH LOCAL TIME ZONE, INTERVAL, ARRAY, MULTISET, MAP,", + "desc":"The DLI SQL syntax supports the following data types:STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP ", "product_code":"dli", "title":"Supported Data Types", "uri":"dli_08_0291.html", "doc_type":"sqlreference", - "p_code":"375", - "code":"376" + "p_code":"479", + "code":"480" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3389,8 +4325,8 @@ "title":"Syntax Definition", "uri":"dli_08_0292.html", "doc_type":"sqlreference", - "p_code":"375", - "code":"377" + "p_code":"479", + "code":"481" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3398,8 +4334,8 @@ "title":"Data Definition Language (DDL)", "uri":"dli_08_0293.html", "doc_type":"sqlreference", - "p_code":"377", - "code":"378" + "p_code":"481", + "code":"482" }, { "desc":"This clause is used to create a table with a specified name.COMPUTED COLUMNA computed column is a virtual column generated using column_name AS computed_column_expression", @@ -3407,8 +4343,8 @@ "title":"CREATE TABLE", "uri":"dli_08_0294.html", "doc_type":"sqlreference", - "p_code":"378", - "code":"379" + "p_code":"482", + "code":"483" }, { "desc":"Create a view with multiple layers nested in it to simplify the development process.IF NOT EXISTSIf the view already exists, nothing happens.Create a view named viewName.", @@ -3416,8 +4352,8 @@ "title":"CREATE VIEW", "uri":"dli_08_0295.html", "doc_type":"sqlreference", - "p_code":"378", - "code":"380" + "p_code":"482", + "code":"484" }, { "desc":"Create a user-defined function.IF NOT EXISTSIf the function already exists, nothing happens.LANGUAGE JAVA|SCALALanguage tag is used to instruct Flink runtime how to execu", @@ -3425,8 +4361,8 @@ "title":"CREATE FUNCTION", "uri":"dli_08_0296.html", "doc_type":"sqlreference", - "p_code":"378", - "code":"381" + "p_code":"482", + "code":"485" }, { "desc":"SyntaxPrecautionsFlink SQL uses a lexical policy for identifier (table, attribute, function names) similar to Java:The case of identifiers is preserved whether they are q", @@ -3434,8 +4370,8 @@ "title":"Data Manipulation Language (DML)", "uri":"dli_08_0297.html", "doc_type":"sqlreference", - "p_code":"377", - "code":"382" + "p_code":"481", + "code":"486" }, { "desc":"This section describes the Flink OpenSource SQL syntax supported by DLI. For details about the parameters and examples, see the syntax description.", @@ -3443,8 +4379,8 @@ "title":"Flink OpenSource SQL 1.10 Syntax", "uri":"dli_08_0298.html", "doc_type":"sqlreference", - "p_code":"374", - "code":"383" + "p_code":"478", + "code":"487" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3452,8 +4388,8 @@ "title":"Data Definition Language (DDL)", "uri":"dli_08_0299.html", "doc_type":"sqlreference", - "p_code":"374", - "code":"384" + "p_code":"478", + "code":"488" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3461,8 +4397,8 @@ "title":"Creating a Source Table", "uri":"dli_08_0300.html", "doc_type":"sqlreference", - "p_code":"384", - "code":"385" + "p_code":"488", + "code":"489" }, { "desc":"Create a source stream to obtain data from Kafka as input data for jobs.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscripti", @@ -3470,8 +4406,8 @@ "title":"Kafka Source Table", "uri":"dli_08_0301.html", "doc_type":"sqlreference", - "p_code":"385", - "code":"386" + "p_code":"489", + "code":"490" }, { "desc":"Create a source stream to read data from DIS. DIS accesses user data and Flink job reads data from the DIS stream as input data for jobs. Flink jobs can quickly remove da", @@ -3479,8 +4415,8 @@ "title":"DIS Source Table", "uri":"dli_08_0302.html", "doc_type":"sqlreference", - "p_code":"385", - "code":"387" + "p_code":"489", + "code":"491" }, { "desc":"The JDBC connector is a Flink's built-in connector to read data from a database.An enhanced datasource connection with the database has been established, so that you can ", @@ -3488,8 +4424,8 @@ "title":"JDBC Source Table", "uri":"dli_08_0303.html", "doc_type":"sqlreference", - "p_code":"385", - "code":"388" + "p_code":"489", + "code":"492" }, { "desc":"DLI reads data of Flink jobs from GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex types an", @@ -3497,8 +4433,8 @@ "title":"GaussDB(DWS) Source Table", "uri":"dli_08_0304.html", "doc_type":"sqlreference", - "p_code":"385", - "code":"389" + "p_code":"489", + "code":"493" }, { "desc":"Create a source stream to obtain data from Redis as input for jobs.An enhanced datasource connection with Redis has been established, so that you can configure security g", @@ -3506,8 +4442,8 @@ "title":"Redis Source Table", "uri":"dli_08_0305.html", "doc_type":"sqlreference", - "p_code":"385", - "code":"390" + "p_code":"489", + "code":"494" }, { "desc":"Create a source stream to obtain data from HBase as input for jobs. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excell", @@ -3515,8 +4451,8 @@ "title":"HBase Source Table", "uri":"dli_08_0306.html", "doc_type":"sqlreference", - "p_code":"385", - "code":"391" + "p_code":"489", + "code":"495" }, { "desc":"You can call APIs to obtain data from the cloud ecosystem or an open source ecosystem and use the obtained data as input of Flink jobs.The customized source class needs t", @@ -3524,8 +4460,8 @@ "title":"userDefined Source Table", "uri":"dli_08_0358.html", "doc_type":"sqlreference", - "p_code":"385", - "code":"392" + "p_code":"489", + "code":"496" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3533,8 +4469,8 @@ "title":"Creating a Result Table", "uri":"dli_08_0307.html", "doc_type":"sqlreference", - "p_code":"384", - "code":"393" + "p_code":"488", + "code":"497" }, { "desc":"DLI exports Flink job data to ClickHouse result tables.ClickHouse is a column-based database oriented to online analysis and processing. It supports SQL query and provide", @@ -3542,8 +4478,8 @@ "title":"ClickHouse Result Table", "uri":"dli_08_0344.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"394" + "p_code":"497", + "code":"498" }, { "desc":"DLI exports the output data of the Flink job to Kafka.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It deli", @@ -3551,8 +4487,8 @@ "title":"Kafka Result Table", "uri":"dli_08_0308.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"395" + "p_code":"497", + "code":"499" }, { "desc":"DLI exports the output data of the Flink job to Kafka in upsert mode.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription ", @@ -3560,8 +4496,8 @@ "title":"Upsert Kafka Result Table", "uri":"dli_08_0309.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"396" + "p_code":"497", + "code":"500" }, { "desc":"DLI writes the Flink job output data into DIS. The data is filtered and imported to the DIS stream for future processing.DIS addresses the challenge of transmitting data ", @@ -3569,8 +4505,8 @@ "title":"DIS Result Table", "uri":"dli_08_0310.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"397" + "p_code":"497", + "code":"501" }, { "desc":"DLI exports the output data of the Flink job to RDS.An enhanced datasource connection with the database has been established, so that you can configure security group rul", @@ -3578,8 +4514,8 @@ "title":"JDBC Result Table", "uri":"dli_08_0311.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"398" + "p_code":"497", + "code":"502" }, { "desc":"DLI outputs the Flink job output data to GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex t", @@ -3587,8 +4523,8 @@ "title":"GaussDB(DWS) Result Table", "uri":"dli_08_0312.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"399" + "p_code":"497", + "code":"503" }, { "desc":"DLI exports the output data of the Flink job to Redis. Redis is a storage system that supports multiple types of data structures such as key-value. It can be used in scen", @@ -3596,8 +4532,8 @@ "title":"Redis Result Table", "uri":"dli_08_0313.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"400" + "p_code":"497", + "code":"504" }, { "desc":"DLI exports Flink job output data to SMN.SMN provides reliable and flexible large-scale message notification services to DLI. It significantly simplifies system coupling ", @@ -3605,8 +4541,8 @@ "title":"SMN Result Table", "uri":"dli_08_0314.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"401" + "p_code":"497", + "code":"505" }, { "desc":"DLI outputs the job data to HBase. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excellent performance, and elastic scal", @@ -3614,8 +4550,8 @@ "title":"HBase Result Table", "uri":"dli_08_0315.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"402" + "p_code":"497", + "code":"506" }, { "desc":"DLI exports Flink job output data to Elasticsearch of Cloud Search Service (CSS). Elasticsearch is a popular enterprise-class Lucene-powered search server and provides th", @@ -3623,8 +4559,8 @@ "title":"Elasticsearch Result Table", "uri":"dli_08_0316.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"403" + "p_code":"497", + "code":"507" }, { "desc":"OpenTSDB is a distributed, scalable time series database based on HBase. OpenTSDB is designed to collect monitoring information of a large-scale cluster and query data in", @@ -3632,8 +4568,8 @@ "title":"OpenTSDB Result Table", "uri":"dli_08_0348.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"404" + "p_code":"497", + "code":"508" }, { "desc":"Write your Java code to insert the processed data into a specified database supported by your cloud service.Implement the custom sink class :The custom sink class is inhe", @@ -3641,8 +4577,8 @@ "title":"User-defined Result Table", "uri":"dli_08_0347.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"405" + "p_code":"497", + "code":"509" }, { "desc":"The print connector exports your data output to the error file or the out file of TaskManager. It is mainly used for code debugging and output viewing.Read data from Kafk", @@ -3650,8 +4586,8 @@ "title":"Print Result Table", "uri":"dli_08_0345.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"406" + "p_code":"497", + "code":"510" }, { "desc":"You can create a file system result table to export data to a file system such as HDFS or OBS. After the data is generated, a non-DLI table can be created directly accord", @@ -3659,8 +4595,8 @@ "title":"File System Result Table", "uri":"dli_08_0346.html", "doc_type":"sqlreference", - "p_code":"393", - "code":"407" + "p_code":"497", + "code":"511" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3668,8 +4604,8 @@ "title":"Creating a Dimension Table", "uri":"dli_08_0317.html", "doc_type":"sqlreference", - "p_code":"384", - "code":"408" + "p_code":"488", + "code":"512" }, { "desc":"Create a JDBC dimension table to connect to the source stream.You have created a JDBC instance for your account.The RDS table is used to connect to the source stream.CREA", @@ -3677,8 +4613,8 @@ "title":"JDBC Dimension Table", "uri":"dli_08_0318.html", "doc_type":"sqlreference", - "p_code":"408", - "code":"409" + "p_code":"512", + "code":"513" }, { "desc":"Create a GaussDB(DWS) dimension table to connect to the input stream.You have created a GaussDB(DWS) instance for your account.Use an RDS table to connect to the source s", @@ -3686,8 +4622,8 @@ "title":"GaussDB(DWS) Dimension Table", "uri":"dli_08_0319.html", "doc_type":"sqlreference", - "p_code":"408", - "code":"410" + "p_code":"512", + "code":"514" }, { "desc":"Create a Hbase dimension table to connect to the source stream.An enhanced datasource connection has been created for DLI to connect to HBase, so that jobs can run on the", @@ -3695,8 +4631,8 @@ "title":"HBase Dimension Table", "uri":"dli_08_0320.html", "doc_type":"sqlreference", - "p_code":"408", - "code":"411" + "p_code":"512", + "code":"515" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3704,8 +4640,8 @@ "title":"Data Manipulation Language (DML)", "uri":"dli_08_0321.html", "doc_type":"sqlreference", - "p_code":"374", - "code":"412" + "p_code":"478", + "code":"516" }, { "desc":"SyntaxDescriptionThis clause is used to select data from a table.ALL indicates that all results are returned.DISTINCT indicates that the duplicated results are removed.Pr", @@ -3713,8 +4649,8 @@ "title":"SELECT", "uri":"dli_08_0322.html", "doc_type":"sqlreference", - "p_code":"412", - "code":"413" + "p_code":"516", + "code":"517" }, { "desc":"SyntaxDescriptionUNION is used to return the union set of multiple query results.INTERSECT is used to return the intersection of multiple query results.EXCEPT is used to ", @@ -3722,17 +4658,17 @@ "title":"Set Operations", "uri":"dli_08_0323.html", "doc_type":"sqlreference", - "p_code":"412", - "code":"414" + "p_code":"516", + "code":"518" }, { - "desc":"DescriptionGroup Window is defined in GROUP BY. One record is generated from each group. Group Window involves the following functions:Array functionsArray functionsGroup", + "desc":"DescriptionGroup Window is defined in GROUP BY. One record is generated from each group. Group Window involves the following functions:Array functionsFor SQL queries in s", "product_code":"dli", "title":"Window", "uri":"dli_08_0324.html", "doc_type":"sqlreference", - "p_code":"412", - "code":"415" + "p_code":"516", + "code":"519" }, { "desc":"SyntaxPrecautionsCurrently, only equi-joins are supported, for example, joins that have at least one conjunctive condition with an equality predicate. Arbitrary cross or ", @@ -3740,8 +4676,8 @@ "title":"JOIN", "uri":"dli_08_0325.html", "doc_type":"sqlreference", - "p_code":"412", - "code":"416" + "p_code":"516", + "code":"520" }, { "desc":"FunctionThis clause is used to sort data in ascending order on a time attribute.PrecautionsCurrently, only sorting by time attribute is supported.ExampleSort data in asce", @@ -3749,8 +4685,8 @@ "title":"OrderBy & Limit", "uri":"dli_08_0326.html", "doc_type":"sqlreference", - "p_code":"412", - "code":"417" + "p_code":"516", + "code":"521" }, { "desc":"Top-N queries ask for the N smallest or largest values ordered by columns. Both smallest and largest values sets are considered Top-N queries. Top-N queries are useful in", @@ -3758,8 +4694,8 @@ "title":"Top-N", "uri":"dli_08_0327.html", "doc_type":"sqlreference", - "p_code":"412", - "code":"418" + "p_code":"516", + "code":"522" }, { "desc":"Deduplication removes rows that duplicate over a set of columns, keeping only the first one or the last one.ROW_NUMBER(): Assigns a unique, sequential number to each row,", @@ -3767,8 +4703,8 @@ "title":"Deduplication", "uri":"dli_08_0328.html", "doc_type":"sqlreference", - "p_code":"412", - "code":"419" + "p_code":"516", + "code":"523" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3776,8 +4712,8 @@ "title":"Functions", "uri":"dli_08_0329.html", "doc_type":"sqlreference", - "p_code":"374", - "code":"420" + "p_code":"478", + "code":"524" }, { "desc":"DLI supports the following three types of user-defined functions (UDFs):Regular UDF: takes in one or more input parameters and returns a single result.User-defined table-", @@ -3785,8 +4721,8 @@ "title":"User-Defined Functions", "uri":"dli_08_0330.html", "doc_type":"sqlreference", - "p_code":"420", - "code":"421" + "p_code":"524", + "code":"525" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3794,8 +4730,8 @@ "title":"Built-In Functions", "uri":"dli_08_0331.html", "doc_type":"sqlreference", - "p_code":"420", - "code":"422" + "p_code":"524", + "code":"526" }, { "desc":"All data types can be compared by using relational operators and the result is returned as a BOOLEAN value.Relationship operators are binary operators. Two compared data ", @@ -3803,17 +4739,17 @@ "title":"Mathematical Operation Functions", "uri":"dli_08_0332.html", "doc_type":"sqlreference", - "p_code":"422", - "code":"423" + "p_code":"526", + "code":"527" }, { - "desc":"SyntaxExampleTest input data.Test the data source kafka. The message content is as follows:\"{name:James,age:24,sex:male,grade:{math:95,science:[80,85],english:100}}\"\n\"{na", + "desc":"SyntaxExampleTest input data.Test the data source kafka. The message content is as follows:\"{name:James,age:24,gender:male,grade:{math:95,science:[80,85],english:100}}\"\n\"", "product_code":"dli", "title":"String Functions", "uri":"dli_08_0333.html", "doc_type":"sqlreference", - "p_code":"422", - "code":"424" + "p_code":"526", + "code":"528" }, { "desc":"Table 1 lists the temporal functions supported by Flink OpenSource SQL.FunctionReturns a date parsed from string in form of yyyy-MM-dd.Returns a date parsed from string i", @@ -3821,8 +4757,8 @@ "title":"Temporal Functions", "uri":"dli_08_0334.html", "doc_type":"sqlreference", - "p_code":"422", - "code":"425" + "p_code":"526", + "code":"529" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3830,8 +4766,8 @@ "title":"Conditional Functions", "uri":"dli_08_0335.html", "doc_type":"sqlreference", - "p_code":"422", - "code":"426" + "p_code":"526", + "code":"530" }, { "desc":"This function is used to forcibly convert types.If the input is NULL, NULL is returned.The following example converts the amount value to an integer.Flink jobs do not sup", @@ -3839,8 +4775,8 @@ "title":"Type Conversion Function", "uri":"dli_08_0336.html", "doc_type":"sqlreference", - "p_code":"422", - "code":"427" + "p_code":"526", + "code":"531" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3848,8 +4784,8 @@ "title":"Collection Functions", "uri":"dli_08_0337.html", "doc_type":"sqlreference", - "p_code":"422", - "code":"428" + "p_code":"526", + "code":"532" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3857,8 +4793,8 @@ "title":"Value Construction Functions", "uri":"dli_08_0338.html", "doc_type":"sqlreference", - "p_code":"422", - "code":"429" + "p_code":"526", + "code":"533" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3866,8 +4802,8 @@ "title":"Value Access Functions", "uri":"dli_08_0339.html", "doc_type":"sqlreference", - "p_code":"422", - "code":"430" + "p_code":"526", + "code":"534" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3875,8 +4811,8 @@ "title":"Hash Functions", "uri":"dli_08_0340.html", "doc_type":"sqlreference", - "p_code":"422", - "code":"431" + "p_code":"526", + "code":"535" }, { "desc":"An aggregate function performs a calculation operation on a set of input values and returns a value. For example, the COUNT function counts the number of rows retrieved b", @@ -3884,8 +4820,8 @@ "title":"Aggregate Function", "uri":"dli_08_0341.html", "doc_type":"sqlreference", - "p_code":"422", - "code":"432" + "p_code":"526", + "code":"536" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3893,8 +4829,8 @@ "title":"Table-Valued Functions", "uri":"dli_08_0342.html", "doc_type":"sqlreference", - "p_code":"422", - "code":"433" + "p_code":"526", + "code":"537" }, { "desc":"The split_cursor function can convert one row of records into multiple rows or convert one column of records into multiple columns. Table-valued functions can only be use", @@ -3902,8 +4838,8 @@ "title":"split_cursor", "uri":"dli_08_0357.html", "doc_type":"sqlreference", - "p_code":"433", - "code":"434" + "p_code":"537", + "code":"538" }, { "desc":"The string_split function splits a target string into substrings based on the specified separator and returns a substring list.Prepare test input data.Source table disSou", @@ -3911,8 +4847,8 @@ "title":"string_split", "uri":"dli_08_0356.html", "doc_type":"sqlreference", - "p_code":"433", - "code":"435" + "p_code":"537", + "code":"539" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3921,7 +4857,7 @@ "uri":"dli_08_0450.html", "doc_type":"sqlreference", "p_code":"", - "code":"436" + "code":"540" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3929,8 +4865,8 @@ "title":"Flink SQL Syntax", "uri":"dli_08_0233.html", "doc_type":"sqlreference", - "p_code":"436", - "code":"437" + "p_code":"540", + "code":"541" }, { "desc":"Currently, Flink SQL only supports the following operations: SELECT, FROM, WHERE, UNION, aggregation, window, JOIN between stream and table data, and JOIN between streams", @@ -3938,8 +4874,8 @@ "title":"SQL Syntax Constraints and Definitions", "uri":"dli_08_0075.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"438" + "p_code":"541", + "code":"542" }, { "desc":"This section describes the Flink SQL syntax list provided by DLI. For details about the parameters and examples, see the syntax description.", @@ -3947,8 +4883,8 @@ "title":"SQL Syntax Overview of Stream Jobs", "uri":"dli_08_0275.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"439" + "p_code":"541", + "code":"543" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -3956,8 +4892,8 @@ "title":"Creating a Source Stream", "uri":"dli_08_0234.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"440" + "p_code":"541", + "code":"544" }, { "desc":"Create a source stream to obtain data from HBase of CloudTable as input data of the job. HBase is a column-oriented distributed cloud storage system that features enhance", @@ -3965,8 +4901,8 @@ "title":"CloudTable HBase Source Stream", "uri":"dli_08_0237.html", "doc_type":"sqlreference", - "p_code":"440", - "code":"441" + "p_code":"544", + "code":"545" }, { "desc":"Create a source stream to read data from DIS. DIS accesses user data and Flink job reads data from the DIS stream as input data for jobs. Flink jobs can quickly remove da", @@ -3974,8 +4910,8 @@ "title":"DIS Source Stream", "uri":"dli_08_0235.html", "doc_type":"sqlreference", - "p_code":"440", - "code":"442" + "p_code":"544", + "code":"546" }, { "desc":"DMS (Distributed Message Service) is a message middleware service based on distributed, high-availability clustering technology. It provides reliable, scalable, fully man", @@ -3983,8 +4919,8 @@ "title":"DMS Source Stream", "uri":"dli_08_0270.html", "doc_type":"sqlreference", - "p_code":"440", - "code":"443" + "p_code":"544", + "code":"547" }, { "desc":"Create a source stream to obtain data from Kafka as input data for jobs.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscripti", @@ -3992,8 +4928,8 @@ "title":"MRS Kafka Source Stream", "uri":"dli_08_0238.html", "doc_type":"sqlreference", - "p_code":"440", - "code":"444" + "p_code":"544", + "code":"548" }, { "desc":"Create a source stream to obtain data from Kafka as input data for jobs.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscripti", @@ -4001,8 +4937,8 @@ "title":"Open-Source Kafka Source Stream", "uri":"dli_08_0239.html", "doc_type":"sqlreference", - "p_code":"440", - "code":"445" + "p_code":"544", + "code":"549" }, { "desc":"Create a source stream to obtain data from OBS. DLI reads data stored by users in OBS as input data for jobs. OBS applies to various scenarios, such as big data analysis,", @@ -4010,8 +4946,8 @@ "title":"OBS Source Stream", "uri":"dli_08_0236.html", "doc_type":"sqlreference", - "p_code":"440", - "code":"446" + "p_code":"544", + "code":"550" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4019,8 +4955,8 @@ "title":"Creating a Sink Stream", "uri":"dli_08_0240.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"447" + "p_code":"541", + "code":"551" }, { "desc":"DLI exports the job output data to HBase of CloudTable. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excellent performa", @@ -4028,8 +4964,8 @@ "title":"CloudTable HBase Sink Stream", "uri":"dli_08_0243.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"448" + "p_code":"551", + "code":"552" }, { "desc":"DLI exports the job output data to OpenTSDB of CloudTable. OpenTSDB is a distributed, scalable time series database based on HBase. It stores time series data. Time serie", @@ -4037,8 +4973,8 @@ "title":"CloudTable OpenTSDB Sink Stream", "uri":"dli_08_0244.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"449" + "p_code":"551", + "code":"553" }, { "desc":"DLI exports the output data of the Flink job to OpenTSDB of MRS.OpenTSDB has been installed in the MRS cluster.In this scenario, jobs must run on the dedicated queue of D", @@ -4046,8 +4982,8 @@ "title":"MRS OpenTSDB Sink Stream", "uri":"dli_08_0286.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"450" + "p_code":"551", + "code":"554" }, { "desc":"DLI exports Flink job output data to Elasticsearch of Cloud Search Service (CSS). Elasticsearch is a popular enterprise-class Lucene-powered search server and provides th", @@ -4055,8 +4991,8 @@ "title":"CSS Elasticsearch Sink Stream", "uri":"dli_08_0252.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"451" + "p_code":"551", + "code":"555" }, { "desc":"DLI exports the Flink job output data to Redis of DCS. Redis is a storage system that supports multiple types of data structures such as key-value. It can be used in scen", @@ -4064,8 +5000,8 @@ "title":"DCS Sink Stream", "uri":"dli_08_0253.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"452" + "p_code":"551", + "code":"556" }, { "desc":"DLI outputs the job output data to Document Database Service (DDS).DDS is compatible with the MongoDB protocol and is secure, highly available, reliable, scalable, and ea", @@ -4073,8 +5009,8 @@ "title":"DDS Sink Stream", "uri":"dli_08_0249.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"453" + "p_code":"551", + "code":"557" }, { "desc":"DLI writes the Flink job output data into DIS. This cloud ecosystem is applicable to scenarios where data is filtered and imported to the DIS stream for future processing", @@ -4082,17 +5018,17 @@ "title":"DIS Sink Stream", "uri":"dli_08_0241.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"454" + "p_code":"551", + "code":"558" }, { - "desc":"DMS (Distributed Message Service) is a message middleware service based on distributed, high-availability clustering technology. It provides reliable, scalable, fully man", + "desc":"DMS for Kafka is a message queuing service based on Apache Kafka. This service provides Kafka premium instances.DLI can write the job output data into the Kafka instance.", "product_code":"dli", "title":"DMS Sink Stream", "uri":"dli_08_0271.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"455" + "p_code":"551", + "code":"559" }, { "desc":"DLI outputs the Flink job output data to Data Warehouse Service (DWS). DWS database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more co", @@ -4100,8 +5036,8 @@ "title":"DWS Sink Stream (JDBC Mode)", "uri":"dli_08_0247.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"456" + "p_code":"551", + "code":"560" }, { "desc":"Create a sink stream to export Flink job data to DWS through OBS-based dumping, specifically, output Flink job data to OBS and then import data from OBS to DWS. For detai", @@ -4109,8 +5045,8 @@ "title":"DWS Sink Stream (OBS-based Dumping)", "uri":"dli_08_0248.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"457" + "p_code":"551", + "code":"561" }, { "desc":"DLI exports the output data of the Flink job to HBase of MRS.An MRS cluster has been created by using your account. DLI can interconnect with HBase clusters with Kerberos", @@ -4118,8 +5054,8 @@ "title":"MRS HBase Sink Stream", "uri":"dli_08_0255.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"458" + "p_code":"551", + "code":"562" }, { "desc":"DLI exports the output data of the Flink job to Kafka.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It deli", @@ -4127,8 +5063,8 @@ "title":"MRS Kafka Sink Stream", "uri":"dli_08_0254.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"459" + "p_code":"551", + "code":"563" }, { "desc":"DLI exports the output data of the Flink job to Kafka.Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It deli", @@ -4136,8 +5072,8 @@ "title":"Open-Source Kafka Sink Stream", "uri":"dli_08_0257.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"460" + "p_code":"551", + "code":"564" }, { "desc":"You can create a sink stream to export data to a file system such as HDFS or OBS. After the data is generated, a non-DLI table can be created directly according to the ge", @@ -4145,8 +5081,8 @@ "title":"File System Sink Stream (Recommended)", "uri":"dli_08_0267.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"461" + "p_code":"551", + "code":"565" }, { "desc":"Create a sink stream to export DLI data to OBS. DLI can export the job analysis results to OBS. OBS applies to various scenarios, such as big data analysis, cloud-native ", @@ -4154,8 +5090,8 @@ "title":"OBS Sink Stream", "uri":"dli_08_0242.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"462" + "p_code":"551", + "code":"566" }, { "desc":"DLI outputs the Flink job output data to RDS. Currently, PostgreSQL and MySQL databases are supported. The PostgreSQL database can store data of more complex types and de", @@ -4163,8 +5099,8 @@ "title":"RDS Sink Stream", "uri":"dli_08_0245.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"463" + "p_code":"551", + "code":"567" }, { "desc":"DLI exports Flink job output data to SMN.SMN provides reliable and flexible large-scale message notification services to DLI. It significantly simplifies system coupling ", @@ -4172,8 +5108,8 @@ "title":"SMN Sink Stream", "uri":"dli_08_0251.html", "doc_type":"sqlreference", - "p_code":"447", - "code":"464" + "p_code":"551", + "code":"568" }, { "desc":"The temporary stream is used to simplify SQL logic. If complex SQL logic is followed, write SQL statements concatenated with temporary streams. The temporary stream is ju", @@ -4181,8 +5117,8 @@ "title":"Creating a Temporary Stream", "uri":"dli_08_0258.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"465" + "p_code":"541", + "code":"569" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4190,8 +5126,8 @@ "title":"Creating a Dimension Table", "uri":"dli_08_0259.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"466" + "p_code":"541", + "code":"570" }, { "desc":"Create a Redis table to connect to the source stream.For details about the JOIN syntax, see JOIN Between Stream Data and Table Data.Redis clusters are not supported.Ensur", @@ -4199,8 +5135,8 @@ "title":"Creating a Redis Table", "uri":"dli_08_0260.html", "doc_type":"sqlreference", - "p_code":"466", - "code":"467" + "p_code":"570", + "code":"571" }, { "desc":"Create an RDS/DWS table to connect to the source stream.For details about the JOIN syntax, see JOIN.Ensure that you have created a PostgreSQL or MySQL RDS instance in RDS", @@ -4208,8 +5144,8 @@ "title":"Creating an RDS Table", "uri":"dli_08_0261.html", "doc_type":"sqlreference", - "p_code":"466", - "code":"468" + "p_code":"570", + "code":"572" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4217,8 +5153,8 @@ "title":"Custom Stream Ecosystem", "uri":"dli_08_0272.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"469" + "p_code":"541", + "code":"573" }, { "desc":"Compile code to obtain data from the desired cloud ecosystem or open-source ecosystem as the input data of Flink jobs.The user-defined source class needs to inherit the R", @@ -4226,8 +5162,8 @@ "title":"Custom Source Stream", "uri":"dli_08_0273.html", "doc_type":"sqlreference", - "p_code":"469", - "code":"470" + "p_code":"573", + "code":"574" }, { "desc":"Compile code to write the data processed by DLI to a specified cloud ecosystem or open-source ecosystem.The user-defined sink class needs to inherit the RichSinkFunction ", @@ -4235,8 +5171,8 @@ "title":"Custom Sink Stream", "uri":"dli_08_0274.html", "doc_type":"sqlreference", - "p_code":"469", - "code":"471" + "p_code":"573", + "code":"575" }, { "desc":"Data type is a basic attribute of data and used to distinguish different types of data. Different data types occupy different storage space and support different operatio", @@ -4244,8 +5180,8 @@ "title":"Data Type", "uri":"dli_08_0207.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"472" + "p_code":"541", + "code":"576" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4253,8 +5189,8 @@ "title":"Built-In Functions", "uri":"dli_08_0086.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"473" + "p_code":"541", + "code":"577" }, { "desc":"All data types can be compared by using relational operators and the result is returned as a BOOLEAN value.Relationship operators are binary operators. Two compared data ", @@ -4262,8 +5198,8 @@ "title":"Mathematical Operation Functions", "uri":"dli_08_0191.html", "doc_type":"sqlreference", - "p_code":"473", - "code":"474" + "p_code":"577", + "code":"578" }, { "desc":"The common string functions of DLI are as follows:FunctionConcatenates two strings.Concatenates two strings.SyntaxVARCHAR VARCHAR a || VARCHAR bParametersa: string.b: str", @@ -4271,8 +5207,8 @@ "title":"String Functions", "uri":"dli_08_0096.html", "doc_type":"sqlreference", - "p_code":"473", - "code":"475" + "p_code":"577", + "code":"579" }, { "desc":"Table 1 lists the time functions supported by Flink SQL.None", @@ -4280,8 +5216,8 @@ "title":"Temporal Functions", "uri":"dli_08_0097.html", "doc_type":"sqlreference", - "p_code":"473", - "code":"476" + "p_code":"577", + "code":"580" }, { "desc":"This function is used to forcibly convert types.If the input is NULL, NULL is returned.Flink jobs do not support the conversion of bigint to timestamp using CAST. You can", @@ -4289,8 +5225,8 @@ "title":"Type Conversion Functions", "uri":"dli_08_0112.html", "doc_type":"sqlreference", - "p_code":"473", - "code":"477" + "p_code":"577", + "code":"581" }, { "desc":"An aggregate function performs a calculation operation on a set of input values and returns a value. For example, the COUNT function counts the number of rows retrieved b", @@ -4298,8 +5234,8 @@ "title":"Aggregate Functions", "uri":"dli_08_0104.html", "doc_type":"sqlreference", - "p_code":"473", - "code":"478" + "p_code":"577", + "code":"582" }, { "desc":"Table-valued functions can convert one row of records into multiple rows or convert one column of records into multiple columns. Table-valued functions can only be used i", @@ -4307,8 +5243,8 @@ "title":"Table-Valued Functions", "uri":"dli_08_0206.html", "doc_type":"sqlreference", - "p_code":"473", - "code":"479" + "p_code":"577", + "code":"583" }, { "desc":"Example:The returned number of elements in the array is 3.HELLO WORLD is returned.", @@ -4316,8 +5252,8 @@ "title":"Other Functions", "uri":"dli_08_0101.html", "doc_type":"sqlreference", - "p_code":"473", - "code":"480" + "p_code":"577", + "code":"584" }, { "desc":"DLI supports the following three types of user-defined functions (UDFs):Regular UDF: takes in one or more input parameters and returns a single result.User-defined table-", @@ -4325,8 +5261,8 @@ "title":"User-Defined Functions", "uri":"dli_08_0099.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"481" + "p_code":"541", + "code":"585" }, { "desc":"Table 1 describes the basic geospatial geometric elements.You can build complex geospatial geometries based on basic geospatial geometric elements. Table 2 describes the ", @@ -4334,8 +5270,8 @@ "title":"Geographical Functions", "uri":"dli_08_0209.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"482" + "p_code":"541", + "code":"586" }, { "desc":"SyntaxDescriptionThe SELECT statement is used to select data from a table or insert constant data into a table.PrecautionsThe table to be queried must exist. Otherwise, a", @@ -4343,17 +5279,17 @@ "title":"SELECT", "uri":"dli_08_0102.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"483" + "p_code":"541", + "code":"587" }, { - "desc":"SyntaxorDescriptionIf the value of value is value1, result1 is returned. If the value is not any of the values listed in the clause, resultZ is returned. If no else state", + "desc":"SyntaxorDescriptionIf the value of value is value1, result1 is returned; otherwise, resultZ is returned. If there is no else statement, null is returned.If the value of c", "product_code":"dli", "title":"Condition Expression", "uri":"dli_08_0103.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"484" + "p_code":"541", + "code":"588" }, { "desc":"DescriptionGroup Window is defined in GROUP BY. One record is generated from each group. Group Window involves the following functions:time_attr can be processing-time or", @@ -4361,8 +5297,8 @@ "title":"Window", "uri":"dli_08_0218.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"485" + "p_code":"541", + "code":"589" }, { "desc":"The JOIN operation allows you to query data from a table and write the query result to the sink stream. Currently, only RDSs and DCS Redis tables are supported. The ON ke", @@ -4370,8 +5306,8 @@ "title":"JOIN Between Stream Data and Table Data", "uri":"dli_08_0106.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"486" + "p_code":"541", + "code":"590" }, { "desc":"Flink provides two time models: processing time and event time.DLI allows you to specify the time model during creation of the source stream and temporary stream.Processi", @@ -4379,8 +5315,8 @@ "title":"Configuring Time Models", "uri":"dli_08_0107.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"487" + "p_code":"541", + "code":"591" }, { "desc":"Complex event processing (CEP) is used to detect complex patterns in endless data streams so as to identify and search patterns in various data rows. Pattern matching is ", @@ -4388,8 +5324,8 @@ "title":"Pattern Matching", "uri":"dli_08_0108.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"488" + "p_code":"541", + "code":"592" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4397,8 +5333,8 @@ "title":"StreamingML", "uri":"dli_08_0109.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"489" + "p_code":"541", + "code":"593" }, { "desc":"Anomaly detection applies to various scenarios, including intrusion detection, financial fraud detection, sensor data monitoring, medical diagnosis, natural data detectio", @@ -4406,8 +5342,8 @@ "title":"Anomaly Detection", "uri":"dli_08_0110.html", "doc_type":"sqlreference", - "p_code":"489", - "code":"490" + "p_code":"593", + "code":"594" }, { "desc":"Modeling and forecasting time series is a common task in many business verticals. Modeling is used to extract meaningful statistics and other characteristics of the data.", @@ -4415,8 +5351,8 @@ "title":"Time Series Forecasting", "uri":"dli_08_0111.html", "doc_type":"sqlreference", - "p_code":"489", - "code":"491" + "p_code":"593", + "code":"595" }, { "desc":"Clustering algorithms belong to unsupervised algorithms. K-Means, a clustering algorithm, partitions data points into related clusters by calculating the distance between", @@ -4424,8 +5360,8 @@ "title":"Real-Time Clustering", "uri":"dli_08_0216.html", "doc_type":"sqlreference", - "p_code":"489", - "code":"492" + "p_code":"593", + "code":"596" }, { "desc":"Deep learning has a wide range of applications in many industries, such as image classification, image recognition, and speech recognition. DLI provides several functions", @@ -4433,8 +5369,8 @@ "title":"Deep Learning Model Prediction", "uri":"dli_08_0088.html", "doc_type":"sqlreference", - "p_code":"489", - "code":"493" + "p_code":"593", + "code":"597" }, { "desc":"Flink SQL reserves some strings as keywords. If you want to use the following strings as field names, ensure that they are enclosed by back quotes, for example, `value` a", @@ -4442,8 +5378,8 @@ "title":"Reserved Keywords", "uri":"dli_08_0125.html", "doc_type":"sqlreference", - "p_code":"437", - "code":"494" + "p_code":"541", + "code":"598" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4452,16 +5388,16 @@ "uri":"dli_08_0001.html", "doc_type":"sqlreference", "p_code":"", - "code":"495" + "code":"599" }, { - "desc":"None.Aggregate function.", + "desc":"NoneAggregate function.aggregate_func is typically used in database queries to perform calculations on a set of values and return a single result.", "product_code":"dli", "title":"aggregate_func", "uri":"dli_08_0002.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"496" + "p_code":"599", + "code":"600" }, { "desc":"None.Alias, which must be STRING type. It can be assigned to a field, table, view, or subquery.", @@ -4469,8 +5405,8 @@ "title":"alias", "uri":"dli_08_0003.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"497" + "p_code":"599", + "code":"601" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4478,8 +5414,8 @@ "title":"attr_expr", "uri":"dli_08_0004.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"498" + "p_code":"599", + "code":"602" }, { "desc":"None.List of attr_expr, which is separated by commas (,).", @@ -4487,8 +5423,8 @@ "title":"attr_expr_list", "uri":"dli_08_0005.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"499" + "p_code":"599", + "code":"603" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4496,8 +5432,8 @@ "title":"attrs_value_set_expr", "uri":"dli_08_0006.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"500" + "p_code":"599", + "code":"604" }, { "desc":"None.Return a boolean expression.", @@ -4505,8 +5441,8 @@ "title":"boolean_expression", "uri":"dli_08_0007.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"501" + "p_code":"599", + "code":"605" }, { "desc":"None.Formal parameter for function call. It is usually a field name, which is the same as col_name.", @@ -4514,8 +5450,8 @@ "title":"col", "uri":"dli_08_0009.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"502" + "p_code":"599", + "code":"606" }, { "desc":"None.Column (field) description, which must be STRING type and cannot exceed 256 bytes.", @@ -4523,8 +5459,8 @@ "title":"col_comment", "uri":"dli_08_0010.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"503" + "p_code":"599", + "code":"607" }, { "desc":"None.Column name, which must be STRING type and cannot exceed 128 bytes.", @@ -4532,8 +5468,8 @@ "title":"col_name", "uri":"dli_08_0011.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"504" + "p_code":"599", + "code":"608" }, { "desc":"None.Field list, which consists of one col_name or more. If there is more than one col_name, separate them by using a comma (,).", @@ -4541,8 +5477,8 @@ "title":"col_name_list", "uri":"dli_08_0012.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"505" + "p_code":"599", + "code":"609" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4550,8 +5486,8 @@ "title":"condition", "uri":"dli_08_0013.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"506" + "p_code":"599", + "code":"610" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4559,8 +5495,8 @@ "title":"condition_list", "uri":"dli_08_0014.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"507" + "p_code":"599", + "code":"611" }, { "desc":"None.Common expression name.", @@ -4568,8 +5504,8 @@ "title":"cte_name", "uri":"dli_08_0015.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"508" + "p_code":"599", + "code":"612" }, { "desc":"None.Data type. Currently, only the primitive data types are supported.", @@ -4577,8 +5513,8 @@ "title":"data_type", "uri":"dli_08_0016.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"509" + "p_code":"599", + "code":"613" }, { "desc":"None.Database description, which must be STRING type and cannot exceed 256 characters.", @@ -4586,8 +5522,8 @@ "title":"db_comment", "uri":"dli_08_0017.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"510" + "p_code":"599", + "code":"614" }, { "desc":"None.Database name, which must be STRING type and cannot exceed 128 bytes.", @@ -4595,8 +5531,8 @@ "title":"db_name", "uri":"dli_08_0018.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"511" + "p_code":"599", + "code":"615" }, { "desc":"None.Returned result for the ELSE clause of the CASE WHEN statement.", @@ -4604,8 +5540,8 @@ "title":"else_result_expression", "uri":"dli_08_0019.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"512" + "p_code":"599", + "code":"616" }, { "desc":"| AVRO| CSV| JSON| ORC| PARQUETCurrently, the preceding formats are supported.Both USING and STORED AS can be used for specifying the data format. You can specify the pre", @@ -4613,8 +5549,8 @@ "title":"file_format", "uri":"dli_08_0020.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"513" + "p_code":"599", + "code":"617" }, { "desc":"None.File path, which is the OBS path", @@ -4622,8 +5558,8 @@ "title":"file_path", "uri":"dli_08_0021.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"514" + "p_code":"599", + "code":"618" }, { "desc":"None.Function name, which must be STRING type.", @@ -4631,8 +5567,8 @@ "title":"function_name", "uri":"dli_08_0022.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"515" + "p_code":"599", + "code":"619" }, { "desc":"None.Expression that includes GROUP BY.", @@ -4640,8 +5576,8 @@ "title":"groupby_expression", "uri":"dli_08_0023.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"516" + "p_code":"599", + "code":"620" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4649,8 +5585,8 @@ "title":"having_condition", "uri":"dli_08_0024.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"517" + "p_code":"599", + "code":"621" }, { "desc":"None.Input expression of the CASE WHEN statement.", @@ -4658,8 +5594,8 @@ "title":"input_expression", "uri":"dli_08_0026.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"518" + "p_code":"599", + "code":"622" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4667,8 +5603,8 @@ "title":"join_condition", "uri":"dli_08_0029.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"519" + "p_code":"599", + "code":"623" }, { "desc":"None.The condition of an inequality join.", @@ -4676,8 +5612,8 @@ "title":"non_equi_join_condition", "uri":"dli_08_0030.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"520" + "p_code":"599", + "code":"624" }, { "desc":"None.Maximum number of output lines specified by LIMIT. Which must be INT type.", @@ -4685,8 +5621,8 @@ "title":"number", "uri":"dli_08_0031.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"521" + "p_code":"599", + "code":"625" }, { "desc":"None.Partition column name, that is, partition field name, which must be STRING type.", @@ -4694,8 +5630,8 @@ "title":"partition_col_name", "uri":"dli_08_0034.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"522" + "p_code":"599", + "code":"626" }, { "desc":"None.Partition column value, that is, partition field value.", @@ -4703,8 +5639,8 @@ "title":"partition_col_value", "uri":"dli_08_0035.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"523" + "p_code":"599", + "code":"627" }, { "desc":"partition_specs : (partition_col_name = partition_col_value, partition_col_name = partition_col_value, ...);Table partition list, which is expressed by using key=value pa", @@ -4712,8 +5648,8 @@ "title":"partition_specs", "uri":"dli_08_0036.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"524" + "p_code":"599", + "code":"628" }, { "desc":"None.Property name, which must be STRING type.", @@ -4721,8 +5657,8 @@ "title":"property_name", "uri":"dli_08_0037.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"525" + "p_code":"599", + "code":"629" }, { "desc":"None.Property value, which must be STRING type.", @@ -4730,8 +5666,8 @@ "title":"property_value", "uri":"dli_08_0038.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"526" + "p_code":"599", + "code":"630" }, { "desc":"None.Pattern matching string, which supports wildcard matching.", @@ -4739,8 +5675,8 @@ "title":"regex_expression", "uri":"dli_08_0039.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"527" + "p_code":"599", + "code":"631" }, { "desc":"None.Returned result for the THEN clause of the CASE WHEN statement.", @@ -4748,8 +5684,8 @@ "title":"result_expression", "uri":"dli_08_0040.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"528" + "p_code":"599", + "code":"632" }, { "desc":"None.Query clause for the basic SELECT statement.", @@ -4757,8 +5693,8 @@ "title":"select_statement", "uri":"dli_08_0042.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"529" + "p_code":"599", + "code":"633" }, { "desc":"None.Separator, which can be customized by users, for example, comma (,), semicolon (;), and colon (:). Which must be CHAR type.", @@ -4766,8 +5702,8 @@ "title":"separator", "uri":"dli_08_0043.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"530" + "p_code":"599", + "code":"634" }, { "desc":"None.SQL statement containing the common expression defined by cte_name.", @@ -4775,8 +5711,8 @@ "title":"sql_containing_cte_name", "uri":"dli_08_0045.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"531" + "p_code":"599", + "code":"635" }, { "desc":"None.Subquery.", @@ -4784,8 +5720,8 @@ "title":"sub_query", "uri":"dli_08_0046.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"532" + "p_code":"599", + "code":"636" }, { "desc":"None.Table description, which must be STRING type and cannot exceed 256 bytes.", @@ -4793,8 +5729,8 @@ "title":"table_comment", "uri":"dli_08_0047.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"533" + "p_code":"599", + "code":"637" }, { "desc":"NoneTable name, which cannot exceed 128 bytes. The string type and \"$\" symbol are supported.", @@ -4802,8 +5738,8 @@ "title":"table_name", "uri":"dli_08_0048.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"534" + "p_code":"599", + "code":"638" }, { "desc":"None.Table property list, which is expressed by using key=value pairs. key represents property_name, and value represents property_value. If there is more than one key=va", @@ -4811,8 +5747,8 @@ "title":"table_properties", "uri":"dli_08_0049.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"535" + "p_code":"599", + "code":"639" }, { "desc":"None.Table or view name, which must be STRING type. It can also be a subquery. If it is subquery, an alias must also be provided.", @@ -4820,8 +5756,8 @@ "title":"table_reference", "uri":"dli_08_0050.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"536" + "p_code":"599", + "code":"640" }, { "desc":"None.When expression of the CASE WHEN statement. It is used for matching with the input expression.", @@ -4829,8 +5765,8 @@ "title":"when_expression", "uri":"dli_08_0053.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"537" + "p_code":"599", + "code":"641" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4838,8 +5774,8 @@ "title":"where_condition", "uri":"dli_08_0054.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"538" + "p_code":"599", + "code":"642" }, { "desc":"NoneAnalysis window function.", @@ -4847,8 +5783,8 @@ "title":"window_function", "uri":"dli_08_0055.html", "doc_type":"sqlreference", - "p_code":"495", - "code":"539" + "p_code":"599", + "code":"643" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4857,7 +5793,7 @@ "uri":"dli_08_0060.html", "doc_type":"sqlreference", "p_code":"", - "code":"540" + "code":"644" }, { "desc":"All data types can be compared by using relational operators and the result is returned as a BOOLEAN value.Relationship operators are binary operators. Two compared data ", @@ -4865,8 +5801,8 @@ "title":"Relational Operators", "uri":"dli_08_0061.html", "doc_type":"sqlreference", - "p_code":"540", - "code":"541" + "p_code":"644", + "code":"645" }, { "desc":"Arithmetic operators include binary operators and unary operators. For both types of operators, the returned results are numbers. Table 1 lists the arithmetic operators s", @@ -4874,8 +5810,8 @@ "title":"Arithmetic Operators", "uri":"dli_08_0062.html", "doc_type":"sqlreference", - "p_code":"540", - "code":"542" + "p_code":"644", + "code":"646" }, { "desc":"Common logical operators include AND, OR, and NOT. The operation result can be TRUE, FALSE, or NULL (which means unknown). The priorities of the operators are as follows:", @@ -4883,8 +5819,8 @@ "title":"Logical Operators", "uri":"dli_08_0063.html", "doc_type":"sqlreference", - "p_code":"540", - "code":"543" + "p_code":"644", + "code":"647" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -4893,6 +5829,6 @@ "uri":"dli_08_00005.html", "doc_type":"sqlreference", "p_code":"", - "code":"544" + "code":"648" } ] \ No newline at end of file diff --git a/docs/dli/sqlreference/dli_08_00005.html b/docs/dli/sqlreference/dli_08_00005.html index 016ac09d0..1fa43ed79 100644 --- a/docs/dli/sqlreference/dli_08_00005.html +++ b/docs/dli/sqlreference/dli_08_00005.html @@ -8,9 +8,22 @@ -

2024-02-27

+

2025-03-04

-

Modified the following section:

+

Added Flink OpenSource SQL 1.15 Syntax Reference.

+ + +

2025-02-08

+ +

Added the following section:

+

Reusing Results of Subqueries

+ + +

2024-02-27

+ +

Added the following section:

+

Spark Open Source Commands

+

Modified the following section:

Added the description that DDS is fully compatible with the MongoDB protocol to Creating a DLI Table and Associating It with DDS.

diff --git a/docs/dli/sqlreference/dli_08_0002.html b/docs/dli/sqlreference/dli_08_0002.html index b448b2ec7..85d3c197b 100644 --- a/docs/dli/sqlreference/dli_08_0002.html +++ b/docs/dli/sqlreference/dli_08_0002.html @@ -1,9 +1,10 @@

aggregate_func

-

Syntax

None.

+

Syntax

None

Description

Aggregate function.

+

aggregate_func is typically used in database queries to perform calculations on a set of values and return a single result.

diff --git a/docs/dli/sqlreference/dli_08_0059.html b/docs/dli/sqlreference/dli_08_0059.html index c7656b9ee..46b58ec9f 100644 --- a/docs/dli/sqlreference/dli_08_0059.html +++ b/docs/dli/sqlreference/dli_08_0059.html @@ -76,19 +76,19 @@ test4 70

Example of STRUCT

Create a struct_test table and set info to the STRUCT<name:STRING, age:INT> data type (the field consists of name and age, where the type of name is STRING and age is INT). After the table is created, insert test data into the struct_test table. The procedure is as follows:

  1. Create a table.

    CREATE TABLE struct_test(id INT, info STRUCT<name:STRING,age:INT>) USING PARQUET;

    -
  2. Run the following statements to insert test data:

    INSERT INTO struct_test VALUES (8, struct('zhang',23));

    -

    INSERT INTO struct_test VALUES (9, struct('li',25));

    -

    INSERT INTO struct_test VALUES (10, struct('wang',26));

    +
  3. Run the following statements to insert test data:

    INSERT INTO struct_test VALUES (8, struct('user1',23));

    +

    INSERT INTO struct_test VALUES (9, struct('user2,25));

    +

    INSERT INTO struct_test VALUES (10, struct('user3',26));

  4. Query the result.

    To query all data in the struct_test table, run the following statement:

    SELECT * FROM struct_test;

    -
    8	{"name":"zhang","age":23}
-10	{"name":"wang","age":26}
-9	{"name":"li","age":25}
    +
    8{"name":"user1,"age":23}
+10{"name":"user2,"age":26}
+9{"name":"user3,"age":25}

    Query name and age in the struct_test table.

    SELECT id,info.name,info.age FROM struct_test;

    -
    8	zhang	23
-10	wang	26
-9	li	25
    +
    8       user1   23
+10      user2   26
+9	user3	25
diff --git a/docs/dli/sqlreference/dli_08_0066.html b/docs/dli/sqlreference/dli_08_0066.html index e13fe7f30..5e18319b8 100644 --- a/docs/dli/sqlreference/dli_08_0066.html +++ b/docs/dli/sqlreference/dli_08_0066.html @@ -29,7 +29,7 @@

TIMESTAMP

-

Returns the current time, for example, 2016-07-04 11:18:11.685.

+

Returns a timestamp of the TIMESTAMP type.

date_add(string startdate, int days)

diff --git a/docs/dli/sqlreference/dli_08_0067.html b/docs/dli/sqlreference/dli_08_0067.html index d78376c1c..1dd5aa0fb 100644 --- a/docs/dli/sqlreference/dli_08_0067.html +++ b/docs/dli/sqlreference/dli_08_0067.html @@ -64,7 +64,7 @@

INT

-

Returns the index of substr that appears earliest in str. Returns NULL if either of the arguments are NULL and returns 0 if substr does not exist in str. Note that the first character in str has index 1.

+

Returns the index of substr that appears earliest in str. It returns NULL if either of the arguments are NULL and returns 0 if substr does not exist in str. Note that the first character in str has index 1.

instr1(string <str1>, string <str2>[, bigint <start_position>[, bigint <nth_appearance>]])

diff --git a/docs/dli/sqlreference/dli_08_0070.html b/docs/dli/sqlreference/dli_08_0070.html index a5e5518e8..0f7296011 100644 --- a/docs/dli/sqlreference/dli_08_0070.html +++ b/docs/dli/sqlreference/dli_08_0070.html @@ -8,9 +8,9 @@
  • Deleting a Database
    • -
  • Viewing a Specified Database
    +
  • Checking a Specified Database
    • -
  • Viewing All Databases
    +
  • Checking All Databases
    • diff --git a/docs/dli/sqlreference/dli_08_0071.html b/docs/dli/sqlreference/dli_08_0071.html index a3e6156ed..9435ca5f1 100644 --- a/docs/dli/sqlreference/dli_08_0071.html +++ b/docs/dli/sqlreference/dli_08_0071.html @@ -9,7 +9,6 @@ [COMMENT db_comment] [WITH DBPROPERTIES (property_name=property_value, ...)];
    -

    Keywords

    • IF NOT EXISTS: Prevents system errors if the database to be created exists.
    • COMMENT: Describes a database.
    @@ -51,7 +50,6 @@

    Example

    1. Create a queue. A queue is the basis for using DLI. Before executing SQL statements, you need to create a queue.
    2. On the DLI management console, click SQL Editor in the navigation pane on the left. The SQL Editor page is displayed.
    3. In the editing window on the right of the SQL Editor page, enter the following SQL statement for creating a database and click Execute. Read and agree to the privacy agreement, and click OK.

      If database testdb does not exist, run the following statement to create database testdb:

      1
      CREATE DATABASE IF NOT EXISTS testdb;
      -
    diff --git a/docs/dli/sqlreference/dli_08_0072.html b/docs/dli/sqlreference/dli_08_0072.html index 39d339a1c..dcf8e41bc 100644 --- a/docs/dli/sqlreference/dli_08_0072.html +++ b/docs/dli/sqlreference/dli_08_0072.html @@ -5,7 +5,6 @@

    Syntax

    1
    DROP [DATABASE | SCHEMA] [IF EXISTS] db_name [RESTRICT|CASCADE];
    -

    Keywords

    IF EXISTS: Prevents system errors if the database to be deleted does not exist.

    @@ -30,7 +29,6 @@

    Example

    1. Create a database, for example, testdb, by referring to Example.
    2. Run the following statement to delete database testdb if it exists:
      1
      DROP DATABASE IF EXISTS testdb;
      -
    diff --git a/docs/dli/sqlreference/dli_08_0073.html b/docs/dli/sqlreference/dli_08_0073.html index f588b2ded..d20413548 100644 --- a/docs/dli/sqlreference/dli_08_0073.html +++ b/docs/dli/sqlreference/dli_08_0073.html @@ -1,11 +1,10 @@ -

    Viewing a Specified Database

    +

    Checking a Specified Database

    Function

    This syntax is used to view the information about a specified database, including the database name and database description.

    Syntax

    1
    DESCRIBE DATABASE [EXTENDED] db_name;
    -

    Keywords

    EXTENDED: Displays the database properties.

    @@ -30,7 +29,6 @@

    Example

    1. Create a database, for example, testdb, by referring to Example.
    2. Run the following statement to query information about the testdb database:
      1
      DESCRIBE DATABASE testdb;
      -
    diff --git a/docs/dli/sqlreference/dli_08_0074.html b/docs/dli/sqlreference/dli_08_0074.html index 5fa60800b..f5f6c6529 100644 --- a/docs/dli/sqlreference/dli_08_0074.html +++ b/docs/dli/sqlreference/dli_08_0074.html @@ -1,11 +1,10 @@ -

    Viewing All Databases

    +

    Checking All Databases

    Function

    This syntax is used to query all current databases.

    Syntax

    1
    SHOW [DATABASES | SCHEMAS] [LIKE regex_expression];
    -

    Keywords

    None

    @@ -31,12 +30,10 @@

    Example

    View all the current databases.

    1
    SHOW DATABASES;
    -

    View all databases whose names start with test.

    1
    SHOW DATABASES LIKE "test.*";
    -
    diff --git a/docs/dli/sqlreference/dli_08_0075.html b/docs/dli/sqlreference/dli_08_0075.html index 3ce7aef8f..eda663b65 100644 --- a/docs/dli/sqlreference/dli_08_0075.html +++ b/docs/dli/sqlreference/dli_08_0075.html @@ -5,7 +5,6 @@

    Data Types Supported by Syntax

    • Basic data types: VARCHAR, STRING, BOOLEAN, TINYINT, SMALLINT, INTEGER/INT, BIGINT, REAL/FLOAT, DOUBLE, DECIMAL, DATE, TIME, and TIMESTAMP
    • Array: Square brackets ([]) are used to quote fields. The following is an example:
      1
      insert into temp select CARDINALITY(ARRAY[1,2,3]) FROM OrderA;
      -
    @@ -109,7 +108,6 @@ | ROLLUP '(' expression [, expression ]* ')' | GROUPING SETS '(' groupItem [, groupItem ]* ')'
    - diff --git a/docs/dli/sqlreference/dli_08_0076.html b/docs/dli/sqlreference/dli_08_0076.html index e5cebdf74..6c9723fa3 100644 --- a/docs/dli/sqlreference/dli_08_0076.html +++ b/docs/dli/sqlreference/dli_08_0076.html @@ -26,7 +26,6 @@ [COMMENT table_comment] [AS select_statement] -

    Keywords

    • IF NOT EXISTS: Prevents system errors when the created table exists.
    • USING: Storage format.
    • OPTIONS: Property name and property value when a table is created.
    • COMMENT: Field or table description.
    • PARTITIONED BY: Partition field.
    • AS: Run the CREATE TABLE AS statement to create a table.
    @@ -281,7 +280,6 @@ USING orc OPTIONS (path 'obs://bucketName/filePath');
    -

    Example 2: Creating an OBS Partitioned Table

    Example description: Create a partitioned table named student. The partitioned table is partitioned using facultyNo and classNo. The student table is partitioned by faculty number (facultyNo) and class number (classNo).

    @@ -300,7 +298,6 @@ OPTIONS (path 'obs://bucketName/filePath') PARTITIONED BY (facultyNo, classNo);
    -

    Example 3: Using CTAS to Create an OBS Non-Partitioned Table Using All or Part of the Data in the Source Table

    Example description: Based on the OBS table table1 created in Example 1: Creating an OBS Non-Partitioned Table, use the CTAS syntax to copy data from table1 to table1_ctas.

    @@ -320,7 +317,6 @@ SELECT * FROM table1;
    -
  • To filter and insert data into table1_ctas in a customized way, you can use the following SELECT statement: SELECT col_1 FROM table1 WHERE col_1 = 'Ann'. This will allow you to select only col_1 from table1 and insert data into table1_ctas where the value equals 'Ann'.
    1
 2
@@ -336,7 +332,6 @@
 FROM    table1
 WHERE   col_1 = 'Ann';
    -
    • @@ -377,12 +372,11 @@ USING parquet OPTIONS (path 'obs://bucketName/filePath'); -

    Example 5: Creating an OBS Partitioned Table and Customizing OPTIONS Parameters

    Example description: When creating an OBS table, you can customize property names and values. For details about OPTIONS parameters, see Table 2.

    In this example, an OBS partitioned table named table3 is created and partitioned based on col_2. Configure path, multiLevelDirEnable, dataDelegated, and compression in OPTIONS.

    -
    • path: OBS storage path. In this example, the value is obs://bucketName/filePath, where bucketName indicates the bucket name and filePath indicates the actual directory name.
    • In big data scenarios, you are advised to use the OBS parallel file system for storage.
    • multiLevelDirEnable: In this example, set this parameter to true, indicating that all files and subdirectories in the table path are read iteratively when the table is queried. If this parameter is not required, set it to false or leave it blank (the default value is false).
    • dataDelegated: In this example, set this parameter to true, indicating that all data in the path is deleted when a table or partition is deleted. If this parameter is not required, set it to false or leave it blank (the default value is false).
    • compression: If the created OBS table needs to be compressed, you can use the keyword compression to configure the compression format. In this example, the zstd compression format is used.
    +
    • path: OBS storage path. In this example, the value is obs://bucketName/filePath, where bucketName indicates the bucket name and filePath indicates the actual directory name.
    • In big data scenarios, you are advised to use the OBS parallel file system for storage.
    • multiLevelDirEnable: In this example, set this parameter to true, indicating that all files and subdirectories in the table path are read iteratively when the table is queried. If this parameter is not required, set it to false or leave it blank (the default value is false).
    • dataDelegated: In this example, set this parameter to true, indicating that all data in the path is deleted when a table or partition is deleted. If this parameter is not required, set it to false or leave it blank (the default value is false).
    • compression: If the created OBS table needs to be compressed, you can use the keyword compression to configure the compression format. In this example, the zstd compression format is used.
     1
  2
  3
@@ -407,7 +401,6 @@
     compression = 'zstd'
 );
    -

    Example 6: Creating an OBS Non-Partitioned Table and Customizing OPTIONS Parameters

    Example description: A CSV table is a file format that uses commas to separate data values in plain text. It is commonly used for storing and sharing data, but it is not ideal for complex data types due to its lack of structured data concepts. So, when file_format is set to csv, more OPTIONS parameters can be configured. For details, see Table 3.

    @@ -453,7 +446,6 @@ encoding = 'utf - 8' );
    - diff --git a/docs/dli/sqlreference/dli_08_0077.html b/docs/dli/sqlreference/dli_08_0077.html index 6b549b231..787294c2b 100644 --- a/docs/dli/sqlreference/dli_08_0077.html +++ b/docs/dli/sqlreference/dli_08_0077.html @@ -6,6 +6,7 @@

    Precautions

    • The size of a table is calculated when the table is created.
    • When data is added, the table size will not be changed.
    • You can check the table size on OBS.
    • Table properties cannot be specified using CTAS table creation statements.
    • Instructions on using partitioned tables:
      • When you create a partitioned table, ensure that the specified column in PARTITIONED BY is not a column in the table and the data type is specified. The partition column supports only the open-source Hive table types including string, boolean, tinyint, smallint, short, int, bigint, long, decimal, float, double, date, and timestamp.
      • Multiple partition fields can be specified. The partition fields need to be specified after the PARTITIONED BY keyword, instead of the table name. Otherwise, an error occurs.
      • A maximum of 200,000 partitions can be created in a single table.
      • CTAS table creation statements cannot be used to create partitioned tables.
      +
    • Instructions on setting the multi-character delimiter during table creation:
      • The field delimiter can contain multiple characters only when ROW FORMAT SERDE is set to org.apache.hadoop.hive.contrib.serde2.MultiDelimitSerDe.
      • You can only specify multi-character delimiters when creating Hive OBS tables.
      • Tables with multi-character delimiters specified do not support data writing statements such as INSERT and IMPORT. To insert data into these tables, save the data file in the OBS path where the tables are located. For example, in Example 7: Creating a Table and Setting a Multi-Character Delimiter, store the data file in obs://bucketName/filePath.

    Syntax

     1
@@ -40,7 +41,6 @@
       [LINES TERMINATED BY char]
       [NULL DEFINED AS char]
    -

    Keywords

    • EXTERNAL: Creates an OBS table.
    • IF NOT EXISTS: Prevents system errors when the created table exists.
    • COMMENT: Field or table description.
    • PARTITIONED BY: Partition field.
    • ROW FORMAT: Row data format.
    • STORED AS: Specifies the format of the file to be stored. Currently, only the TEXTFILE, AVRO, ORC, SEQUENCEFILE, RCFILE, and PARQUET format are supported.
    • LOCATION: Specifies the path of OBS. This keyword is mandatory when you create OBS tables.
    • TBLPROPERTIES: Allows you to add the key/value properties to a table.
      • You can use this statement to enable the multiversion function to back up and restore table data. After the multiversion function is enabled, the system automatically backs up table data when you delete or modify the data using insert overwrite or truncate, and retains the data for a certain period. You can quickly restore data within the retention period. For details about the SQL syntax for the multiversion function, see Enabling or Disabling Multiversion Backup and Backing Up and Restoring Data of Multiple Versions.

        When creating an OBS table, you can use TBLPROPERTIES ("dli.multi.version.enable"="true") to enable multiversion. For details, see the following example.

        @@ -64,7 +64,7 @@

        orc.compress

        An attribute of the ORC table, which specifies the compression mode of the ORC storage. Available values are as follows:

        -
        • ZLIB
        • SNAPPY
        • NONE
        • PARQUET
        +
        • ZLIB
        • SNAPPY
        • NONE

        auto.purge

        @@ -76,7 +76,7 @@
    -
  • AS: You can run the CREATE TABLE AS statement to create a table.
    • +
  • AS: Run the CREATE TABLE AS statement to create a table.
  • The field delimiter can contain multiple characters only when ROW FORMAT SERDE is set to org.apache.hadoop.hive.contrib.serde2.MultiDelimitSerDe. For details, see Example 7: Creating a Table and Setting a Multi-Character Delimiter.

    • Parameters

    Table 2 Parameters

    Parameter

    @@ -179,7 +179,7 @@
    -

    Example 1: Creating an OBS Non-Partitioned Table

    Example description: Create an OBS non-partitioned table named table1 and use the STORED AS keyword to set the storage format of the table to orc.

    +

    Example 1: Creating an OBS Non-Partitioned Table

    Example description: Create an OBS non-partitioned table named table1 and use the STORED AS keyword to set the storage format of the table to orc.

    In practice, you can store OBS tables in textfile, avro, orc, sequencefile, rcfile, or parquet format.

    1
 2
@@ -193,7 +193,6 @@
 STORED AS orc
 LOCATION 'obs://bucketName/filePath';
    -

    Example 2: Creating an OBS Partitioned Table

    Example description: Create a partitioned table named student, which is partitioned using facultyNo and classNo.

    @@ -217,7 +216,6 @@ classNo INT );
    - @@ -238,7 +236,6 @@ SELECT * FROM table1; -
  • To filter and insert data into table1_ctas in a customized way, you can use the following SELECT statement: SELECT col_1 FROM table1 WHERE col_1 = 'Ann'. This will allow you to select only col_1 from table1 and insert data into table1_ctas where the value equals 'Ann'.
    1
 2
@@ -247,14 +244,13 @@
 5
 6
 7
    CREATE TABLE IF NOT EXISTS table1_ctas
-USING parquet
-OPTIONS (path 'obs:// bucketName/filePath')
+STORED AS parquet
+LOCATION  'obs:// bucketName/filePath'
 AS
 SELECT  col_1
 FROM    table1
 WHERE   col_1 = 'Ann';
    -
    • @@ -295,7 +291,6 @@ STORED AS parquet LOCATION 'obs://bucketName/filePath'; -

    Example 5: Creating an OBS Partitioned Table and Customizing TBLPROPERTIES Parameters

    Example description: Create an OBS partitioned table named table3 and partition the table based on col_3. Set dli.multi.version.enable, comment, orc.compress, and auto.purge in TBLPROPERTIES.

    @@ -326,11 +321,10 @@ auto.purge = true );
    -

    Example 6: Creating a Non-Partitioned Table in textfile Format and Setting ROW FORMAT

    Example description: Create a non-partitioned table named table4 in the textfile format and set ROW FORMAT (the ROW FORMAT function is available only for textfile tables).

    -
    • FIELDS: columns in a table. Each field has a name and data type. Fields in a table are separated by slashes (/).
    • COLLECTION ITEMS: A collection item refers to an element in a group of data, which can be an array, a list, or a collection. Collection items in a table are separated by $.
    • MAP KEYS: A map key is a data structure of key-value pairs and is used to store a group of associated data. Map keys in a table are separated by number signs (#).
    • LINES: rows in a table. Each row contains a group of field values. Rows in a table end with \n. (Note that only \n can be used as the row separator.)
    • NULL: a special value that represents a missing or unknown value. In a table, NULL indicates that the field has no value or the value is unknown. When there is a null value in the data, it is represented by the string null.
    +
    • FIELDS: columns in a table. Each field has a name and data type. Fields in a table are separated by slashes (/).
    • COLLECTION ITEMS: A collection item refers to an element in a group of data, which can be an array, a list, or a collection. Collection items in a table are separated by $.
    • MAP KEYS: A map key is a data structure of key-value pairs and is used to store a group of associated data. Map keys in a table are separated by number signs (#).
    • LINES: rows in a table. Each row contains a group of field values. Rows in a table end with \n. (Note that only \n can be used as the row separator.)
    • NULL: a special value that represents a missing or unknown value. In a table, NULL indicates that the field has no value or the value is unknown. When there is a null value in the data, it is represented by the string null.
     1
  2
  3
@@ -355,9 +349,32 @@
 LINES TERMINATED              BY '\n'
 NULL DEFINED                  AS 'null';
    -
    +

    Example 7: Creating a Table and Setting a Multi-Character Delimiter

    Example description: A Hive table named table5 is created. The serialization and deserialization class ROW FORMAT SERDE is specified for the table, with a field delimiter set to /# and data stored in a text file format.

    +
    • The field delimiter can contain multiple characters only when ROW FORMAT SERDE is set to org.apache.hadoop.hive.contrib.serde2.MultiDelimitSerDe.
    • You can only specify multi-character delimiters when creating Hive OBS tables.
    • Tables with multi-character delimiters specified do not support data writing statements such as INSERT and IMPORT. To insert data into these tables, save the data file in the OBS path where the tables are located. For example, in this example, store the data file in obs://bucketName/filePath.
    • In this example, field.delim is set to /#.
    • The ROW FORMAT function is available only for textfile tables.
       1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
      CREATE TABLE IF NOT EXISTS table5 (
+    col_1   STRING,
+    col_2   INT
+)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.MultiDelimitSerDe'
+WITH SERDEPROPERTIES (
+  'field.delim' = '/#'
+)
+STORED AS textfile
+LOCATION 'obs://bucketName/filePath';
+
      +
      +
    +
    diff --git a/docs/dli/sqlreference/dli_08_0079.html b/docs/dli/sqlreference/dli_08_0079.html index 44696c01e..982a1a525 100644 --- a/docs/dli/sqlreference/dli_08_0079.html +++ b/docs/dli/sqlreference/dli_08_0079.html @@ -5,7 +5,6 @@

    Syntax

    1
    MSCK REPAIR TABLE table_name;
    -

    Or

    ALTER TABLE table_name RECOVER PARTITIONS;
    @@ -43,7 +42,6 @@

    Example

    Run the following statements to update the partition information about table ptable in the Metastore:

    1
    MSCK REPAIR TABLE ptable;
    -

    Or

    ALTER TABLE ptable RECOVER PARTITIONS;
    diff --git a/docs/dli/sqlreference/dli_08_0081.html b/docs/dli/sqlreference/dli_08_0081.html index 7ef2fe9a9..6e5a2773e 100644 --- a/docs/dli/sqlreference/dli_08_0081.html +++ b/docs/dli/sqlreference/dli_08_0081.html @@ -15,7 +15,6 @@ PARTITION partition_specs2 [LOCATION 'obs_path2'];
    -

    Keywords

    • IF NOT EXISTS: prevents errors when partitions are repeatedly added.
    • PARTITION: specifies a partition.
    • LOCATION: specifies the partition path.
    @@ -67,7 +66,6 @@ dt varchar(16) ) using JSON OPTIONS (path 'obs://bucketName/datapath') PARTITIONED by (external_data, dt);
    -
  • Copy the partition directories to obs://bucketName/datapath. In this example, copy files in external_data=22 and its subdirectory dt=2021-07-27 to obs://bucketName/datapath.
  • Run the following command to add partition data:
    1
 2
@@ -77,13 +75,11 @@
 ADD
   PARTITION (external_data = '22', dt = '2021-07-27') LOCATION 'obs://bucketName/datapath/external_data=22/dt=2021-07-27';
    -
  • After the partition data is added successfully, you can perform operations such as data query based on the partition columns.
    1
 2
    select * from testobstable where external_data = '22';
 select * from testobstable where external_data = '22' and dt='2021-07-27';
    -
    • diff --git a/docs/dli/sqlreference/dli_08_0082.html b/docs/dli/sqlreference/dli_08_0082.html index 1886164a8..418c1d662 100644 --- a/docs/dli/sqlreference/dli_08_0082.html +++ b/docs/dli/sqlreference/dli_08_0082.html @@ -9,7 +9,6 @@ PARTITION partition_specs RENAME TO PARTITION partition_specs; -

    Keywords

    • PARTITION: a specified partition
    • RENAME: new name of the partition
    @@ -44,7 +43,6 @@ PARTITION (city='xxx',dt='2008-08-08') RENAME TO PARTITION (city='xxx',dt='2009-09-09');
    - diff --git a/docs/dli/sqlreference/dli_08_0083.html b/docs/dli/sqlreference/dli_08_0083.html index 6d93665c7..508e6c20d 100644 --- a/docs/dli/sqlreference/dli_08_0083.html +++ b/docs/dli/sqlreference/dli_08_0083.html @@ -12,7 +12,6 @@ DROP [IF EXISTS] PARTITION partition_spec1[,PARTITION partition_spec2,...]; -

    Keywords

    • DROP: deletes a partition.
    • IF EXISTS: The partition to be deleted must exist. Otherwise, an error is reported.
    • PARTITION: specifies the partition to be deleted
    @@ -61,7 +60,6 @@ options (path 'obs://bucketName/filePath') partitioned by (faculytNo, classNo);
    -

  • Insert partition data into the table.

    You can insert the following data:

     1
@@ -118,7 +116,6 @@
 partition (facultyNo = 30, classNo = 102)
 values (3010213, "student13"), (3010214, "student14");
    -

  • View the partitions.

    You can view all partitions in the table.

    The example code is as follows:

    diff --git a/docs/dli/sqlreference/dli_08_0084.html b/docs/dli/sqlreference/dli_08_0084.html index 78e2d6170..b18a83097 100644 --- a/docs/dli/sqlreference/dli_08_0084.html +++ b/docs/dli/sqlreference/dli_08_0084.html @@ -9,7 +9,6 @@ PARTITION partition_specs SET LOCATION obs_path; -

    Keywords

    • PARTITION: a specified partition
    • LOCATION: path of the partition
    @@ -49,7 +48,6 @@ PARTITION(dt='2008-08-08',city='xxx') SET LOCATION 'obs://bucketName/fileName/student/dt=2008-08-08/city=xxx';
    - diff --git a/docs/dli/sqlreference/dli_08_0087.html b/docs/dli/sqlreference/dli_08_0087.html index a33bf1d88..32100ca23 100644 --- a/docs/dli/sqlreference/dli_08_0087.html +++ b/docs/dli/sqlreference/dli_08_0087.html @@ -5,7 +5,6 @@

    Syntax

    1
    DROP TABLE [IF EXISTS] [db_name.]table_name;
    -

    Keywords

    • If the table is stored in OBS, only the metadata is deleted. The data stored on OBS is not deleted.
    • If the table is stored in DLI, the data and the corresponding metadata are all deleted.
    @@ -35,7 +34,6 @@

    Example

    1. Create a table. For details, see Creating an OBS Table or Creating a DLI Table.
    2. Run the following statement to delete table test from the current database:
      1
      DROP TABLE IF EXISTS test;
      -
    diff --git a/docs/dli/sqlreference/dli_08_0088.html b/docs/dli/sqlreference/dli_08_0088.html index 4d1570c19..2a762aada 100644 --- a/docs/dli/sqlreference/dli_08_0088.html +++ b/docs/dli/sqlreference/dli_08_0088.html @@ -17,7 +17,6 @@ DL_TEXT_MAX_PREDICTION_INDEX(field_name, model_path, is_dl4j_model) -- Use the default word2vec model. DL_TEXT_MAX_PREDICTION_INDEX(field_name, word2vec_path, model_path, is_dl4j_model) -

    Models and configuration files must be stored on OBS. The path format is obs://your_ak:your_sk@obs.your_obs_region.xxx.com:443/your_model_path.

    @@ -93,7 +92,6 @@ SELECT DL_IMAGE_MAX_PREDICTION_INDEX(image, 'your_keras_model_path', true) FROM Mnist SELECT DL_IMAGE_MAX_PREDICTION_INDEX(image, 'your_keras_model_config_path', 'keras_weights_path') FROM Mnist -

    For prediction in text classification, use data of a group of news titles as the input and load the pre-trained Deeplearning4j model or Keras model to predict the category of each news title in real time, such as economy, sports, and entertainment.

    1
@@ -110,7 +108,6 @@
 SELECT DL_TEXT_MAX_PREDICTION_INDEX(title, 'your_dl4j_model_path', false) FROM New
 SELECT DL_TEXT_MAX_PREDICTION_INDEX(title, 'your_keras_model_path', true) FROM New
    -
    diff --git a/docs/dli/sqlreference/dli_08_0089.html b/docs/dli/sqlreference/dli_08_0089.html index cb6e1ad39..238bd42f1 100644 --- a/docs/dli/sqlreference/dli_08_0089.html +++ b/docs/dli/sqlreference/dli_08_0089.html @@ -1,20 +1,20 @@ -

    Viewing Tables

    +

    Checking Tables

    diff --git a/docs/dli/sqlreference/dli_08_0090.html b/docs/dli/sqlreference/dli_08_0090.html index 114ce2e68..1fda72536 100644 --- a/docs/dli/sqlreference/dli_08_0090.html +++ b/docs/dli/sqlreference/dli_08_0090.html @@ -1,11 +1,10 @@ -

    Viewing All Tables

    -

    Function

    This statement is used to view all tables and views in the current database.

    +

    Checking All Tables

    +

    Function

    This statement is used to check all tables and views in the current database.

    Syntax

    1
    SHOW TABLES [IN | FROM db_name] [LIKE regex_expression];
    -

    Keywords

    FROM/IN: followed by the name of a database whose tables and views will be displayed.

    @@ -35,18 +34,16 @@

    Example

    1. Create a table. For details, see Creating an OBS Table or Creating a DLI Table.
    2. To show all tables and views in the current database, run the following statement:
      1
      SHOW TABLES;
      -
    3. To show all tables started with test in the testdb database, run the following statement:
      1
      SHOW TABLES IN testdb LIKE "test*";
      -
    -
    Parent topic: Viewing Tables
    +
    Parent topic: Checking Tables
    diff --git a/docs/dli/sqlreference/dli_08_0091.html b/docs/dli/sqlreference/dli_08_0091.html index 12e931689..63af6f5f5 100644 --- a/docs/dli/sqlreference/dli_08_0091.html +++ b/docs/dli/sqlreference/dli_08_0091.html @@ -1,25 +1,28 @@ -

    Viewing Table Creation Statements

    +

    Checking Table Creation Statements

    Function

    This statement is used to show the statements for creating a table.

    Syntax

    1
    SHOW CREATE TABLE table_name;
    -
    +

    Use the following syntax to view table creation statements when using Spark 3.3.1 (this applies only to querying table creation statements for Hive tables):

    +
    1
    SHOW CREATE TABLE table_name AS SERDE;
+
    +

    Keywords

    CREATE TABLE: statement for creating a table

    Parameters

    -
    -
    Table 1 Parameter

    Parameter

    +
    - - - @@ -45,11 +48,28 @@ TBLPROPERTIES ( ) +
    Example of Spark 3.3.1:
    • Run the following command to return the statement for creating the testDB02.testTable5 table:

      SHOW CREATE TABLE testDB02.testTable5 AS SERDE

      +
    • Return the statement for creating the test table.
      createtab_stmt
+ CREATE TABLE testDB02.testTable5 (
+  id INT,
+  age INT,
+  money DOUBLE)
+ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+WITH SERDEPROPERTIES (
+  'serialization.format' = '1')
+STORED AS
+  INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+  OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+TBLPROPERTIES (
+  'hive.serialization.extend.nesting.levels' = 'true',
+  'transient_lastDdlTime' = '1707201874')
      +
    +
    -
    Parent topic: Viewing Tables
    +
    Parent topic: Checking Tables
    diff --git a/docs/dli/sqlreference/dli_08_0092.html b/docs/dli/sqlreference/dli_08_0092.html index 5f5294cf1..04f9ff674 100644 --- a/docs/dli/sqlreference/dli_08_0092.html +++ b/docs/dli/sqlreference/dli_08_0092.html @@ -1,11 +1,10 @@ -

    Viewing Table Properties

    +

    Checking Table Properties

    Function

    Check the properties of a table.

    Syntax

    Table 1 Parameter

    Parameter

    Description

    +

    Description

    table_name

    +

    table_name

    Table name

    +

    Table name

    		 
    1
    SHOW TBLPROPERTIES table_name [('property_name')];
    -

    Keywords

    TBLPROPERTIES: This statement allows you to add a key/value property to a table.

    @@ -36,13 +35,12 @@

    Example

    To return the value of property_key1 in the test table, run the following statement:

    1
    SHOW TBLPROPERTIES test ('property_key1');
    -
    -
    Parent topic: Viewing Tables
    +
    Parent topic: Checking Tables
    diff --git a/docs/dli/sqlreference/dli_08_0093.html b/docs/dli/sqlreference/dli_08_0093.html index 05bd1c308..632c546b2 100644 --- a/docs/dli/sqlreference/dli_08_0093.html +++ b/docs/dli/sqlreference/dli_08_0093.html @@ -1,11 +1,10 @@ -

    Viewing All Columns in a Specified Table

    +

    Checking All Columns in a Specified Table

    Function

    This statement is used to query all columns in a specified table.

    Syntax

    1
    SHOW COLUMNS {FROM | IN} table_name [{FROM | IN} db_name];
    -

    Keywords

    • COLUMNS: columns in the current table
    • FROM/IN: followed by the name of a database whose tables and views will be displayed. Keyword FROM is equivalent to IN. You can use either of them in a statement.
    @@ -36,13 +35,12 @@

    Example

    Run the following statement to view all columns in the student table.

    1
    SHOW COLUMNS IN student;
    -
    -
    Parent topic: Viewing Tables
    +
    Parent topic: Checking Tables
    diff --git a/docs/dli/sqlreference/dli_08_0094.html b/docs/dli/sqlreference/dli_08_0094.html index d1d373153..1bf45371b 100644 --- a/docs/dli/sqlreference/dli_08_0094.html +++ b/docs/dli/sqlreference/dli_08_0094.html @@ -1,13 +1,12 @@ -

    Viewing All Partitions in a Specified Table

    -

    Function

    This statement is used to view all partitions in a specified table.

    +

    Checking All Partitions in a Specified Table

    +

    Function

    This statement is used to check all partitions in a specified table.

    Syntax

    1
 2
    SHOW PARTITIONS [db_name.]table_name
   [PARTITION partition_specs];
    -

    Keywords

    • PARTITIONS: partitions in a specified table
    • PARTITION: a specified partition
    @@ -26,7 +25,8 @@

    table_name

    Table name of a database that contains letters, digits, and underscores (_). It cannot contain only digits and cannot start with an underscore (_). The matching rule is ^(?!_)(?![0-9]+$)[A-Za-z0-9_$]*$. If special characters are required, use single quotation marks ('') to enclose them.

    +

    Table name of a database that contains letters, digits, and underscores (_). The name cannot contain only digits or start with an underscore (_).

    +

    The matching rule is ^(?!_)(?![0-9]+$)[A-Za-z0-9_$]*$. If special characters are required, use single quotation marks ('') to enclose them.

    partition_specs

    @@ -42,18 +42,16 @@

    Example

    • To show all partitions in the student table, run the following statement:
      1
      SHOW PARTITIONS student;
      -
      -
    • Check the dt='2010-10-10' partition in the student table, run the following statement:
      1
      SHOW PARTITIONS student PARTITION(dt='2010-10-10')
+
    • Check the dt='2010-10-10' partition in the student table, run the following statement:
      1
      SHOW PARTITIONS student PARTITION(dt='2010-10-10');
 
      
-
 
      
 
      •
      -
      Parent topic: Viewing Tables
      +
      Parent topic: Checking Tables
      diff --git a/docs/dli/sqlreference/dli_08_0095.html b/docs/dli/sqlreference/dli_08_0095.html index 9b68ede1a..173d56bdd 100644 --- a/docs/dli/sqlreference/dli_08_0095.html +++ b/docs/dli/sqlreference/dli_08_0095.html @@ -3,17 +3,20 @@

      Inserting Data

      Function

      This statement is used to insert the SELECT query result or a certain data record into a table.

      +

      Notes and Constraints

      • The INSERT OVERWRITE syntax is not suitable for "read-write" scenarios, where data is continuously processed and updated. Using this syntax in such scenarios may result in data loss.

        "Read-write" refers to the ability to read data while generating new data or modifying existing data during data processing.

        +
      • When using Hive and Datasource tables (excluding Hudi), executing data modification commands (such as insert into and load data) may result in data duplication or inconsistency if the data source does not support transactions and there is a system failure or queue restart.

        To avoid this situation, you are advised to prioritize data sources that support transactions, such as Hudi data sources. This type of data source has Atomicity, Consistency, Isolation, Durability (ACID) capabilities, which helps ensure data consistency and accuracy.

        +

        To learn more, refer to How Do I Handle Duplicate Records After Executing the INSERT INTO Statement?

        +
      +

      Syntax

      • Insert the SELECT query result into a table.
        1
 2
        INSERT INTO [TABLE] [db_name.]table_name
   [PARTITION part_spec] select_statement;
        - 
        1
 2
        INSERT OVERWRITE TABLE [db_name.]table_name
   [PARTITION part_spec] select_statement;
        - 
        part_spec:  
   : (part_col_name1=val1 [, part_col_name2=val2, ...])
        @@ -21,57 +24,54 @@ 2
      INSERT INTO [TABLE] [db_name.]table_name
   [PARTITION part_spec] VALUES values_row [, values_row ...];
      - 
      1
 2
      INSERT OVERWRITE TABLE [db_name.]table_name
   [PARTITION part_spec] VALUES values_row [, values_row ...];
      - 
      values_row:
   : (val1 [, val2, ...])

    Keywords

    -
    Table 1 INSERT keywords

    Parameter

    +
    - - - - - - - - - - -
    Table 1 INSERT keywords

    Parameter

    Description

    +

    Description

    db_name

    +

    db_name

    Name of the database where the target table resides.

    +

    Name of the database where the target table resides.

    table_name

    +

    table_name

    Name of the target table.

    +

    Name of the target table.

    part_spec

    +

    part_spec

    Detailed partition information. If there are multiple partition fields, all fields must be contained, but the corresponding values are optional. The system matches the corresponding partition. A maximum of 100,000 partitions can be created in a single table.

    +

    Detailed partition information. If there are multiple partition fields, all fields must be contained, but the corresponding values are optional. The system matches the corresponding partition. A maximum of 100,000 partitions can be created in a single table.

    select_statement

    +

    select_statement

    SELECT query on the source table (DLI and OBS tables).

    +

    SELECT query on the source table (DLI and OBS tables).

    values_row

    +

    values_row

    Value to be inserted to a table. Use commas (,) to separate columns.

    +

    Value to be inserted to a table. Use commas (,) to separate columns.
    -

    Precautions

    • The target DLI table must exist.
    • If no partition needs to be specified for dynamic partitioning, place part_spec in the SELECT statement as a common field.
    • During creation of the target OBS table, only the folder path can be specified.
    • The source table and the target table must have the same data types and column field quantity. Otherwise, data insertion fails.
    • You are advised not to concurrently insert data into a table. If you concurrently insert data into a table, there is a possibility that conflicts occur, leading to failed data insertion.
    • The INSERT INTO statement is used to add the query result to the target table.
    • The INSERT OVERWRITE statement is used to overwrite existing data in the source table.
    • The INSERT INTO statement can be batch executed, but the INSERT OVERWRITE statement can be batch executed only when data of different partitioned tables is inserted to different static partitions.
    • The INSERT INTO and INSERT OVERWRITE statements can be executed at the same time. However, the result is unknown.
    • When you insert data of the source table to the target table, you cannot import or update data of the source table.
    • The dynamic INSERT OVERWRITE statement of Hive partitioned tables can overwrite the involved partition data but cannot overwrite the entire table data.
    • To overwrite data in a specified partition of the datasource table, set dli.sql.dynamicPartitionOverwrite.enabled to true and run the insert overwrite statement. The default value of dli.sql.dynamicPartitionOverwrite.enabled is false, indicating that data in the entire table is overwritten. The following is an example:
      1
      insert overwrite table tb1 partition(part1='v1', part2='v2') select * from ...
+
      Precautions
      • The target DLI table must exist.
      • If no partition needs to be specified for dynamic partitioning, place part_spec in the SELECT statement as a common field.
      • During creation of the target OBS table, only the folder path can be specified.
      • The source table and the target table must have the same data types and column field quantity. Otherwise, data insertion fails.
      • You are not advised to insert data concurrently into the same table as it may result in abnormal data insertion due to concurrency conflicts.
      • The INSERT INTO statement is used to add the query result to the target table.
      • The INSERT OVERWRITE statement is used to overwrite existing data in the source table.
      • The INSERT INTO statement can be batch executed, but the INSERT OVERWRITE statement can be batch executed only when data of different partitioned tables is inserted to different static partitions.
      • The INSERT INTO and INSERT OVERWRITE statements can be executed at the same time. However, the result is unknown.
      • When you insert data of the source table to the target table, you cannot import or update data of the source table.
      • The dynamic INSERT OVERWRITE statement of Hive partitioned tables can overwrite the involved partition data but cannot overwrite the entire table data.
      • To overwrite data in a specified partition of the datasource table, set dli.sql.dynamicPartitionOverwrite.enabled to true and run the insert overwrite statement. The default value of dli.sql.dynamicPartitionOverwrite.enabled is false, indicating that data in the entire table is overwritten. The following is an example:
        1
        insert overwrite table tb1 partition(part1='v1', part2='v2') select * from ...
 
        
-
 
        
 
         
        On the DLI management console, click SQL Editor. In the upper right corner of the editing window, click Settings to configure parameters.
        
 
        
@@ -96,6 +96,11 @@
 
      
 
 
      
+
      How Do I Handle Duplicate Records After Executing the INSERT INTO Statement?
      • Symptom
        When using Hive and Datasource tables (excluding Hudi), executing data modification commands (such as insert into and load data) may result in data duplication or inconsistency if the data source does not support transactions and there is a system failure or queue restart.
        
+
      • Possible causes
        If queue resources are restarted in the data commit phase, data may have been restored to a formal directory. If an insert into statement is executed and a retry is triggered after a resource restart, there is a possibility that data will be repeatedly written.
        
+
      • Solution
        1. Hudi data sources that support ACID properties are recommended.
        2. Use idempotent syntax such as insert overwrite instead of non-idempotent syntax such as insert into to insert data.
        3. If it is strictly required that data cannot be duplicated, you are advised to perform deduplication on the table data after executing the insert into statement to prevent duplicate data.
        
+
      
+
      diff --git a/docs/dli/sqlreference/dli_08_0098.html b/docs/dli/sqlreference/dli_08_0098.html index dc3550277..2e5b53f86 100644 --- a/docs/dli/sqlreference/dli_08_0098.html +++ b/docs/dli/sqlreference/dli_08_0098.html @@ -20,7 +20,6 @@ [COMMENT table_comment] [AS select_statement];
      -

    Keywords

    • IF NOT EXISTS: Prevents system errors when the created table exists.
    • USING: Storage format.
    • OPTIONS: Property name and property value when a table is created.
    • COMMENT: Field or table description.
    • PARTITIONED BY: Partition field.
    • AS: Run the CREATE TABLE AS statement to create a table.
    @@ -132,7 +131,7 @@
    -

    Example 1: Creating a DLI Non-Partitioned Table

    Example description: Create a DLI non-partitioned table named table1 and use the USING keyword to set the storage format of the table to orc.

    +

    Example 1: Creating a DLI Non-Partitioned Table

    Example description: Create a DLI non-partitioned table named table1 and use the USING keyword to set the storage format of the table to orc.

    You can save DLI tables in the parquet format.

    1
 2
@@ -142,7 +141,6 @@
     col_2 INT)
 USING orc;
    -

    Example 2: Creating a DLI Partitioned Table

    Example description: Create a partitioned table named student, which is partitioned using facultyNo and classNo.

    @@ -161,7 +159,6 @@ USING orc PARTITIONED BY (facultyNo, classNo);
    -

    Example 3: Using CTAS to Create a DLI Table Using All or Part of the Data in the Source Table

    Example description: Based on the DLI table table1 created in Example 1: Creating a DLI Non-Partitioned Table, use the CTAS syntax to copy data from table1 to table1_ctas.

    @@ -179,7 +176,6 @@ SELECT * FROM table1;
    -
  • To filter and insert data into table1_ctas in a customized way, you can use the following SELECT statement: SELECT col_1 FROM table1 WHERE col_1 = 'Ann'. This will allow you to select only col_1 from table1 and insert data into table1_ctas where the value equals 'Ann'.
    1
 2
@@ -193,7 +189,6 @@
 FROM    table1
 WHERE   col_1 = 'Ann';
    -
    • @@ -232,7 +227,6 @@ ) USING parquet; -

    Example 5: Creating a DLI Partitioned Table and Customizing OPTIONS Parameters

    Example description: When creating a DLI table, you can customize property names and values. For details about OPTIONS parameters, see Table 2.

    @@ -257,7 +251,6 @@ compression = 'zstd' );
    - diff --git a/docs/dli/sqlreference/dli_08_0099.html b/docs/dli/sqlreference/dli_08_0099.html index 70372b895..9c7ef9b82 100644 --- a/docs/dli/sqlreference/dli_08_0099.html +++ b/docs/dli/sqlreference/dli_08_0099.html @@ -4,8 +4,6 @@

    Overview

    DLI supports the following three types of user-defined functions (UDFs):

    • Regular UDF: takes in one or more input parameters and returns a single result.
    • User-defined table-generating function (UDTF): takes in one or more input parameters and returns multiple rows or columns.
    • User-defined aggregate function (UDAF): aggregates multiple records into one value.
    -

    UDFs can only be used in dedicated queues.

    -

    POM Dependency

    <dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-table_2.11</artifactId>
@@ -65,7 +63,6 @@ public class UdfScalarFunction extends ScalarFunction {
 2
    CREATE FUNCTION udf_test AS 'com.xxx.udf.UdfScalarFunction';
 INSERT INTO sink_stream select udf_test(attr) FROM source_stream;
    -

    UDTF

    The UDTF must inherit the TableFunction function and implement the eval method. The open and close functions are optional. If the UDTF needs to return multiple columns, you only need to declare the returned value as Tuple or Row. If Row is used, you need to overload the getResultType method to declare the returned field type.

    @@ -124,11 +121,10 @@ public class UdfTableFunction extends TableFunction<Row> { INSERT INTO sink_stream select subValue, length FROM source_stream LEFT JOIN LATERAL TABLE(udtf_test(attr, ',')) as T(subValue, length) ON TRUE; -

    UDAF

    The UDAF must inherit the AggregateFunction function. You need to create an accumulator for storing the computing result, for example, WeightedAvgAccum in the following example code.

    -

    Example code

    +

    Example code

    public class WeightedAvgAccum {
 public long sum = 0;
 public int count = 0;
@@ -180,12 +176,11 @@ acc.count = 0;
 acc.sum = 0L;
 }
 }
    -

    Example

    +

    Example

    1
 2
    CREATE FUNCTION udaf_test AS 'com.xxx.udf.UdfAggFunction';
 INSERT INTO sink_stream SELECT udaf_test(attr2) FROM source_stream GROUP BY attr1;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0100.html b/docs/dli/sqlreference/dli_08_0100.html index d6fc59204..d3437d1b1 100644 --- a/docs/dli/sqlreference/dli_08_0100.html +++ b/docs/dli/sqlreference/dli_08_0100.html @@ -7,7 +7,6 @@ 2
    LOAD DATA INPATH 'folder_path' INTO TABLE [db_name.]table_name
   OPTIONS(property_name=property_value, ...);
    -

    Keywords

    • INPATH: path of data to be imported
    • OPTIONS: list of properties
    @@ -214,13 +213,11 @@ 2
    LOAD DATA INPATH 'obs://dli/data.csv' INTO TABLE t
   OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','COMMENTCHAR'='#','HEADER'='false');
    -
  • To import a JSON file to a DLI table named jsontb, run the following statement:
    1
 2
    LOAD DATA INPATH 'obs://dli/alltype.json' into table jsontb
   OPTIONS('DATA_TYPE'='json','DATEFORMAT'='yyyy/MM/dd','TIMESTAMPFORMAT'='yyyy/MM/dd HH:mm:ss');
    -
    • diff --git a/docs/dli/sqlreference/dli_08_0102.html b/docs/dli/sqlreference/dli_08_0102.html index 77adc0b72..fc8f6f16a 100644 --- a/docs/dli/sqlreference/dli_08_0102.html +++ b/docs/dli/sqlreference/dli_08_0102.html @@ -12,7 +12,6 @@ [ GROUP BY { groupItem [, groupItem ]* } ] [ HAVING booleanExpression ] -

    Description

    The SELECT statement is used to select data from a table or insert constant data into a table.

    @@ -22,12 +21,10 @@

    Select the order which contains more than 3 pieces of data.

    1
    insert into temp SELECT  * FROM Orders WHERE units > 3;
    -

    Insert a group of constant data.

    1
    insert into temp select 'Lily', 'male', 'student', 17;
    -

    WHERE Filtering Clause

    Syntax

    @@ -37,7 +34,6 @@ FROM tableExpression [ WHERE booleanExpression ]
    -

    Description

    This statement is used to filter the query results using the WHERE clause.

    @@ -49,7 +45,6 @@ 2
    insert into temp SELECT  * FROM Orders
   WHERE units > 3 and units < 10;
    -

    HAVING Filtering Clause

    Function

    @@ -65,7 +60,6 @@ [ GROUP BY { groupItem [, groupItem ]* } ] [ HAVING booleanExpression ]
    -

    Description

    Generally, HAVING and GROUP BY are used together. GROUP BY applies first for grouping and HAVING then applies for filtering. The arithmetic operation and aggregate function are supported by the HAVING clause.

    @@ -79,7 +73,6 @@ GROUP BY name HAVING max(score) >95 -

    Column-Based GROUP BY

    Function

    @@ -93,7 +86,6 @@ [ WHERE booleanExpression ] [ GROUP BY { groupItem [, groupItem ]* } ]
    -

    Description

    Column-based GROUP BY can be categorized into single-column GROUP BY and multi-column GROUP BY.

    @@ -106,7 +98,6 @@ 2
    insert into temp SELECT name,score, max(score) FROM student 
   GROUP BY name,score;
    -

    Expression-Based GROUP BY

    Function

    @@ -120,7 +111,6 @@ [ WHERE booleanExpression ] [ GROUP BY { groupItem [, groupItem ]* } ]
    -

    Description

    groupItem can have one or more fields. The fields can be called by string functions, but cannot be called by aggregate functions.

    @@ -132,7 +122,6 @@ 2
    insert into temp SELECT substring(name,6),count(name) FROM student
   GROUP BY substring(name,6);
    -

    GROUP BY Using HAVING

    Function

    @@ -148,7 +137,6 @@ [ GROUP BY { groupItem [, groupItem ]* } ] [ HAVING booleanExpression ]
    -

    Description

    Generally, HAVING and GROUP BY are used together. GROUP BY applies first for grouping and HAVING then applies for filtering.

    @@ -164,13 +152,11 @@ GROUP BY num HAVING max(price*amount)>5000; -

    UNION

    Syntax

    1
    query UNION [ ALL ] query
    -

    Description

    This statement is used to return the union set of multiple query results.

    @@ -182,7 +168,6 @@ 2
    insert into temp SELECT  * FROM Orders1
   UNION SELECT  * FROM Orders2;
    - diff --git a/docs/dli/sqlreference/dli_08_0103.html b/docs/dli/sqlreference/dli_08_0103.html index fcc0340a8..d7d627082 100644 --- a/docs/dli/sqlreference/dli_08_0103.html +++ b/docs/dli/sqlreference/dli_08_0103.html @@ -8,7 +8,6 @@ [ WHEN valueN [, valueN1 ]* THEN resultN ]* [ ELSE resultZ ] END -

    or

    1
@@ -17,10 +16,9 @@
   [ WHEN conditionN THEN resultN ]* [ ELSE resultZ ]
   END
    -

    Description

    -
    • If the value of value is value1, result1 is returned. If the value is not any of the values listed in the clause, resultZ is returned. If no else statement is specified, null is returned.
    • If the value of condition1 is true, result1 is returned. If the value does not match any condition listed in the clause, resultZ is returned. If no else statement is specified, null is returned.
    +
    • If the value of value is value1, result1 is returned; otherwise, resultZ is returned. If there is no else statement, null is returned.
    • If the value of condition1 is true, result1 is returned; otherwise, resultZ is returned. If there is no else statement, null is returned.

    Precautions

    • All results must be of the same type.
    • All conditions must be of the Boolean type.
    • If the value does not match any condition, the value of ELSE is returned when the else statement is specified, and null is returned when no else statement is specified.

    Example

    @@ -28,34 +26,29 @@

    Example 1:

    1
    insert into temp SELECT  CASE units WHEN 5 THEN 1 ELSE 0 END FROM Orders;
    -

    Example 2:

    1
    insert into temp SELECT CASE WHEN units = 5 THEN 1 ELSE 0 END FROM Orders;
    -

    NULLIF Expression

    Syntax

    1
    NULLIF(value, value)
    -

    Description

    If the values are the same, NULL is returned. For example, NULL is returned from NULLIF (5,5) and 5 is returned from NULLIF (5,0).

    Precautions

    None

    Example

    -

    If the value of units equals 3, null is returned. Otherwise, the value of units is returned.

    +

    If the value of units equals 3, null is returned. Otherwise, the value of units is returned.

    1
    insert into temp SELECT  NULLIF(units, 3) FROM Orders;
    -

    COALESCE Expression

    Syntax

    1
    COALESCE(value, value [, value ]* )
    -

    Description

    Return the first value that is not NULL, counting from left to right.

    @@ -65,7 +58,6 @@

    5 is returned from the following example:

    1
    insert into temp SELECT  COALESCE(NULL, 5) FROM Orders;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0105.html b/docs/dli/sqlreference/dli_08_0105.html index 3957d04a7..1c1309ddf 100644 --- a/docs/dli/sqlreference/dli_08_0105.html +++ b/docs/dli/sqlreference/dli_08_0105.html @@ -1,11 +1,10 @@ -

    Viewing Table Statistics

    -

    Function

    This statement is used to view the table statistics. The names and data types of all columns in a specified table will be returned.

    +

    Checking Table Statistics

    +

    Function

    This statement is used to check table statistics. The names and data types of all columns in a specified table will be returned.

    Syntax

    1
    DESCRIBE [EXTENDED|FORMATTED] [db_name.]table_name;
    -

    Keywords

    • EXTENDED: displays all metadata of the specified table. It is used during debugging in general.
    • FORMATTED: displays all metadata of the specified table in a form.
    @@ -36,13 +35,12 @@

    Example

    To query the names and data types of all columns in the student table, run the following statement:

    1
    DESCRIBE student;
    -
    -
    Parent topic: Viewing Tables
    +
    Parent topic: Checking Tables
    diff --git a/docs/dli/sqlreference/dli_08_0106.html b/docs/dli/sqlreference/dli_08_0106.html index 6adce8889..77e2187a4 100644 --- a/docs/dli/sqlreference/dli_08_0106.html +++ b/docs/dli/sqlreference/dli_08_0106.html @@ -8,7 +8,6 @@ 2
    FROM tableExpression JOIN tableExpression
   ON value11 = value21 [ AND value12 = value22]
    -

    Syntax Description

    The ON keyword only supports equivalent query of table attributes. If level-2 keys exist (specifically, the Redis value type is HASH), the AND keyword needs to be used to express the equivalent query between Key and Hash Key.

    @@ -95,7 +94,7 @@ * Reconfigure the following options according to actual conditions: * value_type: indicates the value type of the Redis key value. The value can be STRING, HASH, SET, ZSET, or LIST. For the HASH type, you need to specify hash_key_column as the layer-2 primary key. For the SET type, you need to concatenate all queried values using commas (,). * key_column: indicates the column name corresponding to the primary key of the dimension table. - * hash_key_column: indicates the column name corresponding to the KEY of the HASHMAP when value_type is HASH. If value_type is not HASH, you do not need to set this option. + * hash_key_column: indicates the column name that corresponds to the KEY in the HASHMAP when value_type is set to HASH. If value_type is not HASH, you do not need to set this option. * cluster_address: indicates the DCS Redis cluster address. * password: indicates the DCS Redis cluster password. **/ @@ -135,7 +134,6 @@ ON t2.car_brand = t1.car_brand and t2.car_detail_type = t1.car_detail_type WHERE t1.car_brand = "audi";
    -
    diff --git a/docs/dli/sqlreference/dli_08_0107.html b/docs/dli/sqlreference/dli_08_0107.html index b1a75c833..c1c45e59a 100644 --- a/docs/dli/sqlreference/dli_08_0107.html +++ b/docs/dli/sqlreference/dli_08_0107.html @@ -13,7 +13,6 @@ CREATE TEMP STREAM stream_name(...) TIMESTAMP BY proctime.proctime;
    -

    Description

    To set the processing time, you only need to add proctime.proctime following TIMESTAMP BY. You can directly use the proctime field later.

    @@ -56,7 +55,6 @@ SELECT student_name, sum(score) over (order by proctime RANGE UNBOUNDED PRECEDING) FROM student_scores; -

    Configuring Event Time

    Event Time refers to the time when an event is generated, that is, the timestamp generated during data generation.

    @@ -67,7 +65,6 @@ TIMESTAMP BY {attr_name}.rowtime SET WATERMARK (RANGE {time_interval} | ROWS {literal}, {time_interval});
    -

    Description

    To set the event time, you need to select a certain attribute in the stream as the timestamp and set the watermark policy.

    @@ -75,11 +72,9 @@

    The following two watermark policies are available:

    • By time interval
      1
      SET WATERMARK(range interval {time_unit}, interval {time_unit})
      -
    • By event quantity
      1
      SET WATERMARK(rows literal, interval {time_unit})
      -

    Parameters are separated by commas (,). The first parameter indicates the watermark sending interval and the second indicates the maximum event delay.

    @@ -129,7 +124,6 @@ SELECT student_name, sum(score) over (order by time2 RANGE UNBOUNDED PRECEDING) FROM student_scores;
    -
  • Send the watermark every time when 10 pieces of data are received, and the maximum event latency is 20s.
     1
  2
@@ -173,7 +167,6 @@
 SELECT student_name, sum(score) over (order by time2 RANGE UNBOUNDED PRECEDING) 
 FROM student_scores;
    -
    • diff --git a/docs/dli/sqlreference/dli_08_0108.html b/docs/dli/sqlreference/dli_08_0108.html index 20f412cf6..447b5389e 100644 --- a/docs/dli/sqlreference/dli_08_0108.html +++ b/docs/dli/sqlreference/dli_08_0108.html @@ -35,7 +35,6 @@ DEFINE variable AS condition [, variable AS condition ]* ) MR -

    Pattern matching in SQL is performed using the MATCH_RECOGNIZE clause. MATCH_RECOGNIZE enables you to do the following tasks:

    • Logically partition and order the data that is used in the MATCH_RECOGNIZE clause with its PARTITION BY and ORDER BY clauses.
    • Define patterns of rows to seek using the PATTERN clause of the MATCH_RECOGNIZE clause. These patterns use regular expression syntax.
    • Specify the logical conditions required to map a row to a row pattern variable in the DEFINE clause.
    • Define measures, which are expressions usable in other parts of the SQL query, in the MEASURES clause.
    @@ -194,7 +193,6 @@ C AS C.camera_zone_number = A.camera_zone_number ) MR;
    -

    According to this rule, if a vehicle of a license plate number drives from area A to area B but another vehicle of the same license plate number is detected in area A within 5 minutes, then the vehicle in area A is considered to carry a fake license plate.

    diff --git a/docs/dli/sqlreference/dli_08_0110.html b/docs/dli/sqlreference/dli_08_0110.html index 9d669425e..3eb33e64e 100644 --- a/docs/dli/sqlreference/dli_08_0110.html +++ b/docs/dli/sqlreference/dli_08_0110.html @@ -6,7 +6,6 @@
    • The one-pass algorithm is used with O(1) amortized time complexity and O(1) space complexity.
    • The random forest structure is constructed only once. The model update operation only updates the node data distribution values.
    • The node stores data distribution information of multiple windows, and the algorithm can detect data distribution changes.
    • Anomaly detection and model updates are completed in the same code framework.

    Syntax

    1
    SRF_UNSUP(ARRAY[Field 1, Field 2, ...], 'Optional parameter list')
    -
    • The anomaly score returned by the function is a DOUBLE value in the range of [0, 1].
    • The field names must be of the same type. If the field types are different, you can use the CAST function to escape the field names, for example, [a, CAST(b as DOUBLE)].
    • The syntax of the optional parameter list is as follows: "key1=value,key2=value2,..."
    @@ -103,7 +102,6 @@ END FROM MyTable
    - diff --git a/docs/dli/sqlreference/dli_08_0111.html b/docs/dli/sqlreference/dli_08_0111.html index b3de9557b..4739fb291 100644 --- a/docs/dli/sqlreference/dli_08_0111.html +++ b/docs/dli/sqlreference/dli_08_0111.html @@ -17,7 +17,6 @@ ARMA_COEF(field, degree): Return the weight of the ARMA model. ARIMA_PRED(field, degree, derivativeOrder): Use ARIMA to forecast new data. -
    Table 1 Parameters

    Parameter

    @@ -72,14 +71,12 @@ ARIMA_PRED(b) OVER (ORDER BY rowtime ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS arima FROM MyTable
    -

    Holt Winters

    The Holt-Winters algorithm is one of the Exponential smoothing methods used to forecast seasonal data in time series.

    Syntax

    1
    HOLT_WINTERS(field, seasonality, forecastOrder)
    -
    Table 2 Parameters

    Parameter

    @@ -127,7 +124,6 @@ HOLT_WINTERS(b, 5, 2) OVER (ORDER BY rowtime ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS a2 FROM MyTable
    -
    diff --git a/docs/dli/sqlreference/dli_08_0112.html b/docs/dli/sqlreference/dli_08_0112.html index 443476413..4bd0248d3 100644 --- a/docs/dli/sqlreference/dli_08_0112.html +++ b/docs/dli/sqlreference/dli_08_0112.html @@ -188,7 +188,6 @@ FROM car_infos;
    -

    Returned data

    {"case_string_to_timestamp":1514736001000,"cast_int_to_varchar":"5","case_timestamp_to_date":"2018-01-01","cast_String_to_int":100}
    diff --git a/docs/dli/sqlreference/dli_08_0119.html b/docs/dli/sqlreference/dli_08_0119.html index 2d65a6f0f..2b613d4a1 100644 --- a/docs/dli/sqlreference/dli_08_0119.html +++ b/docs/dli/sqlreference/dli_08_0119.html @@ -5,7 +5,7 @@

    In Spark cross-source development scenarios, there is a risk of password leakage if datasource authentication information is directly configured. You are advised to use the datasource authentication provided by DLI.

    -

    Prerequisites

    • Before creating a DLI table and associating it with HBase, you need to create a datasource connection. For details about operations on the management console, see
    • Ensure that the /etc/hosts information of the master node in the MRS cluster is added to the host file of the DLI queue.

      For details about how to add an IP-domain mapping, see Enhanced Datasource Connection in the Data Lake Insight User Guide.

      +

      Prerequisites

      • Before creating a DLI table and associating it with HBase, you need to create a datasource connection.
      • Ensure that the /etc/hosts information of the master node in the MRS cluster is added to the host file of the DLI queue.

        For details about how to add an IP-domain mapping, see Enhanced Datasource Connection in the Data Lake Insight User Guide.

      • The syntax is not supported for security clusters.

      Syntax

      • Single row key
        1
@@ -26,7 +26,6 @@
   'RowKey'='ATTR1',
   'Cols'='ATTR2:CF1.C1, ATTR3:CF1.C2');
        -
      • Combined row key
        1
 2
@@ -46,7 +45,6 @@
   'RowKey'='ATTR1:2, ATTR2:10',
   'Cols'='ATTR2:CF1.C1, ATTR3:CF1.C2'
        -
      @@ -65,7 +63,7 @@

      ZKHost

      ZooKeeper IP address of the HBase cluster.

      -

      Before obtaining the ZooKeeper IP address, you need to create a datasource connection first..

      +

      Create a datasource connection before you can obtain this IP address.

      • Access the CloudTable cluster and enter the ZooKeeper IP address (internal network).
      • To access the MRS cluster, enter the IP address of the node where the ZooKeeper is located and the external port number of the ZooKeeper. The format is ZK_IP1:ZK_PORT1,ZK_IP2:ZK_PORT2.
      NOTE:
      @@ -110,7 +108,6 @@ 'RowKey'='ATTR1', 'Cols'='ATTR2:CF1.C1, ATTR3:CF1.C2');
    - diff --git a/docs/dli/sqlreference/dli_08_0120.html b/docs/dli/sqlreference/dli_08_0120.html index 61b051860..b6256468c 100644 --- a/docs/dli/sqlreference/dli_08_0120.html +++ b/docs/dli/sqlreference/dli_08_0120.html @@ -17,13 +17,11 @@ [GROUP BY field] [ORDER BY field] ...; -
  • Insert a data record into a table.
    1
 2
    INSERT INTO DLI_TABLE
   VALUES values_row [, values_row ...];
    -
    • @@ -84,7 +82,6 @@ LIMIT 3 GROUP BY user_age -
  • Insert data 1 into the test table.
    INSERT INTO test 
   VALUES (1);
    diff --git a/docs/dli/sqlreference/dli_08_0121.html b/docs/dli/sqlreference/dli_08_0121.html index 0aa7592a4..76351c124 100644 --- a/docs/dli/sqlreference/dli_08_0121.html +++ b/docs/dli/sqlreference/dli_08_0121.html @@ -4,17 +4,15 @@

    This statement is used to query data in an HBase table.

    Syntax

    1
    SELECT * FROM table_name LIMIT number;
    -

    Keywords

    LIMIT is used to limit the query results. Only INT type is supported by the number parameter.

    Precautions

    The table to be queried must exist. Otherwise, an error is reported.

    -

    Example

    Query data in the test_ct table.

    +

    Example

    Query data in the table.

    1
    SELECT * FROM test_hbase limit 100;
    -

    Query Pushdown

    Query pushdown implements data filtering using HBase. Specifically, the HBase Client sends filtering conditions to the HBase server, and the HBase server returns only the required data, speeding up your Spark SQL queries. For the filter criteria that HBase does not support, for example, query with the composite row key, Spark SQL performs data filtering.

    @@ -24,32 +22,26 @@
  • Query pushdown is not supported for the following filter criteria:
    • >, <, >=, <=, =, !=, and, or

      The following is an example:

      1
      select * from tableName where (column1 >= value1 and column2<= value2) or column3 != value3
      -
    • The filtering conditions are like and not like. The prefix, suffix, and inclusion match are supported.

      The following is an example:

      1
      select * from tableName where column1 like "%value" or column2 like "value%" or column3 like "%value%"
      -
    • IsNotNull()

      The following is an example:

      1
      select * from tableName where IsNotNull(column)
      -
    • in and not in

      The following is an example:

      1
      select * from tableName where column1 in (value1,value2,value3)  and column2 not in (value4,value5,value6)
      -
    • between _ and _

      The following is an example:

      1
      select * from tableName where column1 between value1 and value2
      -
    • Filtering of the row sub-keys in the composite row key

      For example, to perform row sub-key query on the composite row key column1+column2+column3, run the following statement:

      1
      select * from tableName where column1= value1
      -
    • @@ -57,7 +49,6 @@
  • Query pushdown is not supported for the following filter criteria:
    • Length, count, max, min, join, groupby, orderby, limit, and avg
    • Column comparison

      The following is an example:

      1
      select * from tableName where column1 > (column2+column3)
      -
    • diff --git a/docs/dli/sqlreference/dli_08_0122.html b/docs/dli/sqlreference/dli_08_0122.html index c07d2ece1..60af8f800 100644 --- a/docs/dli/sqlreference/dli_08_0122.html +++ b/docs/dli/sqlreference/dli_08_0122.html @@ -3,7 +3,7 @@

    Creating a DLI Table and Associating It with OpenTSDB

    Function

    Run the CREATE TABLE statement to create the DLI table and associate it with the existing metric in OpenTSDB. This syntax supports the OpenTSDB of CloudTable and MRS.

    -

    Prerequisites

    Before creating a DLI table and associating it with OpenTSDB, you need to create a datasource connection. For details about operations on the management console, see

    +

    Prerequisites

    Before creating a DLI table and associating it with OpenTSDB, you need to create a datasource connection.

    Syntax

    1
 2
@@ -15,7 +15,6 @@
   'metric' = 'METRIC_NAME',
   'tags' = 'TAG1,TAG2');
    -

    Keywords

    @@ -28,8 +27,8 @@

    host

    OpenTSDB IP address.

    -

    Before obtaining the OpenTSDB IP address, you need to create a datasource connection first..

    -
    • After successfully created a connection, you can access the CloudTable OpenTSDB by entering the IP address of the OpenTSDB.
    • You can also access the MRS OpenTSDB. If you have created an enhanced datasource connection, enter the IP address and port number of the node where the OpenTSDB is located. The format is IP:PORT. If the OpenTSDB has multiple nodes, enter one of the node IP addresses.
    +

    Create a datasource connection before you can obtain this IP address.

    +
    • After successfully created a connection, you can access the CloudTable OpenTSDB by entering the IP address of the OpenTSDB.
    • You can also access the MRS OpenTSDB. If you have created an enhanced datasource connection, enter the IP address and port number of the node where the OpenTSDB is located. The format is IP:PORT. If the OpenTSDB has multiple nodes, enter one of the node IP addresses.

    metric

    @@ -59,7 +58,6 @@ 'metric' = 'city.temp', 'tags' = 'city,location');
    -
    diff --git a/docs/dli/sqlreference/dli_08_0123.html b/docs/dli/sqlreference/dli_08_0123.html index 9daafad46..552ab3c2d 100644 --- a/docs/dli/sqlreference/dli_08_0123.html +++ b/docs/dli/sqlreference/dli_08_0123.html @@ -7,11 +7,9 @@

    Syntax

    1
    INSERT INTO TABLE TABLE_NAME SELECT * FROM DLI_TABLE;
    - 
    1
    INSERT INTO TABLE TABLE_NAME VALUES(XXX);
    -

    Keywords

    @@ -39,7 +37,6 @@

    Example

    1
    INSERT INTO TABLE opentsdb_table VALUES('xxx','xxx','2018-05-03 00:00:00',21);
    -
    diff --git a/docs/dli/sqlreference/dli_08_0124.html b/docs/dli/sqlreference/dli_08_0124.html index cc133c74a..0618899f4 100644 --- a/docs/dli/sqlreference/dli_08_0124.html +++ b/docs/dli/sqlreference/dli_08_0124.html @@ -6,7 +6,6 @@

    Syntax

    1
    SELECT * FROM table_name LIMIT number;
    -

    Keywords

    LIMIT is used to limit the query results. Only INT type is supported by the number parameter.

    @@ -16,7 +15,6 @@

    Example

    Query data in the opentsdb_table table.

    1
    SELECT * FROM opentsdb_table limit 100;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0130.html b/docs/dli/sqlreference/dli_08_0130.html index ba6cafafb..5c91e62ca 100644 --- a/docs/dli/sqlreference/dli_08_0130.html +++ b/docs/dli/sqlreference/dli_08_0130.html @@ -5,7 +5,6 @@

    Syntax

    1
    CREATE [OR REPLACE] VIEW view_name AS select_statement;
    -

    Keywords

    • CREATE VIEW: creates views based on the given select statement. The result of the select statement will not be written into the disk.
    • OR REPLACE: updates views using the select statement. No error is reported and the view definition is updated using the SELECT statement if a view exists.
    @@ -16,7 +15,6 @@

    Example

    To create a view named student_view for the queried ID and name of the student table, run the following statement:

    1
    CREATE VIEW student_view AS SELECT id, name FROM student;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0131.html b/docs/dli/sqlreference/dli_08_0131.html index a788c064e..5de1deee8 100644 --- a/docs/dli/sqlreference/dli_08_0131.html +++ b/docs/dli/sqlreference/dli_08_0131.html @@ -5,7 +5,6 @@

    Syntax

    1
    DROP VIEW [IF EXISTS] [db_name.]view_name;
    -

    Keywords

    DROP: Deletes the metadata of a specified view. Although views and tables have many common points, the DROP TABLE statement cannot be used to delete views.

    @@ -15,7 +14,6 @@

    Example

    To delete a view named student_view, run the following statement: 

    1
    DROP VIEW student_view;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0138.html b/docs/dli/sqlreference/dli_08_0138.html index aeaeb47c2..2274740db 100644 --- a/docs/dli/sqlreference/dli_08_0138.html +++ b/docs/dli/sqlreference/dli_08_0138.html @@ -1,11 +1,10 @@ -

    Viewing the Execution Plan

    +

    Checking the Execution Plan

    Function

    This statement returns the logical plan and physical execution plan for the SQL statement.

    Syntax

    1
    EXPLAIN [EXTENDED | CODEGEN] statement;
    -

    Keywords

    EXTENDED: After this keyword is specified, the logical and physical plans are outputted at the same time.

    @@ -16,7 +15,6 @@

    Example

    To return the logical and physical plans of SELECT * FROM test, run the following statement:

    1
    EXPLAIN EXTENDED select * from test;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0140.html b/docs/dli/sqlreference/dli_08_0140.html index 604c6be5e..d27fe4a41 100644 --- a/docs/dli/sqlreference/dli_08_0140.html +++ b/docs/dli/sqlreference/dli_08_0140.html @@ -75,7 +75,7 @@

    SELECT count(*) FROM tb1

    -

    The SELECT permission of database.db1.tables.tb1

    +

    The SELECT permission of database.db1.tables.tb1

    -

    @@ -89,7 +89,7 @@

    SELECT count(*) FROM view1

    -

    The SELECT permission of database.db1.tables.view1

    +

    The SELECT permission of database.db1.tables.view1

    -

    @@ -103,7 +103,7 @@

    INSERT INTO TABLE

    -

    The INSERT_INTO_TABLE permission of database.db1.tables.tb1

    +

    The INSERT_INTO_TABLE permission of database.db1.tables.tb1

    -

    @@ -191,8 +191,8 @@

    For privilege granting or revocation on databases and tables, DLI supports the following permissions:

    -
    • Permissions that can be assigned or revoked on databases are as follows:
      • DROP_DATABASE (Deleting a database)
      • CREATE_TABLE (Creating a table)
      • CREATE_VIEW (Creating a view)
      • EXPLAIN (Explaining a SQL statement as an execution plan)
      • CREATE_ROLE (Creating a role)
      • DROP_ROLE (Deleting a role)
      • SHOW_ROLES (Displaying a role)
      • GRANT_ROLE (Bounding a role)
      • REVOKE_ROLE (Unbinding a role)
      • DESCRIBE_TABLE (Describing a table)
      • DROP_TABLE (Deleting a table)
      • Select (Querying a table)
      • INSERT_INTO_TABLE (Inserting)
      • INSERT_OVERWRITE_TABLE (Overwriting)
      • GRANT_PRIVILEGE (Granting permissions to a database)
      • REVOKE_PRIVILEGE (Revoking permissions from a database)
      • SHOW_PRIVILEGES (Viewing the database permissions of other users)
      • ALTER_TABLE_ADD_PARTITION (Adding partitions to a partitioned table)
      • ALTER_TABLE_DROP_PARTITION (Deleting partitions from a partitioned table)
      • ALTER_TABLE_RENAME_PARTITION (Renaming table partitions)
      • ALTER_TABLE_RECOVER_PARTITION (Restoring table partitions)
      • ALTER_TABLE_SET_LOCATION (Setting the path of a partition)
      • SHOW_PARTITIONS (Displaying all partitions)
      • SHOW_CREATE_TABLE (Viewing table creation statements)
      -
    • Permissions that can be assigned or revoked on tables are as follows:
      • DESCRIBE_TABLE (Describing a table)
      • DROP_TABLE (Deleting a table)
      • Select (Querying a table)
      • INSERT_INTO_TABLE (Inserting)
      • INSERT_OVERWRITE_TABLE (Overwriting)
      • GRANT_PRIVILEGE (Granting permissions to a table)
      • REVOKE_PRIVILEGE (Revoking permissions from a table)
      • SHOW_PRIVILEGES (Viewing the table permissions of other users)
      • ALTER_TABLE_ADD_COLUMNS (Adding a column)
      • ALTER_TABLE_RENAME (Renaming a table)
      • ALTER_TABLE_ADD_PARTITION (Adding partitions to a partitioned table)
      • ALTER_TABLE_DROP_PARTITION (Deleting partitions from a partitioned table)
      • ALTER_TABLE_RENAME_PARTITION (Renaming table partitions)
      • ALTER_TABLE_RECOVER_PARTITION (Restoring table partitions)
      • ALTER_TABLE_SET_LOCATION (Setting the path of a partition)
      • SHOW_PARTITIONS (Displaying all partitions)
      • SHOW_CREATE_TABLE (Viewing table creation statements)
      +
      • Permissions that can be assigned or revoked on databases are as follows:
        • DROP_DATABASE (Deleting a database)
        • CREATE_TABLE (Creating a table)
        • CREATE_VIEW (Creating a view)
        • EXPLAIN (Explaining a SQL statement as an execution plan)
        • CREATE_ROLE (Creating a role)
        • DROP_ROLE (Deleting a role)
        • SHOW_ROLES (Displaying a role)
        • GRANT_ROLE (Bounding a role)
        • REVOKE_ROLE (Unbinding a role)
        • DESCRIBE_TABLE (Describing a table)
        • DROP_TABLE (Deleting a table)
        • Select (Querying a table)
        • INSERT_INTO_TABLE (Inserting)
        • INSERT_OVERWRITE_TABLE (Overwriting)
        • GRANT_PRIVILEGE (Granting permissions to a database)
        • REVOKE_PRIVILEGE (Revoking permissions from a database)
        • SHOW_PRIVILEGES (Checking the database permissions of other users)
        • ALTER_TABLE_ADD_PARTITION (Adding partitions to a partitioned table)
        • ALTER_TABLE_DROP_PARTITION (Deleting partitions from a partitioned table)
        • ALTER_TABLE_RENAME_PARTITION (Renaming table partitions)
        • ALTER_TABLE_RECOVER_PARTITION (Restoring table partitions)
        • ALTER_TABLE_SET_LOCATION (Setting the path of a partition)
        • SHOW_PARTITIONS (Displaying all partitions)
        • SHOW_CREATE_TABLE (Checking table creation statements)
        +
      • Permissions that can be assigned or revoked on tables are as follows:
        • DESCRIBE_TABLE (Describing a table)
        • DROP_TABLE (Deleting a table)
        • Select (Querying a table)
        • INSERT_INTO_TABLE (Inserting)
        • INSERT_OVERWRITE_TABLE (Overwriting)
        • GRANT_PRIVILEGE (Granting permissions to a table)
        • REVOKE_PRIVILEGE (Revoking permissions from a table)
        • SHOW_PRIVILEGES (Checking the table permissions of other users)
        • ALTER_TABLE_ADD_COLUMNS (Adding a column)
        • ALTER_TABLE_RENAME (Renaming a table)
        • ALTER_TABLE_ADD_PARTITION (Adding partitions to a partitioned table)
        • ALTER_TABLE_DROP_PARTITION (Deleting partitions from a partitioned table)
        • ALTER_TABLE_RENAME_PARTITION (Renaming table partitions)
        • ALTER_TABLE_RECOVER_PARTITION (Restoring table partitions)
        • ALTER_TABLE_SET_LOCATION (Setting the path of a partition)
        • SHOW_PARTITIONS (Displaying all partitions)
        • SHOW_CREATE_TABLE (Checking table creation statements)
      diff --git a/docs/dli/sqlreference/dli_08_0141.html b/docs/dli/sqlreference/dli_08_0141.html index f694fe1d3..3899fdcc0 100644 --- a/docs/dli/sqlreference/dli_08_0141.html +++ b/docs/dli/sqlreference/dli_08_0141.html @@ -5,7 +5,6 @@

      Syntax

      1
      CREATE ROLE [db_name].role_name;
      -

      Keywords

      None

      @@ -14,7 +13,6 @@

      Example

      1
      CREATE ROLE role1;
      -
      diff --git a/docs/dli/sqlreference/dli_08_0142.html b/docs/dli/sqlreference/dli_08_0142.html index aa8094ff3..5324ea889 100644 --- a/docs/dli/sqlreference/dli_08_0142.html +++ b/docs/dli/sqlreference/dli_08_0142.html @@ -5,7 +5,6 @@

      Syntax

      1
      GRANT ([db_name].role_name,...) TO (user_name,...);
      -

      Keywords

      None

      @@ -14,7 +13,6 @@

      Example

      1
      GRANT role1 TO user_name1;
      -
      diff --git a/docs/dli/sqlreference/dli_08_0143.html b/docs/dli/sqlreference/dli_08_0143.html index fcc36a590..5618bcfa4 100644 --- a/docs/dli/sqlreference/dli_08_0143.html +++ b/docs/dli/sqlreference/dli_08_0143.html @@ -5,7 +5,6 @@

      Syntax

      1
      SHOW [ALL] ROLES [user_name];
      -

      Keywords

      ALL: Displays all roles.

      @@ -14,17 +13,14 @@

      Example

      • To display all roles bound to the user, run the following statement:
        1
        SHOW ROLES;
        -
      • To display all roles in the project, run the following statement:
        1
        SHOW ALL ROLES;
        -

        Only the administrator has the permission to run the show all roles statement.

      • To display all roles bound to the user named user_name1, run the following statement:
        1
        SHOW ROLES user_name1;
        -
      diff --git a/docs/dli/sqlreference/dli_08_0144.html b/docs/dli/sqlreference/dli_08_0144.html index 81a7441f7..ef60dd347 100644 --- a/docs/dli/sqlreference/dli_08_0144.html +++ b/docs/dli/sqlreference/dli_08_0144.html @@ -5,7 +5,6 @@

      Syntax

      1
      GRANT (privilege,...) ON (resource,..) TO ((ROLE [db_name].role_name) | (USER user_name)),...);
      -

      Keywords

      ROLE: The subsequent role_name must be a role.

      @@ -56,7 +55,7 @@

      SHOW_PRIVILEGES

      -

      Viewing queue permissions of other users

      +

      Checking queue permissions of other users

      @@ -72,17 +71,14 @@

      Example

      Run the following statement to grant user_name1 the permission to delete the db1 database:

      1
      GRANT DROP_DATABASE ON databases.db1 TO USER user_name1;
      -

      Run the following statement to grant user_name1 the SELECT permission of data table tb1 in the db1 database:

      1
      GRANT SELECT ON databases.db1.tables.tb1 TO USER user_name1;
      -

      Run the following statement to grant role_name the SELECT permission of data table tb1 in the db1 database:

      1
      GRANT SELECT ON databases.db1.tables.tb1 TO ROLE role_name;
      -
      diff --git a/docs/dli/sqlreference/dli_08_0145.html b/docs/dli/sqlreference/dli_08_0145.html index a35d10d4b..68f257798 100644 --- a/docs/dli/sqlreference/dli_08_0145.html +++ b/docs/dli/sqlreference/dli_08_0145.html @@ -5,7 +5,6 @@

      Syntax

      1
      SHOW GRANT USER user_name ON resource;
      -

      Keywords

      USER: The subsequent user_name must be a user.

      @@ -16,7 +15,6 @@

      Example

      Run the following statement to show permissions of user_name1 in the db1 database:

      1
      SHOW GRANT USER user_name1 ON databases.db1;
      -
      diff --git a/docs/dli/sqlreference/dli_08_0146.html b/docs/dli/sqlreference/dli_08_0146.html index 8492e8af2..ec1f36874 100644 --- a/docs/dli/sqlreference/dli_08_0146.html +++ b/docs/dli/sqlreference/dli_08_0146.html @@ -5,7 +5,6 @@

      Syntax

      1
      REVOKE (privilege,...) ON (resource,..) FROM ((ROLE [db_name].role_name) | (USER user_name)),...);
      -

      Keywords

      ROLE: The subsequent role_name must be a role.

      @@ -17,17 +16,14 @@

      Example

      To revoke the permission of user user_name1 to delete database db1, run the following statement:

      1
      REVOKE DROP_DATABASE ON databases.db1 FROM USER user_name1;
      -

      To revoke the SELECT permission of user user_name1 on table tb1 in database db1, run the following statement:

      1
      REVOKE SELECT ON databases.db1.tables.tb1 FROM USER user_name1;
      -

      To revoke the SELECT permission of role role_name on table tb1 in database db1, run the following statement:

      1
      REVOKE SELECT ON databases.db1.tables.tb1 FROM ROLE role_name;
      -
      diff --git a/docs/dli/sqlreference/dli_08_0147.html b/docs/dli/sqlreference/dli_08_0147.html index bfbb9072c..564dcc5d7 100644 --- a/docs/dli/sqlreference/dli_08_0147.html +++ b/docs/dli/sqlreference/dli_08_0147.html @@ -5,7 +5,6 @@

      Syntax

      1
      REVOKE ([db_name].role_name,...) FROM (user_name,...);
      -

      Keywords

      None

      @@ -15,7 +14,6 @@

      Example

      To unbind the user_name1 from role1, run the following statement: 

      1
      REVOKE role1 FROM user_name1;
      -
      diff --git a/docs/dli/sqlreference/dli_08_0148.html b/docs/dli/sqlreference/dli_08_0148.html index 22e047cc3..67a908a1c 100644 --- a/docs/dli/sqlreference/dli_08_0148.html +++ b/docs/dli/sqlreference/dli_08_0148.html @@ -5,7 +5,6 @@

      Syntax

      1
      DROP ROLE [db_name].role_name;
      -

      Keywords

      None

      @@ -14,7 +13,6 @@

      Example

      1
      DROP ROLE role1;
      -
      diff --git a/docs/dli/sqlreference/dli_08_0149.html b/docs/dli/sqlreference/dli_08_0149.html index b81f597b5..701e99230 100644 --- a/docs/dli/sqlreference/dli_08_0149.html +++ b/docs/dli/sqlreference/dli_08_0149.html @@ -5,7 +5,6 @@

      Syntax

      1
      SHOW PRINCIPALS ROLE;
      -

      Keywords

      None

      @@ -14,7 +13,6 @@

      Example

      1
      SHOW PRINCIPALS role1;
      -
      diff --git a/docs/dli/sqlreference/dli_08_0150.html b/docs/dli/sqlreference/dli_08_0150.html index c66c8af50..705ecaf71 100644 --- a/docs/dli/sqlreference/dli_08_0150.html +++ b/docs/dli/sqlreference/dli_08_0150.html @@ -17,7 +17,6 @@ [SORT BY col_name_list]] [LIMIT number]; -

      Keywords

      @@ -29,12 +28,13 @@

      ALL

      -

      Returns duplicate rows. By default, all repeated rows are returned. It is followed by asterisks (*) only. Otherwise, an error will occur.

      +

      The keyword ALL is used to return all matching rows from a database, including duplicates. It can only be followed by an asterisk (*); otherwise, an error will occur.

      +

      This is the default behavior in SQL statements and is typically not written explicitly. If neither ALL nor DISTINCT is specified, the query result will include all rows, even if they are duplicates.

      DISTINCT

      -

      Removes duplicate rows from the result set.

      +

      When the DISTINCT keyword is used in a SELECT statement, the system removes duplicate data from the query results to ensure uniqueness.

      WHERE

      @@ -91,7 +91,7 @@
      -

      Precautions

      The table to be queried must exist. Otherwise, an error is reported.

      +

      Precautions

      • The table to be queried must already exist, or an error message will be displayed.
      • When submitting SQL statements to display binary data on the DLI management console, the binary data is converted to Base64.

      Example

      To filter the record, in which the name is Mike, from the student table and sort the results in ascending order of score, run the following statement:

      1
@@ -100,7 +100,6 @@
   WHERE name = 'Mike'
   ORDER BY score;
      -
      diff --git a/docs/dli/sqlreference/dli_08_0152.html b/docs/dli/sqlreference/dli_08_0152.html index f6a762c4c..e62b21890 100644 --- a/docs/dli/sqlreference/dli_08_0152.html +++ b/docs/dli/sqlreference/dli_08_0152.html @@ -7,7 +7,6 @@ 2
      SELECT [ALL | DISTINCT] attr_expr_list FROM table_reference
   WHERE where_condition;
      -

      Keyword

      • All is used to return repeated rows. By default, all repeated rows are returned. It is followed by asterisks (*) only. Otherwise, an error will occur.
      • DISTINCT is used to remove the repeated line from the result.
      • WHERE is used to filter out records that do not meet the condition and return records that meet the condition.
      @@ -19,7 +18,6 @@ 2
      SELECT * FROM student
   WHERE score > 90 AND score < 95;
      - diff --git a/docs/dli/sqlreference/dli_08_0153.html b/docs/dli/sqlreference/dli_08_0153.html index 753693674..ba2bd9363 100644 --- a/docs/dli/sqlreference/dli_08_0153.html +++ b/docs/dli/sqlreference/dli_08_0153.html @@ -11,7 +11,6 @@ [GROUP BY col_name_list] HAVING having_condition; -

      Keyword

      • All is used to return repeated rows. By default, all repeated rows are returned. It is followed by asterisks (*) only. Otherwise, an error will occur.
      • DISTINCT is used to remove the repeated line from the result.
      • Generally, HAVING and GROUP BY are used together. GROUP BY applies first for grouping and HAVING then applies for filtering. The arithmetic operation and aggregate function are supported by the HAVING clause.
      @@ -25,7 +24,6 @@ GROUP BY name HAVING max(score) >95;
      - diff --git a/docs/dli/sqlreference/dli_08_0155.html b/docs/dli/sqlreference/dli_08_0155.html index 9987946f7..b2ddff030 100644 --- a/docs/dli/sqlreference/dli_08_0155.html +++ b/docs/dli/sqlreference/dli_08_0155.html @@ -9,7 +9,6 @@ ORDER BY col_name [ASC | DESC] [,col_name [ASC | DESC],...]; -

      Keyword

      • ASC/DESC: ASC sorts from the lowest value to the highest value. DESC sorts from the highest value to the lowest value. ASC is the default sort order.
      • ORDER BY: specifies that the values in one or more columns should be sorted globally. When ORDER BY is used with GROUP BY, ORDER BY can be followed by the aggregate function.
      @@ -21,7 +20,6 @@ 2
      SELECT * FROM student
   ORDER BY score;
      - diff --git a/docs/dli/sqlreference/dli_08_0156.html b/docs/dli/sqlreference/dli_08_0156.html index 70cda4fdb..968e72345 100644 --- a/docs/dli/sqlreference/dli_08_0156.html +++ b/docs/dli/sqlreference/dli_08_0156.html @@ -9,7 +9,6 @@ SORT BY col_name [ASC | DESC] [,col_name [ASC | DESC],...]; -

      Keyword

      • ASC/DESC: ASC sorts from the lowest value to the highest value. DESC sorts from the highest value to the lowest value. ASC is the default sort order.
      • SORT BY: Used together with GROUP BY to perform local sorting of a single column or multiple columns for PARTITION.
      @@ -21,7 +20,6 @@ 2
      SELECT * FROM student
   SORT BY score;
      - diff --git a/docs/dli/sqlreference/dli_08_0157.html b/docs/dli/sqlreference/dli_08_0157.html index 577fa6850..9f8e0b9c0 100644 --- a/docs/dli/sqlreference/dli_08_0157.html +++ b/docs/dli/sqlreference/dli_08_0157.html @@ -7,7 +7,6 @@ 2
      SELECT attr_expr_list FROM table_reference
   CLUSTER BY col_name [,col_name ,...];
      -

      Keyword

      CLUSTER BY: Buckets are created based on specified fields. Single fields and multiple fields are supported, and data is sorted in buckets.

      @@ -19,7 +18,6 @@ 2
      SELECT * FROM student
   CLUSTER BY score;
      - diff --git a/docs/dli/sqlreference/dli_08_0158.html b/docs/dli/sqlreference/dli_08_0158.html index c84ba13e6..687091625 100644 --- a/docs/dli/sqlreference/dli_08_0158.html +++ b/docs/dli/sqlreference/dli_08_0158.html @@ -7,7 +7,6 @@ 2
      SELECT attr_expr_list FROM table_reference
   DISTRIBUTE BY col_name [,col_name ,...];
      -

      Keyword

      DISTRIBUTE BY: Buckets are created based on specified fields. A single field or multiple fields are supported, and the fields are not sorted in the bucket. This parameter is used together with SORT BY to sort data after bucket division.

      @@ -19,7 +18,6 @@ 2
      SELECT * FROM student
   DISTRIBUTE BY score;
      - diff --git a/docs/dli/sqlreference/dli_08_0160.html b/docs/dli/sqlreference/dli_08_0160.html index 58e856a44..6de5cd757 100644 --- a/docs/dli/sqlreference/dli_08_0160.html +++ b/docs/dli/sqlreference/dli_08_0160.html @@ -7,7 +7,6 @@ 2
      SELECT attr_expr_list FROM table_reference
   GROUP BY col_name_list;
      -

      Keywords

      Column-based GROUP BY can be categorized into single-column GROUP BY and multi-column GROUP BY.

      @@ -20,7 +19,6 @@ 2
      SELECT score, count(name) FROM student
   GROUP BY score,name;
      - diff --git a/docs/dli/sqlreference/dli_08_0161.html b/docs/dli/sqlreference/dli_08_0161.html index 0c6ead91e..865e9c98e 100644 --- a/docs/dli/sqlreference/dli_08_0161.html +++ b/docs/dli/sqlreference/dli_08_0161.html @@ -7,7 +7,6 @@ 2
      SELECT attr_expr_list FROM table_reference
   GROUP BY groupby_expression [, groupby_expression, ...];
      -

      Keywords

      The groupby_expression can contain a single field or multiple fields, and also can call aggregate functions or string functions.

      @@ -19,7 +18,6 @@ 2
      SELECT substr(name,6),count(name) FROM student
   GROUP BY substr(name,6);
      - diff --git a/docs/dli/sqlreference/dli_08_0162.html b/docs/dli/sqlreference/dli_08_0162.html index c712014d5..d070e734b 100644 --- a/docs/dli/sqlreference/dli_08_0162.html +++ b/docs/dli/sqlreference/dli_08_0162.html @@ -9,7 +9,6 @@ GROUP BY groupby_expression [, groupby_expression...] HAVING having_expression; -

      Keyword

      The groupby_expression can contain a single field or multiple fields, and can also call aggregate functions or string functions.

      @@ -25,7 +24,6 @@ GROUP BY num HAVING max(price*amount)>5000;
      - diff --git a/docs/dli/sqlreference/dli_08_0163.html b/docs/dli/sqlreference/dli_08_0163.html index 377cf2ebf..841a15024 100644 --- a/docs/dli/sqlreference/dli_08_0163.html +++ b/docs/dli/sqlreference/dli_08_0163.html @@ -9,30 +9,25 @@ GROUP BY col_name_list WITH ROLLUP; -

      Keyword

      ROLLUP is the expansion of GROUP BY. For example, SELECT a, b, c, SUM(expression) FROM table GROUP BY a, b, c WITH ROLLUP; can be transformed into the following query statements:
      • Counting the (a, b, c) combinations
        1
 2
        SELECT a, b, c, sum(expression) FROM table
   GROUP BY a, b, c;
        -
      • Counting the (a, b) combinations
        1
 2
        SELECT a, b, NULL, sum(expression) FROM table
   GROUP BY a, b;
        -
      • Counting the (a) combinations
        1
 2
        SELECT a, NULL, NULL, sum(expression) FROM table
   GROUP BY a;
        -
      • Total
        1
        SELECT NULL, NULL, NULL, sum(expression) FROM table;
        -
      @@ -46,7 +41,6 @@ GROUP BY group_id, job WITH ROLLUP;
      - diff --git a/docs/dli/sqlreference/dli_08_0164.html b/docs/dli/sqlreference/dli_08_0164.html index 14c8eb2e5..e02b1c592 100644 --- a/docs/dli/sqlreference/dli_08_0164.html +++ b/docs/dli/sqlreference/dli_08_0164.html @@ -9,7 +9,6 @@ GROUP BY col_name_list GROUPING SETS(col_name_list); -

      Keyword

      GROUPING SETS is the expansion of GROUP BY. For example:

      @@ -17,7 +16,6 @@ 2
      SELECT a, b, sum(expression) FROM table
   GROUP BY a, b;
      -
    • SELECT a, b, sum(expression) FROM table GROUP BY a, b GROUPING SETS(a,b);

      It can be converted to the following two queries:

      @@ -27,7 +25,6 @@ UNION SELECT NULL, b, sum(expression) FROM table GROUP BY b; -
    • SELECT a, b, sum(expression) FROM table GROUP BY a, b GROUPING SETS((a,b), a);
      It can be converted to the following two queries:
      1
 2
@@ -35,7 +32,6 @@
 UNION
 SELECT a, NULL, sum(expression) FROM table GROUP BY a;
      -
    • SELECT a, b, sum(expression) FROM table GROUP BY a, b GROUPING SETS((a,b), a, b, ());
      It can be converted to the following four queries:
      1
@@ -52,7 +48,6 @@
 UNION
 SELECT NULL, NULL, sum(expression) FROM table;
      -
    @@ -66,7 +61,6 @@ GROUP BY group_id, job GROUPING SETS (group_id, job); - diff --git a/docs/dli/sqlreference/dli_08_0166.html b/docs/dli/sqlreference/dli_08_0166.html index 0eded3364..4b9d3f082 100644 --- a/docs/dli/sqlreference/dli_08_0166.html +++ b/docs/dli/sqlreference/dli_08_0166.html @@ -7,7 +7,6 @@ 2
    SELECT attr_expr_list FROM table_reference
   {JOIN | INNER JOIN} table_reference ON join_condition;
    -

    Keyword

    JOIN/INNER JOIN: Only the records that meet the JOIN conditions in joined tables will be displayed.

    @@ -19,7 +18,6 @@ 2
    SELECT student_info.name, course_info.courseName FROM student_info
   JOIN course_info ON (student_info.courseId = course_info.courseId);
    - diff --git a/docs/dli/sqlreference/dli_08_0167.html b/docs/dli/sqlreference/dli_08_0167.html index 1961db9bb..5d275d6a6 100644 --- a/docs/dli/sqlreference/dli_08_0167.html +++ b/docs/dli/sqlreference/dli_08_0167.html @@ -7,7 +7,6 @@ 2
    SELECT attr_expr_list FROM table_reference
   LEFT OUTER JOIN table_reference ON join_condition;
    -

    Keyword

    LEFT OUTER JOIN: Returns all joined records of the left table. If no record is matched, NULL is returned.

    @@ -19,7 +18,6 @@ 2
    SELECT student_info.name, course_info.courseName FROM student_info
   LEFT OUTER JOIN course_info ON (student_info.courseId = course_info.courseId);
    - diff --git a/docs/dli/sqlreference/dli_08_0168.html b/docs/dli/sqlreference/dli_08_0168.html index 143801a46..9a8c7908b 100644 --- a/docs/dli/sqlreference/dli_08_0168.html +++ b/docs/dli/sqlreference/dli_08_0168.html @@ -7,7 +7,6 @@ 2
    SELECT attr_expr_list FROM table_reference
   RIGHT OUTER JOIN table_reference ON join_condition;
    -

    Keyword

    RIGHT OUTER JOIN: Return all matched records of the right table. If no record is matched, NULL is returned.

    @@ -19,7 +18,6 @@ 2
    SELECT student_info.name, course_info.courseName FROM student_info
   RIGHT OUTER JOIN course_info ON (student_info.courseId = course_info.courseId);
    - diff --git a/docs/dli/sqlreference/dli_08_0169.html b/docs/dli/sqlreference/dli_08_0169.html index 43a87218c..d26b6eff2 100644 --- a/docs/dli/sqlreference/dli_08_0169.html +++ b/docs/dli/sqlreference/dli_08_0169.html @@ -7,7 +7,6 @@ 2
    SELECT attr_expr_list FROM table_reference
   FULL OUTER JOIN table_reference ON join_condition;
    -

    Keyword

    FULL OUTER JOIN: Matches all records in the left and right tables. If no record is matched, NULL is returned.

    @@ -19,7 +18,6 @@ 2
    SELECT student_info.name, course_info.courseName FROM student_info
   FULL OUTER JOIN course_info ON (student_info.courseId = course_info.courseId);
    - diff --git a/docs/dli/sqlreference/dli_08_0170.html b/docs/dli/sqlreference/dli_08_0170.html index 6117aa613..dc6aefbca 100644 --- a/docs/dli/sqlreference/dli_08_0170.html +++ b/docs/dli/sqlreference/dli_08_0170.html @@ -7,7 +7,6 @@ 2
    SELECT table_reference.col_name, table_reference.col_name, ... FROM table_reference, table_reference
   WHERE table_reference.col_name = table_reference.col_name;
    -

    Keyword

    The keyword WHERE achieves the same function as JOIN...ON... and the mapped records will be returned. Syntax shows the WHERE filtering according to an equation. The WHERE filtering according to an inequation is also supported.

    @@ -19,7 +18,6 @@ 2
    SELECT student_info.name, course_info.courseName FROM student_info,course_info
   WHERE student_info.courseId = course_info.courseId;
    - diff --git a/docs/dli/sqlreference/dli_08_0171.html b/docs/dli/sqlreference/dli_08_0171.html index 3d55fceaf..e99a57b53 100644 --- a/docs/dli/sqlreference/dli_08_0171.html +++ b/docs/dli/sqlreference/dli_08_0171.html @@ -7,7 +7,6 @@ 2
    SELECT attr_expr_list FROM table_reference
   CROSS JOIN table_reference ON join_condition;
    -

    Keyword

    The join_condition is the condition for joining. If join_condition is always true, for example 1=1, the join is Cartesian JOIN. Therefore, the number of records output by Cartesian join is equal to the product of the number of records in the joined table. If Cartesian join is required, use the special keyword CROSS JOIN. CROSS JOIN is the standard way to calculate Cartesian product.

    @@ -19,7 +18,6 @@ 2
    SELECT student_info.name, course_info.courseName FROM student_info
   CROSS JOIN course_info ON (1 = 1);
    - diff --git a/docs/dli/sqlreference/dli_08_0172.html b/docs/dli/sqlreference/dli_08_0172.html index 8a20c9dbb..41ced385c 100644 --- a/docs/dli/sqlreference/dli_08_0172.html +++ b/docs/dli/sqlreference/dli_08_0172.html @@ -7,7 +7,6 @@ 2
    SELECT attr_expr_list FROM table_reference
   LEFT SEMI JOIN table_reference ON join_condition;
    -

    Keyword

    LEFT SEMI JOIN: Indicates to only return the records from the left table. LEFT SEMI JOIN can be achieved by nesting subqueries in LEFT SEMI JOIN, WHERE...IN, or WHERE EXISTS. LEFT SEMI JOIN returns the records that meet the JOIN condition from the left table, while LEFT OUTER JOIN returns all the records from the left table or NULL if no records that meet the JOIN condition are found.

    @@ -19,7 +18,6 @@ 2
    SELECT student_info.name, student_info.courseId FROM student_info
   LEFT SEMI JOIN course_info ON (student_info.courseId = course_info.courseId);
    - diff --git a/docs/dli/sqlreference/dli_08_0173.html b/docs/dli/sqlreference/dli_08_0173.html index d1edb2373..485a23bed 100644 --- a/docs/dli/sqlreference/dli_08_0173.html +++ b/docs/dli/sqlreference/dli_08_0173.html @@ -7,7 +7,6 @@ 2
    SELECT attr_expr_list FROM table_reference
   JOIN table reference ON non_equi_join_condition;
    -

    Keyword

    The non_equi_join_condition is similar to join_condition. The only difference is that the JOIN condition is inequation.

    @@ -19,7 +18,6 @@ 2
    SELECT student_info_1.name, student_info_2.name FROM student_info_1
   JOIN student_info_2 ON (student_info_1. name <> student_info_2. name);
    - diff --git a/docs/dli/sqlreference/dli_08_0175.html b/docs/dli/sqlreference/dli_08_0175.html index c546922c2..6d2a101f1 100644 --- a/docs/dli/sqlreference/dli_08_0175.html +++ b/docs/dli/sqlreference/dli_08_0175.html @@ -7,7 +7,6 @@ 2
    SELECT [ALL | DISTINCT] attr_expr_list FROM table_reference
   WHERE {col_name operator (sub_query) | [NOT] EXISTS sub_query};
    -

    Keyword

    • All is used to return repeated rows. By default, all repeated rows are returned. It is followed by asterisks (*) only. Otherwise, an error will occur.
    • DISTINCT is used to remove the repeated line from the result.
    • The subquery results are used as the filter condition in the subquery nested by WHERE.
    • The operator includes the equation and inequation operators, and IN, NOT IN, EXISTS, and NOT EXISTS operators.
      • If the operator is IN or NOT IN, the returned records are in a single column.
      • If the operator is EXISTS or NOT EXISTS, the subquery must contain WHERE. If any a field in the subquery is the same as that in the external query, add the table name before the field in the subquery.
      @@ -20,7 +19,6 @@ 2
    SELECT name FROM student_info
   WHERE courseId = (SELECT courseId FROM course_info WHERE courseName = 'Biology');
    - diff --git a/docs/dli/sqlreference/dli_08_0176.html b/docs/dli/sqlreference/dli_08_0176.html index 6cf6da827..235a2c5fe 100644 --- a/docs/dli/sqlreference/dli_08_0176.html +++ b/docs/dli/sqlreference/dli_08_0176.html @@ -5,7 +5,6 @@

    Syntax

    1
    SELECT [ALL | DISTINCT] attr_expr_list FROM (sub_query) [alias];
    -

    Keyword

    • All is used to return repeated rows. By default, all repeated rows are returned. It is followed by asterisks (*) only. Otherwise, an error will occur.
    • DISTINCT is used to remove the repeated line from the result.
    @@ -17,7 +16,6 @@ 2
    SELECT DISTINCT name FROM (SELECT name FROM student_info
   JOIN course_info ON student_info.courseId = course_info.courseId) temp;
    - diff --git a/docs/dli/sqlreference/dli_08_0177.html b/docs/dli/sqlreference/dli_08_0177.html index 68a36a0b1..baab44e0a 100644 --- a/docs/dli/sqlreference/dli_08_0177.html +++ b/docs/dli/sqlreference/dli_08_0177.html @@ -9,7 +9,6 @@ GROUP BY groupby_expression HAVING aggregate_func(col_name) operator (sub_query); -

    Keyword

    • All is used to return repeated rows. By default, all repeated rows are returned. It is followed by asterisks (*) only. Otherwise, an error will occur.
    • DISTINCT is used to remove the repeated line from the result.
    @@ -24,7 +23,6 @@ GROUP BY name HAVING count(name) = (SELECT count(*) FROM course_info);
    - diff --git a/docs/dli/sqlreference/dli_08_0178.html b/docs/dli/sqlreference/dli_08_0178.html index b19e1757c..e3c3f23dc 100644 --- a/docs/dli/sqlreference/dli_08_0178.html +++ b/docs/dli/sqlreference/dli_08_0178.html @@ -5,7 +5,6 @@

    Syntax

    1
    SELECT attr_expr FROM ( SELECT attr_expr FROM ( SELECT attr_expr FROM... ... ) [alias] ) [alias];
    -

    Keyword

    • All is used to return repeated rows. By default, all repeated rows are returned. It is followed by asterisks (*) only. Otherwise, an error will occur.
    • DISTINCT is used to remove the repeated line from the result.
    @@ -15,7 +14,6 @@

    Example

    To return the name field from the user_info table after three queries, run the following statement:

    1
    SELECT name FROM ( SELECT name, acc_num FROM ( SELECT name, acc_num, password FROM ( SELECT name, acc_num, password, bank_acc FROM user_info) a ) b ) c;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0180.html b/docs/dli/sqlreference/dli_08_0180.html index 7b79352cc..d5ab13e3a 100644 --- a/docs/dli/sqlreference/dli_08_0180.html +++ b/docs/dli/sqlreference/dli_08_0180.html @@ -5,7 +5,6 @@

    Syntax

    1
    SELECT attr_expr_list FROM table_reference [AS] alias;
    -

    Keyword

    • table_reference: Can be a table, view, or subquery.
    • As: Is used to connect to table_reference and alias. Whether this keyword is added or not does not affect the command execution result.
    @@ -14,11 +13,9 @@

    Example

    • To specify alias n for table simple_table and visit the name field in table simple_table by using n.name, run the following statement:
      1
      SELECT n.score FROM simple_table n WHERE n.name = "leilei";
      -
    • To specify alias m for the subquery result and return all the query results using SELECT * FROM m, run the following statement:
      1
      SELECT * FROM (SELECT * FROM simple_table WHERE score > 90) AS m;
      -
    diff --git a/docs/dli/sqlreference/dli_08_0181.html b/docs/dli/sqlreference/dli_08_0181.html index 19a689ed6..66c780899 100644 --- a/docs/dli/sqlreference/dli_08_0181.html +++ b/docs/dli/sqlreference/dli_08_0181.html @@ -5,7 +5,6 @@

    Syntax

    1
    SELECT attr_expr [AS] alias, attr_expr [AS] alias, ... FROM table_reference;
    -

    Keyword

    • alias: gives an alias for the attr_expr field.
    • AS: Whether to add AS does not affect the result.
    @@ -15,7 +14,6 @@

    Example

    Run SELECT name AS n FROM simple_table WHERE score > 90 to obtain the subquery result. The alias n for name can be used by external SELECT statement. 

    1
    SELECT n FROM (SELECT name AS n FROM simple_table WHERE score > 90) m WHERE n = "xiaoming";
    -
    diff --git a/docs/dli/sqlreference/dli_08_0183.html b/docs/dli/sqlreference/dli_08_0183.html index 4609a28c3..de623d27d 100644 --- a/docs/dli/sqlreference/dli_08_0183.html +++ b/docs/dli/sqlreference/dli_08_0183.html @@ -5,7 +5,6 @@

    Syntax

    1
    select_statement UNION [ALL] select_statement;
    -

    Keyword

    UNION: The set operation is used to join the head and tail of a table based on certain conditions. The number of columns returned by each SELECT statement must be the same. The column type and column name may not be the same.

    @@ -15,7 +14,6 @@

    Example

    To return the union set of the query results of the SELECT * FROM student _1 and SELECT * FROM student _2 commands with the repeated records removed, run the following statement:

    1
    SELECT * FROM student_1 UNION SELECT * FROM student_2;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0184.html b/docs/dli/sqlreference/dli_08_0184.html index 668a45cae..e659f1b9f 100644 --- a/docs/dli/sqlreference/dli_08_0184.html +++ b/docs/dli/sqlreference/dli_08_0184.html @@ -5,7 +5,6 @@

    Syntax

    1
    select_statement INTERSECT select_statement;
    -

    Keyword

    INTERSECT returns the intersection of multiple query results. The number of columns returned by each SELECT statement must be the same. The column type and column name may not be the same. By default, INTERSECT deduplication is used.

    @@ -15,7 +14,6 @@

    Example

    To return the intersection set of the query results of the SELECT * FROM student _1 and SELECT * FROM student _2 commands with the repeated records removed, run the following statement:

    1
    SELECT * FROM student _1 INTERSECT SELECT * FROM student _2;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0185.html b/docs/dli/sqlreference/dli_08_0185.html index a987703b7..64f08d274 100644 --- a/docs/dli/sqlreference/dli_08_0185.html +++ b/docs/dli/sqlreference/dli_08_0185.html @@ -5,7 +5,6 @@

    Syntax

    1
    select_statement EXCEPT select_statement;
    -

    Keywords

    EXCEPT minus the sets. A EXCEPT B indicates to remove the records that exist in both A and B from A and return the results. The repeated records returned by EXCEPT are not removed by default. The number of columns returned by each SELECT statement must be the same. The types and names of columns do not have to be the same.

    @@ -15,7 +14,6 @@

    Example

    To remove the records that exist in both SELECT * FROM student_1 and SELECT * FROM student_2 from SELECT * FROM student_1 and return the results, run the following statement:

    1
    SELECT * FROM student_1 EXCEPT SELECT * FROM student_2;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0186.html b/docs/dli/sqlreference/dli_08_0186.html index 87cdef549..48ca73d60 100644 --- a/docs/dli/sqlreference/dli_08_0186.html +++ b/docs/dli/sqlreference/dli_08_0186.html @@ -5,7 +5,6 @@

    Syntax

    1
    WITH cte_name AS (select_statement) sql_containing_cte_name;
    -

    Keyword

    • cte_name: Name of a public expression. The name must be unique.
    • select_statement: complete SELECT clause.
    • sql_containing_cte_name: SQL statement containing the defined common expression.
    @@ -15,7 +14,6 @@

    Example

    Define SELECT courseId FROM course_info WHERE courseName = 'Biology' as CTE nv and use nv as the SELECT statement in future queries.

    1
    WITH nv AS (SELECT courseId FROM course_info WHERE courseName = 'Biology') SELECT DISTINCT courseId FROM nv;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0188.html b/docs/dli/sqlreference/dli_08_0188.html index 55d029a9f..d14d2e8f4 100644 --- a/docs/dli/sqlreference/dli_08_0188.html +++ b/docs/dli/sqlreference/dli_08_0188.html @@ -5,7 +5,6 @@

    Syntax

    1
    CASE input_expression WHEN when_expression THEN result_expression [...n] [ELSE else_result_expression] END;
    -

    Keyword

    CASE: Subquery is supported in basic CASE statement. However, input_expression and when_expression must be joinable.

    @@ -16,7 +15,6 @@
    • If id is 1, 'a' is returned.
    • If id is 2, 'b' is returned.
    • If id is 3, 'c' is returned.
    • Otherwise, NULL is returned.
    1
    SELECT name, CASE id WHEN 1 THEN 'a' WHEN 2 THEN 'b' WHEN 3 THEN 'c' ELSE NULL END FROM student;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0189.html b/docs/dli/sqlreference/dli_08_0189.html index 4f1591745..f183412b9 100644 --- a/docs/dli/sqlreference/dli_08_0189.html +++ b/docs/dli/sqlreference/dli_08_0189.html @@ -5,7 +5,6 @@

    Syntax

    1
    CASE WHEN boolean_expression THEN result_expression [...n] [ELSE else_result_expression] END;
    -

    Keyword

    boolean_expression: can include subquery. However, the return value of boolean_expression can only be of Boolean type.

    @@ -15,7 +14,6 @@

    Example

    To query the student table and return the related results for the name and score fields: EXCELLENT if the score is higher than 90, GOOD if the score ranges from 80 to 90, and BAD if the score is lower than 80, run the following statement:

    1
    SELECT name, CASE WHEN score >= 90 THEN 'EXCELLENT' WHEN 80 < score AND score < 90 THEN 'GOOD' ELSE 'BAD' END AS level FROM student;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0190.html b/docs/dli/sqlreference/dli_08_0190.html index e501233b5..481c4b6c8 100644 --- a/docs/dli/sqlreference/dli_08_0190.html +++ b/docs/dli/sqlreference/dli_08_0190.html @@ -13,29 +13,23 @@ [ROWS | RANGE BETWEEN (CURRENT ROW | (UNBOUNDED |[num]) PRECEDING) AND (CURRENT ROW | ( UNBOUNDED | [num]) FOLLOWING)]); -

    Keywords

    • PARTITION BY: used to partition a table with one or multiple fields. Similar to GROUP BY, PARTITION BY is used to partition table by fields and each partition is a window. The window function can apply to the entire table or specific partitions. A maximum of 7,000 partitions can be created in a single table.
    • ORDER BY: used to specify the order for the window function to obtain the value. ORDER BY can be used to sort table with one or multiple fields. The sorting order can be ascending (specified by ASC) or descending (specified by DESC). The window is specified by WINDOW. If the window is not specified, the default window is ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW. In other words, the window starts from the head of the table or partition (if PARTITION BY is used in the OVER clause) to the current row.
    • WINDOW: used to define the window by specifying a range of rows.
    • CURRENT ROW: indicates the current row.
    • num PRECEDING: used to specify the start of the defined window. The window starts from the num row precedes the current row.
    • UNBOUNDED PRECEDING: used to indicate that there is no start of the window.
    • num FOLLOWING: used to specify the end of the defined window. The window ends from the num row following the current row.
    • UNBOUNDED FOLLOWING: used to indicate that there is no end of the window.
    • The differences between ROWS BETWEEN... and RANGE BETWEEN... are as follows:
      • ROWS refers to the physical window. After the data is sorted, the physical window starts at the nth row in front of the current row and ends at the mth row following the current row.
      • RANGE refers to the logic window. The column of the logic window is determined by the values rather than the location of rows.
    • The scenarios of the window are as follows:
      • The window only contains the current row.
        1
        ROWS BETWEEN CURRENT ROW AND CURRENT ROW
        -
      • The window starts from three rows precede the current row and ends at the fifth row follows the current row.
        1
        ROWS BETWEEN 3 PRECEDING AND 5 FOLLOWING
        -
      • The window starts from the beginning of the table or partition and ends at the current row.
        1
        ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
        -
      • The window starts from the current window and ends at the end of the table or partition.
        1
        ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING
        -
      • The window starts from the beginning of the table or partition and ends at the end of the table or partition.
        1
        ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
        -
    @@ -46,7 +40,6 @@ 
    1
    SELECT id, count(id) OVER (ORDER BY id ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM over_test;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0192.html b/docs/dli/sqlreference/dli_08_0192.html index bf54f7837..1d631d1bf 100644 --- a/docs/dli/sqlreference/dli_08_0192.html +++ b/docs/dli/sqlreference/dli_08_0192.html @@ -4,7 +4,7 @@
      -
    • Creating a DLI Table and Associating It with DWS
      +
    • Creating a DLI Table and Associating It with GaussDB(DWS)
    • Inserting Data to the DWS Table
      • diff --git a/docs/dli/sqlreference/dli_08_0193.html b/docs/dli/sqlreference/dli_08_0193.html index b82ebe109..9842eec1e 100644 --- a/docs/dli/sqlreference/dli_08_0193.html +++ b/docs/dli/sqlreference/dli_08_0193.html @@ -1,11 +1,11 @@ -

      Creating a DLI Table and Associating It with DWS

      -

      Function

      This statement is used to create a DLI table and associate it with an existing DWS table.

      +

      Creating a DLI Table and Associating It with GaussDB(DWS)

      +

      Function

      This statement is used to create a DLI table and associate it with an existing GaussDB(DWS) table.

      In Spark cross-source development scenarios, there is a risk of password leakage if datasource authentication information is directly configured. You are advised to use the datasource authentication provided by DLI.

      -

      Prerequisites

      Before creating a DLI table and associating it with DWS, you need to create a datasource connection. For details about operations on the management console, see

      +

      Prerequisites

      Before creating a DLI table and associating it with DWS, you need to create a datasource connection.

      Syntax

      1
 2
@@ -19,7 +19,6 @@
   'passwdauth' = 'xxx',
   'encryption' = 'true');
      -

      Keywords

      @@ -31,7 +30,7 @@

      url

      -

      Before obtaining the DWS IP address, you need to create a datasource connection first..

      +

      Create a datasource connection before you can obtain this GaussDB(DWS) connection URL.

      If you have created an enhanced datasource connection, you can use the JDBC Connection String (intranet) provided by DWS or the intranet address and port number to access DWS. The format is protocol header: //Internal IP address:Internal network port/Database name, for example: jdbc:postgresql://192.168.0.77:8000/postgres.

      NOTE:

      The DWS IP address is in the following format: protocol header://IP address:port number/database name

      The following is an example:

      @@ -136,7 +135,6 @@ 'passwdauth' = 'xxx', 'encryption' = 'true');
      -
      diff --git a/docs/dli/sqlreference/dli_08_0194.html b/docs/dli/sqlreference/dli_08_0194.html index d69c5036a..989f924e1 100644 --- a/docs/dli/sqlreference/dli_08_0194.html +++ b/docs/dli/sqlreference/dli_08_0194.html @@ -17,13 +17,11 @@ [GROUP BY field] [ORDER BY field] ...;
      -
  • Insert a data record into a table.
    1
 2
    INSERT INTO DLI_TABLE
   VALUES values_row [, values_row ...];
    -
    • @@ -84,7 +82,6 @@ LIMIT 3 GROUP BY user_age -
  • Insert data 1 into the test table.
    INSERT INTO test 
   VALUES (1);
    diff --git a/docs/dli/sqlreference/dli_08_0195.html b/docs/dli/sqlreference/dli_08_0195.html index d3bc2a653..4647a1cd2 100644 --- a/docs/dli/sqlreference/dli_08_0195.html +++ b/docs/dli/sqlreference/dli_08_0195.html @@ -4,7 +4,6 @@

    This statement is used to query data in a DWS table.

    Syntax

    1
    SELECT * FROM table_name LIMIT number;
    -

    Keywords

    LIMIT is used to limit the query results. Only INT type is supported by the number parameter.

    @@ -14,7 +13,6 @@

    Example

    To query data in the dli_to_dws table, enter the following statement:

    1
    SELECT * FROM dli_to_dws limit 100;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0197.html b/docs/dli/sqlreference/dli_08_0197.html index 4717f3aff..499bc7cc0 100644 --- a/docs/dli/sqlreference/dli_08_0197.html +++ b/docs/dli/sqlreference/dli_08_0197.html @@ -5,7 +5,7 @@

    In Spark cross-source development scenarios, there is a risk of password leakage if datasource authentication information is directly configured. You are advised to use the datasource authentication provided by DLI.

    -

    Prerequisites

    Before creating a DLI table and associating it with RDS, you need to create a datasource connection. For details about operations on the management console, see

    +

    Prerequisites

    Before creating a DLI table and associating it with RDS, you need to create a datasource connection.

    Syntax

    1
 2
@@ -21,7 +21,6 @@
   'passwdauth' = 'xxx',
   'encryption' = 'true');
    -

    Keywords

    @@ -33,7 +32,7 @@

    url

    -

    Before obtaining the RDS IP address, you need to create a datasource connection first..

    +

    Create a datasource connection before you can obtain this RDS connection URL.

    After an enhanced datasource connection is created, use the internal network domain name or internal network address and database port number provided by RDS to connect to DLI. If MySQL is used, the format is protocol header://internal IP address:internal network port number. If PostgreSQL is used, the format is protocol header://internal IP address:internal network port number/database name.

    For example: jdbc:mysql://192.168.0.193:3306 or jdbc:postgresql://192.168.0.193:3306/postgres.

    @@ -64,7 +63,7 @@

    passwdauth

    -

    Datasource password authentication name. For details about how to create datasource authentication, see Datasource Authentication in the Data Lake Insight User Guide.

    +

    Datasource password authentication name. For how to create datasource authentication, see "Datasource Authentication" in Data Lake Insight User Guide.

    encryption

    @@ -146,7 +145,6 @@ 'passwdauth' = 'xxx', 'encryption' = 'true');
    -

    Accessing PostgreSQL

    @@ -164,7 +162,6 @@ 'passwdauth' = 'xxx', 'encryption' = 'true'); - diff --git a/docs/dli/sqlreference/dli_08_0198.html b/docs/dli/sqlreference/dli_08_0198.html index f678c5428..f90a94cb5 100644 --- a/docs/dli/sqlreference/dli_08_0198.html +++ b/docs/dli/sqlreference/dli_08_0198.html @@ -17,13 +17,11 @@ [GROUP BY field] [ORDER BY field] ...; -
  • Insert a data record into a table.
    1
 2
    INSERT INTO DLI_TABLE
   VALUES values_row [, values_row ...];
    -
    • @@ -84,7 +82,6 @@ LIMIT 3 GROUP BY user_age -
  • Insert data 1 into the test table.
    INSERT INTO test 
   VALUES (1);
    diff --git a/docs/dli/sqlreference/dli_08_0199.html b/docs/dli/sqlreference/dli_08_0199.html index a6c2a28d9..8e673b686 100644 --- a/docs/dli/sqlreference/dli_08_0199.html +++ b/docs/dli/sqlreference/dli_08_0199.html @@ -4,7 +4,6 @@

    This statement is used to query data in an RDS table.

    Syntax

    1
    SELECT * FROM table_name LIMIT number;
    -

    Keywords

    LIMIT is used to limit the query results. Only INT type is supported by the number parameter.

    @@ -14,7 +13,6 @@

    Example

    Query data in the test_ct table.

    1
    SELECT * FROM dli_to_rds limit 100;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0201.html b/docs/dli/sqlreference/dli_08_0201.html index 381e7df3f..898e8df92 100644 --- a/docs/dli/sqlreference/dli_08_0201.html +++ b/docs/dli/sqlreference/dli_08_0201.html @@ -5,7 +5,7 @@

    In Spark cross-source development scenarios, there is a risk of password leakage if datasource authentication information is directly configured. You are advised to use the datasource authentication provided by DLI.

    -

    Prerequisites

    Before creating a DLI table and associating it with CSS, you need to create a datasource connection. For details about operations on the management console, see

    +

    Prerequisites

    Before creating a DLI table and associating it with CSS, you need to create a datasource connection.

    Syntax

     1
  2
@@ -31,7 +31,6 @@
   'es.nodes.wan.only' = 'true',
   'es.mapping.id' = 'FIELDNAME');
    -

    Keywords

    @@ -43,7 +42,7 @@

    es.nodes

    -

    Before obtaining the CSS IP address, you need to create a datasource connection first..

    +

    Create a datasource connection before you can obtain this CSS connection address.

    If you have created an enhanced datasource connection, you can use the internal IP address provided by CSS. The format is IP1:PORT1,IP2:PORT2.

    @@ -111,14 +110,13 @@ 7 8
    CREATE TABLE IF NOT EXISTS dli_to_css (doc_id String, name string, age int)
   USING CSS OPTIONS (
-  es.nodes 'to-css-1174404703-LzwpJEyx.datasource.com:9200',
-  resource '/dli_index/dli_type',
-  pushdown 'false',
-  strict 'true',
-  es.nodes.wan.only 'true',
-  es.mapping.id 'doc_id');
+  'es.nodes' ='to-css-1174404703-LzwpJEyx.datasource.com:9200',
+  'resource' ='/dli_index/dli_type',
+  'pushdown' ='false',
+  'strict' ='true',
+  'es.nodes.wan.only' ='true',
+  'es.mapping.id' ='doc_id');
    - diff --git a/docs/dli/sqlreference/dli_08_0202.html b/docs/dli/sqlreference/dli_08_0202.html index a07513c26..253b3428c 100644 --- a/docs/dli/sqlreference/dli_08_0202.html +++ b/docs/dli/sqlreference/dli_08_0202.html @@ -17,13 +17,11 @@ [GROUP BY field] [ORDER BY field] ...; -
  • Insert a data record into a table.
    1
 2
    INSERT INTO DLI_TABLE
   VALUES values_row [, values_row ...];
    -
    • @@ -84,7 +82,6 @@ LIMIT 3 GROUP BY user_age -
  • Insert data 1 into the test table.
    INSERT INTO test 
   VALUES (1);
    diff --git a/docs/dli/sqlreference/dli_08_0203.html b/docs/dli/sqlreference/dli_08_0203.html index e9934c58c..70d43ac31 100644 --- a/docs/dli/sqlreference/dli_08_0203.html +++ b/docs/dli/sqlreference/dli_08_0203.html @@ -4,7 +4,6 @@

    This statement is used to query data in a CSS table.

    Syntax

    1
    SELECT * FROM table_name LIMIT number;
    -

    Keywords

    LIMIT is used to limit the query results. Only INT type is supported by the number parameter.

    @@ -14,7 +13,6 @@

    Example

    To query data in the dli_to_css table, enter the following statement:

    1
    SELECT * FROM dli_to_css limit 100;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0204.html b/docs/dli/sqlreference/dli_08_0204.html index b37c303ba..58e52b426 100644 --- a/docs/dli/sqlreference/dli_08_0204.html +++ b/docs/dli/sqlreference/dli_08_0204.html @@ -3,7 +3,7 @@

    Creating a DLI Table Using the Hive Syntax

    Function

    This Hive syntax is used to create a DLI table. The main differences between the DataSource and the Hive syntax lie in the supported data formats and the number of supported partitions. For details, see syntax and precautions.

    -

    Precautions

    • Table properties cannot be specified using CTAS table creation statements.
    • Instructions on using partitioned tables:
      • When you create a partitioned table, ensure that the specified column in PARTITIONED BY is not a column in the table and the data type is specified. The partition column supports only the open-source Hive table types including string, boolean, tinyint, smallint, short, int, bigint, long, decimal, float, double, date, and timestamp.
      • Multiple partition fields can be specified. The partition fields need to be specified after the PARTITIONED BY keyword, instead of the table name. Otherwise, an error occurs.
      • A maximum of 200,000 partitions can be created in a single table.
      • CTAS table creation statements cannot be used to create partitioned tables.
      +

      Precautions

      • Table properties cannot be specified using CTAS table creation statements.
      • You cannot specify multi-character delimiters when creating Hive DLI tables.
      • Instructions on using partitioned tables:
        • When you create a partitioned table, ensure that the specified column in PARTITIONED BY is not a column in the table and the data type is specified. The partition column supports only the open-source Hive table types including string, boolean, tinyint, smallint, short, int, bigint, long, decimal, float, double, date, and timestamp.
        • Multiple partition fields can be specified. The partition fields need to be specified after the PARTITIONED BY keyword, instead of the table name. Otherwise, an error occurs.
        • A maximum of 200,000 partitions can be created in a single table.
        • CTAS table creation statements cannot be used to create partitioned tables.

      Syntax

       1
@@ -38,7 +38,6 @@
       [LINES TERMINATED BY char]
       [NULL DEFINED AS char]
      -

      Keywords

      • IF NOT EXISTS: Prevents system errors when the created table exists.
      • COMMENT: Field or table description.
      • PARTITIONED BY: Partition field.
      • ROW FORMAT: Row data format.
      • STORED AS: Specifies the format of the file to be stored. Currently, only the TEXTFILE, AVRO, ORC, SEQUENCEFILE, RCFILE, and PARQUET format are supported. This keyword is mandatory when you create DLI tables.
      • TBLPROPERTIES: This keyword is used to add a key/value property to a table.
        • If the table storage format is Parquet, you can use TBLPROPERTIES(parquet.compression = 'zstd') to set the table compression format to zstd.
        @@ -96,14 +95,14 @@

        Yes

        -

        Row data format

        +

        Row data format The ROW FORMAT function is available only for textfile tables.

        file_format

        Yes

        -

        DLI table storage format, which can be TEXTFILE, AVRO, ORC, SEQUENCEFILE, RCFILE, or PARQUET.

        +

        Data storage format of DLI tables. The options include textfile, avro, orc, sequencefile, rcfile, and parquet.

        table_comment

        @@ -132,7 +131,7 @@
    -

    Example 1: Creating a DLI Non-Partitioned Table

    Example description: Create a DLI non-partitioned table named table1 and use the STORED AS keyword to set the storage format of the table to orc.

    +

    Example 1: Creating a DLI Non-Partitioned Table

    Example description: Create a DLI non-partitioned table named table1 and use the STORED AS keyword to set the storage format of the table to orc.

    You can save DLI tables in the textfile, avro, orc, sequencefile, rcfile, or parquet format.

    1
 2
@@ -144,7 +143,6 @@
 )
 STORED AS orc;
    -

    Example 2: Creating a DLI Partitioned Table

    Example description: Create a partitioned table named student, which is partitioned using facultyNo and classNo.

    @@ -167,7 +165,6 @@ classNo INT );
    -

    Example 3: Using CTAS to Create a DLI Table Using All or Part of the Data in the Source Table

    Example description: Based on the DLI table table1 created in Example 1: Creating a DLI Non-Partitioned Table, use the CTAS syntax to copy data from table1 to table1_ctas.

    @@ -185,7 +182,6 @@ SELECT * FROM table1;
    -
  • If you do not need all data in table1, change AS SELECT * FROM table1 to AS SELECT col_1 FROM table1 WHERE col_1 = Ann. In this way, you can run the SELECT statement to insert all rows whose col_1 column is Ann from table1 to table1_ctas.
    1
 2
@@ -199,7 +195,6 @@
 FROM table1
 WHERE col_1 = 'Ann';
    -
    • @@ -238,7 +233,6 @@ ) STORED AS parquet; -

    Example 5: Creating a DLI Partitioned Table and Customizing TBLPROPERTIES Parameters

    Example description: Create a DLI partitioned table named table3 and partition the table based on col_3. Set dli.multi.version.enable, comment, orc.compress, and auto.purge in TBLPROPERTIES.

    @@ -267,11 +261,10 @@ auto.purge = true );
    - -

    Example 6: Creating a Non-Partitioned Table in Textfile Format and Setting ROW FORMAT

    Example description: In this example, create a non-partitioned table named table4 in the textfile format and set ROW FORMAT (the ROW FORMAT function is available only for textfile tables).

    -
    • Fields: columns in a table. Each field has a name and data type. Fields in a table are separated by slashes (/).
    • COLLECTION ITEMS: A collection item refers to an element in a group of data, which can be an array, a list, or a collection. Collection items in table4 are separated by $.
    • MAP KEYS: A map key is a data structure of key-value pairs and is used to store a group of associated data. Map keys in a table are separated by number signs (#).
    • Rows: rows in a table. Each row contains a group of field values. Rows in a table end with \n. (Note that only \n can be used as the row separator.)
    • NULL: a special value that represents a missing or unknown value. In a table, NULL indicates that the field has no value or the value is unknown. When there is a null value in the data, it is represented by the string null.
    +

    Example 6: Creating a Non-Partitioned Table in Textfile Format and Setting ROW FORMAT

    Example description: In this example, create a non-partitioned table named table4 in the textfile format and set ROW FORMAT (the ROW FORMAT function is available only for textfile tables).

    +
    • Fields: columns in a table. Each field has a name and data type. Fields in a table are separated by slashes (/).
    • COLLECTION ITEMS: A collection item refers to an element in a group of data, which can be an array, a list, or a collection. Collection items in table4 are separated by $.
    • MAP KEYS: A map key is a data structure of key-value pairs and is used to store a group of associated data. Map keys in a table are separated by number signs (#).
    • Rows: rows in a table. Each row contains a group of field values. Rows in a table end with \n. (Note that only \n can be used as the row separator.)
    • NULL: a special value that represents a missing or unknown value. In a table, NULL indicates that the field has no value or the value is unknown. When there is a null value in the data, it is represented by the string null.
     1
  2
  3
@@ -294,7 +287,6 @@
 LINES TERMINATED              BY '\n'
 NULL DEFINED                  AS 'NULL';
    -
    diff --git a/docs/dli/sqlreference/dli_08_0205.html b/docs/dli/sqlreference/dli_08_0205.html index 48fa2aaff..c27aad790 100644 --- a/docs/dli/sqlreference/dli_08_0205.html +++ b/docs/dli/sqlreference/dli_08_0205.html @@ -11,7 +11,6 @@ [OPTIONS(key1=value1)] select_statement; -

    Keywords

    • USING: Specifies the storage format.
    • OPTIONS: Specifies the list of attributes to be exported. This parameter is optional.
    @@ -41,7 +40,7 @@

    Precautions

    • You can configure the spark.sql.shuffle.partitions parameter to set the number of files to be inserted into the OBS bucket in the non-DLI table. In addition, to avoid data skew, you can add distribute by rand() to the end of the INSERT statement to increase the number of concurrent jobs. The following is an example:
      insert into table table_target select * from table_source distribute by cast(rand() * N as int);
    • When the configuration item is OPTIONS('DELIMITER'=','), you can specify a separator. The default value is ,.

      For CSV data, the following delimiters are supported:

      -
      • Tab character, for example, 'DELIMITER'='\t'.
      • Any binary character, for example, 'DELIMITER'='\u0001(^A)'.
      • Single quotation mark ('). A single quotation mark must be enclosed in double quotation marks (" "). For example, 'DELIMITER'= "'".
      • \001(^A) and \017(^Q) are also supported, for example, 'DELIMITER'='\001(^A)' and 'DELIMITER'='\017(^Q)'.
      +
      • Tab character, for example, 'DELIMITER'='\t'.
      • You can specify a delimiter using Unicode encoding, for example: 'DELIMITER'='\u0001'.
      • Single quotation mark ('). A single quotation mark must be enclosed in double quotation marks (" "). For example, 'DELIMITER'= "'".

    Example

    1
@@ -52,7 +51,6 @@
   OPTIONS(key1=value1)
   select * from db1.tb1;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0207.html b/docs/dli/sqlreference/dli_08_0207.html index a226a0e85..6c8cd0e0d 100644 --- a/docs/dli/sqlreference/dli_08_0207.html +++ b/docs/dli/sqlreference/dli_08_0207.html @@ -260,7 +260,6 @@ speeds[1] FROM car_infos; - diff --git a/docs/dli/sqlreference/dli_08_0209.html b/docs/dli/sqlreference/dli_08_0209.html index 2b48273fa..b88cf5d87 100644 --- a/docs/dli/sqlreference/dli_08_0209.html +++ b/docs/dli/sqlreference/dli_08_0209.html @@ -295,7 +295,6 @@ FROM driver_behavior WHERE NOT ST_WITHIN(ST_POINT(cast(Longitude as DOUBLE), cast(Latitude as DOUBLE)), ST_BUFFER(ST_LINE(ARRAY[ST_POINT(34.585555,105.725221),ST_POINT(34.586729,105.735974),ST_POINT(34.586492,105.740538),ST_POINT(34.586388,105.741651),ST_POINT(34.586135,105.748712),ST_POINT(34.588691,105.74997)]),0.001)); -

    IP Functions

    Currently, only IPv4 addresses are supported.

    diff --git a/docs/dli/sqlreference/dli_08_0216.html b/docs/dli/sqlreference/dli_08_0216.html index da68ad2db..b7cc850f9 100644 --- a/docs/dli/sqlreference/dli_08_0216.html +++ b/docs/dli/sqlreference/dli_08_0216.html @@ -11,7 +11,6 @@ ALL_POINTS_OF_CLUSTER(ARRAY[field_names], distance_threshold): Compute all data points in the cluster where the current data point is assigned. ALL_CLUSTERS_POINTS(ARRAY[field_names], distance_threshold): Computers all data points in each cluster after the current data point is assigned.
    -
    • Clustering algorithms can be applied in unbounded streams.
    @@ -64,7 +63,6 @@ ALL_CLUSTERS_POINTS(ARRAY[c,e], 1.0) OVER (ORDER BY proctime RANGE BETWEEN INTERVAL '60' MINUTE PRECEDING AND CURRENT ROW) AS listoflistofpoints FROM MyTable - diff --git a/docs/dli/sqlreference/dli_08_0217.html b/docs/dli/sqlreference/dli_08_0217.html index 43c4247cc..b9e1ef762 100644 --- a/docs/dli/sqlreference/dli_08_0217.html +++ b/docs/dli/sqlreference/dli_08_0217.html @@ -5,7 +5,6 @@

    Syntax

    1
    TRUNCATE TABLE tablename [PARTITION (partcol1=val1, partcol2=val2 ...)];
    -

    Keywords

    @@ -33,7 +32,6 @@

    Example

    1
    truncate table test PARTITION (class = 'test');
    -
    diff --git a/docs/dli/sqlreference/dli_08_0218.html b/docs/dli/sqlreference/dli_08_0218.html index 9c0f718f3..4309487c2 100644 --- a/docs/dli/sqlreference/dli_08_0218.html +++ b/docs/dli/sqlreference/dli_08_0218.html @@ -125,7 +125,6 @@ FROM Orders GROUP BY SESSION(ts, INTERVAL '12' HOUR), name; -

    @@ -139,7 +138,6 @@ ORDER BY proctime|rowtime(ROWS number PRECEDING) |(RANGE (BETWEEN INTERVAL '1' SECOND PRECEDING AND CURRENT ROW | UNBOUNDED preceding)) ) -

    Description

    @@ -209,7 +207,6 @@ sum(amount) OVER (PARTITION BY name ORDER BY timeattr RANGE BETWEEN INTERVAL '60' SECOND PRECEDING AND CURRENT ROW) as cnt2 FROM Orders; -

    diff --git a/docs/dli/sqlreference/dli_08_0219.html b/docs/dli/sqlreference/dli_08_0219.html index d47804add..80a0acdb1 100644 --- a/docs/dli/sqlreference/dli_08_0219.html +++ b/docs/dli/sqlreference/dli_08_0219.html @@ -17,10 +17,10 @@

    Deleting a Database

    -

    Viewing a Specified Database

    +

    Checking a Specified Database

    -

    Viewing All Databases

    +

    Checking All Databases

    Syntax for Creating an OBS Table

    @@ -44,24 +44,24 @@

    Deleting a Table

    -

    Syntax for Viewing a Table

    +

    Syntax for Checking a Table

    -

    Viewing All Tables

    +

    Checking All Tables

    -

    Viewing Table Creation Statements

    +

    Checking Table Creation Statements

    -

    Viewing Table Properties

    +

    Checking Table Properties

    -

    Viewing All Columns in a Specified Table

    +

    Checking All Columns in a Specified Table

    -

    Viewing All Partitions in a Specified Table

    +

    Checking All Partitions in a Specified Table

    -

    Viewing Table Statistics

    +

    Checking Table Statistics

    Syntax for Modifying a Table

    @@ -195,9 +195,9 @@

    Deleting a View

    -

    Syntax for Viewing the Execution Plan

    +

    Syntax for Checking the Execution Plan

    -

    Viewing the Execution Plan

    +

    Checking the Execution Plan

    Syntax Related to Data Permissions

    @@ -251,7 +251,7 @@

    Enabling Multiversion Backup When Creating an OBS Table

    Enabling or Disabling Multiversion Backup When Modifying a Table

    Setting the Retention Period for Multiversion Backup Data

    -

    Viewing Multiversion Backup Data

    +

    Checking Multiversion Backup Data

    Restoring Multiversion Backup Data

    Configuring the Trash Bin for Expired Multiversion Data

    Deleting Multiversion Backup Data

    diff --git a/docs/dli/sqlreference/dli_08_0221.html b/docs/dli/sqlreference/dli_08_0221.html index d8a9a6487..ee2fa13e7 100644 --- a/docs/dli/sqlreference/dli_08_0221.html +++ b/docs/dli/sqlreference/dli_08_0221.html @@ -19,7 +19,7 @@
  • Deleting a Table
    • -
  • Viewing Tables
    +
  • Checking Tables
  • Modifying a Table
    • @@ -29,13 +29,15 @@
  • Inserting Data
    • +
  • Reusing Results of Subqueries
    +
  • Clearing Data
  • Exporting Search Results
  • Backing Up and Restoring Data of Multiple Versions
    • -
  • Table Lifecycle Management
    +
  • Table Lifecycle Management
  • Creating a Datasource Connection with an HBase Table
    • @@ -55,7 +57,7 @@
  • Views
    • -
  • Viewing the Execution Plan
    +
  • Checking the Execution Plan
  • Data Permissions Management
    • diff --git a/docs/dli/sqlreference/dli_08_0226.html b/docs/dli/sqlreference/dli_08_0226.html index e74dfc377..f207e3b57 100644 --- a/docs/dli/sqlreference/dli_08_0226.html +++ b/docs/dli/sqlreference/dli_08_0226.html @@ -29,7 +29,6 @@ 'key.column'= 'FIELDNAME1' ); -
  • Wildcard key
     1
  2
@@ -53,7 +52,6 @@
   'key.column'= 'FIELDNAME1'
 );
    -
    • @@ -151,7 +149,6 @@ 'table' = 'person' ); -
    • Wildcarding the table name
    1
@@ -170,7 +167,6 @@
   'key.column' = 'id'
 );
    -
    diff --git a/docs/dli/sqlreference/dli_08_0227.html b/docs/dli/sqlreference/dli_08_0227.html index eaabd89da..9d23d0a03 100644 --- a/docs/dli/sqlreference/dli_08_0227.html +++ b/docs/dli/sqlreference/dli_08_0227.html @@ -17,13 +17,11 @@ [GROUP BY field] [ORDER BY field] ...; -
  • Insert a data record into a table.
    1
 2
    INSERT INTO DLI_TABLE
   VALUES values_row [, values_row ...];
    -
    • @@ -86,7 +84,6 @@ ); insert into test_redis values("James", 35), ("Michael", 22); -

    The Redis database contains two tables, naming test_with_key_column:James and test_with_key_column:Michael respectively.

    @@ -105,7 +102,6 @@ ); insert into test_redis values("James", 35), ("Michael", 22); -

    In Redis, there are two tables named test_without_key_column:uuid.

    @@ -116,7 +112,6 @@ 2
    INSERT INTO test_redis
   VALUES("James", 35), ("Michael", 22);
    - diff --git a/docs/dli/sqlreference/dli_08_0228.html b/docs/dli/sqlreference/dli_08_0228.html index defcd6934..de7d2301c 100644 --- a/docs/dli/sqlreference/dli_08_0228.html +++ b/docs/dli/sqlreference/dli_08_0228.html @@ -4,7 +4,6 @@

    This statement is used to query data in a DCS table.

    Syntax

    1
    SELECT * FROM table_name LIMIT number;
    -

    Keywords

    LIMIT is used to limit the query results. Only INT type is supported by the number parameter.

    @@ -12,7 +11,6 @@

    Example

    Query data in the test_redis table.

    1
    SELECT * FROM test_redis limit 100;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0230.html b/docs/dli/sqlreference/dli_08_0230.html index b70fdd086..50228966a 100644 --- a/docs/dli/sqlreference/dli_08_0230.html +++ b/docs/dli/sqlreference/dli_08_0230.html @@ -27,7 +27,6 @@ 'encryption' = 'true' );
    -

    Document Database Service (DDS) is fully compatible with the MongoDB protocol. Therefore, the syntax used is using mongo options.

    @@ -88,14 +87,13 @@ 3 4 5 -6
    create table 1_datasource.test_table1(id string, name string, age int) using mongo options(
+6
    create table 1_datasource_mongo.test_momgo(id string, name string, age int) using mongo options(
   'url' = '192.168.4.62:8635,192.168.5.134:8635/test?authSource=admin',
   'database' = 'test',
   'collection' = 'test',
   'passwdauth' = 'xxx',
   'encryption' = 'true');
    - diff --git a/docs/dli/sqlreference/dli_08_0231.html b/docs/dli/sqlreference/dli_08_0231.html index 037659241..08985e278 100644 --- a/docs/dli/sqlreference/dli_08_0231.html +++ b/docs/dli/sqlreference/dli_08_0231.html @@ -17,13 +17,11 @@ [GROUP BY field] [ORDER BY field] ...; -
  • Insert a data record into a table.
    1
 2
    INSERT INTO DLI_TABLE
   VALUES values_row [, values_row ...];
    -
    • Overwriting the inserted data
      1
@@ -40,7 +38,6 @@
   [GROUP BY field]
   [ORDER BY field] ...;
      -
    @@ -101,7 +98,6 @@ LIMIT 3 GROUP BY user_age -
  • Insert data 1 into the test table.
    INSERT INTO test 
   VALUES (1);
    diff --git a/docs/dli/sqlreference/dli_08_0232.html b/docs/dli/sqlreference/dli_08_0232.html index 29ceae7b4..7e75b4769 100644 --- a/docs/dli/sqlreference/dli_08_0232.html +++ b/docs/dli/sqlreference/dli_08_0232.html @@ -4,7 +4,6 @@

    This statement is used to query data in a DDS table.

    Syntax

    1
    SELECT * FROM table_name LIMIT number;
    -

    Keywords

    LIMIT is used to limit the query results. Only INT type is supported by the number parameter.

    @@ -14,7 +13,6 @@

    Example

    Query data in the test_table1 table.

    1
    SELECT * FROM test_table1 limit 100;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0235.html b/docs/dli/sqlreference/dli_08_0235.html index 3910817c9..c8b5f4382 100644 --- a/docs/dli/sqlreference/dli_08_0235.html +++ b/docs/dli/sqlreference/dli_08_0235.html @@ -69,7 +69,7 @@

    No

    -

    Range of partitions of a DIS stream, data in which is ingested by the DLI job. This parameter and partition_count cannot be configured at the same time. If this parameter is not specified, data of all partitions is read by default.

    +

    Range of partitions of a DIS stream, data in which is ingested by the DLI job. This parameter and partition_count cannot be configured at the same time. If this parameter is not specified, data of all partitions is read by default.

    If you set this parameter to [0:2], data will be read from partitions 1, 2, and 3.

    @@ -206,7 +206,6 @@ field_delimiter = "," );
    -
  • In JSON encoding format, DLI reads data from the DIS stream and records it as codes in JSON format. For example, {"car":{"car_id":"ZJA710XC", "car_owner":"coco", "car_age":5, "average_speed":80, "total_miles":15000, "car_timestamp":1526438880}}
     1
  2
@@ -238,7 +237,6 @@
     json_config = "car_id=car.car_id;car_owner =car.car_owner;car_age=car.car_age;average_speed =car.average_speed ;total_miles=car.total_miles;"
 );
    -
  • In XML encoding format, DLI reads data from the DIS stream and records it as codes in XML format.
     1
  2
@@ -274,7 +272,6 @@
     xml_config = "pid=person.pid;page=person.page;pname=person.pname;plocation=person.plocation;pbir=person.pbir;pgrade=person.pgrade;phealthy=person.phealthy"
 );
    -
    An example of XML data is as follows:
     1
  2
@@ -302,7 +299,6 @@
   </person>
 </root>
    -
  • In EMAIL encoding format, DLI reads data from the DIS stream and records it as a complete Email.
     1
@@ -347,25 +343,24 @@
     email_key = "Message-ID, Date, Subject, From, To, CC, BCC, Body, Mime-Version, Content-Type, charset, Content_Transfer_Encoding"  
 );
    -

    An example of email data is as follows:

    Message-ID: <200906291839032504254@sample.com>
 Date: Fri, 11 May 2001 09:54:00 -0700 (PDT)
-From: zhangsan@sample.com
-To: lisi@sample.com, wangwu@sample.com 
+From: user1@sample.com
+To: user2@sample.com, user3@sample.com 
 Subject:  "Hello World" 
-Cc: lilei@sample.com, hanmei@sample.com
+Cc: user4@sample.com, user5@sample.com
 Mime-Version: 1.0
 Content-Type: text/plain; charset=us-ascii
 Content-Transfer-Encoding: 7bit
-Bcc: jack@sample.com, lily@sample.com
-X-From: Zhang San
-X-To: Li Si, Wang Wu
-X-cc: Li Lei, Han Mei
+Bcc: user6@sample.com, user7@sample.com
+X-From: user1
+X-To: user2, user3
+X-cc: user4, user5
 X-bcc: 
-X-Folder: \Li_Si_June2001\Notes Folders\Notes inbox
-X-Origin: Lucy
+X-Folder: \user2_June2001\Notes Folders\Notes inbox
+X-Origin: user8
 X-FileName: sample.nsf
 
 Dear Associate / Analyst Committee:
@@ -375,7 +370,7 @@ Hello World!
 Thank you,
 
 Associate / Analyst Program 
-zhangsan
    +user1
    • diff --git a/docs/dli/sqlreference/dli_08_0236.html b/docs/dli/sqlreference/dli_08_0236.html index 8c4826dd9..3c841b2be 100644 --- a/docs/dli/sqlreference/dli_08_0236.html +++ b/docs/dli/sqlreference/dli_08_0236.html @@ -24,7 +24,6 @@ version_id = "" ); -

    Keywords

    @@ -156,7 +155,6 @@ field_delimiter = "," );
    -
  • The input.json file is read from the OBS bucket. Rows are separated by '\n'.
    CREATE SOURCE STREAM obs_source (
diff --git a/docs/dli/sqlreference/dli_08_0237.html b/docs/dli/sqlreference/dli_08_0237.html
index d95e7745a..2216022f2 100644
--- a/docs/dli/sqlreference/dli_08_0237.html
+++ b/docs/dli/sqlreference/dli_08_0237.html
@@ -24,7 +24,6 @@
     table_columns = ""
   );
    -

    Keywords

    @@ -55,7 +54,7 @@

    Yes

    ID of the cluster to which the data table to be read belongs.

    -

    For details about how to view the ID of the CloudTable cluster, see section "Viewing Basic Cluster Information" in the CloudTable Service User Guide.

    +

    For how to check the ID of the CloudTable cluster, see "Checking Basic Cluster Information" in the CloudTable Service User Guide.

    table_name

    @@ -107,7 +106,6 @@ table_columns = "rowKey,info:owner,info:age,car:speed,car:miles" );
    - diff --git a/docs/dli/sqlreference/dli_08_0238.html b/docs/dli/sqlreference/dli_08_0238.html index bd4376370..76e4b441e 100644 --- a/docs/dli/sqlreference/dli_08_0238.html +++ b/docs/dli/sqlreference/dli_08_0238.html @@ -24,7 +24,6 @@ encode = "json" ); -

    Keywords

    @@ -174,7 +173,6 @@ encode = "json" );
    -
    • Read the topic whose object is test from Kafka and use json_config to map JSON data to table fields.

      The data encoding format is non-nested JSON.

      @@ -197,7 +195,6 @@ json_config = "name=attr1;age=attr2" ); -
    diff --git a/docs/dli/sqlreference/dli_08_0239.html b/docs/dli/sqlreference/dli_08_0239.html index ca3e34f4d..e7410e56d 100644 --- a/docs/dli/sqlreference/dli_08_0239.html +++ b/docs/dli/sqlreference/dli_08_0239.html @@ -26,7 +26,6 @@ json_config="" ); -

    Keywords

    @@ -166,7 +165,6 @@ json_config = "name=attr1;age=attr2" );
    -
  • Read Kafka topic test. The data is encoded in JSON format and nested. This example uses the complex data type ROW. For details about the syntax of ROW, see Data Type.

    The test data is as follows:

    {
diff --git a/docs/dli/sqlreference/dli_08_0241.html b/docs/dli/sqlreference/dli_08_0241.html
index 027a643b1..aa65c2671 100644
--- a/docs/dli/sqlreference/dli_08_0241.html
+++ b/docs/dli/sqlreference/dli_08_0241.html
@@ -22,7 +22,6 @@
     field_delimiter= ""
   );
    -

    Keywords

    @@ -154,7 +153,6 @@ field_delimiter = "," );
    -
    • JSON: Data is written to the DIS stream and encoded using JSON. If there are multiple partitions, car_owner and car_brand are used as the keys to distribute data to different partitions. If enableOutputNull is set to true, an empty field (the value is null) is generated. If set to false, no empty field is generated. An example is as follows: "car_id ":"ZJA710XC", "car_owner ":"lilei", "car_brand ":"BMW", "car_price ":700000.
       1
@@ -185,7 +183,6 @@
     enable_output_null = "false"
 );
      -
    diff --git a/docs/dli/sqlreference/dli_08_0242.html b/docs/dli/sqlreference/dli_08_0242.html index 021f536ff..5cb614008 100644 --- a/docs/dli/sqlreference/dli_08_0242.html +++ b/docs/dli/sqlreference/dli_08_0242.html @@ -44,7 +44,6 @@ dis_notice_channel = "" ) -

    Keywords

    @@ -236,7 +235,6 @@ rolling_size = "100m" );
    -
    • Example of the ORC encoding format
       1
@@ -271,7 +269,6 @@
     dump_interval = "60"
 );
      -
    • For details about the parquet encoding example, see the example in File System Sink Stream (Recommended).
    diff --git a/docs/dli/sqlreference/dli_08_0243.html b/docs/dli/sqlreference/dli_08_0243.html index 496822dd9..a3b2520af 100644 --- a/docs/dli/sqlreference/dli_08_0243.html +++ b/docs/dli/sqlreference/dli_08_0243.html @@ -26,7 +26,6 @@ create_if_not_exist = "" ) -

    Keywords

    @@ -135,7 +134,6 @@ batch_insert_data_num = "20" );
    - diff --git a/docs/dli/sqlreference/dli_08_0244.html b/docs/dli/sqlreference/dli_08_0244.html index 40b0861d5..99afea19b 100644 --- a/docs/dli/sqlreference/dli_08_0244.html +++ b/docs/dli/sqlreference/dli_08_0244.html @@ -30,7 +30,6 @@ batch_insert_data_num = "" ) -

    Keywords

    @@ -146,7 +145,6 @@ batch_insert_data_num = "10" );
    - diff --git a/docs/dli/sqlreference/dli_08_0245.html b/docs/dli/sqlreference/dli_08_0245.html index c24c26a9a..7b1af6870 100644 --- a/docs/dli/sqlreference/dli_08_0245.html +++ b/docs/dli/sqlreference/dli_08_0245.html @@ -27,7 +27,6 @@ table_name = "" ); -

    @@ -152,7 +151,6 @@ primary_key = "c_timeminute"); table_name = "audi_cheaper_than_30w" ); - diff --git a/docs/dli/sqlreference/dli_08_0247.html b/docs/dli/sqlreference/dli_08_0247.html index 7181f3663..b41b2bbb6 100644 --- a/docs/dli/sqlreference/dli_08_0247.html +++ b/docs/dli/sqlreference/dli_08_0247.html @@ -25,7 +25,6 @@ table_name = "" ); -

    Keywords

    @@ -96,7 +95,6 @@ table_name = "t1" );
    -

    In the example, student_name corresponds to the name attribute in the database, and student_age corresponds to the age attribute in the database.

    NOTE:
    • If db_columns is not configured, it is normal that the number of attributes in the output stream is less than that of attributes in the database table and the extra attributes in the database table are all nullable or have default values.
    @@ -127,7 +125,6 @@ primary_key = "c_timeminute" );
    - @@ -168,7 +165,6 @@ insert into audi_cheaper_than_30w select "1","2","3",4; - diff --git a/docs/dli/sqlreference/dli_08_0248.html b/docs/dli/sqlreference/dli_08_0248.html index 593a0f159..612881230 100644 --- a/docs/dli/sqlreference/dli_08_0248.html +++ b/docs/dli/sqlreference/dli_08_0248.html @@ -49,7 +49,6 @@ dump_interval = "" ); -

    Keywords

    @@ -231,7 +230,6 @@ dump_interval = "10" );
    -
  • Dump files in ORC format.
     1
  2
@@ -277,7 +275,6 @@
     dump_interval = "10"
   );
    -
    • diff --git a/docs/dli/sqlreference/dli_08_0249.html b/docs/dli/sqlreference/dli_08_0249.html index 591976e94..1ecf9a36a 100644 --- a/docs/dli/sqlreference/dli_08_0249.html +++ b/docs/dli/sqlreference/dli_08_0249.html @@ -26,7 +26,6 @@ field_names = "" ); -

    Keywords

    @@ -117,7 +116,6 @@ batch_insert_data_num = "10" );
    - diff --git a/docs/dli/sqlreference/dli_08_0251.html b/docs/dli/sqlreference/dli_08_0251.html index 65b52b318..57375d945 100644 --- a/docs/dli/sqlreference/dli_08_0251.html +++ b/docs/dli/sqlreference/dli_08_0251.html @@ -23,7 +23,6 @@ message_column = "" ) -

    @@ -109,7 +108,6 @@ message_column = "over_speed_message" ); - 
     1
  2
@@ -135,7 +133,6 @@
     message_column = "over_speed_message"
   );
    -
    diff --git a/docs/dli/sqlreference/dli_08_0252.html b/docs/dli/sqlreference/dli_08_0252.html index 06ca58a53..b55021e12 100644 --- a/docs/dli/sqlreference/dli_08_0252.html +++ b/docs/dli/sqlreference/dli_08_0252.html @@ -32,7 +32,6 @@ batch_insert_data_num= "" ); -

    @@ -164,7 +163,6 @@ batch_insert_data_num = "10" ); - diff --git a/docs/dli/sqlreference/dli_08_0253.html b/docs/dli/sqlreference/dli_08_0253.html index 621f98e61..212e39aa3 100644 --- a/docs/dli/sqlreference/dli_08_0253.html +++ b/docs/dli/sqlreference/dli_08_0253.html @@ -28,7 +28,6 @@ value_type= "",key_value= "" ); -

    Keywords

    @@ -118,7 +117,6 @@ key_value = "${car_id}_str: ${car_owner}; name_list: ${car_owner}; ${car_id}_hash: {name:${car_owner}, age: ${car_age}}; name_set: ${car_owner}; math_zset: {${car_owner}:${average_speed}}" );
    - diff --git a/docs/dli/sqlreference/dli_08_0254.html b/docs/dli/sqlreference/dli_08_0254.html index 466dee049..aed980642 100644 --- a/docs/dli/sqlreference/dli_08_0254.html +++ b/docs/dli/sqlreference/dli_08_0254.html @@ -22,7 +22,6 @@ encode = "json" ) -

    @@ -122,7 +121,6 @@ encode = "json" ); -
  • Example 2:
     1
  2
@@ -154,7 +152,6 @@
     kafka_properties = "sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username=\"xxx\" password=\"xxx\";,sasl.mechanism=PLAIN,security.protocol=SASL_SSL"
   );
    -
    • diff --git a/docs/dli/sqlreference/dli_08_0255.html b/docs/dli/sqlreference/dli_08_0255.html index b1663ecbe..f3104cecc 100644 --- a/docs/dli/sqlreference/dli_08_0255.html +++ b/docs/dli/sqlreference/dli_08_0255.html @@ -30,7 +30,6 @@ action = "" ) -

    Keywords

    @@ -151,7 +150,6 @@ krb_auth = "KRB_AUTH_NAME" );
    - diff --git a/docs/dli/sqlreference/dli_08_0257.html b/docs/dli/sqlreference/dli_08_0257.html index 8fec1b8e2..685420840 100644 --- a/docs/dli/sqlreference/dli_08_0257.html +++ b/docs/dli/sqlreference/dli_08_0257.html @@ -22,7 +22,6 @@ encode = "json" ) -

    Keywords

    @@ -121,7 +120,6 @@ encode = "json" );
    - diff --git a/docs/dli/sqlreference/dli_08_0258.html b/docs/dli/sqlreference/dli_08_0258.html index 0b1ac951a..160353eee 100644 --- a/docs/dli/sqlreference/dli_08_0258.html +++ b/docs/dli/sqlreference/dli_08_0258.html @@ -3,14 +3,14 @@

    Creating a Temporary Stream

    Function

    The temporary stream is used to simplify SQL logic. If complex SQL logic is followed, write SQL statements concatenated with temporary streams. The temporary stream is just a logical concept and does not generate any data.

    -

    Syntax

    1
    CREATE TEMP STREAM stream_id (attr_name attr_type (',' attr_name attr_type)* )
+
    Syntax
    The syntax for creating a temporary stream is as follows:
    
+
    1
    CREATE TEMP STREAM stream_id (attr_name attr_type (',' attr_name attr_type)* )
 
    
-
 
    
 
    
-
    Example
    1
    create temp stream a2(attr1 int, attr2 string);
+
    Example
    The following is an example of creating a temporary stream:
    
+
    1
    create temp stream a2(attr1 int, attr2 string);
 
    
-
 
    
 
    
 
    
diff --git a/docs/dli/sqlreference/dli_08_0260.html b/docs/dli/sqlreference/dli_08_0260.html
index 5a05c9069..149d6a9c4 100644
--- a/docs/dli/sqlreference/dli_08_0260.html
+++ b/docs/dli/sqlreference/dli_08_0260.html
@@ -15,7 +15,6 @@
     value_type= "",
     key_column= ""(,hash_key_column="")?);
 
    
-
 
    
 
    
 
    Keywords
    
diff --git a/docs/dli/sqlreference/dli_08_0261.html b/docs/dli/sqlreference/dli_08_0261.html
index 14f34e43f..ed080ae98 100644
--- a/docs/dli/sqlreference/dli_08_0261.html
+++ b/docs/dli/sqlreference/dli_08_0261.html
@@ -34,7 +34,6 @@
     table_name = ""
   );
    -

    @@ -203,7 +202,6 @@ SELECT a.car_id, b.car_owner, b.car_brand, b.car_price FROM car_infos as a join db_info as b on a.car_id = b.car_id;
    -

    To create a DWS dimension table, set the database connection address to a DWS database address. If the DWS database version is later than 8.1.0, the open-source PostgreSQL driver cannot be used for connection. You need to use the GaussDB driver for connection.

    diff --git a/docs/dli/sqlreference/dli_08_0263.html b/docs/dli/sqlreference/dli_08_0263.html index 4fc189a55..59f05500b 100644 --- a/docs/dli/sqlreference/dli_08_0263.html +++ b/docs/dli/sqlreference/dli_08_0263.html @@ -5,7 +5,6 @@

    Syntax

    1
    ALTER TABLE [db_name.]table_name ADD COLUMNS (col_name1 col_type1 [COMMENT col_comment1], ...);
    -

    Keywords

    • ADD COLUMNS: columns to add
    • COMMENT: column description
    @@ -50,7 +49,6 @@

    Example

    1
    ALTER TABLE t1 ADD COLUMNS (column2 int, column3 string);
    -
    diff --git a/docs/dli/sqlreference/dli_08_0266.html b/docs/dli/sqlreference/dli_08_0266.html index af4417553..713d6a298 100644 --- a/docs/dli/sqlreference/dli_08_0266.html +++ b/docs/dli/sqlreference/dli_08_0266.html @@ -18,15 +18,6 @@

    Maximum number of records to be written into a single file. If the value is zero or negative, there is no limit.

    -

    spark.sql.autoBroadcastJoinThreshold

    - -

    209715200

    - -

    Maximum size of the table that displays all working nodes when a connection is executed. You can set this parameter to -1 to disable the display.

    -
    NOTE:

    Currently, only the configuration unit metastore table that runs the ANALYZE TABLE COMPUTE statistics noscan command and the file-based data source table that directly calculates statistics based on data files are supported.

    -
    - -

    spark.sql.shuffle.partitions

    200

    @@ -56,6 +47,47 @@

    Path of bad records.

    +

    dli.sql.sqlasync.enabled

    + +

    true

    + +

    Whether DDL and DCL statements are executed asynchronously. The value true indicates that asynchronous execution is enabled.

    + + +

    dli.sql.job.timeout

    + +

    -

    + +

    Job running timeout interval, in seconds. If the job times out, it will be canceled.

    + + +

    spark.sql.keep.distinct.expandThreshold

    + +

    -

    + +
    • Parameter description:

      When running queries with multidimensional analysis that include the count(distinct) function using the cube structure in Spark, the typical execution plan involves using the expand operator. However, this operation can cause query inflation. To avoid this issue, you are advised to configure the following settings:

      +
      • spark.sql.keep.distinct.expandThreshold:

        Default value: -1, indicating that Spark's default expand operator is used.

        +

        Setting the parameter to a specific value, such as 512, defines the threshold for query inflation. If the threshold is exceeded, the count(distinct) function will use the distinct aggregation operator to execute the query instead of the expand operator.

        +
      • spark.sql.distinct.aggregator.enabled: whether to forcibly use the distinct aggregation operator. If set to true, spark.sql.keep.distinct.expandThreshold is not used.
      +
    • Use case: Queries with multidimensional analysis that use the cube structure and may include multiple count(distinct) functions, as well as the cube or rollup operator.
    • Example of a typical use case:
      SELECT a1, a2, count(distinct b), count(distinct c) FROM test_distinct group by a1, a2 with cube
      +
    + + +

    spark.sql.distinct.aggregator.enabled

    + +

    false

    + + +

    spark.sql.optimizer.dynamicPartitionPruning.enabled

    + +

    true

    + +

    This parameter is used to control whether to enable dynamic partition pruning. Dynamic partition pruning can help reduce the amount of data that needs to be scanned and improve query performance when executing SQL queries.

    +
    • When set to true, dynamic partition pruning is enabled. SQL automatically detects and deletes partitions that do not meet the WHERE clause conditions during query. This is useful for tables that have a large number of partitions.
    • If SQL queries contain a large number of nested left join operations and the table has a large number of dynamic partitions, a large number of memory resources may be consumed during data parsing. As a result, the memory of the driver node is insufficient and there are frequent Full GCs.

      To avoid such issues, you can disable dynamic partition pruning by setting this parameter to false.

      +

      However, disabling this optimization may reduce query performance. Once disabled, Spark does not automatically prun the partitions that do not meet the requirements.

      +
    + + diff --git a/docs/dli/sqlreference/dli_08_0267.html b/docs/dli/sqlreference/dli_08_0267.html index e0ecba5d3..56bfa1659 100644 --- a/docs/dli/sqlreference/dli_08_0267.html +++ b/docs/dli/sqlreference/dli_08_0267.html @@ -22,7 +22,6 @@ sk = "" ); -

    Keywords

    @@ -129,7 +128,6 @@ sk = "{{mySk}}" );
    -

    The data is ultimately stored in OBS. Directory: obs://obs-sink/car_infos/buyday=xx/part-x-x.

    After the data is generated, the OBS partitioned table can be established for subsequent batch processing through the following SQL statements:

    @@ -149,11 +147,9 @@ stored as parquet location 'obs://obs-sink/car_infos'; -
  • Restore partition information from the associated OBS path.
    1
    alter table car_infos recover partitions;
    -
  • Example 2:

    The following example dumps the car_info data to HDFS, with the buyday field as the partition field and csv as the encoding format.

    @@ -181,7 +177,6 @@ field_delimiter = "," ); -

    The data is ultimately stored in HDFS. Directory: /user/car_infos/buyday=xx/part-x-x.

    • diff --git a/docs/dli/sqlreference/dli_08_0270.html b/docs/dli/sqlreference/dli_08_0270.html index 9595940f0..70f9d819c 100644 --- a/docs/dli/sqlreference/dli_08_0270.html +++ b/docs/dli/sqlreference/dli_08_0270.html @@ -1,8 +1,10 @@

    DMS Source Stream

    -

    DMS (Distributed Message Service) is a message middleware service based on distributed, high-availability clustering technology. It provides reliable, scalable, fully managed queues for sending, receiving, and storing messages. DMS for Kafka is a message queuing service based on Apache Kafka. This service provides Kafka premium instances.

    -

    The source stream can read data from a Kafka instance as the input data of jobs. The syntax for creating a Kafka source stream is the same as that for creating an open source Apache Kafka source stream. For details, see Open-Source Kafka Source Stream.

    +

    DMS (Distributed Message Service) is a message middleware service based on distributed, high-availability clustering technology. It provides reliable, scalable, fully managed queues for sending, receiving, and storing messages.

    +

    DMS for Kafka is a message queuing service based on Apache Kafka. This service provides Kafka premium instances.

    +

    The source stream can read data from a Kafka instance as the input data of jobs.

    +

    The syntax for creating a Kafka source stream is the same as that for creating an open source Apache Kafka source stream. For details, see Open-Source Kafka Source Stream.

    diff --git a/docs/dli/sqlreference/dli_08_0271.html b/docs/dli/sqlreference/dli_08_0271.html index 98d87e464..c0a0db981 100644 --- a/docs/dli/sqlreference/dli_08_0271.html +++ b/docs/dli/sqlreference/dli_08_0271.html @@ -1,8 +1,9 @@

    DMS Sink Stream

    -

    DMS (Distributed Message Service) is a message middleware service based on distributed, high-availability clustering technology. It provides reliable, scalable, fully managed queues for sending, receiving, and storing messages. DMS for Kafka is a message queuing service based on Apache Kafka. This service provides Kafka premium instances.

    -

    DLI can write the job output data into the Kafka instance. The syntax for creating a Kafka sink stream is the same as that for creating an open source Apache Kafka sink stream. For details, see MRS Kafka Sink Stream.

    +

    DMS for Kafka is a message queuing service based on Apache Kafka. This service provides Kafka premium instances.

    +

    DLI can write the job output data into the Kafka instance.

    +

    The syntax for creating a Kafka sink stream is the same as that for creating an open source Apache Kafka sink stream. For details, see MRS Kafka Sink Stream.

    diff --git a/docs/dli/sqlreference/dli_08_0273.html b/docs/dli/sqlreference/dli_08_0273.html index 54f5e88d4..99dd4631e 100644 --- a/docs/dli/sqlreference/dli_08_0273.html +++ b/docs/dli/sqlreference/dli_08_0273.html @@ -16,7 +16,6 @@ ) (TIMESTAMP BY timeindicator (',' timeindicator)?);timeindicator:PROCTIME '.' PROCTIME| ID '.' ROWTIME
    -

    Keywords

    @@ -88,7 +87,6 @@ ) TIMESTAMP BY car_timestamp.rowtime;
    -

    To customize the implementation of the source class, you need to pack the class in a JAR package and upload the UDF function on the SQL editing page.

    diff --git a/docs/dli/sqlreference/dli_08_0274.html b/docs/dli/sqlreference/dli_08_0274.html index 8dc22e301..cc856c109 100644 --- a/docs/dli/sqlreference/dli_08_0274.html +++ b/docs/dli/sqlreference/dli_08_0274.html @@ -76,7 +76,6 @@ type_class_parameter = "" ); -

    To customize the implementation of the sink class, you need to pack the class in a JAR package and upload the UDF function on the SQL editing page.

    diff --git a/docs/dli/sqlreference/dli_08_0281.html b/docs/dli/sqlreference/dli_08_0281.html index 8428e83bf..24bebcca1 100644 --- a/docs/dli/sqlreference/dli_08_0281.html +++ b/docs/dli/sqlreference/dli_08_0281.html @@ -5,7 +5,6 @@

    Syntax

    1
    DESCRIBE FUNCTION [EXTENDED] [db_name.] function_name;
    -

    Keywords

    EXTENDED: displays extended usage information.

    @@ -15,7 +14,6 @@

    Example

    Displays information about the mergeBill function.

    1
    DESCRIBE FUNCTION mergeBill;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0283.html b/docs/dli/sqlreference/dli_08_0283.html index bde9bdd7b..ccfa9373e 100644 --- a/docs/dli/sqlreference/dli_08_0283.html +++ b/docs/dli/sqlreference/dli_08_0283.html @@ -13,7 +13,6 @@ resource: : JAR file_uri -

    Or

    1
@@ -26,7 +25,6 @@
 resource: 
   : JAR file_uri
    -

    Precautions

    • If a function with the same name exists in the database, the system reports an error.
    • Only the Hive syntax can be used to create functions.
    • If you specify the same class name for two UDFs, the functions conflict though the package names are different. Avoid this problem because it causes failure of job execution.
    @@ -38,7 +36,6 @@ 2
    CREATE FUNCTION mergeBill AS 'com.xxx.hiveudf.MergeBill'
   using jar 'obs://onlyci-7/udf/MergeBill.jar';
    - diff --git a/docs/dli/sqlreference/dli_08_0284.html b/docs/dli/sqlreference/dli_08_0284.html index 07f32b281..9350d291b 100644 --- a/docs/dli/sqlreference/dli_08_0284.html +++ b/docs/dli/sqlreference/dli_08_0284.html @@ -5,7 +5,6 @@

    Syntax

    1
    DROP [TEMPORARY] FUNCTION [IF EXISTS] [db_name.] function_name;
    -

    Keywords

    • TEMPORARY: Indicates whether the function to be deleted is a temporary function.
    • IF EXISTS: Used when the function to be deleted does not exist to avoid system errors.
    @@ -15,7 +14,6 @@

    Example

    The mergeBill function is deleted.

    1
    DROP FUNCTION mergeBill;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0285.html b/docs/dli/sqlreference/dli_08_0285.html index a3c10bae1..67db9f512 100644 --- a/docs/dli/sqlreference/dli_08_0285.html +++ b/docs/dli/sqlreference/dli_08_0285.html @@ -5,7 +5,6 @@

    Syntax

    1
    SHOW [USER|SYSTEM|ALL] FUNCTIONS ([LIKE] regex | [db_name.] function_name);
    -

    In the preceding statement, regex is a regular expression. For details about its parameters, see Table 1.

    @@ -45,7 +44,6 @@

    Example

    This statement is used to view all functions.

    1
    SHOW FUNCTIONS;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0286.html b/docs/dli/sqlreference/dli_08_0286.html index 0427e2223..a7ed7e704 100644 --- a/docs/dli/sqlreference/dli_08_0286.html +++ b/docs/dli/sqlreference/dli_08_0286.html @@ -27,7 +27,6 @@ batch_insert_data_num = "" ) -

    Keywords

    @@ -136,7 +135,6 @@ batch_insert_data_num = "10" );
    - diff --git a/docs/dli/sqlreference/dli_08_0291.html b/docs/dli/sqlreference/dli_08_0291.html index 6c00f5505..63c8db4a9 100644 --- a/docs/dli/sqlreference/dli_08_0291.html +++ b/docs/dli/sqlreference/dli_08_0291.html @@ -1,7 +1,9 @@

    Supported Data Types

    -

    STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP WITH LOCAL TIME ZONE, INTERVAL, ARRAY, MULTISET, MAP, ROW

    +

    The DLI SQL syntax supports the following data types:

    +

    STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP WITH LOCAL TIME ZONE, INTERVAL, ARRAY, MULTISET, MAP, ROW

    +

    In the SQL syntax, these types are used to define the data types of columns within a table.

    diff --git a/docs/dli/sqlreference/dli_08_0301.html b/docs/dli/sqlreference/dli_08_0301.html index 337750004..f55e834d4 100644 --- a/docs/dli/sqlreference/dli_08_0301.html +++ b/docs/dli/sqlreference/dli_08_0301.html @@ -38,7 +38,6 @@ 'format.type' = '' );
    -

    Parameters

    @@ -171,7 +170,6 @@ 'format.type' = 'csv' );
    -
  • Create table kafkaSource and read data in non-nested JSON strings from Kafka.
    Assume that the non-nested JSON strings are as follows:
    {"car_id": 312, "car_owner": "wang", "car_brand": "tang"}
 {"car_id": 313, "car_owner": "li", "car_brand": "lin"}
@@ -205,7 +203,6 @@
   'format.type' = 'json'
 );
    -
  • Create table kafkaSource and read the nested JSON data from Kafka.

    Assume that the JSON data is as follows:

    diff --git a/docs/dli/sqlreference/dli_08_0302.html b/docs/dli/sqlreference/dli_08_0302.html index 7acb4a5c5..ea7a829ce 100644 --- a/docs/dli/sqlreference/dli_08_0302.html +++ b/docs/dli/sqlreference/dli_08_0302.html @@ -168,7 +168,6 @@ with ( 'format.type' = 'csv' ); - diff --git a/docs/dli/sqlreference/dli_08_0304.html b/docs/dli/sqlreference/dli_08_0304.html index 4b9c0111d..bc784f19d 100644 --- a/docs/dli/sqlreference/dli_08_0304.html +++ b/docs/dli/sqlreference/dli_08_0304.html @@ -33,7 +33,6 @@ 'connector.password' = '' ); -

    Parameters

    @@ -160,7 +159,6 @@ 'connector.password' = 'xx' );
    -

    Create table dwsSource with data fetched from GaussDB(DWS) table test that is in a schema named test_schema:

     1
@@ -187,7 +185,6 @@
   'connector.password' = 'xx'
 );
    -
    • diff --git a/docs/dli/sqlreference/dli_08_0305.html b/docs/dli/sqlreference/dli_08_0305.html index e801132e9..7b56638fd 100644 --- a/docs/dli/sqlreference/dli_08_0305.html +++ b/docs/dli/sqlreference/dli_08_0305.html @@ -26,7 +26,6 @@ 'connector.port' = '' ); -

    Parameters

    diff --git a/docs/dli/sqlreference/dli_08_0308.html b/docs/dli/sqlreference/dli_08_0308.html index e11aa5408..f3348850c 100644 --- a/docs/dli/sqlreference/dli_08_0308.html +++ b/docs/dli/sqlreference/dli_08_0308.html @@ -32,7 +32,6 @@ 'format.type' = '' );
    -

    Parameters

    @@ -144,7 +143,6 @@ 'format.type' = 'csv' );
    - diff --git a/docs/dli/sqlreference/dli_08_0309.html b/docs/dli/sqlreference/dli_08_0309.html index 3cecdd743..44bb6738f 100644 --- a/docs/dli/sqlreference/dli_08_0309.html +++ b/docs/dli/sqlreference/dli_08_0309.html @@ -32,7 +32,6 @@ 'format.type' = '' ); -

    Parameters

    diff --git a/docs/dli/sqlreference/dli_08_0310.html b/docs/dli/sqlreference/dli_08_0310.html index c580673cd..159a7ffeb 100644 --- a/docs/dli/sqlreference/dli_08_0310.html +++ b/docs/dli/sqlreference/dli_08_0310.html @@ -26,7 +26,6 @@ 'format.type' = '' );
    -

    Parameters

    @@ -127,7 +126,6 @@ 'format.type' = 'csv' );
    - diff --git a/docs/dli/sqlreference/dli_08_0311.html b/docs/dli/sqlreference/dli_08_0311.html index 28506a8bd..660984ae1 100644 --- a/docs/dli/sqlreference/dli_08_0311.html +++ b/docs/dli/sqlreference/dli_08_0311.html @@ -31,7 +31,6 @@ 'connector.password' = '' ); -

    Parameters

    @@ -148,7 +147,6 @@ 'connector.password' = 'xxxxxx' );
    - diff --git a/docs/dli/sqlreference/dli_08_0312.html b/docs/dli/sqlreference/dli_08_0312.html index 3ffccda5c..981520840 100644 --- a/docs/dli/sqlreference/dli_08_0312.html +++ b/docs/dli/sqlreference/dli_08_0312.html @@ -7,7 +7,9 @@

    Prerequisites

    • Ensure that you have created a GaussDB(DWS) cluster using your account.

      For details about how to create a GaussDB(DWS) cluster, see "Creating a Cluster" in Data Warehouse Service Management Guide.

    • A GaussDB(DWS) database table has been created.
    • An enhanced datasource connection has been created for DLI to connect to GaussDB(DWS) clusters, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required.
    -

    Syntax

     1
+
    Syntax
     
    Do not set all attributes in a GaussDB(DWS) result table to PRIMARY KEY.
    
+
    
+
     1
  2
  3
  4
@@ -33,7 +35,6 @@
   'connector.password' = ''
 );
 
    
-
 
    
 
    
 
    Parameters
    
@@ -166,7 +167,6 @@
   'connector.write.flush.interval' = '30s'
 );
    -
    Create table dwsSource with data fetched from GaussDB(DWS) table test that is in a schema named ads_game_sdk_base:
    CREATE TABLE ads_rpt_game_sdk_realtime_ada_reg_user_pay_mm (
   ddate DATE,
diff --git a/docs/dli/sqlreference/dli_08_0313.html b/docs/dli/sqlreference/dli_08_0313.html
index 96a15133a..8d45d511f 100644
--- a/docs/dli/sqlreference/dli_08_0313.html
+++ b/docs/dli/sqlreference/dli_08_0313.html
@@ -32,7 +32,6 @@
   'connector.key-column' = ''
 );
    -

    Parameters

    @@ -187,7 +186,6 @@ insert into redisSink VALUES ("car_id","A1234");
    -
  • Set type
    The table contains two columns: key and value.
    create table redisSink(
diff --git a/docs/dli/sqlreference/dli_08_0318.html b/docs/dli/sqlreference/dli_08_0318.html
index 0ee1840d5..32acb3343 100644
--- a/docs/dli/sqlreference/dli_08_0318.html
+++ b/docs/dli/sqlreference/dli_08_0318.html
@@ -26,7 +26,6 @@
   'connector.password' = ''
 );
    -

    Parameters

    @@ -129,7 +128,7 @@

    No

    -

    Maximum number of cached rows in a dimension table. If the number of cached rows exceeds the value , old data will be deleted. The value -1 indicates that data cache disabled.

    +

    Maximum number of cached rows in a dimension table. When the rows exceed this value, the data that is added first will be marked as expired. The value -1 indicates that data cache disabled.

    connector.lookup.cache.ttl

    @@ -242,7 +241,6 @@ SELECT a.car_id, b.car_owner, b.car_brand, b.car_price FROM car_infos as a join db_info FOR SYSTEM_TIME AS OF a.proctime AS b on a.car_id = b.car_id;
    - diff --git a/docs/dli/sqlreference/dli_08_0319.html b/docs/dli/sqlreference/dli_08_0319.html index 739359be0..5895f698f 100644 --- a/docs/dli/sqlreference/dli_08_0319.html +++ b/docs/dli/sqlreference/dli_08_0319.html @@ -26,7 +26,6 @@ 'connector.password' = '' ); -

    Parameters

    @@ -128,7 +127,7 @@

    No

    -

    Maximum number of cached rows in a dimension table. If the number of cached rows exceeds the value , old data will be deleted. The value -1 indicates that data cache disabled.

    +

    Maximum number of cached rows in a dimension table. When the rows exceed this value, the data that is added first will be marked as expired. The value -1 indicates that data cache disabled.

    connector.lookup.cache.ttl

    @@ -246,7 +245,6 @@ SELECT a.car_id, b.car_owner, b.car_brand, b.car_price FROM car_infos as a join db_info FOR SYSTEM_TIME AS OF a.proctime AS b on a.car_id = b.car_id;
    - diff --git a/docs/dli/sqlreference/dli_08_0322.html b/docs/dli/sqlreference/dli_08_0322.html index 795e749cd..d6a648dde 100644 --- a/docs/dli/sqlreference/dli_08_0322.html +++ b/docs/dli/sqlreference/dli_08_0322.html @@ -14,7 +14,6 @@ [ GROUP BY { groupItem [, groupItem ]* } ] [ HAVING booleanExpression ] -

    Description

    This clause is used to select data from a table.

    @@ -26,12 +25,10 @@

    Select the order which contains more than 3 pieces of data.

    1
    insert into temp SELECT  * FROM Orders WHERE units > 3;
    -

    Insert a group of constant data.

    1
    insert into temp select 'Lily', 'male', 'student', 17;
    -

    WHERE Filtering Clause

    Syntax

    @@ -41,7 +38,6 @@ FROM tableExpression [ WHERE booleanExpression ]
    -

    Description

    This clause is used to filter the query results using the WHERE clause.

    @@ -53,7 +49,6 @@ 2
    insert into temp SELECT  * FROM Orders
   WHERE units > 3 and units < 10;
    -

    HAVING Filtering Clause

    Function

    @@ -69,7 +64,6 @@ [ GROUP BY { groupItem [, groupItem ]* } ] [ HAVING booleanExpression ]
    -

    Description

    Generally, HAVING and GROUP BY are used together. GROUP BY applies first for grouping and HAVING then applies for filtering. The arithmetic operation and aggregate function are supported by the HAVING clause.

    @@ -83,7 +77,6 @@ GROUP BY name HAVING max(score) >95; -

    Column-Based GROUP BY

    Function

    @@ -97,7 +90,6 @@ [ WHERE booleanExpression ] [ GROUP BY { groupItem [, groupItem ]* } ]
    -

    Description

    Column-based GROUP BY can be categorized into single-column GROUP BY and multi-column GROUP BY.

    @@ -110,7 +102,6 @@ 2
    insert into temp SELECT name,score, max(score) FROM student 
   GROUP BY name,score;
    -

    Expression-Based GROUP BY

    Function

    @@ -124,7 +115,6 @@ [ WHERE booleanExpression ] [ GROUP BY { groupItem [, groupItem ]* } ]
    -

    Description

    groupItem can have one or more fields. The fields can be called by string functions, but cannot be called by aggregate functions.

    @@ -136,7 +126,6 @@ 2
    insert into temp SELECT substring(name,6),count(name) FROM student
   GROUP BY substring(name,6);
    -

    Grouping sets, Rollup, Cube

    Function

    @@ -169,7 +158,6 @@ GROUP BY GROUPING SETS ((user), (product)); [ GROUP BY { groupItem [, groupItem ]* } ] [ HAVING booleanExpression ]
    -

    Description

    Generally, HAVING and GROUP BY are used together. GROUP BY applies first for grouping and HAVING then applies for filtering.

    @@ -185,7 +173,6 @@ GROUP BY GROUPING SETS ((user), (product)); GROUP BY num HAVING max(price*amount)>5000; - diff --git a/docs/dli/sqlreference/dli_08_0323.html b/docs/dli/sqlreference/dli_08_0323.html index 56de63822..0d77f2768 100644 --- a/docs/dli/sqlreference/dli_08_0323.html +++ b/docs/dli/sqlreference/dli_08_0323.html @@ -4,7 +4,6 @@

    UNION/UNION ALL/INTERSECT/EXCEPT

    Syntax

    1
    query UNION [ ALL ] | Intersect | Except query
    -

    Description

    • UNION is used to return the union set of multiple query results.
    • INTERSECT is used to return the intersection of multiple query results.
    • EXCEPT is used to return the difference set of multiple query results.
    @@ -16,7 +15,6 @@ 2
    insert into temp SELECT  * FROM Orders1
   UNION SELECT  * FROM Orders2;
    -

    IN

    Syntax

    @@ -26,7 +24,6 @@ FROM tableExpression WHERE column_name IN (value (, value)* ) | query
    -

    Description

    The IN operator allows multiple values to be specified in the WHERE clause. It returns true if the expression exists in the given table subquery.

    @@ -44,7 +41,6 @@ SELECT product FROM NewProducts ); - diff --git a/docs/dli/sqlreference/dli_08_0324.html b/docs/dli/sqlreference/dli_08_0324.html index a5cc4e306..2f21a61e6 100644 --- a/docs/dli/sqlreference/dli_08_0324.html +++ b/docs/dli/sqlreference/dli_08_0324.html @@ -3,75 +3,75 @@

    Window

    GROUP WINDOW

    Description

    Group Window is defined in GROUP BY. One record is generated from each group. Group Window involves the following functions:

    -
    • Array functions -
      Table 1 Array functions

      Grouping Window Function

      +
      • Array functions
        • For SQL queries in stream processing tables, the time_attr parameter of the grouping window function must reference a valid time attribute, which needs to specify the processing time or event time of the row.
        • For batch SQL queries, the time_attr parameter of the grouping window function must be a TIMESTAMP type attribute.
        +
        + +
        - - - - - - -
        Table 1 Array functions

        Grouping Window Function

        Description

        +

        Description

        TUMBLE(time_attr, interval)

        +

        TUMBLE(time_attr, interval)

        Defines a tumbling time window. A tumbling time window assigns rows to non-overlapping, continuous windows with a fixed duration (interval). For example, a tumbling window of 5 minutes groups rows in 5 minutes intervals. Tumbling windows can be defined on event-time (stream + batch) or processing-time (stream).

        +

        Defines a tumbling time window. A tumbling time window assigns rows to non-overlapping, continuous windows with a fixed duration (interval). For example, a tumbling window of 5 minutes groups rows in 5 minutes intervals. Tumbling windows can be defined on event-time (stream + batch) or processing-time (stream).

        HOP(time_attr, interval, interval)

        +

        HOP(time_attr, interval, interval)

        Defines a hopping time window (called sliding window in the Table API). A hopping time window has a fixed duration (second interval parameter) and hops by a specified hop interval (first interval parameter). If the hop interval is smaller than the window size, hopping windows are overlapping. Thus, rows can be assigned to multiple windows. For example, a hopping window of 15 minutes size and 5 minute hop interval assigns each row to 3 different windows of 15 minute size, which are evaluated in an interval of 5 minutes. Hopping windows can be defined on event-time (stream + batch) or processing-time (stream).

        +

        Defines a hopping time window (called sliding window in the Table API). A hopping time window has a fixed duration (second interval parameter) and hops by a specified hop interval (first interval parameter). If the hop interval is smaller than the window size, hopping windows are overlapping. Thus, rows can be assigned to multiple windows. For example, a hopping window of 15 minutes size and 5 minute hop interval assigns each row to 3 different windows of 15 minute size, which are evaluated in an interval of 5 minutes. Hopping windows can be defined on event-time (stream + batch) or processing-time (stream).

        SESSION(time_attr, interval)

        +

        SESSION(time_attr, interval)

        Defines a session time window. Session time windows do not have a fixed duration but their bounds are defined by a time interval of inactivity, i.e., a session window is closed if no event appears for a defined gap period. For example a session window with a 30 minute gap starts when a row is observed after 30 minutes inactivity (otherwise the row would be added to an existing window) and is closed if no row is added within 30 minutes. Session windows can work on event-time (stream + batch) or processing-time (stream).

        +

        Defines a session time window. Session time windows do not have a fixed duration but their bounds are defined by a time interval of inactivity, i.e., a session window is closed if no event appears for a defined gap period. For example a session window with a 30 minute gap starts when a row is observed after 30 minutes inactivity (otherwise the row would be added to an existing window) and is closed if no row is added within 30 minutes. Session windows can work on event-time (stream + batch) or processing-time (stream).
        -

        Notes:

        -

        In streaming mode, the time_attr argument of the group window function must refer to a valid time attribute that specifies the processing time or event time of rows.

        -

        In batch mode, the time_attr argument of the group window function must be an attribute of type TIMESTAMP.

        -
      • Window auxiliary functions
        The start and end timestamps of group windows as well as time attributes can be selected with the following auxiliary functions. -
        Table 2 Window auxiliary functions

        Auxiliary Function

        +
      • Window auxiliary functions
        You can use the following helper functions to select the start and end timestamps, as well as the time attribute, for grouping windows.

        When calling helper functions, it is important to use the same parameters as those used in the GROUP BY clause for grouping window functions.

        +
        + +
        - - - - - - - - -
        Table 2 Window auxiliary functions

        Auxiliary Function

        Description

        +

        Description

        TUMBLE_START(time_attr, interval)

        +

        TUMBLE_START(time_attr, interval)

        HOP_START(time_attr, interval, interval)

        SESSION_START(time_attr, interval)

        Returns the timestamp of the inclusive lower bound of the corresponding tumbling, hopping, or session window.

        +

        Returns the timestamp of the inclusive lower bound of the corresponding tumbling, hopping, or session window.

        TUMBLE_END(time_attr, interval)

        +

        TUMBLE_END(time_attr, interval)

        HOP_END(time_attr, interval, interval)

        SESSION_END(time_attr, interval)

        Returns the timestamp of the exclusive upper bound of the corresponding tumbling, hopping, or session window.

        +

        Returns the timestamp of the exclusive upper bound of the corresponding tumbling, hopping, or session window.

        Note: The exclusive upper bound timestamp cannot be used as a rowtime attribute in subsequent time-based operations, such as interval joins and group window or over window aggregations.

        TUMBLE_ROWTIME(time_attr, interval)

        +

        TUMBLE_ROWTIME(time_attr, interval)

        HOP_ROWTIME(time_attr, interval, interval)

        SESSION_ROWTIME(time_attr, interval)

        Returns the timestamp of the inclusive upper bound of the corresponding tumbling, hopping, or session window. The resulting attribute is a rowtime attribute that can be used in subsequent time-based operations such as interval joins and group window or over window aggregations.

        +

        Returns the timestamp of the inclusive upper bound of the corresponding tumbling, hopping, or session window. The resulting attribute is a rowtime attribute that can be used in subsequent time-based operations such as interval joins and group window or over window aggregations.

        TUMBLE_PROCTIME(time_attr, interval)

        +

        TUMBLE_PROCTIME(time_attr, interval)

        HOP_PROCTIME(time_attr, interval, interval)

        SESSION_PROCTIME(time_attr, interval)

        Returns a proctime attribute that can be used in subsequent time-based operations such as interval joins and group window or over window aggregations.

        +

        Returns a proctime attribute that can be used in subsequent time-based operations such as interval joins and group window or over window aggregations.
        -

        Note: Auxiliary functions must be called with exactly same arguments as the group window function in the GROUP BY clause.

        • Example

         1
@@ -126,7 +126,6 @@
     FROM Orders
     GROUP BY SESSION(ts, INTERVAL '12' HOUR), name;
        -

        TUMBLE WINDOW Extension

        Function

        @@ -211,7 +210,6 @@ RANGE BETWEEN (UNBOUNDED|timeInterval) PRECEDING AND CURRENT ROW FROM TABLENAME
        -

        Description

        @@ -281,7 +279,6 @@ sum(amount) OVER (PARTITION BY name ORDER BY timeattr RANGE BETWEEN INTERVAL '60' SECOND PRECEDING AND CURRENT ROW) as cnt2 FROM Orders;
      -
    diff --git a/docs/dli/sqlreference/dli_08_0325.html b/docs/dli/sqlreference/dli_08_0325.html index dbc57033e..565d663d3 100644 --- a/docs/dli/sqlreference/dli_08_0325.html +++ b/docs/dli/sqlreference/dli_08_0325.html @@ -6,7 +6,6 @@ 2
    FROM tableExpression INNER | LEFT | RIGHT | FULL JOIN tableExpression
   ON value11 = value21 [ AND value12 = value22]
    -

    Precautions

    diff --git a/docs/dli/sqlreference/dli_08_0330.html b/docs/dli/sqlreference/dli_08_0330.html index 5ff303ecc..82bce6bff 100644 --- a/docs/dli/sqlreference/dli_08_0330.html +++ b/docs/dli/sqlreference/dli_08_0330.html @@ -4,8 +4,6 @@

    Overview

    DLI supports the following three types of user-defined functions (UDFs):

    • Regular UDF: takes in one or more input parameters and returns a single result.
    • User-defined table-generating function (UDTF): takes in one or more input parameters and returns multiple rows or columns.
    • User-defined aggregate function (UDAF): aggregates multiple records into one value.
    -

    UDFs can only be used in dedicated queues.

    -

    POM Dependency

    <dependency>
         <groupId>org.apache.flink</groupId>
         <artifactId>flink-table-common</artifactId>
@@ -56,7 +54,6 @@ public class UdfScalarFunction extends ScalarFunction {
 2
    CREATE FUNCTION udf_test AS 'com.company.udf.UdfScalarFunction';
 INSERT INTO sink_stream select udf_test(attr) FROM source_stream;
    -

    UDTF

    The UDTF must inherit the TableFunction function and implement the eval method. The open and close functions are optional. If the UDTF needs to return multiple columns, you only need to declare the returned value as Tuple or Row. If Row is used, you need to overload the getResultType method to declare the returned field type.

    @@ -115,11 +112,10 @@ public class UdfTableFunction extends TableFunction<Row> { INSERT INTO sink_stream select subValue, length FROM source_stream LEFT JOIN LATERAL TABLE(udtf_test(attr, ',')) as T(subValue, length) ON TRUE; -

    UDAF

    The UDAF must inherit the AggregateFunction function. You need to create an accumulator for storing the computing result, for example, WeightedAvgAccum in the following example code.

    -

    Example code

    +

    Example code

    public class WeightedAvgAccum {
 public long sum = 0;
 public int count = 0;
@@ -171,12 +167,11 @@ acc.count = 0;
 acc.sum = 0L;
 }
 }
    -

    Example

    +

    Example

    1
 2
    CREATE FUNCTION udaf_test AS 'com.company.udf.UdfAggFunction';
 INSERT INTO sink_stream SELECT udaf_test(attr2) FROM source_stream GROUP BY attr1;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0333.html b/docs/dli/sqlreference/dli_08_0333.html index 5a6601207..96454488a 100644 --- a/docs/dli/sqlreference/dli_08_0333.html +++ b/docs/dli/sqlreference/dli_08_0333.html @@ -384,8 +384,8 @@
    -
    • Example
      1. Test input data.
        Test the data source kafka. The message content is as follows:
        "{name:James,age:24,sex:male,grade:{math:95,science:[80,85],english:100}}"
-"{name:James,age:24,sex:male,grade:{math:95,science:[80,85],english:100}]"
        +
        • Example
          1. Test input data.
            Test the data source kafka. The message content is as follows:
            "{name:James,age:24,gender:male,grade:{math:95,science:[80,85],english:100}}"
+"{name:James,age:24,gender:male,grade:{math:95,science:[80,85],english:100}]"
          2. Use JSON_VAL in SQL statements.
            create table kafkaSource(
   message STRING
diff --git a/docs/dli/sqlreference/dli_08_0334.html b/docs/dli/sqlreference/dli_08_0334.html
index f4f0f130e..fb1ebc883 100644
--- a/docs/dli/sqlreference/dli_08_0334.html
+++ b/docs/dli/sqlreference/dli_08_0334.html
@@ -224,7 +224,7 @@
 
            TIMESTAMP
            
 
 
            Convert a datetime string1 from time zone string2 to time zone string3. 
            
-
            For example, CONVERT_TZ('1970-01-01 00:00:00', 'UTC', 'America/Los_Angeles') returns '1969-12-31 16:00:00'.
            
+
            For example, CONVERT_TZ('1970-01-01 00:00:00', 'UTC', 'Country A/City A') returns '1969-12-31 16:00:00'.
            
 
 
 
            FROM_UNIXTIME(numeric[, string])
            
@@ -1421,21 +1421,21 @@ FROM	testtable;

            STRING

            -

            Time zone before conversion. The format of time zone should be either an abbreviation such as PST, a full name such as America/Los_Angeles, or a custom ID such as GMT-08:00.

            +

            Time zone before conversion. The format of time zone should be either an abbreviation such as PST, a full name such as Country A/City A, or a custom ID such as GMT-08:00.

            string3

            STRING

            -

            Time zone after conversion. The format of time zone should be either an abbreviation such as PST, a full name such as America/Los_Angeles, or a custom ID such as GMT-08:00.

            +

            Time zone after conversion. The format of time zone should be either an abbreviation such as PST, a full name such as Country A/City A, or a custom ID such as GMT-08:00.

      2. Example
        • Test statement
          SELECT 
-	CONVERT_TZ(1970-01-01 00:00:00, UTC, America/Los_Angeles) AS `result`,
+	CONVERT_TZ(1970-01-01 00:00:00, UTC, Country A/City A) AS `result`,
         CONVERT_TZ(1997-04-25 10:00:00, UTC, GMT-08:00) AS `result2`
 FROM	testtable;
        • Test result diff --git a/docs/dli/sqlreference/dli_08_0343.html b/docs/dli/sqlreference/dli_08_0343.html index a64667fbb..706280516 100644 --- a/docs/dli/sqlreference/dli_08_0343.html +++ b/docs/dli/sqlreference/dli_08_0343.html @@ -11,7 +11,6 @@ DROP [IF EXISTS] PARTITIONS partition_filtercondition; -

          Keywords

          • DROP: deletes specified partitions.
          • IF EXISTS: Partitions to be deleted must exist. Otherwise, an error is reported.
          • PARTITIONS: specifies partitions meeting the conditions
          @@ -66,7 +65,6 @@ options (path 'path 'obs://bucketName/filePath'') partitioned by (faculytNo, classNo);
          -

        • Insert partition data into the table.

          You can insert the following data:

           1
@@ -123,7 +121,6 @@
 partition (facultyNo = 30, classNo = 102)
 values (3010213, "student13"), (3010214, "student14");
          -

        • View the partitions.

          You can view all partitions in the table.

          The example code is as follows:

          diff --git a/docs/dli/sqlreference/dli_08_0344.html b/docs/dli/sqlreference/dli_08_0344.html index 20d672791..48076d66f 100644 --- a/docs/dli/sqlreference/dli_08_0344.html +++ b/docs/dli/sqlreference/dli_08_0344.html @@ -4,8 +4,8 @@

          Function

          DLI exports Flink job data to ClickHouse result tables.

          ClickHouse is a column-based database oriented to online analysis and processing. It supports SQL query and provides good query performance. The aggregation analysis and query performance based on large and wide tables is excellent, which is one order of magnitude faster than other analytical databases.

          -

          Prerequisites

          • Ensure your jobs run on an exclusive queue (non-shared queue) of DLI.
          • You have established an enhanced datasource connection to ClickHouse and set the port in the security group rule of the ClickHouse cluster as needed.

            For details about how to set up an enhanced datasource connection. For details, see "Enhanced Datasource Connection" in the Data Lake Insight User Guide.

            -
          +

          Prerequisites

          You have established an enhanced datasource connection to ClickHouse and set the port in the security group rule of the ClickHouse cluster as needed.

          +

          For details about how to set up an enhanced datasource connection. For details, see "Enhanced Datasource Connection" in the Data Lake Insight User Guide.

          Precautions

          • When you create a ClickHouse cluster for MRS, set the cluster version to MRS 3.1.0 and do not enable Kerberos authentication.
          • Do not define a primary key in Flink SQL statements. Do not use any syntax that generates primary keys, such as insert into clickhouseSink select id, cout(*) from sourceName group by id.
          • Flink supports the following data types: string, tinyint, smallint, int, long, float, double, date, timestamp, decimal, and Array.

            The array supports only the int, bigint, string, float, and double data types.

          @@ -28,7 +28,6 @@ 'connector.table' = '' );
          -

          Parameters

          @@ -155,7 +154,6 @@ "format.type" = 'csv' );
          -
        • Create ClickHouse result table clickhouse and insert the data from the disSource table to the result table.
          create table clickhouse(
   attr0 string,
diff --git a/docs/dli/sqlreference/dli_08_0345.html b/docs/dli/sqlreference/dli_08_0345.html
index 555f18ddd..a58e157a7 100644
--- a/docs/dli/sqlreference/dli_08_0345.html
+++ b/docs/dli/sqlreference/dli_08_0345.html
@@ -17,7 +17,6 @@
   'standard-error' = ''
 );
          -

          Parameters

          diff --git a/docs/dli/sqlreference/dli_08_0346.html b/docs/dli/sqlreference/dli_08_0346.html index fb18acc16..2848f8585 100644 --- a/docs/dli/sqlreference/dli_08_0346.html +++ b/docs/dli/sqlreference/dli_08_0346.html @@ -17,7 +17,6 @@ 'format.type' = '' );
          -

          Important Notes

          • If the data output directory in the table creation syntax is OBS, the directory must be a parallel file system and cannot be an OBS bucket.
          • When using a file system table, you must enable checkpointing to ensure job consistency.
          • When format.type is parquet, the supported data type is string, boolean, tinyint, smallint, int, bigint, float, double, map<string, string>, timestamp(3), and time.
          • To avoid data loss or data coverage, you need to enable automatic restart upon job exceptions. Enable the Restore Job from Checkpoint.
          • Set the checkpoint interval after weighing between real-time output file, file size, and recovery time, such as 10 minutes.
          • When using HDFS, you need to bind the data source and enter the host information.
          • When using HDFS, you need to configure information about the node where the active NameNode locates.
          diff --git a/docs/dli/sqlreference/dli_08_0349.html b/docs/dli/sqlreference/dli_08_0349.html index 1a567ee56..2106d520c 100644 --- a/docs/dli/sqlreference/dli_08_0349.html +++ b/docs/dli/sqlreference/dli_08_0349.html @@ -10,7 +10,7 @@ 
          ALTER TABLE [db_name.]table_name 
 SET TBLPROPERTIES ("dli.multi.version.retention.days"="days");
          -

          Keywords

          • TBLPROPERTIES: This keyword is used to add a key/value property to a table.
          @@ -46,7 +45,6 @@ 2
          ALTER TABLE test_table 
 SET TBLPROPERTIES ("dli.multi.version.retention.days"="5");
          - diff --git a/docs/dli/sqlreference/dli_08_0351.html b/docs/dli/sqlreference/dli_08_0351.html index 906362bde..7660b8224 100644 --- a/docs/dli/sqlreference/dli_08_0351.html +++ b/docs/dli/sqlreference/dli_08_0351.html @@ -1,6 +1,6 @@ -

          Viewing Multiversion Backup Data

          +

          Checking Multiversion Backup Data

          Function

          After the multiversion function is enabled, you can run the SHOW HISTORY command to view the backup data of a table. For details about the syntax for enabling or disabling the multiversion function, see Enabling or Disabling Multiversion Backup.

          Currently, the multiversion function supports only OBS tables created using the Hive syntax. For details about the syntax for creating a table, see Creating an OBS Table Using the Hive Syntax.

          @@ -45,12 +45,10 @@

          Example

          • View multiversion backup data of the test_table table.
            1
            SHOW HISTORY FOR TABLE test_table;
            -
          • View multiversion backup data of the dt partition in the test_table partitioned table.
            1
            SHOW HISTORY FOR TABLE test_table PARTITION (dt='2021-07-27');
            -
          diff --git a/docs/dli/sqlreference/dli_08_0352.html b/docs/dli/sqlreference/dli_08_0352.html index f23abcd08..a572f2683 100644 --- a/docs/dli/sqlreference/dli_08_0352.html +++ b/docs/dli/sqlreference/dli_08_0352.html @@ -8,7 +8,7 @@
        • Restore the data of a single partition in a partitioned table to the backup data of a specified version.
          RESTORE TABLE [db_name.]table_name PARTITION (column = value, ...) TO VERSION 'version_id';
        -

        Keywords

        • RESTORE TABLE: Used to restore backup data
        • PARTITION: Used to specify the partition column
        • TO VERSION: Used to specify the version number You can run the SHOW HISTORY command to obtain the version number. For details, see Viewing Multiversion Backup Data.
        +

        Keywords

        • RESTORE TABLE: Used to restore backup data
        • PARTITION: Used to specify the partition column
        • TO VERSION: Used to specify the version number You can run the SHOW HISTORY command to obtain the version number. For details, see Checking Multiversion Backup Data.

        Parameters

        - @@ -50,12 +50,10 @@

        Example

        • Restore the data in non-partitioned table test_table to version 20210930.
        Table 1 Parameters

        Parameter

        @@ -39,7 +39,7 @@

        version_id

        Target version of the backup data to be restored You can run the SHOW HISTORY command to obtain the version number. For details, see Viewing Multiversion Backup Data.

        +

        Target version of the backup data to be restored You can run the SHOW HISTORY command to obtain the version number. For details, see Checking Multiversion Backup Data.

        		 
        1
        RESTORE TABLE test_table TO VERSION '20210930';
        -
    • Restore the data of partition dt in partitioned table test_table to version 20210930.
      1
      RESTORE TABLE test_table PARTITION (dt='2021-07-27') TO VERSION '20210930';
      -
    diff --git a/docs/dli/sqlreference/dli_08_0353.html b/docs/dli/sqlreference/dli_08_0353.html index f858a3be5..a48ff4b39 100644 --- a/docs/dli/sqlreference/dli_08_0353.html +++ b/docs/dli/sqlreference/dli_08_0353.html @@ -8,7 +8,6 @@ 2
    ALTER TABLE [db_name.]table_name 
 SET TBLPROPERTIES ("dli.multi.version.trash.dir"="OBS bucket for expired multiversion backup data");
    -

    Keywords

    • TBLPROPERTIES: This keyword is used to add a key/value property to a table.
    @@ -46,7 +45,6 @@ 2
    ALTER TABLE test_table 
 SET TBLPROPERTIES ("dli.multi.version.trash.dir"="/.Trash");
    - diff --git a/docs/dli/sqlreference/dli_08_0354.html b/docs/dli/sqlreference/dli_08_0354.html index 99a5ce3a4..a63a97d92 100644 --- a/docs/dli/sqlreference/dli_08_0354.html +++ b/docs/dli/sqlreference/dli_08_0354.html @@ -10,7 +10,6 @@ 2
    ALTER TABLE [db_name.]table_name 
 UNSET TBLPROPERTIES ("dli.multi.version.enable");
    -
    After multiversion is enabled, data of different versions is automatically stored in the OBS storage directory when insert overwrite or truncate is executed. After multiversion is disabled, run the following statement to restore the multiversion backup data directory:
    RESTORE TABLE [db_name.]table_name TO initial layout;
    @@ -45,13 +44,11 @@ 2
    ALTER TABLE test_table 
 SET TBLPROPERTIES ("dli.multi.version.enable"="true");
    -
  • Modify the test_table table to disable multiversion.
    1
 2
    ALTER TABLE test_table 
 UNSET TBLPROPERTIES ("dli.multi.version.enable");
    -
    Restore the multiversion backup data directory.
    RESTORE TABLE test_table TO initial layout;
    diff --git a/docs/dli/sqlreference/dli_08_0359.html b/docs/dli/sqlreference/dli_08_0359.html index 85277e0c8..72c40a920 100644 --- a/docs/dli/sqlreference/dli_08_0359.html +++ b/docs/dli/sqlreference/dli_08_0359.html @@ -7,7 +7,6 @@

    Syntax

    1
    REFRESH TABLE [db_name.]table_name;
    -

    Keywords

    None

    @@ -38,7 +37,6 @@

    Example

    Update metadata of the test table.

    1
    REFRESH TABLE test;
    -
    diff --git a/docs/dli/sqlreference/dli_08_0372.html b/docs/dli/sqlreference/dli_08_0372.html index 14418a450..2ec826b0f 100644 --- a/docs/dli/sqlreference/dli_08_0372.html +++ b/docs/dli/sqlreference/dli_08_0372.html @@ -1,7 +1,9 @@

    Supported Data Types

    -

    STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP WITH LOCAL TIME ZONE, INTERVAL, ARRAY, MULTISET, MAP, ROW

    +

    The DLI SQL syntax supports the following data types:

    +

    STRING, BOOLEAN, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP WITH LOCAL TIME ZONE, INTERVAL, ARRAY, MULTISET, MAP, ROW

    +

    In the SQL syntax, these types are used to define the data types of columns within a table.

    diff --git a/docs/dli/sqlreference/dli_08_0382.html b/docs/dli/sqlreference/dli_08_0382.html index dce74a795..eea75cd3b 100644 --- a/docs/dli/sqlreference/dli_08_0382.html +++ b/docs/dli/sqlreference/dli_08_0382.html @@ -17,7 +17,7 @@ with ( );

    Parameters

    -
    Table 1 Parameter description

    Parameter

    +
    @@ -131,7 +131,7 @@ with (

    Example

    Create a Flink OpenSource SQL job. Run the following script to generate random data through the DataGen table and output the data to the Print result table.

    When you create a job, set Flink Version to 1.12 on the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.

    -
    create table dataGenSOurce(
+
    create table dataGenSource(
   user_id string,
   amount int
 ) with (
@@ -148,7 +148,7 @@ create table printSink(
   'connector' = 'print'
 );
 
-insert into printSink select * from dataGenSOurce;
    
+insert into printSink select * from dataGenSource;

    After the job is submitted, the job status changes to Running. You can perform the following operations to view the output result:

    • Method 1:
      1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
      2. Locate the row that contains the target Flink job, and choose More > FlinkUI in the Operation column.
      3. On the Flink UI, choose Task Managers, click the task name, and select Stdout to view job logs.
    • Method 2: If you select Save Job Log on the Running Parameters tab before submitting the job, perform the following operations:
      1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
      2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
      3. Go to the folder of the date, find the folder whose name contains taskmanager, download the taskmanager.out file, and view result logs.
      diff --git a/docs/dli/sqlreference/dli_08_0383.html b/docs/dli/sqlreference/dli_08_0383.html index 81824fb6e..6a8bdc874 100644 --- a/docs/dli/sqlreference/dli_08_0383.html +++ b/docs/dli/sqlreference/dli_08_0383.html @@ -36,7 +36,6 @@ 'password' = '' );
    Table 1 Parameters

    Parameter

    Mandatory
    -

    Parameters

    diff --git a/docs/dli/sqlreference/dli_08_0385.html b/docs/dli/sqlreference/dli_08_0385.html index 43827dedc..73bf154b6 100644 --- a/docs/dli/sqlreference/dli_08_0385.html +++ b/docs/dli/sqlreference/dli_08_0385.html @@ -324,7 +324,7 @@

    Example

    This example uses JDBC as the data source and Print as the sink to read data from the RDS MySQL database and write the data to the Print result table.

    -
    1. Create an enhanced datasource connection in the VPC and subnet where RDS MySQL locates, and bind the connection to the required Flink elastic resource pool.
    2. Set RDS MySQL security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the RDS address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
    3. Log in to the RDS MySQL database, create table orders in the Flink database, and insert data.

      Create table orders in the Flink database.

      +
      1. Create an enhanced datasource connection in the VPC and subnet where RDS MySQL locates, and bind the connection to the required Flink elastic resource pool.
      2. Set RDS MySQL security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the RDS address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
      3. Log in to the RDS MySQL database, create table orders in the Flink database, and insert data.

        Create table orders in the Flink database.

        CREATE TABLE `flink`.`orders` (
 	`order_id` VARCHAR(32) NOT NULL,
 	`order_channel` VARCHAR(32) NULL,
diff --git a/docs/dli/sqlreference/dli_08_0386.html b/docs/dli/sqlreference/dli_08_0386.html
index 0f82c3a84..f43f78972 100644
--- a/docs/dli/sqlreference/dli_08_0386.html
+++ b/docs/dli/sqlreference/dli_08_0386.html
@@ -37,7 +37,6 @@
   'format' = ''
 );
    -

    Parameters

    @@ -530,7 +529,7 @@ CREATE TABLE ordersSink ( ); insert into ordersSink select * from ordersSource; -
  • Example 2: Enable Kafka SASL_SSL authentication for the MRS cluster.
    • Enable Kerberos authentication for the MRS cluster.
    • Click the Components tab and click Kafka. In the displayed page, click the Service Configuration tab, locate the security.protocol, and set it to SASL_SSL.
    • Log in to the FusionInsight Manager of the MRS cluster and download the user credential. Choose System > Permission > User. Locate the row that contains the target user, choose More > Download Authentication Credential.

      Obtain the truststore.jks file using the authentication credential and store the credential and truststore.jks file in OBS.

      +
    • Example 2: Enable Kafka SASL_SSL authentication for the MRS cluster.
      • Enable Kerberos authentication for the MRS cluster.
      • Click the Components tab and click Kafka. In the displayed page, click the Service Configuration tab, locate the security.protocol, and set it to SASL_SSL.
      • Download the user credential. Log in to the MRS Manager of the MRS cluster and choose System > Permission > User. Locate the row that contains the target user, click More, and select Download Authentication Credential.

        Obtain the truststore.jks file using the authentication credential and store the credential and truststore.jks file in OBS.

      • If "Message stream modified (41)" is displayed, the JDK version may be incorrect. Change the JDK version in the sample code to a version earlier than 8u_242 or delete the renew_lifetime = 0m configuration item from the krb5.conf configuration file.
      • Set the port to the sasl_ssl.port configured in the Kafka service configuration.
      • In the following statements, set security.protocol to SASL_SSL.
      CREATE TABLE ordersSource (
   order_id string,
@@ -587,7 +586,7 @@ CREATE TABLE ordersSink (
 );
  
 insert into ordersSink select * from ordersSource;
      -
    • Example 3: Enable Kerberos SASL_PAINTEXT authentication for the MRS cluster
      • Enable Kerberos authentication for the MRS cluster.
      • Click the Components tab and click Kafka. In the displayed page, click the Service Configuration tab, locate the security.protocol, and set it to SASL_PLAINTEXT.
      • Log in to the FusionInsight Manager of the MRS cluster and download the user credential. Choose System > Permission > User. Locate the row that contains the target user, choose More > Download Authentication Credential. Upload the credential to OBS.
      • If error message "Message stream modified (41)" is displayed, the JDK version may be incorrect. Change the JDK version in the sample code to a version earlier than 8u_242 or delete the renew_lifetime = 0m configuration item from the krb5.conf configuration file.
      • Set the port to the sasl.port configured in the Kafka service configuration.
      • In the following statements, set security.protocol to SASL_PLAINTEXT.
      +
    • Example 3: Enable Kerberos SASL_PAINTEXT authentication for the MRS cluster
      • Enable Kerberos authentication for the MRS cluster.
      • Click the Components tab and click Kafka. In the displayed page, click the Service Configuration tab, locate the security.protocol, and set it to SASL_PLAINTEXT.
      • Log in to the MRS Manager of the MRS cluster and download the user credential. Choose System > Permission > User. Locate the row that contains the target user, choose More > Download Authentication Credential. Upload the credential to OBS.
      • If error message "Message stream modified (41)" is displayed, the JDK version may be incorrect. Change the JDK version in the sample code to a version earlier than 8u_242 or delete the renew_lifetime = 0m configuration item from the krb5.conf configuration file.
      • Set the port to the sasl.port configured in the Kafka service configuration.
      • In the following statements, set security.protocol to SASL_PLAINTEXT.
      CREATE TABLE ordersSources (
   order_id string,
   order_channel string,
@@ -639,7 +638,7 @@ CREATE TABLE ordersSink (
 );
  
 insert into ordersSink select * from ordersSource;
      -
    • Example 4: Use SSL for the MRS cluster
      • Do not enable Kerberos authentication for the MRS cluster.
      • Log in to the FusionInsight Manager of the MRS cluster and download the user credential. Choose System > Permission > User. Locate the row that contains the target user, choose More > Download Authentication Credential.

        Obtain the truststore.jks file using the authentication credential and store the credential and truststore.jks file in OBS.

        +
      • Example 4: Use SSL for the MRS cluster
        • Do not enable Kerberos authentication for the MRS cluster.
        • Download the user credential. Log in to the MRS Manager of the MRS cluster and choose System > Permission > User. Locate the row that contains the target user, click More, and select Download Authentication Credential.

          Obtain the truststore.jks file using the authentication credential and store the credential and truststore.jks file in OBS.

        • Set the port to the ssl.port configured in the Kafka service configuration.
        • In the following statements, set security.protocol to SSL.
        • Set ssl.mode.enable to true.
          CREATE TABLE ordersSource (
   order_id string,
   order_channel string,
diff --git a/docs/dli/sqlreference/dli_08_0387.html b/docs/dli/sqlreference/dli_08_0387.html
index 4cbf58f6a..d4d51db61 100644
--- a/docs/dli/sqlreference/dli_08_0387.html
+++ b/docs/dli/sqlreference/dli_08_0387.html
@@ -9,7 +9,8 @@
 
          Precautions
          • When creating a Flink OpenSource SQL job, you need to set Flink Version to 1.12 on the Running Parameters tab of the job editing page, select Save Job Log, and set the OBS bucket for saving job logs.
          • Each client that synchronizes database data has a unique ID, that is, the server ID. You are advised to configure a unique server ID for each MySQL CDC job in the same database.
            Main reasons are as follows:
            • The MySQL server maintains the network connection and Binlog location based on the ID. Therefore, if a large number of clients with the same server ID connect to the MySQL server, the CPU usage of the MySQL server may increase sharply, affecting the stability of online services.
            • If multiple jobs share the same server ID, Binlog locations will be disordered, making data read inaccurate. Therefore, you are advised to configure different server IDs for each MySQL CDC job.
            
 
            
 
          
-
          • Watermarks cannot be defined for MySQL CDC source tables. For details about window aggregation, see FAQ.
          • If you connect to a sink source that supports upsert, such as GaussDB(DWS) and MySQL, you need to define the primary key in the statement for creating the sink table. For details, see the printSink table creation statement in Example.
          
+
          • Watermarks cannot be defined for MySQL CDC source tables. For details about window aggregation, see FAQ.
          • If you connect to a sink source that supports upsert, such as GaussDB(DWS) and MySQL, you need to define the primary key in the statement for creating the sink table. For details, see the printSink table creation statement in Example.
          • When using the MySQL CDM source table, do not manually disable debezium.connect.keep.alive in the source table parameters. Make sure that debezium.connect.keep.alive is set to true (default value).
            If debezium.connect.keep.alive is manually disabled and there is a connection exception between the Binlog thread and the MySQL server, the Binlog thread will not attempt to reconnect automatically. This may result in the inability to properly pull binlog logs from the source.
            
+
          
 
          
 
          Syntax
          create table mySqlCdcSource (
   attr_name attr_type 
@@ -26,7 +27,7 @@ with (
 );
          
 
          
 
          Parameters
          
-
          Table 1 Parameter description
          Parameter
          
+
          
diff --git a/docs/dli/sqlreference/dli_08_0389.html b/docs/dli/sqlreference/dli_08_0389.html
index 2749cf801..0af92685c 100644
--- a/docs/dli/sqlreference/dli_08_0389.html
+++ b/docs/dli/sqlreference/dli_08_0389.html
@@ -75,7 +75,6 @@
   'host' = '');
          Table 1 Parameters
          Parameter
          
           		
 
          Mandatory
          
           		
 
 
          
-
 
 
 
          Parameters
          
diff --git a/docs/dli/sqlreference/dli_08_0390.html b/docs/dli/sqlreference/dli_08_0390.html
index 7df252f14..a7fa5a039 100644
--- a/docs/dli/sqlreference/dli_08_0390.html
+++ b/docs/dli/sqlreference/dli_08_0390.html
@@ -33,7 +33,6 @@
   'value.format' = ''
 );
 
          
-
 
          
 
 
          Parameters
          
@@ -142,7 +141,7 @@
 
 
          This option can set and pass arbitrary Kafka configurations.
          
 
          The suffix to properties. must match the parameter defined in Kafka Configuration documentation. Flink will remove the properties. key prefix and pass the transformed key and value to the underlying KafkaClient.
          
-
          For example, you can disable automatic topic creation via 'properties.allow.auto.create.topics' = 'false'.
          
+
          For example, you can disable automatic topic creation via 'properties.allow.auto.create.topics' = 'false'.
          
 
          But there are some configurations that do not support to set, because Flink will override them, for example, 'key.deserializer' and 'value.deserializer'.
          
 
 
diff --git a/docs/dli/sqlreference/dli_08_0393.html b/docs/dli/sqlreference/dli_08_0393.html
index 3abdf7365..0225bc359 100644
--- a/docs/dli/sqlreference/dli_08_0393.html
+++ b/docs/dli/sqlreference/dli_08_0393.html
@@ -3,7 +3,7 @@
 
          ClickHouse Result Table
          
 
          Function
          DLI can output Flink job data to the ClickHouse database. ClickHouse is a column-based database oriented to online analysis and processing. It supports SQL query and provides good query performance. The aggregation analysis and query performance based on large and wide tables is excellent, which is one order of magnitude faster than other analytical databases. 
          
 
          
-
          Prerequisites
          • Your jobs are running on a dedicated queue (non-shared queue) of DLI.
          • You have established an enhanced datasource connection to ClickHouse and set the port in the security group rule of the ClickHouse cluster as needed.
          
+
          Prerequisites
          • You have established an enhanced datasource connection to ClickHouse and set the port in the security group rule of the ClickHouse cluster as needed.
          
 
          
 
          Precautions
          • When creating a Flink OpenSource SQL job, you need to set Flink Version to 1.12 on the Running Parameters tab of the job editing page, select Save Job Log, and set the OBS bucket for saving job logs.
          • When you create a ClickHouse cluster for MRS, set the cluster version to MRS 3.1.0 or later and do not enable Kerberos authentication.
          • The ClickHouse result table does not support table data deletion.
          • Flink supports the following data types: string, tinyint, smallint, int, long, float, double, date, timestamp, decimal, and array.
            The array supports only the int, bigint, string, float, and double data types.
            
 
          
@@ -26,7 +26,6 @@
   'connector.table' = ''
 );
          -

          Parameters

          diff --git a/docs/dli/sqlreference/dli_08_0394.html b/docs/dli/sqlreference/dli_08_0394.html index 8bd71d511..3f77c7586 100644 --- a/docs/dli/sqlreference/dli_08_0394.html +++ b/docs/dli/sqlreference/dli_08_0394.html @@ -33,7 +33,6 @@ 'write.mode' = 'upsert' );
          -
        @@ -66,7 +65,6 @@ 'password' = '' ); -

        Parameters

        diff --git a/docs/dli/sqlreference/dli_08_0395.html b/docs/dli/sqlreference/dli_08_0395.html index 11f83ab1d..4539239f5 100644 --- a/docs/dli/sqlreference/dli_08_0395.html +++ b/docs/dli/sqlreference/dli_08_0395.html @@ -236,7 +236,7 @@ with (

        Example

        In this example, data is read from the Kafka data source and written to the Elasticsearch result table. The procedure is as follows:

        -
        1. Create an enhanced datasource connection in the VPC and subnet where Elasticsearch and Kafka locate, and bind the connection to the required Flink elastic resource pool.
        2. Set Elasticsearch and Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Elasticsearch and Kafka address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
        3. Log in to Kibana of the Elasticsearch cluster, select Dev Tools, enter and execute the following statement to create an index whose value is orders:
          PUT /orders
+
          1. Create an enhanced datasource connection in the VPC and subnet where Elasticsearch and Kafka locate, and bind the connection to the required Flink elastic resource pool.
          2. Set Elasticsearch and Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Elasticsearch and Kafka address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
          3. Log in to Kibana of the Elasticsearch cluster, select Dev Tools, enter and execute the following statement to create an index whose value is orders:
            PUT /orders
 {
   "settings": {
     "number_of_shards": 1
diff --git a/docs/dli/sqlreference/dli_08_0396.html b/docs/dli/sqlreference/dli_08_0396.html
index 982e18ac7..8f24c4863 100644
--- a/docs/dli/sqlreference/dli_08_0396.html
+++ b/docs/dli/sqlreference/dli_08_0396.html
@@ -139,8 +139,8 @@
 
 
            Integer
            
 
-
            Defines the parallelism of the HBase sink operator.
            
-
            By default, the parallelism is determined by the framework using the same parallelism of the upstream chained operator.
            
+
            Defines the parallelism for the HBase sink operator.
            
+
            By default, the parallelism is determined by the framework: using the same parallelism as the upstream join operator.
            
 
 
 
diff --git a/docs/dli/sqlreference/dli_08_0397.html b/docs/dli/sqlreference/dli_08_0397.html
index 7ca3bec77..41bb9ac1d 100644
--- a/docs/dli/sqlreference/dli_08_0397.html
+++ b/docs/dli/sqlreference/dli_08_0397.html
@@ -35,7 +35,6 @@
   'password' = ''
 );
        -

        Parameters

        @@ -310,7 +309,7 @@

        Example

        In this example, Kafka is used to send data, and Kafka data is written to the MySQL database through the JDBC result table.

        -
        1. Create an enhanced datasource connection in the VPC and subnet where MySQL and Kafka locate, and bind the connection to the required Flink elastic resource pool.
        2. Set MySQL and Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the MySQL and Kafka address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
        3. Log in to the MySQL database and create table orders in database flink.
          CREATE TABLE `flink`.`orders` (
+
          1. Create an enhanced datasource connection in the VPC and subnet where MySQL and Kafka locate, and bind the connection to the required Flink elastic resource pool.
          2. Set MySQL and Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the MySQL and Kafka address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
          3. Log in to the MySQL database and create table orders in database flink.
            CREATE TABLE `flink`.`orders` (
 	`order_id` VARCHAR(32) NOT NULL,
 	`order_channel` VARCHAR(32) NULL,
 	`order_time` VARCHAR(32) NULL,
diff --git a/docs/dli/sqlreference/dli_08_0398.html b/docs/dli/sqlreference/dli_08_0398.html
index f774e418a..8a4f0e241 100644
--- a/docs/dli/sqlreference/dli_08_0398.html
+++ b/docs/dli/sqlreference/dli_08_0398.html
@@ -31,7 +31,6 @@
   'format' = ''
 );
        -

        Parameters

        @@ -77,7 +76,7 @@

        string

        -

        Kafka broker address. The value is in the format of host:port,host:port,host:port. Multiple host:port pairs are separated with commas (,).

        +

        Kafka broker address. The value is in the format of host:port,host:port,host:port. Multiple host:port pairs are separated with commas (,).

        format

        @@ -335,7 +334,7 @@ CREATE TABLE ordersSink ( ); insert into ordersSink select * from ordersSource; -
      • Example 2: Enable Kafka SASL_SSL authentication for the MRS cluster.
        • Enable Kerberos authentication for the MRS cluster.
        • Click the Components tab and click Kafka. In the displayed page, click the Service Configuration tab, locate the security.protocol, and set it to SASL_SSL.
        • Log in to the FusionInsight Manager of the MRS cluster and download the user credential. Choose System > Permission > User. Locate the row that contains the target user, choose More > Download Authentication Credential.

          Obtain the truststore.jks file using the authentication credential and store the credential and truststore.jks file in OBS.

          +
        • Example 2: Enable Kafka SASL_SSL authentication for the MRS cluster.
          • Enable Kerberos authentication for the MRS cluster.
          • Click the Components tab and click Kafka. In the displayed page, click the Service Configuration tab, locate the security.protocol, and set it to SASL_SSL.
          • Download the user credential. Log in to the MRS Manager of the MRS cluster and choose System > Permission > User. Locate the row that contains the target user, click More, and select Download Authentication Credential.

            Obtain the truststore.jks file using the authentication credential and store the credential and truststore.jks file in OBS.

          • If "Message stream modified (41)" is displayed, the JDK version may be incorrect. Change the JDK version in the sample code to a version earlier than 8u_242 or delete the renew_lifetime = 0m configuration item from the krb5.conf configuration file.
          • Set the port to the sasl_ssl.port configured in the Kafka service configuration.
          • In the following statements, set security.protocol to SASL_SSL.
          CREATE TABLE ordersSource (
   order_id string,
@@ -392,7 +391,7 @@ CREATE TABLE ordersSink (
 );
  
 insert into ordersSink select * from ordersSource;
          -
        • Example 3: Enable Kerberos SASL_PAINTEXT authentication for the MRS cluster
          • Enable Kerberos authentication for the MRS cluster.
          • Click the Components tab and click Kafka. In the displayed page, click the Service Configuration tab, locate the security.protocol, and set it to SASL_PLAINTEXT.
          • Log in to the FusionInsight Manager of the MRS cluster and download the user credential. Choose System > Permission > User. Locate the row that contains the target user, choose More > Download Authentication Credential. Upload the credential to OBS.
          • If error message "Message stream modified (41)" is displayed, the JDK version may be incorrect. Change the JDK version in the sample code to a version earlier than 8u_242 or delete the renew_lifetime = 0m configuration item from the krb5.conf configuration file.
          • Set the port to the sasl.port configured in the Kafka service configuration.
          • In the following statements, set security.protocol to SASL_PLAINTEXT.
          +
        • Example 3: Enable Kerberos SASL_PAINTEXT authentication for the MRS cluster
          • Enable Kerberos authentication for the MRS cluster.
          • Click the Components tab and click Kafka. In the displayed page, click the Service Configuration tab, locate the security.protocol, and set it to SASL_PLAINTEXT.
          • Log in to the MRS Manager of the MRS cluster and download the user credential. Choose System > Permission > User. Locate the row that contains the target user, choose More > Download Authentication Credential. Upload the credential to OBS.
          • If error message "Message stream modified (41)" is displayed, the JDK version may be incorrect. Change the JDK version in the sample code to a version earlier than 8u_242 or delete the renew_lifetime = 0m configuration item from the krb5.conf configuration file.
          • Set the port to the sasl.port configured in the Kafka service configuration.
          • In the following statements, set security.protocol to SASL_PLAINTEXT.
          CREATE TABLE ordersSources (
   order_id string,
   order_channel string,
@@ -444,7 +443,7 @@ CREATE TABLE ordersSink (
 );
  
 insert into ordersSink select * from ordersSource;
          -
        • Example 4: Use SSL for the MRS cluster
          • Do not enable Kerberos authentication for the MRS cluster.
          • Log in to the FusionInsight Manager of the MRS cluster and download the user credential. Choose System > Permission > User. Locate the row that contains the target user, choose More > Download Authentication Credential.

            Obtain the truststore.jks file using the authentication credential and store the credential and truststore.jks file in OBS.

            +
          • Example 4: Use SSL for the MRS cluster
            • Do not enable Kerberos authentication for the MRS cluster.
            • Download the user credential. Log in to the MRS Manager of the MRS cluster and choose System > Permission > User. Locate the row that contains the target user, click More, and select Download Authentication Credential.

              Obtain the truststore.jks file using the authentication credential and store the credential and truststore.jks file in OBS.

            • Set the port to the ssl.port configured in the Kafka service configuration.
            • In the following statements, set security.protocol to SSL.
            • Set ssl.mode.enable to true.
              CREATE TABLE ordersSource (
   order_id string,
   order_channel string,
diff --git a/docs/dli/sqlreference/dli_08_0399.html b/docs/dli/sqlreference/dli_08_0399.html
index 37aa7d837..77e6e35d7 100644
--- a/docs/dli/sqlreference/dli_08_0399.html
+++ b/docs/dli/sqlreference/dli_08_0399.html
@@ -66,7 +66,6 @@
   'standard-error' = ''
 );
              -

              Parameters

              @@ -121,15 +120,15 @@

              Example

              Create a Flink OpenSource SQL job. Run the following script to generate random data through the DataGen table and output the data to the Print result table.

              -

              When you create a job, set Flink Version to 1.12 on the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.

              -
              create table dataGenSOurce(
+
              When you create a job, set Flink Version to 1.12 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              
+
              create table dataGenSource(
   user_id string,
   amount int
 ) with (
   'connector' = 'datagen',
-  'rows-per-second' = '1', --Generate a piece of data per second.
-  'fields.user_id.kind' = 'random', --Specify a random generator for the user_id field.
-  'fields.user_id.length' = '3' --Limit the length of user_id to 3.
+  'rows-per-second' = '1', --Generates a piece of data per second.
+  'fields.user_id.kind' = 'random', --Specifies a random generator for the user_id field.
+  'fields.user_id.length' = '3' --Limits the length of user_id to 3.
 );
 
 create table printSink(
@@ -139,10 +138,10 @@ create table printSink(
   'connector' = 'print'
 );
 
-insert into printSink select * from dataGenSOurce;
              
-
              After the job is submitted, the job status changes to Running. You can perform the following operations of either method to view the output result:
              
-
              • Method 1:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Locate the row that contains the target Flink job, and choose More > FlinkUI in the Operation column.
                3. On the Flink UI, choose Task Managers, click the task name, and select Stdout to view job logs.
                
-
              • Method 2: If you select Save Job Log on the Running Parameters tab before submitting the job, perform the following operations:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the taskmanager.out file, and view result logs.
                
+insert into printSink select * from dataGenSource;
              +

              After the job is submitted, the job status changes to Running. You can perform the following operations to view the output result:

              +
              • Method 1:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Locate the row that contains the target Flink job, click More in the Operation column, and select FlinkUI.
                3. On the Flink UI, choose Task Managers, click the task name, and select Stdout to view job logs.
                +
              • Method 2: If you select Save Job Log on the Running Parameters tab before submitting the job, perform the following operations:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the taskmanager.out file, and view result logs.
              diff --git a/docs/dli/sqlreference/dli_08_0400.html b/docs/dli/sqlreference/dli_08_0400.html index d56c9aa27..d62c25ac5 100644 --- a/docs/dli/sqlreference/dli_08_0400.html +++ b/docs/dli/sqlreference/dli_08_0400.html @@ -62,7 +62,6 @@ 'host' = '' ); -

              Parameters

              diff --git a/docs/dli/sqlreference/dli_08_0401.html b/docs/dli/sqlreference/dli_08_0401.html index fc57477b7..42eb3895a 100644 --- a/docs/dli/sqlreference/dli_08_0401.html +++ b/docs/dli/sqlreference/dli_08_0401.html @@ -34,7 +34,6 @@ 'value.format' = '' );
              -

              Parameters

              @@ -154,7 +153,7 @@

              This option can set and pass arbitrary Kafka configurations.

              The suffix of this parameter must match the parameter defined in Kafka Configuration documentation. Flink will remove the properties. key prefix and pass the transformed key and value to the underlying KafkaClient.

              -

              For example, you can disable automatic topic creation via 'properties.allow.auto.create.topics' = 'false'. But there are some configurations that do not support to set, because Flink will override them, for example, 'key.deserializer' and 'value.deserializer'.

              +

              For example, you can disable automatic topic creation via 'properties.allow.auto.create.topics' = 'false'. But there are some configurations that do not support to set, because Flink will override them, for example, 'key.deserializer' and 'value.deserializer'.

              diff --git a/docs/dli/sqlreference/dli_08_0403.html b/docs/dli/sqlreference/dli_08_0403.html index d85cd575c..234f76665 100644 --- a/docs/dli/sqlreference/dli_08_0403.html +++ b/docs/dli/sqlreference/dli_08_0403.html @@ -30,7 +30,6 @@ 'password' = '' );
              -

              Parameters

              @@ -193,7 +192,7 @@

              Integer

              -

              The max number of rows of lookup cache. Caches exceeding the TTL will be expired.

              +

              Maximum number of cached rows in a dimension table. When the rows exceed this value, the data that is added first will be marked as expired.

              Lookup cache is disabled by default.

              diff --git a/docs/dli/sqlreference/dli_08_0404.html b/docs/dli/sqlreference/dli_08_0404.html index 8609665ca..321c01725 100644 --- a/docs/dli/sqlreference/dli_08_0404.html +++ b/docs/dli/sqlreference/dli_08_0404.html @@ -98,7 +98,7 @@ with (

              Long

              -

              The max number of rows of lookup cache. Caches exceeding the TTL will be expired.

              +

              Maximum number of cached rows in a dimension table. When the rows exceed this value, the data that is added first will be marked as expired.

              Lookup cache is disabled by default.

              diff --git a/docs/dli/sqlreference/dli_08_0405.html b/docs/dli/sqlreference/dli_08_0405.html index a90e818ca..0637f73f2 100644 --- a/docs/dli/sqlreference/dli_08_0405.html +++ b/docs/dli/sqlreference/dli_08_0405.html @@ -30,7 +30,6 @@ 'password' = '' );
              -

              Parameters

              @@ -127,7 +126,7 @@

              No

              -

              Maximum number of cached rows in a dimension table. If the number of cached rows exceeds the value , old data will be deleted. The value -1 indicates that data cache disabled.

              +

              Maximum number of cached rows in a dimension table. When the rows exceed this value, the data that is added first will be marked as expired. The value -1 indicates that data cache disabled.

              lookup.cache.ttl

              @@ -443,7 +442,6 @@ area.area_street_name, area.region_name from orders left join area_info for system_time as of orders.proctime as area on orders.area_id = area.area_id;
              -
            • Connect to the Kafka cluster and insert the following test data into the source topic in Kafka:
              {"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
diff --git a/docs/dli/sqlreference/dli_08_0408.html b/docs/dli/sqlreference/dli_08_0408.html
index ef162c9bf..aa34a7024 100644
--- a/docs/dli/sqlreference/dli_08_0408.html
+++ b/docs/dli/sqlreference/dli_08_0408.html
@@ -6,7 +6,7 @@
 
              Supported Connectors
              • Kafka
              • Upsert Kafka
              
 
              
 
              Parameters
              
-
              Table 1 Parameter
              Parameter
              
+
              
@@ -46,133 +46,132 @@
 
              Data Type Mapping
              Currently, the Avro schema is derived from the table schema and cannot be explicitly defined. The following table lists mappings between Flink to Avro types.
              
 
              In addition to the following types, Flink supports reading/writing nullable types. Flink maps nullable types to Avro union(something, null), where something is an Avro type converted from Flink type.
              
-
              You can refer to Apache Avro 1.11.0 Specification for more information about Avro types.
              
 
-
              Table 1 Parameters
              Parameter
              
               		
 
              Mandatory
              
               		
 
              Table 2 Data Type Mapping
              Flink SQL Type
              
+
              
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/docs/dli/sqlreference/dli_08_0410.html b/docs/dli/sqlreference/dli_08_0410.html
index e474a4140..bf070f2e1 100644
--- a/docs/dli/sqlreference/dli_08_0410.html
+++ b/docs/dli/sqlreference/dli_08_0410.html
@@ -1,9 +1,9 @@
 
              Confluent Avro
              
-
              Function
              The Avro Schema Registry (avro-confluent) format allows you to read records that were serialized by the io.confluent.kafka.serializers.KafkaAvroSerializer and to write records that can in turn be read by the io.confluent.kafka.serializers.KafkaAvroDeserializer.
              
+
              Function
              The Avro Schema Registry (avro-confluent) format allows you to read records that were serialized by the io.confluent.kafka.serializers.KafkaAvroSerializer and to write records that can in turn be read by the io.confluent.kafka.serializers.KafkaAvroDeserializer.
              
 
              When reading (deserializing) a record with this format the Avro writer schema is fetched from the configured Confluent Schema Registry based on the schema version ID encoded in the record while the reader schema is inferred from table schema.
              
-
              When writing (serializing) a record with this format the Avro schema is inferred from the table schema and used to retrieve a schema ID to be encoded with the data The lookup is performed with in the configured Confluent Schema Registry under the subject. The subject is specified by avro-confluent.schema-registry.subject.
              
+
              When writing (serializing) a record with this format the Avro schema is inferred from the table schema and used to retrieve a schema ID to be encoded with the data The lookup is performed with in the configured Confluent Schema Registry under the subject. The subject is specified by avro-confluent.schema-registry.subject.
              
 
              
 
              Supported Connectors
              • kafka
              • upsert kafka
              
 
              
diff --git a/docs/dli/sqlreference/dli_08_0411.html b/docs/dli/sqlreference/dli_08_0411.html
index cd93d189d..abbecef97 100644
--- a/docs/dli/sqlreference/dli_08_0411.html
+++ b/docs/dli/sqlreference/dli_08_0411.html
@@ -37,7 +37,7 @@
 
 
              
-
              Table 2 Data type mapping
              Flink SQL Type
              
 
              Avro Type
              
+
              Avro Type
              
 
              Avro Logical Type
              
+
              Avro Logical Type
              
               		
 
              
 
              CHAR / VARCHAR / STRING
              
+
              CHAR / VARCHAR / STRING
              
 
              string
              
+
              string
              
 
              -
              
+
              -
              
               		
 
              BOOLEAN
              
+
              BOOLEAN
              
 
              boolean
              
+
              boolean
              
 
              -
              
+
              -
              
               		
 
              BINARY / VARBINARY
              
+
              BINARY / VARBINARY
              
 
              bytes
              
+
              bytes
              
 
              -
              
+
              -
              
               		
 
              DECIMAL
              
+
              DECIMAL
              
 
              fixed
              
+
              fixed
              
 
              decimal
              
+
              decimal
              
               		
 
              TINYINT
              
+
              TINYINT
              
 
              int
              
+
              int
              
 
              -
              
+
              -
              
               		
 
              SMALLINT
              
+
              SMALLINT
              
 
              int
              
+
              int
              
 
              -
              
+
              -
              
               		
 
              INT
              
+
              INT
              
 
              int
              
+
              int
              
 
              -
              
+
              -
              
               		
 
              BIGINT
              
+
              BIGINT
              
 
              long
              
+
              long
              
 
              -
              
+
              -
              
               		
 
              FLOAT
              
+
              FLOAT
              
 
              float
              
+
              float
              
 
              -
              
+
              -
              
               		
 
              DOUBLE
              
+
              DOUBLE
              
 
              double
              
+
              double
              
 
              -
              
+
              -
              
               		
 
              DATE
              
+
              DATE
              
 
              int
              
+
              int
              
 
              date
              
+
              date
              
               		
 
              TIME
              
+
              TIME
              
 
              int
              
+
              int
              
 
              time-millis
              
+
              time-millis
              
               		
 
              TIMESTAMP
              
+
              TIMESTAMP
              
 
              long
              
+
              long
              
 
              timestamp-millis
              
+
              timestamp-millis
              
               		
 
              ARRAY
              
+
              ARRAY
              
 
              array
              
+
              array
              
 
              -
              
+
              -
              
               		
 
              MAP (keys must be of the string, char, or varchar type.)
              
+
              MAP (keys must be of the string, char, or varchar type.)
              
 
              map
              
+
              map
              
 
              -
              
+
              -
              
               		
 
              MULTISET (elements must be of the string, char, or varchar type.)
              
+
              MULTISET (elements must be of the string, char, or varchar type.)
              
 
              map
              
+
              map
              
 
              -
              
+
              -
              
               		
 
              ROW
              
+
              ROW
              
 
              record
              
+
              record
              
 
              -
              
+
              -
              
               		
 
              
               
 
 
              String
              
 
              Field delimiter character, which must be a single character. You can use backslash to specify special characters, for example, \t represents the tab character. You can also use unicode to specify them in plain SQL, for example, 'csv.field-delimiter' = '\u0001' represents the 0x01 character.
              
+
              Field delimiter character, which must be a single character. You can use backslash to specify special characters, for example, \t represents the tab character. You can also use unicode to specify them in plain SQL, for example, 'csv.field-delimiter' = '\u0001' represents the 0x01 character.
              
               		
 
              
 
              csv.disable-quote-character
              
diff --git a/docs/dli/sqlreference/dli_08_0414.html b/docs/dli/sqlreference/dli_08_0414.html
index 2ba09be44..196a98824 100644
--- a/docs/dli/sqlreference/dli_08_0414.html
+++ b/docs/dli/sqlreference/dli_08_0414.html
@@ -7,76 +7,76 @@
 
              • Synchronizing incremental data from databases to other systems
              • Auditing logs
              • Real-time materialized views on databases
              • Temporal join changing history of a database table and so on
              
 
              Flink also supports to encode the INSERT/UPDATE/DELETE messages in Flink SQL as Maxwell JSON messages, and emit to external systems like Kafka. However, currently Flink cannot combine UPDATE_BEFORE and UPDATE_AFTER into a single UPDATE message. Therefore, Flink encodes UPDATE_BEFORE and UDPATE_AFTER as DELETE and INSERT Maxwell messages.
              
 
              Parameters
              
-
              Table 1 
              Parameter
              
+
              
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/docs/dli/sqlreference/dli_08_0417.html b/docs/dli/sqlreference/dli_08_0417.html
index c635a3be6..7c6c485c2 100644
--- a/docs/dli/sqlreference/dli_08_0417.html
+++ b/docs/dli/sqlreference/dli_08_0417.html
@@ -14,7 +14,6 @@
   [ GROUP BY { groupItem [, groupItem ]* } ]  [ HAVING booleanExpression ]
              Parameter
              
 
              Mandatory
              
+
              Mandatory
              
 
              Default Value
              
+
              Default Value
              
 
              Type
              
+
              Type
              
 
              Description
              
+
              Description
              
               		
 
              
 
              format
              
+
              format
              
 
              Yes
              
+
              Yes
              
 
              None
              
+
              None
              
 
              String
              
+
              String
              
 
              Format to be used. Set this parameter to maxwell-json.
              
+
              Format to be used. Set this parameter to maxwell-json.
              
               		
 
              maxwell-json.ignore-parse-errors
              
+
              maxwell-json.ignore-parse-errors
              
 
              No
              
+
              No
              
 
              false
              
+
              false
              
 
              Boolean
              
+
              Boolean
              
 
              Whether fields and rows with parse errors will be skipped or failed. Fields are set to null in case of errors.
              
+
              Whether fields and rows with parse errors will be skipped or failed. Fields are set to null in case of errors.
              
               		
 
              maxwell-json.timestamp-format.standard
              
+
              maxwell-json.timestamp-format.standard
              
 
              No
              
+
              No
              
 
              'SQL'
              
+
              'SQL'
              
 
              String
              
+
              String
              
 
              Input and output timestamp formats. Currently supported values are SQL and ISO-8601: 
              
+
              Input and output timestamp formats. Currently supported values are SQL and ISO-8601: 
              
 
              SQL will parse input timestamp in "yyyy-MM-dd HH:mm:ss.s{precision}" format, for example, 2020-12-30 12:13:14.123 and output timestamp in the same format.
              
 
              ISO-8601 will parse input timestamp in "yyyy-MM-ddTHH:mm:ss.s{precision}" format, for example 2020-12-30T12:13:14.123 and output timestamp in the same format.
              
               		
 
              maxwell-json.map-null-key.mode
              
+
              maxwell-json.map-null-key.mode
              
 
              No
              
+
              No
              
 
              'FAIL'
              
+
              'FAIL'
              
 
              String
              
+
              String
              
 
              Handling mode when serializing null keys for map data. Currently supported values are 'FAIL', 'DROP' and 'LITERAL': 
              
+
              Handling mode when serializing null keys for map data. Currently supported values are 'FAIL', 'DROP' and 'LITERAL': 
              
 
              FAIL will throw exception when encountering map with null key.
              
 
              DROP will drop null key entries for map data.
              
 
              LITERAL will replace null key with string literal. The string literal is defined by maxwell-json.map-null-key.literal option.
              
               		
 
              maxwell-json.map-null-key.literal
              
+
              maxwell-json.map-null-key.literal
              
 
              No
              
+
              No
              
 
              'null'
              
+
              'null'
              
 
              String
              
+
              String
              
 
              String literal to replace null key when maxwell-json.map-null-key.mode is LITERAL.
              
+
              String literal to replace null key when maxwell-json.map-null-key.mode is LITERAL.
              
               		
 
              
               
 
 
              
-
 
 
              Description
              
 
              SELECT is used to select data from a table.
              
@@ -26,12 +25,10 @@
 
              Select the order which contains more than 3 pieces of data.
              
 
              1
              insert into temp SELECT  * FROM Orders WHERE units > 3; 
 
              
-
 
              
 
              Insert a group of constant data.
              
 
              1
              insert into temp select 'Lily', 'male', 'student', 17;
 
              
-
 
              
 
 
              WHERE
              Syntax
              
@@ -41,7 +38,6 @@
   FROM tableExpression
   [ WHERE booleanExpression ]
 
              
-
 
              
 
              Description
              
 
              This clause is used to filter the query results using the WHERE clause.
              
@@ -53,7 +49,6 @@
 2
              insert into temp SELECT  * FROM Orders
   WHERE units > 3 and units < 10; 
 
              
-
 
 
 
              HAVING
              Function
              
@@ -69,7 +64,6 @@
   [ GROUP BY { groupItem [, groupItem ]* } ]
   [ HAVING booleanExpression ]
 
              
-
 
 
              Description
              
 
              Generally, HAVING and GROUP BY are used together. You can use GROUP BY for grouping and then use HAVING for filtering. Arithmetic operations and aggregate functions are supported in the HAVING clause.
              
@@ -83,7 +77,6 @@
   GROUP BY name
   HAVING max(score) >95;
 
              
-
 
              
 
 
              Column-Based GROUP BY
              Function
              
@@ -97,7 +90,6 @@
   [ WHERE booleanExpression ]
   [ GROUP BY { groupItem [, groupItem ]* } ]
              -

              Description

              Column-based GROUP BY can be categorized into single-column GROUP BY and multi-column GROUP BY.

              @@ -110,7 +102,6 @@ 2
              insert into temp SELECT name,score, max(score) FROM student 
   GROUP BY name,score;
              -

              Expression-Based GROUP BY

              Function

              @@ -124,7 +115,6 @@ [ WHERE booleanExpression ] [ GROUP BY { groupItem [, groupItem ]* } ]
              -

              Description

              groupItem can have one or more fields. The fields can be called by string functions, but cannot be called by aggregate functions.

              @@ -136,7 +126,6 @@ 2
              insert into temp SELECT substring(name,6),count(name) FROM student
   GROUP BY substring(name,6);
              -

              Grouping sets, Rollup, Cube

              Function

              @@ -169,7 +158,6 @@ GROUP BY GROUPING SETS ((user), (product)); [ GROUP BY { groupItem [, groupItem ]* } ] [ HAVING booleanExpression ]
              -

              Description

              Generally, HAVING and GROUP BY are used together. You can use GROUP BY for grouping and the HAVING for filtering.

              @@ -185,7 +173,6 @@ GROUP BY GROUPING SETS ((user), (product)); GROUP BY num HAVING max(price*amount)>5000; - diff --git a/docs/dli/sqlreference/dli_08_0418.html b/docs/dli/sqlreference/dli_08_0418.html index c49aa952a..9a6dd2f20 100644 --- a/docs/dli/sqlreference/dli_08_0418.html +++ b/docs/dli/sqlreference/dli_08_0418.html @@ -4,7 +4,6 @@

              Union/Union ALL/Intersect/Except

              Syntax

              1
              query UNION [ ALL ] | Intersect | Except query
              -

              Description

              • UNION is used to return the union set of multiple query results.
              • INTERSECT is used to return the intersection of multiple query results.
              • EXCEPT is used to return the difference set of multiple query results.
              @@ -16,7 +15,6 @@ 2
              insert into temp SELECT  * FROM Orders1
   UNION SELECT  * FROM Orders2;
              -

              IN

              Syntax

              @@ -26,7 +24,6 @@ FROM tableExpression WHERE column_name IN (value (, value)* ) | query
              -

              Description

              The IN operator allows multiple values to be specified in the WHERE clause. It returns true if the expression exists in the given table subquery.

              @@ -44,7 +41,6 @@ SELECT product FROM NewProducts ); - diff --git a/docs/dli/sqlreference/dli_08_0419.html b/docs/dli/sqlreference/dli_08_0419.html index 9b2041dc8..e9cbcd995 100644 --- a/docs/dli/sqlreference/dli_08_0419.html +++ b/docs/dli/sqlreference/dli_08_0419.html @@ -22,19 +22,18 @@

              SESSION(time_attr, interval)

              -

              Defines a session time window. Session time windows do not have a fixed duration but their bounds are defined by a time interval of inactivity, that is, a session window is closed if no event appears for a defined gap period. For example a session window with a 30 minute gap starts when a row is observed after 30 minutes inactivity (otherwise the row would be added to an existing window) and is closed if no row is added within 30 minutes. Session windows can work on event-time (stream + batch) or processing-time (stream).

              +

              Defines a session time window. Session time windows do not have a fixed duration but their bounds are defined by a time interval of inactivity, i.e., a session window is closed if no event appears for a defined gap period. For example a session window with a 30 minute gap starts when a row is observed after 30 minutes inactivity (otherwise the row would be added to an existing window) and is closed if no row is added within 30 minutes. Session windows can work on event-time (stream + batch) or processing-time (stream).

              -

              Notes:

              In streaming mode, the time_attr argument of the group window function must refer to a valid time attribute that specifies the processing time or event time of rows.

              • event-time: The type is timestamp(3).
              • processing-time: No need to specify the type.

              In batch mode, the time_attr argument of the group window function must be an attribute of type timestamp.

              -
            • Window auxiliary functions
              The start and end timestamps of group windows as well as time attributes can be selected with the following auxiliary functions. -
              Table 2 Window auxiliary functions

              Auxiliary Function

              +
            • Window helper functions
              You can use the following helper functions to select the start and end timestamps, as well as the time attribute, for grouping windows. +
              @@ -73,7 +72,7 @@
              Table 2 Window helper functions

              Helper Function

              Description

              -

              Note: Auxiliary functions must be called with exactly same arguments as the group window function in the GROUP BY clause.

              +

              Note: When calling helper functions, it is important to use the same parameters as those used in the GROUP BY clause for grouping window functions.

              • Example

               1
@@ -128,7 +127,6 @@
     FROM Orders
     GROUP BY SESSION(ts, INTERVAL '12' HOUR), name;
              -

              TUMBLE WINDOW Extension

              Function

              @@ -137,7 +135,7 @@

              Precautions

              -
              • If you use the INSERT statement to write results to a sink, it must support the upsert mode. Ensure that the result table supports upsert operations and the primary key is defined.
              • Latency settings only take effect for event time and not for proctime.
              • Auxiliary functions must be called with the same parameters as the grouping window functions in the GROUP BY clause.
              • If event time is used, watermark must be used. The code is as follows (order_time is identified as the event time column and watermark is set to 3 seconds):
                CREATE TABLE orders (
+
                • If you use the INSERT statement to write results to a sink, it must support the upsert mode. Ensure that the result table supports upsert operations and the primary key is defined.
                • Latency settings only take effect for event time and not for proctime.
                • When calling helper functions, it is important to use the same parameters as those used in the GROUP BY clause for grouping window functions.
                • If event time is used, watermark must be used. The code is as follows (order_time is identified as the event time column and watermark is set to 3 seconds):
                  CREATE TABLE orders (
   order_id string,
   order_channel string,
   order_time timestamp(3),
@@ -228,9 +226,9 @@
 
                   
                  Values of period_interval and lateness_interval cannot be negative numbers.
                  • If period_interval is set to 0, periodic triggering is disabled for the window.
                  • If lateness_interval is set to 0, the latency after the window ends is disabled.
                  • If neither of the two parameters is set, both periodic triggering and latency are disabled and only the regular tumbling window functions are available .
                  • If only the latency function needs to be used, set period_interval INTERVAL '0' SECOND.
                  
 
                  
 
                  
-
                  Auxiliary Functions
                  
+
                  Helper Functions
                  
 
-
                  Table 4 Auxiliary function
                  Auxiliary Function
                  
+
                  
@@ -337,11 +335,10 @@ insert into jdbcSink select
   RANGE    BETWEEN (UNBOUNDED|timeInterval) PRECEDING AND CURRENT ROW FROM TABLENAME
                  Table 4 Helper functions
                  Helper Function
                  
                   		
 
                  Description
                  
                   		
 
 
                  
-
 
 
                  Description
                  
 
-
                  Table 5 Parameter description
                  Parameter
                  
+
                  
@@ -407,7 +404,6 @@ insert into jdbcSink select
     sum(amount) OVER (PARTITION BY name ORDER BY timeattr RANGE BETWEEN INTERVAL '60' SECOND PRECEDING AND CURRENT ROW) as cnt2    FROM Orders;
                  Table 5 Parameters
                  Parameter
                  
                   		
 
                  Description
                  
                   		
 
 
                  
-
 
 
 
diff --git a/docs/dli/sqlreference/dli_08_0420.html b/docs/dli/sqlreference/dli_08_0420.html
index 46adac89e..44ea4a5be 100644
--- a/docs/dli/sqlreference/dli_08_0420.html
+++ b/docs/dli/sqlreference/dli_08_0420.html
@@ -6,7 +6,6 @@
 2
                  FROM tableExpression INNER | LEFT | RIGHT | FULL JOIN tableExpression
   ON value11 = value21 [ AND value12 = value22]
 
                  
-
 
 
 
                  Precautions
                  
diff --git a/docs/dli/sqlreference/dli_08_0423.html b/docs/dli/sqlreference/dli_08_0423.html
index 5ae5f1cca..2460acd81 100644
--- a/docs/dli/sqlreference/dli_08_0423.html
+++ b/docs/dli/sqlreference/dli_08_0423.html
@@ -11,7 +11,7 @@ FROM (
    FROM table_name)
 WHERE rownum = 1
 
-
                  Description
                  • ROW_NUMBER(): Assigns a unique, sequential number to each row, starting with one.
                  • PARTITION BY col1[, col2...]: Specifies the partition columns, i.e. the deduplicate key.
                  • ORDER BY time_attr [asc|desc]: Specifies the ordering column, it must be a time attribute. Currently Flink supports proctime only. Ordering by ASC means keeping the first row, ordering by DESC means keeping the last row.
                  • WHERE rownum = 1: The rownum = 1 is required for Flink to recognize this query is deduplication.
                  
+
                  Description
                  • ROW_NUMBER(): Assigns a unique, sequential number to each row, starting with one.
                  • PARTITION BY col1[, col2...]: Specifies the partition columns, i.e. the deduplicate key.
                  • ORDER BY time_attr [asc|desc]: Specifies the ordering column, it must be a time attribute. Currently Flink supports proctime only. Ascending (ASC) sorting keeps only the first row, while descending (DESC) sorting keeps only the last row.
                  • WHERE rownum = 1: The rownum = 1 is required for Flink to recognize this query is deduplication.
                  
 
                  
 
                  Precautions
                  None
                  
 
                  
diff --git a/docs/dli/sqlreference/dli_08_0425.html b/docs/dli/sqlreference/dli_08_0425.html
index d1ab4d7ea..ebc6d1568 100644
--- a/docs/dli/sqlreference/dli_08_0425.html
+++ b/docs/dli/sqlreference/dli_08_0425.html
@@ -4,7 +4,7 @@
 
                  Overview
                  DLI supports the following three types of user-defined functions (UDFs):
                  
 
                  
 
                  • Regular UDF: takes in one or more input parameters and returns a single result.
                  • User-defined table-generating function (UDTF): takes in one or more input parameters and returns multiple rows or columns.
                  • User-defined aggregate function (UDAF): aggregates multiple records into one value.
                  
-
                   
                  • UDFs can only be used in dedicated queues.
                  • Currently, Python is not supported for programming UDFs, UDTFs, and UDAFs.
                  
+
                   
                  • Currently, UDF, UDTF, or UDAF custom functions cannot be written using Python.
                  • If you use a UDF in a Flink OpenSource SQL job, it is not possible to generate a static stream graph.
                  
 
                  
 
                  POM Dependency
                  <dependency>
         <groupId>org.apache.flink</groupId>
@@ -13,7 +13,7 @@
         <scope>provided</scope>
 </dependency>
                  
 
                  
-
                  Using UDFs
                  1. Encapsulate the implemented UDFs into a JAR package and upload the package to OBS.
                  2. In the navigation pane of the DLI management console, choose Data Management > Package Management. On the displayed page, click Create and use the JAR package uploaded to OBS to create a package.
                  3. In the left navigation, choose Job Management and click Flink Jobs. Locate the row where the target resides and click Edit in the Operation column to switch to the page where you can edit the job.
                  4. Click the Running Parameters tab of your job, select the UDF JAR and click Save.
                  5. Add the following statement to the SQL statements to use the functions:
                  
+
                  Using UDFs
                  1. Encapsulate the implemented UDFs into a JAR package and upload the package to OBS.
                  2. In the navigation pane of the DLI management console, choose Data Management > Package Management. On the displayed page, click Create and use the JAR package uploaded to OBS to create a package.
                  3. In the left navigation, choose Job Management and click Flink Jobs. Locate the row where the target resides and click Edit in the Operation column to switch to the page where you can edit the job.
                  4. Click the Running Parameters tab of your job, select the UDF JAR and click Save.
                  5. Add the following statement to the SQL statements to use the functions:
                  
 
                  
 
                  UDF
                  The regular UDF must inherit the ScalarFunction function and implement the eval method. The open and close functions are optional.
                  
 
                  
@@ -50,7 +50,6 @@ public class UdfScalarFunction extends ScalarFunction {
 2
                  CREATE FUNCTION udf_test AS 'com.company.udf.UdfScalarFunction';
 INSERT INTO sink_stream select udf_test(attr) FROM source_stream;
 
                  
-
 
 
                  UDTF
                  The UDTF must inherit the TableFunction function and implement the eval method. The open and close functions are optional. If the UDTF needs to return multiple columns, you only need to declare the returned value as Tuple or Row. If Row is used, you need to overload the getResultType method to declare the returned field type.
                  
 
                  
@@ -109,7 +108,6 @@ public class UdfTableFunction extends TableFunction<Row> {
 INSERT INTO sink_stream select subValue, length FROM source_stream LEFT JOIN LATERAL
 TABLE(udtf_test(attr, ',')) as T(subValue, length) ON TRUE;
              -

              UDAF

              The UDAF must inherit the AggregateFunction function. You need to create an accumulator for storing the computing result, for example, WeightedAvgAccum in the following example code.

              @@ -170,7 +168,6 @@ acc.sum = 0L; 2
              CREATE FUNCTION udaf_test AS 'com.company.udf.UdfAggFunction';
 INSERT INTO sink_stream SELECT udaf_test(attr2) FROM source_stream GROUP BY attr1;
              -
              diff --git a/docs/dli/sqlreference/dli_08_0428.html b/docs/dli/sqlreference/dli_08_0428.html index 91c5d67fa..7fdaa007f 100644 --- a/docs/dli/sqlreference/dli_08_0428.html +++ b/docs/dli/sqlreference/dli_08_0428.html @@ -1,153 +1,155 @@

              String Functions

              -
              -
              diff --git a/docs/dli/sqlreference/dli_08_0461.html b/docs/dli/sqlreference/dli_08_0461.html index 918c5bca3..4db2d87e1 100644 --- a/docs/dli/sqlreference/dli_08_0461.html +++ b/docs/dli/sqlreference/dli_08_0461.html @@ -4,7 +4,7 @@

              Function

              This statement is used to create a DLI table and associate it with an existing Oracle table.

              Prerequisites

              • Before creating a DLI table and associating it with Oracle, you need to create an enhanced datasource connection.

                For details about operations on the management console, see .

                -
              • Only enhanced datasource connections can be used to connect to Oracle, and only pay-per-use and yearly/monthly queues support enhanced datasource connections. So, only SQL jobs on pay-per-use and yearly/monthly queues can be connected to Oracle databases.
              +

              Syntax

              Table 1 String Functions

              Function

              +

              String Functions

              DLI offers a wide range of string functions for processing and transforming string data. These functions include concatenation, case conversion, substring extraction, replacement, regex matching, encoding and decoding, format conversion, and more. Additionally, it supports string length calculation, position searching, padding, reversing, and even extracting values from JSON strings using the JSON_VAL function. These features are widely used in data cleansing, text processing, and data analysis scenarios, providing developers with powerful tool support.

              +

              For detailed string functions, see Table 1. For more information, see Apache Flink.

              + +
              - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
              Table 1 String Functions

              Function

              Return Type

              +

              Return Type

              Description

              +

              Description

              string1 || string2

              +

              string1 || string2

              STRING

              +

              STRING

              Returns the concatenation of string1 and string2.

              +

              Returns the concatenation of string1 and string2.

              CHAR_LENGTH(string)

              +

              CHAR_LENGTH(string)

              CHARACTER_LENGTH(string)

              INT

              +

              INT

              Returns the number of characters in the string.

              +

              Returns the number of characters in the string.

              UPPER(string)

              +

              UPPER(string)

              STRING

              +

              STRING

              Returns the string in uppercase.

              +

              Returns the string in uppercase.

              LOWER(string)

              +

              LOWER(string)

              STRING

              +

              STRING

              Returns the string in lowercase.

              +

              Returns the string in lowercase.

              POSITION(string1 IN string2)

              +

              POSITION(string1 IN string2)

              INT

              +

              INT

              Returns the position (start from 1) of the first occurrence of string1 in string2; returns 0 if string1 cannot be found in string2.

              +

              Returns the position (start from 1) of the first occurrence of string1 in string2; returns 0 if string1 cannot be found in string2.

              TRIM([ BOTH | LEADING | TRAILING ] string1 FROM string2)

              +

              TRIM([ BOTH | LEADING | TRAILING ] string1 FROM string2)

              STRING

              +

              STRING

              Returns a string that removes leading and/or trailing characters string2 from string1.

              +

              Returns a string that removes leading and/or trailing characters string2 from string1.

              LTRIM(string)

              +

              LTRIM(string)

              STRING

              +

              STRING

              Returns a string that removes the left whitespaces from the specified string.

              +

              Returns a string that removes the left whitespaces from the specified string.

              For example, LTRIM(' This is a test String.') returns "This is a test String.".

              RTRIM(string)

              +

              RTRIM(string)

              STRING

              +

              STRING

              Returns a string that removes the right whitespaces from the specified string.

              +

              Returns a string that removes the right whitespaces from the specified string.

              For example, RTRIM('This is a test String. ') returns "This is a test String.".

              REPEAT(string, integer)

              +

              REPEAT(string, integer)

              STRING

              +

              STRING

              Returns a string that repeats the base string integer times.

              +

              Returns a string that repeats the base string integer times.

              For example, REPEAT('This is a test String.', 2) returns "This is a test String.This is a test String.".

              REGEXP_REPLACE(string1, string2, string3)

              +

              REGEXP_REPLACE(string1, string2, string3)

              STRING

              +

              STRING

              Returns a string from string1 with all the substrings that match a regular expression string2 consecutively being replaced with string3.

              +

              Returns a string from string1 with all the substrings that match a regular expression string2 consecutively being replaced with string3.

              For example, REGEXP_REPLACE('foobar', 'oo|ar', '') returns "fb".

              REGEXP_REPLACE('ab\ab', '\\', 'e') returns "abeab".

              OVERLAY(string1 PLACING string2 FROM integer1 [ FOR integer2 ])

              +

              OVERLAY(string1 PLACING string2 FROM integer1 [ FOR integer2 ])

              STRING

              +

              STRING

              Returns a string that replaces integer2 characters of STRING1 with STRING2 from position integer1.

              +

              Returns a string that replaces integer2 characters of STRING1 with STRING2 from position integer1.

              The default value of integer2 is the length of string2.

              For example, OVERLAY('This is an old string' PLACING ' new' FROM 10 FOR 5) returns "This is a new string".

              SUBSTRING(string FROM integer1 [ FOR integer2 ])

              +

              SUBSTRING(string FROM integer1 [ FOR integer2 ])

              STRING

              +

              STRING

              Returns a substring of the specified string starting from position integer1 with length integer2 (to the end by default). If integer2 is not configured, the substring from integer1 to the end is returned by default.

              +

              Returns a substring of the specified string starting from position integer1 with length integer2 (to the end by default). If integer2 is not configured, the substring from integer1 to the end is returned by default.

              REPLACE(string1, string2, string3)

              +

              REPLACE(string1, string2, string3)

              STRING

              +

              STRING

              Returns a new string which replaces all the occurrences of string2 with string3 (non-overlapping) from string1.

              +

              Returns a new string which replaces all the occurrences of string2 with string3 (non-overlapping) from string1.

              For example, REPLACE('hello world', 'world', 'flink') returns "hello flink"; REPLACE('ababab', 'abab', 'z') returns "zab".

              REPLACE('ab\\ab', '\\', 'e') returns "abeab".

              REGEXP_EXTRACT(string1, string2[, integer])

              +

              REGEXP_EXTRACT(string1, string2[, integer])

              STRING

              +

              STRING

              Returns a string from string1 which extracted with a specified regular expression string2 and a regex match group index integer.

              +

              Returns a string from string1 which extracted with a specified regular expression string2 and a regex match group index integer.

              Returns NULL, if the parameter is NULL or the regular expression is invalid.

              For example, REGEXP_EXTRACT('foothebar', 'foo(.*?)(bar)', 2)" returns "bar".

              INITCAP(string)

              +

              INITCAP(string)

              STRING

              +

              STRING

              Returns a new form of STRING with the first character of each word converted to uppercase and the rest characters to lowercase.

              +

              Returns a new form of STRING with the first character of each word converted to uppercase and the rest characters to lowercase.

              CONCAT(string1, string2,...)

              +

              CONCAT(string1, string2,...)

              STRING

              +

              STRING

              Returns a string that concatenates string1, string2, ….

              +

              Returns a string that concatenates string1, string2, ….

              For example, CONCAT('AA', 'BB', 'CC') returns "AABBCC".

              CONCAT_WS(string1, string2, string3,...)

              +

              CONCAT_WS(string1, string2, string3,...)

              STRING

              +

              STRING

              Returns a string that concatenates string2, string3, … with a separator string1. The separator is added between the strings to be concatenated. Returns NULL if string1 is NULL. If other arguments are NULL, this function automatically skips NULL arguments.

              +

              Returns a string that concatenates string2, string3, … with a separator string1. The separator is added between the strings to be concatenated. Returns NULL if string1 is NULL. If other arguments are NULL, this function automatically skips NULL arguments.

              For example, CONCAT_WS('~', 'AA', NULL, 'BB', '', 'CC') returns "AA~BB~~CC".

              LPAD(string1, integer, string2)

              +

              LPAD(string1, integer, string2)

              STRING

              +

              STRING

              Returns a new string from string1 left-padded with string2 to a length of integer characters.

              +

              Returns a new string from string1 left-padded with string2 to a length of integer characters.

              If any argument is NULL, NULL is returned.

              If integer is negative, NULL is returned.

              If the length of string1 is shorter than integer, returns string1 shortened to integer characters.

              @@ -155,11 +157,11 @@

              LPAD('hi',1,'??') returns "h".

              RPAD(string1, integer, string2)

              +

              RPAD(string1, integer, string2)

              STRING

              +

              STRING

              Returns a new string from string1 right-padded with string2 to a length of integer characters.

              +

              Returns a new string from string1 right-padded with string2 to a length of integer characters.

              If any argument is NULL, NULL is returned.

              If integer is negative, NULL is returned.

              If the length of string1 is shorter than integer, returns string1 shortened to integer characters.

              @@ -167,102 +169,102 @@

              RPAD('hi',1,'??') returns "h".

              FROM_BASE64(string)

              +

              FROM_BASE64(string)

              STRING

              +

              STRING

              Returns the base64-decoded result from string.

              +

              Returns the base64-decoded result from string.

              Returns NULL if string is NULL.

              For example, FROM_BASE64('aGVsbG8gd29ybGQ=') returns "hello world".

              TO_BASE64(string)

              +

              TO_BASE64(string)

              STRING

              +

              STRING

              Returns the base64-encoded result from string; f string is NULL.

              +

              Returns the base64-encoded result from string; f string is NULL.

              Returns NULL if string is NULL.

              For example, TO_BASE64(hello world) returns "aGVsbG8gd29ybGQ=".

              ASCII(string)

              +

              ASCII(string)

              INT

              +

              INT

              Returns the numeric value of the first character of string.

              +

              Returns the numeric value of the first character of string.

              Returns NULL if string is NULL.

              For example, ascii('abc') returns 97.

              ascii(CAST(NULL AS VARCHAR)) returns NULL.

              CHR(integer)

              +

              CHR(integer)

              STRING

              +

              STRING

              Returns the ASCII character having the binary equivalent to integer.

              +

              Returns the ASCII character having the binary equivalent to integer.

              If integer is larger than 255, we will get the modulus of integer divided by 255 first, and returns CHR of the modulus.

              Returns NULL if integer is NULL.

              chr(97) returns a.

              chr(353) Return a.

              DECODE(binary, string)

              +

              DECODE(binary, string)

              STRING

              +

              STRING

              Decodes the first argument into a String using the provided character set (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').

              +

              Decodes the first argument into a String using the provided character set (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').

              If either argument is NULL, the result will also be NULL.

              ENCODE(strinh1, string2)

              +

              ENCODE(strinh1, string2)

              STRING

              +

              STRING

              Encodes the string1 into a BINARY using the provided string2 character set (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').

              +

              Encodes the string1 into a BINARY using the provided string2 character set (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').

              If either argument is NULL, the result will also be NULL.

              INSTR(string1, string2)

              +

              INSTR(string1, string2)

              INT

              +

              INT

              Returns the position of the first occurrence of string2 in string1.

              +

              Returns the position of the first occurrence of string2 in string1.

              Returns NULL if any argument is NULL.

              LEFT(string, integer)

              +

              LEFT(string, integer)

              STRING

              +

              STRING

              Returns the leftmost integer characters from the string.

              +

              Returns the leftmost integer characters from the string.

              Returns EMPTY String if integer is negative.

              Returns NULL if any argument is NULL.

              RIGHT(string, integer)

              +

              RIGHT(string, integer)

              STRING

              +

              STRING

              Returns the rightmost integer characters from the string.

              +

              Returns the rightmost integer characters from the string.

              Returns EMPTY String if integer is negative.

              Returns NULL if any argument is NULL.

              LOCATE(string1, string2[, integer])

              +

              LOCATE(string1, string2[, integer])

              INT

              +

              INT

              Returns the position of the first occurrence of string1 in string2 after position integer.

              +

              Returns the position of the first occurrence of string1 in string2 after position integer.

              Returns 0 if not found.

              The value of integer defaults to 0.

              Returns NULL if any argument is NULL.

              PARSE_URL(string1, string2[, string3])

              +

              PARSE_URL(string1, string2[, string3])

              STRING

              +

              STRING

              Returns the specified part from the URL.

              +

              Returns the specified part from the URL.

              Valid values for string2 include 'HOST', 'PATH', 'QUERY', 'REF', 'PROTOCOL', 'AUTHORITY', 'FILE', and 'USERINFO'.

              Returns NULL if any argument is NULL.

              If string2 is QUERY, the key in QUERY can be specified as string3.

              @@ -271,55 +273,56 @@

              parse_url('http://facebook.com/path1/p.php?k1=v1&k2=v2#Ref1', 'QUERY', 'k1') returns 'v1'.

              REGEXP(string1, string2)

              +

              REGEXP(string1, string2)

              BOOLEAN

              +

              BOOLEAN

              Performs a regular expression search on the specified string and returns a BOOLEAN value indicating whether the specified match pattern is found. If it is found, TRUE is returned. string1 indicates the specified string, and string2 indicates the regular expression.

              +

              Performs a regular expression search on the specified string and returns a BOOLEAN value indicating whether the specified match pattern is found. If it is found, TRUE is returned. string1 indicates the specified string, and string2 indicates the regular expression.

              Returns NULL if any argument is NULL.

              REVERSE(string)

              +

              REVERSE(string)

              STRING

              +

              STRING

              Returns the reversed string.

              +

              Returns the reversed string.

              Returns NULL if any argument is NULL.

              NOTE:

              Note that backquotes must be added to this function, for example, `REVERSE`.

              SPLIT_INDEX(string1, string2, integer1)

              +

              SPLIT_INDEX(string1, string2, integer1)

              STRING

              +

              STRING

              Splits string1 by the delimiter string2, returns the integerth (zero-based) string of the split strings. Returns NULL if integer is negative.

              +

              Splits string1 by the delimiter string2, returns the integerth (zero-based) string of the split strings. Returns NULL if integer is negative.

              Returns NULL if integer is negative.

              Returns NULL if any argument is NULL.

              +

              For details about this function, see SPLIT_INDEX Function.

              STR_TO_MAP(string1[, string2, string3]])

              +

              STR_TO_MAP(string1[, string2, string3]])

              MAP

              +

              MAP

              Returns a map after splitting the string1 into key/value pairs using delimiters.

              +

              Returns a map after splitting the string1 into key/value pairs using delimiters.

              The default value of string2 is ','.

              The default value of string3 is '='.

              SUBSTR(string[, integer1[, integer2]])

              +

              SUBSTR(string[, integer1[, integer2]])

              STRING

              +

              STRING

              Returns a substring of string starting from position integer1 with length integer2.

              +

              Returns a substring of string starting from position integer1 with length integer2.

              If integer2 is not specified, the string is truncated to the end.

              JSON_VAL(STRING json_string, STRING json_path)

              +

              JSON_VAL(STRING json_string, STRING json_path)

              STRING

              +

              STRING

              Returns the value of the specified json_path from the json_string. For details about how to use the functions, see JSON_VAL Function.

              +

              Returns the value of the specified json_path from the json_string. For details about how to use the functions, see JSON_VAL Function.

              NOTE:

              The following rules are listed in descending order of priority.

              1. The two arguments json_string and json_path cannot be NULL.
              2. The value of json_string must be a valid JSON string. Otherwise, the function returns NULL.
              3. If json_string is an empty string, the function returns an empty string.
              4. If json_path is an empty string or the path does not exist, the function returns NULL.
              @@ -328,66 +331,83 @@
              +
              +

              SPLIT_INDEX Function

              SPLIT_INDEX(string1, string2, integer1)

              +
              • Parameter description
                • string1:
                  • Type: STRING
                  • Description: Original string to be split.
                  • Example: 'a,b,c,d' or 'a\bc\bd'
                  +
                • string2:
                  • Type: STRING
                  • Description: Delimiter that splits strings.
                  • Special character processing:
                    • If the delimiter contains special characters (such as \\.*), you need to use a double backslash (\\) for escaping.
                    • If the delimiter is a period (.), escape is not required.
                    • If the delimiter is a double backslash (\\), enter \\\\.
                    +
                  +
                • integer1:
                  • Type: INT
                  • Description: Specifies the index of the extracted split part, starting from 0.
                  • Example: 0 indicates the extraction of the first part, 1 indicates the extraction of the second part, and so on.
                  +
                +
              • Return values

                Returns the part of the split string at the specified index.

                +

                If the index is out of range or the input parameter is NULL, NULL is returned.

                +
              • Example 1

                SELECT SPLIT_INDEX('a,b,c,d', ',', 1); -- Returns b.

                +
              • Example 2

                If the delimiter is a period (.), escape is not required.

                +

                SELECT SPLIT_INDEX('a.b.c.d', '.', 2); -- Returns c.

                +
              • Example 3

                Using a backslash (\) as a delimiter:

                +

                The backslash itself also needs to be escaped, using \\\\:

                +

                SELECT SPLIT_INDEX('a\\bc\\bd', '\\\\', 1); -- Returns bc.

                +
              +

              JSON_VAL Function

              • Syntax
              STRING JSON_VAL(STRING json_string, STRING json_path)
              -
              Table 2 Parameters

              Parameter

              +
              - - - - - - - -
              Table 2 Parameters

              Parameter

              Data Types

              +

              Data Types

              Description

              +

              Description

              json_string

              +

              json_string

              STRING

              +

              STRING

              JSON object to be parsed

              +

              JSON object to be parsed

              json_path

              +

              json_path

              STRING

              +

              STRING

              Path expression for parsing the JSON string For the supported expressions, see Table 3.

              +

              Path expression for parsing the JSON string For the supported expressions, see Table 3.
              -
              - -
              Table 3 Expressions supported

              Expression

              +
              - - - - - - - - -
              Table 3 Expressions supported

              Expression

              Description

              +

              Description

              $

              +

              $

              Root node in the path

              +

              Root node in the path

              []

              +

              []

              Access array elements

              +

              Access array elements

              *

              +

              *

              Array wildcard

              +

              Array wildcard

              .

              +

              .

              Access child elements

              +

              Access child elements
              -
              • Example
                1. Test input data.
                  Test the data source kafka. The message content is as follows:
                  {name:James,age:24,sex:male,grade:{math:95,science:[80,85],english:100}}
-{name:James,age:24,sex:male,grade:{math:95,science:[80,85],english:100}]
                  +
                  • Example
                    1. Test input data.
                      Test the data source kafka. The message content is as follows:
                      {name:James,age:24,gender:male,grade:{math:95,science:[80,85],english:100}}
+{name:James,age:24,gender:male,grade:{math:95,science:[80,85],english:100}]
                    2. Use JSON_VAL in SQL statements.
                      CREATE TABLE kafkaSource (
   `message` string
diff --git a/docs/dli/sqlreference/dli_08_0429.html b/docs/dli/sqlreference/dli_08_0429.html
index ec843133e..94ea06780 100644
--- a/docs/dli/sqlreference/dli_08_0429.html
+++ b/docs/dli/sqlreference/dli_08_0429.html
@@ -224,7 +224,7 @@

              TIMESTAMP

              Convert a datetime string1 from time zone string2 to time zone string3.

              -

              For example, CONVERT_TZ('1970-01-01 00:00:00', 'UTC', 'America/Los_Angeles') returns '1969-12-31 16:00:00'.

              +

              For example, CONVERT_TZ('1970-01-01 00:00:00', 'UTC', 'Country A/City A') returns '1969-12-31 16:00:00'.

              FROM_UNIXTIME(numeric[, string])

              @@ -1397,21 +1397,21 @@ FROM testtable;

              STRING

              Time zone before conversion. The format of time zone should be either an abbreviation such as PST, a full name such as America/Los_Angeles, or a custom ID such as GMT-08:00.

              +

              Time zone before conversion. The format of time zone should be either an abbreviation such as PST, a full name such as Country A/City A, or a custom ID such as GMT-08:00.

              string3

              STRING

              Time zone after conversion. The format of time zone should be either an abbreviation such as PST, a full name such as America/Los_Angeles, or a custom ID such as GMT-08:00.

              +

              Time zone after conversion. The format of time zone should be either an abbreviation such as PST, a full name such as Country A/City A, or a custom ID such as GMT-08:00.
            • Example
              • Test statement
                SELECT 
-	CONVERT_TZ(1970-01-01 00:00:00, UTC, America/Los_Angeles) AS `result`,
+	CONVERT_TZ(1970-01-01 00:00:00, UTC, Country A/City A) AS `result`,
         CONVERT_TZ(1997-04-25 10:00:00, UTC, GMT-08:00) AS `result2`
 FROM	testtable;
              • Test Result diff --git a/docs/dli/sqlreference/dli_08_0439.html b/docs/dli/sqlreference/dli_08_0439.html index 91b0b42e0..af1784a31 100644 --- a/docs/dli/sqlreference/dli_08_0439.html +++ b/docs/dli/sqlreference/dli_08_0439.html @@ -29,10 +29,9 @@ 'auto-compaction' = 'true' );
              • -
              -

              Usage

              • Rolling Policy

                The Rolling Policy defines when a given in-progress part file will be closed and moved to the pending and later to finished state. Part files in the "finished" state are the ones that are ready for viewing and are guaranteed to contain valid data that will not be reverted in case of failure.

                +

                Usage

                • Rolling Policy

                  The Rolling Policy determines when to close the current in-progress part file and transition it from the in-progress state to the pending state, and then to the finished state. Part files in the "finished" state are the ones that are ready for viewing and are guaranteed to contain valid data that will not be reverted in case of failure.

                  In STREAMING mode, the Rolling Policy in combination with the checkpointing interval (pending files become finished on the next checkpoint) control how quickly part files become available for downstream readers and also the size and number of these parts. For details, see Parameters.

                • Part File Lifecycle

                  To use the output of the FileSink in downstream systems, we need to understand the naming and lifecycle of the output files produced.

                  Part files can be in one of three states:

                  @@ -113,7 +112,7 @@

              MemorySize

              Maximum size of a part file. If the size of a part file exceeds this value, a new file will be generated.

              -
              NOTE:

              The Rolling Policy defines when a given in-progress part file will be closed and moved to the pending and later to finished state. Part files in the "finished" state are the ones that are ready for viewing and are guaranteed to contain valid data that will not be reverted in case of failure. In STREAMING mode, the Rolling Policy in combination with the checkpointing interval (pending files become finished on the next checkpoint) control how quickly part files become available for downstream readers and also the size and number of these parts.

              +
              NOTE:

              The Rolling Policy determines when to close the current in-progress part file and transition it from the in-progress state to the pending state, and then to the finished state. Part files in the "finished" state are the ones that are ready for viewing and are guaranteed to contain valid data that will not be reverted in case of failure. In STREAMING mode, the Rolling Policy in combination with the checkpointing interval (pending files become finished on the next checkpoint) control how quickly part files become available for downstream readers and also the size and number of these parts.

              		 
              1
 2
@@ -24,7 +24,6 @@
   'resource' = 'obs://rest-authinfo/tools/oracle/driver/ojdbc6.jar' 
 );
              -

              Keywords

              @@ -91,7 +90,6 @@ 'resource' = 'obs://rest-authinfo/tools/oracle/driver/ojdbc6.jar' );
              - diff --git a/docs/dli/sqlreference/dli_08_0462.html b/docs/dli/sqlreference/dli_08_0462.html index cc1d4b954..acb318a56 100644 --- a/docs/dli/sqlreference/dli_08_0462.html +++ b/docs/dli/sqlreference/dli_08_0462.html @@ -17,13 +17,11 @@ [GROUP BY field] [ORDER BY field] ...; -
            • Insert a data record into a table.
              1
 2
              INSERT INTO DLI_TABLE
   VALUES values_row [, values_row ...];
              -
            • Overwriting the inserted data
              1
@@ -40,7 +38,6 @@
   [GROUP BY field]
   [ORDER BY field] ...;
              -
            @@ -101,7 +98,6 @@ LIMIT 3 GROUP BY user_age -
          • Insert data 1 into the test table.
            INSERT INTO test 
   VALUES (1);
            diff --git a/docs/dli/sqlreference/dli_08_0463.html b/docs/dli/sqlreference/dli_08_0463.html index 494e64cb0..adc778739 100644 --- a/docs/dli/sqlreference/dli_08_0463.html +++ b/docs/dli/sqlreference/dli_08_0463.html @@ -5,7 +5,6 @@

            Syntax

            1
            SELECT * FROM table_name LIMIT number;
            -

            Keywords

            LIMIT is used to limit the query results. Only INT type is supported by the number parameter.

            @@ -15,7 +14,6 @@

            Example

            Querying data in the test_oracle table

            1
            SELECT * FROM test_oracle limit 100;
            -
            diff --git a/docs/dli/sqlreference/dli_08_0465.html b/docs/dli/sqlreference/dli_08_0465.html new file mode 100644 index 000000000..ef958e150 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_0465.html @@ -0,0 +1,19 @@ + + +

            Table Lifecycle Management

            +
            + + diff --git a/docs/dli/sqlreference/dli_08_0466.html b/docs/dli/sqlreference/dli_08_0466.html new file mode 100644 index 000000000..a16f69137 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_0466.html @@ -0,0 +1,124 @@ + + +

            Specifying the Lifecycle of a Table When Creating the Table

            +

            Function

            DLI provides table lifecycle management to allow you to specify the lifecycle of a table when creating the table. DLI determines whether to reclaim a table based on the table's last modification time and its lifecycle. By setting the lifecycle of a table, you can better manage a large number of tables, automatically delete data tables that are no longer used for a long time, and simplify the process of reclaiming data tables. Additionally, data restoration settings are supported to prevent data loss caused by misoperations.

            +
            +

            Table Reclamation Rules

            • When creating a table, use TBLPROPERTIES to specify the lifecycle of the table.
              • Non-partitioned table

                If the table is not a partitioned table, the system determines whether to reclaim the table after the lifecycle time based on the last modification time of the table.

                +
              • Partitioned table

                If the table is a partitioned table, the system determines whether the partition needs to be reclaimed based on the last modification time (LAST_ACCESS_TIME) of the partition. After the last partition of a partitioned table is reclaimed, the table is not deleted.

                +

                Only table-level lifecycle management is supported for partitioned tables.

                +
              +
            • Lifecycle reclamation starts at a specified time every day to scan all partitions.

              Lifecycle reclamation starts at a specified time every day. Reclamation only occurs if the last modification time of the table data (LAST_ACCESS_TIME) detected when scanning complete partitions exceeds the time specified by the lifecycle.

              +

              Assume that the lifecycle of a partitioned table is one day and the last modification time of the partitioned data is 15:00 on May 20, 2023. If the table is scanned before 15:00 on May 20, 2023 (less than one day), the partitions in the table will not be reclaimed. If the last data modification time (LAST_ACCESS_TIME) of a table partition exceeds the time specified by the lifecycle during reclamation scan on May 20, 2023, the partition will be reclaimed.

              +
            • The lifecycle function periodically reclaims tables or partitions, which are reclaimed irregularly every day depending on the level of busyness of the service. It cannot ensure that a table or partition will be reclaimed immediately after its lifecycle expires.
            • After a table is deleted, all properties of the table, including the lifecycle, will be deleted. After a table with the same name is created again, the lifecycle of the table will be determined by the new property.
            +
            +

            Notes and Constraints

            • Before using the lifecycle function, log in to the DLI console, choose Global Configuration > Service Authorization, select dli_admin_agency, and click Update.
            • The table lifecycle function currently only supports creating tables and versioning tables using Hive and Datasource syntax.
            • The unit of the lifecycle is in days. The value should be a positive integer.
            • The lifecycle can be set only at the table level. The lifecycle specified for a partitioned table applies to all partitions of the table.
            • After the lifecycle is set, DLI and OBS tables will support data backup. The backup directory for OBS tables needs to be set manually. The backup directory must be in the parallel file system and in the same bucket as the original table directory. It cannot have the same directory or subdirectory name as the original table.
            +
            +

            Syntax

            • Creating a DLI table using the Datasource syntax
              CREATE TABLE table_name(name string, id int) 
+USING parquet 
+TBLPROPERTIES( "dli.lifecycle.days"=1 );
              +
            • Creating a DLI table using the Hive syntax
              CREATE TABLE table_name(name string, id int) 
+stored as parquet 
+TBLPROPERTIES( "dli.lifecycle.days"=1 );
              +
            • Creating an OBS table using the Datasource syntax
              CREATE TABLE table_name(name string, id int) 
+USING parquet 
+OPTIONS (path "obs://dli-test/table_name")
+TBLPROPERTIES( "dli.lifecycle.days"=1, "external.table.purge"='true', "dli.lifecycle.trash.dir"='obs://dli-test/Lifecycle-Trash' );
              +
            • Creating an OBS table using the Hive syntax
              1
+2
+3
+4
              CREATE TABLE table_name(name string, id int) 
+STORED AS parquet 
+LOCATION 'obs://dli-test/table_name'
+TBLPROPERTIES( "dli.lifecycle.days"=1, "external.table.purge"='true', "dli.lifecycle.trash.dir"='obs://dli-test/Lifecycle-Trash' );
+
              +
              +
            +
            +

            Keywords

            • TBLPROPERTIES: Table properties, which can be used to extend the lifecycle of a table.
            • OPTIONS: path of the new table, which is applicable to OBS tables created using the Datasource syntax.
            • LOCATION: path of the new table, which is applicable to OBS tables created using the Hive syntax.
            +
            +

            Parameters

            +
            + + + + + + + + + + + + + + + + + + + + + +
            Table 1 Parameters

            Parameter

            +

            Mandatory

            +

            Description

            +

            table_name

            +

            Yes

            +

            Name of the table whose lifecycle needs to be set

            +

            dli.lifecycle.days

            +

            Yes

            +

            Lifecycle duration. The value must be a positive integer, in days.

            +

            external.table.purge

            +

            No

            +

            This parameter is available only for OBS tables.

            +

            Whether to clear data in the path when deleting a table or partition. The data is not cleared by default.

            +

            When this parameter is set to true:

            +
            • After a file is deleted from a non-partitioned OBS table, the table directory is also deleted.
            • The custom partition data in the partitioned OBS table is also deleted.
            +

            dli.lifecycle.trash.dir

            +

            No

            +

            This parameter is available only for OBS tables.

            +

            When external.table.purge is set to true, the backup directory will be deleted. By default, backup data is deleted seven days later.

            +
            +
            +
            +

            Example

            • Create the test_datasource_lifecycle table using the Datasource syntax. The lifecycle is set to 100 days.
              1
+2
+3
              CREATE TABLE test_datasource_lifecycle(id int) 
+USING parquet 
+TBLPROPERTIES( "dli.lifecycle.days"=100);
+
              +
              +
            • Create the test_hive_lifecycle table using the Hive syntax. The lifecycle is set to 100 days.
              1
+2
+3
              CREATE TABLE test_hive_lifecycle(id int) 
+stored as parquet 
+TBLPROPERTIES( "dli.lifecycle.days"=100);
+
              +
              +
            • Create the test_datasource_lifecycle_obs table using the Datasource syntax. The lifecycle is set to 100 days. When the lifecycle expires, data is deleted by default and backed up to the obs://dli-test/ directory.
              1
+2
+3
+4
              CREATE TABLE test_datasource_lifecycle_obs(name string, id int) 
+USING parquet 
+OPTIONS (path "obs://dli-test/xxx")
+TBLPROPERTIES( "dli.lifecycle.days"=100, "external.table.purge"='true', "dli.lifecycle.trash.dir"='obs://dli-test/Lifecycle-Trash' );
+
              +
              +
            • Create the test_hive_lifecycle_obs table using the Hive syntax. The lifecycle is set to 100 days. When the lifecycle expires, data is deleted by default and backed up to the obs://dli-test/ directory.
              1
+2
+3
+4
              CREATE TABLE test_hive_lifecycle_obs(name string, id int) 
+STORED AS parquet 
+LOCATION 'obs://dli-test/xxx'
+TBLPROPERTIES( "dli.lifecycle.days"=100, "external.table.purge"='true', "dli.lifecycle.trash.dir"='obs://dli-test/Lifecycle-Trash' );
+
              +
              +
            +
            +
            +
            +
            +
            Parent topic: Table Lifecycle Management
            +
            +
            + diff --git a/docs/dli/sqlreference/dli_08_0467.html b/docs/dli/sqlreference/dli_08_0467.html new file mode 100644 index 000000000..0eebe06a9 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_0467.html @@ -0,0 +1,56 @@ + + +

            Modifying the Lifecycle of a Table

            +

            Function

            This section describes how to modify the lifecycle of an existing partitioned or non-partitioned table.

            +

            When the lifecycle function is enabled for the first time, the system scans tables or partitions, scans table data files in the path, and updates LAST_ACCESS_TIME of tables or partitions. The time required depends on the number of partitions and files.

            +
            +

            Notes and Constraints

            • The table lifecycle function currently only supports creating tables and versioning tables using Hive and Datasource syntax.
            • The unit of the lifecycle is in days. The value should be a positive integer.
            • The lifecycle can be set only at the table level. The lifecycle specified for a partitioned table applies to all partitions of the table.
            +
            +

            Syntax

            ALTER TABLE table_name
+SET TBLPROPERTIES("dli.lifecycle.days"='N')
            +
            +

            Keywords

            TBLPROPERTIES: Table properties, which can be used to extend the lifecycle of a table.

            +
            +

            Parameters

            +
            + + + + + + + + + + + + + +
            Table 1 Parameters

            Parameter

            +

            Mandatory

            +

            Description

            +

            table_name

            +

            Yes

            +

            Name of the table whose lifecycle needs to be modified

            +

            dli.lifecycle.days

            +

            Yes

            +

            Lifecycle duration after the modification. The value must be a positive integer, in days.

            +
            +
            +
            +

            Example

            • Example 1: Enable the lifecycle function for the test_lifecycle_exists table and set the lifecycle to 50 days.
              alter table test_lifecycle_exists
+SET TBLPROPERTIES("dli.lifecycle.days"='50');
              +
            • Example 2: Enable the lifecycle function for an existing partitioned or non-partitioned table for which lifecycle is not set, for example, for the test_lifecycle_exists table, and set the lifecycle to 50 days.
              alter table test_lifecycle_exists
+SET TBLPROPERTIES(
+    "dli.lifecycle.days"='50',
+    "dli.table.lifecycle.status"='enable'
+);
              +
            +
            +
            +
            +
            +
            Parent topic: Table Lifecycle Management
            +
            +
            + diff --git a/docs/dli/sqlreference/dli_08_0468.html b/docs/dli/sqlreference/dli_08_0468.html new file mode 100644 index 000000000..502fa91be --- /dev/null +++ b/docs/dli/sqlreference/dli_08_0468.html @@ -0,0 +1,81 @@ + + +

            Disabling or Restoring the Lifecycle of a Table

            +

            Function

            This section describes how to disable or restore the lifecycle of a specified table or partition.

            +
            You can disable or restore the lifecycle of a table in either of the following scenarios:
            1. If the lifecycle function has been enabled for a table or partitioned table, the system allows you to disable or restore the lifecycle of the table by changing the value of dli.table.lifecycle.status.
            2. If the lifecycle function is not enabled for a table or partitioned table, the system will add the dli.table.lifecycle.status property to allow you to disable or restore the lifecycle function of the table.
            +
            +
            +

            Notes and Constraints

            • The table lifecycle function currently only supports creating tables and versioning tables using Hive and Datasource syntax.
            • The unit of the lifecycle is in days. The value should be a positive integer.
            • The lifecycle can be set only at the table level. The lifecycle specified for a partitioned table applies to all partitions of the table.
            +
            +

            Syntax

            • This syntax can be used to disable or restore the lifecycle of a table at the table level.
              1
              ALTER TABLE table_name SET TBLPROPERTIES("dli.table.lifecycle.status"={enable|disable});
+
              +
              +
            +
            • This syntax can be used to disable or restore the lifecycle of a specified table at the table or partition table level.
              1
              ALTER TABLE table_name [pt_spec] LIFECYCLE {enable|disable};
+
              +
              +
            +
            +

            Keywords

            TBLPROPERTIES: Table properties, which can be used to extend the lifecycle of a table.

            +
            +

            Parameters

            +
            + + + + + + + + + + + + + + + + + + + + + +
            Table 1 Parameters

            Parameter

            +

            Mandatory

            +

            Description

            +

            table_name

            +

            Yes

            +

            Name of the table whose lifecycle is to be disabled or restored

            +

            pt_spec

            +

            No

            +

            Partition information of the table whose lifecycle is to be disabled or restored. The format is partition_col1=col1_value1, partition_col2=col2_value1.... For a table with multi-level partitions, all partition values must be specified.

            +

            enable

            +

            No

            +
            Restores the lifecycle function of a table or a specified partition.
            • The table and its partitions participate in lifecycle reclamation again. By default, the lifecycle configuration of the current table and its partitions is used.
            • Before enabling the table lifecycle function, it is recommended to modify the lifecycle configuration of the table and its partitions. This will help prevent any accidental data reclamation caused by the previous configuration once the table lifecycle function is enabled.
            +
            +

            disable

            +

            No

            +

            Disables the lifecycle function of a table or a specified partition.

            +
            • Prevents a table and all its partitions from being reclaimed by the lifecycle. It takes priority over restoring the lifecycle of a table and its partitions. That is, when the lifecycle function of a table or a specified partition is disabled, the partition information of the table whose lifecycle is to be disabled or restored is invalid.
            • After the lifecycle function of a table is disabled, the lifecycle configuration of the table and the enable and disable flags of its partitions are retained.
            • After the lifecycle function of a table is disabled, the lifecycle configuration of the table and partitioned table can still be modified.
            +
            +
            +
            +

            Example

            • Example 1: Disable the lifecycle function of the test_lifecycle table.
              1
              alter table test_lifecycle SET TBLPROPERTIES("dli.table.lifecycle.status"='disable');
+
              +
              +
            +
            • Example 2: Disable the lifecycle function for the partition whose time is 20230520 in the test_lifecycle table.
              1
              alter table test_lifecycle partition (dt='20230520') LIFECYCLE 'disable';
+
              +
              +
              • After the lifecycle function of a partitioned table is disabled, the lifecycle function of all partitions within the table will also be disabled.
              +
              +
            +
            +
            +
            +
            +
            Parent topic: Table Lifecycle Management
            +
            +
            + diff --git a/docs/dli/sqlreference/dli_08_0470.html b/docs/dli/sqlreference/dli_08_0470.html index 51f94216c..f6c73d533 100644 --- a/docs/dli/sqlreference/dli_08_0470.html +++ b/docs/dli/sqlreference/dli_08_0470.html @@ -58,7 +58,6 @@

            Example

            Change the comment of the c1 column in the t1 table to the new comment.

            1
            ALTER TABLE t1 CHANGE COLUMN c1 c1 STRING COMMENT 'the new comment';
            -
            diff --git a/docs/dli/sqlreference/dli_08_0475.html b/docs/dli/sqlreference/dli_08_0475.html index 36cd06507..f5253fe24 100644 --- a/docs/dli/sqlreference/dli_08_0475.html +++ b/docs/dli/sqlreference/dli_08_0475.html @@ -1,8 +1,11 @@ -

            Window Functions

            -

            + +

            Window Functions

            + +

            +
            • Overview
              diff --git a/docs/dli/sqlreference/dli_08_15001.html b/docs/dli/sqlreference/dli_08_15001.html new file mode 100644 index 000000000..20c9ab524 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15001.html @@ -0,0 +1,21 @@ + + +

              Constraints and Definitions

              +
              + + diff --git a/docs/dli/sqlreference/dli_08_15002.html b/docs/dli/sqlreference/dli_08_15002.html new file mode 100644 index 000000000..20511278a --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15002.html @@ -0,0 +1,13 @@ + + +

              Supported Data Types

              +

              DLI supports the following data types:

              +

              CHAR, VARCHAR, STRING, BOOLEAN, BINARY, VARBINARY, BYTES, DECIMAL, TINYINT, SMALLINT, INTEGER, BIGINT, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, TIMESTAMP_LTZ, INTERVAL, ARRAY, MULTISET, MAP, ROW, and RAW.

              +

              In the SQL syntax, these types are used to define the data types of columns within a table.

              +
              +
              +
              +
              Parent topic: Constraints and Definitions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15003.html b/docs/dli/sqlreference/dli_08_15003.html new file mode 100644 index 000000000..5080163e1 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15003.html @@ -0,0 +1,13 @@ + + +

              Reserved Keywords

              +

              Certain combinations of strings have been reserved as keywords for future use.

              +

              If you use any of the following strings as field names, enclose them in backticks when using them, for example: `value`, `count`.

              +

              A, ABS, ABSOLUTE, ACTION, ADA, ADD, ADMIN, AFTER, ALL, ALLOCATE, ALLOW, ALTER, ALWAYS, AND, ANY, ARE, ARRAY, AS, ASC, ASENSITIVE, ASSERTION, ASSIGNMENT, ASYMMETRIC, AT, ATOMIC, ATTRIBUTE, ATTRIBUTES, AUTHORIZATION, AVG, BEFORE, BEGIN, BERNOULLI, BETWEEN, BIGINT, BINARY, BIT, BLOB, BOOLEAN, BOTH, BREADTH, BY, BYTES, C, CALL, CALLED, CARDINALITY, CASCADE, CASCADED, CASE, CAST, CATALOG, CATALOG_NAME, CEIL, CEILING, CENTURY, CHAIN, CHAR, CHARACTER, CHARACTERISTICS, CHARACTERS, CHARACTER_LENGTH, CHARACTER_SET_CATALOG, CHARACTER_SET_NAME, CHARACTER_SET_SCHEMA, CHAR_LENGTH, CHECK, CLASS_ORIGIN, CLOB, CLOSE, COALESCE, COBOL, COLLATE, COLLATION, COLLATION_CATALOG, COLLATION_NAME, COLLATION_SCHEMA, COLLECT, COLUMN, COLUMNS, COLUMN_NAME, COMMAND_FUNCTION, COMMAND_FUNCTION_CODE, COMMIT, COMMITTED, CONDITION, CONDITION_NUMBER, CONNECT, CONNECTION, CONNECTION_NAME, CONSTRAINT, CONSTRAINTS, CONSTRAINT_CATALOG, CONSTRAINT_NAME, CONSTRAINT_SCHEMA, CONSTRUCTOR, CONTAINS, CONTINUE, CONVERT, CORR, CORRESPONDING, COUNT, COVAR_POP, COVAR_SAMP, CREATE, CROSS, CUBE, CUME_DIST, CURRENT, CURRENT_CATALOG, CURRENT_DATE, CURRENT_DEFAULT_TRANSFORM_GROUP, CURRENT_PATH, CURRENT_ROLE, CURRENT_SCHEMA, CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_TRANSFORM_GROUP_FOR_TYPE, CURRENT_USER, CURSOR, CURSOR_NAME, CYCLE, DATA, DATABASE, DATE, DATETIME_INTERVAL_CODE, DATETIME_INTERVAL_PRECISION, DAY, DEALLOCATE, DEC, DECADE, DECIMAL, DECLARE, DEFAULT, DEFAULTS, DEFERRABLE, DEFERRED, DEFINED, DEFINER, DEGREE, DELETE, DENSE_RANK, DEPTH, DEREF, DERIVED, DESC, DESCRIBE, DESCRIPTION, DESCRIPTOR, DETERMINISTIC, DIAGNOSTICS, DISALLOW, DISCONNECT, DISPATCH, DISTINCT, DOMAIN, DOUBLE, DOW, DOY, DROP, DYNAMIC, DYNAMIC_FUNCTION, DYNAMIC_FUNCTION_CODE, EACH, ELEMENT, ELSE, END, END-EXEC, EPOCH, EQUALS, ESCAPE, EVERY, EXCEPT, EXCEPTION, EXCLUDE, EXCLUDING, EXEC, EXECUTE, EXISTS, EXP, EXPLAIN, EXTEND, EXTERNAL, EXTRACT, FALSE, FETCH, FILTER, FINAL, FIRST, FIRST_VALUE, FLOAT, FLOOR, FOLLOWING, FOR, FOREIGN, FORTRAN, FOUND, FRAC_SECOND, FREE, FROM, FULL, FUNCTION, FUSION, G, GENERAL, GENERATED, GET, GLOBAL, GO, GOTO, GRANT, GRANTED, GROUP, GROUPING, HAVING, HIERARCHY, HOLD, HOUR, IDENTITY, IMMEDIATE, IMPLEMENTATION, IMPORT, IN, INCLUDING, INCREMENT, INDICATOR, INITIALLY, INNER, INOUT, INPUT, INSENSITIVE, INSERT, INSTANCE, INSTANTIABLE, INT, INTEGER, INTERSECT, INTERSECTION, INTERVAL, INTO, INVOKER, IS, ISOLATION, JAVA, JOIN, K, KEY, KEY_MEMBER, KEY_TYPE, LABEL, LANGUAGE, LARGE, LAST, LAST_VALUE, LATERAL, LEADING, LEFT, LENGTH, LEVEL, LIBRARY, LIKE, LIMIT, LN, LOCAL, LOCALTIME, LOCALTIMESTAMP, LOCATOR, LOWER, M, MAP, MATCH, MATCHED, MAX, MAXVALUE, MEMBER, MERGE, MESSAGE_LENGTH, MESSAGE_OCTET_LENGTH, MESSAGE_TEXT, METHOD, MICROSECOND, MILLENNIUM, MIN, MINUTE, MINVALUE, MOD, MODIFIES, MODULE, MODULES, MONTH, MORE, MULTISET, MUMPS, NAME, NAMES, NATIONAL, NATURAL, NCHAR, NCLOB, NESTING, NEW, NEXT, NO, NONE, NORMALIZE, NORMALIZED, NOT, NULL, NULLABLE, NULLIF, NULLS, NUMBER, NUMERIC, OBJECT, OCTETS, OCTET_LENGTH, OF, OFFSET, OLD, ON, ONLY, OPEN, OPTION, OPTIONS, OR, ORDER, ORDERING, ORDINALITY, OTHERS, OUT, OUTER, OUTPUT, OVER, OVERLAPS, OVERLAY, OVERRIDING, PAD, PARAMETER, PARAMETER_MODE, PARAMETER_NAME, PARAMETER_ORDINAL_POSITION, PARAMETER_SPECIFIC_CATALOG, PARAMETER_SPECIFIC_NAME, PARAMETER_SPECIFIC_SCHEMA, PARTIAL, PARTITION, PASCAL, PASSTHROUGH, PATH, PERCENTILE_CONT, PERCENTILE_DISC, PERCENT_RANK, PLACING, PLAN, PLI, POSITION, POWER, PRECEDING, PRECISION, PREPARE, PRESERVE, PRIMARY, PRIOR, PRIVILEGES, PROCEDURE, PUBLIC, QUARTER, RANGE, RANK, RAW, READ, READS, REAL, RECURSIVE, REF, REFERENCES, REFERENCING, REGR_AVGX, REGR_AVGY, REGR_COUNT, REGR_INTERCEPT, REGR_R2, REGR_SLOPE, REGR_SXX, REGR_SXY, REGR_SYY, RELATIVE, RELEASE, REPEATABLE, RESET, RESTART, RESTRICT, RESULT, RETURN, RETURNED_CARDINALITY, RETURNED_LENGTH, RETURNED_OCTET_LENGTH, RETURNED_SQLSTATE, RETURNS, REVOKE, RIGHT, ROLE, ROLLBACK, ROLLUP, ROUTINE, ROUTINE_CATALOG, ROUTINE_NAME, ROUTINE_SCHEMA, ROW, ROWS, ROW_COUNT, ROW_NUMBER, SAVEPOINT, SCALE, SCHEMA, SCHEMA_NAME, SCOPE, SCOPE_CATALOGS, SCOPE_NAME, SCOPE_SCHEMA, SCROLL, SEARCH, SECOND, SECTION, SECURITY, SELECT, SELF, SENSITIVE, SEQUENCE, SERIALIZABLE, SERVER, SERVER_NAME, SESSION, SESSION_USER, SET, SETS, SIMILAR, SIMPLE, SIZE, SMALLINT, SOME, SOURCE, SPACE, SPECIFIC, SPECIFICTYPE, SPECIFIC_NAME, SQL, SQLEXCEPTION, SQLSTATE, SQLWARNING, SQL_TSI_DAY, SQL_TSI_FRAC_SECOND, SQL_TSI_HOUR, SQL_TSI_MICROSECOND, SQL_TSI_MINUTE, SQL_TSI_MONTH, SQL_TSI_QUARTER, SQL_TSI_SECOND, SQL_TSI_WEEK, SQL_TSI_YEAR, SQRT, START, STATE, STATEMENT, STATIC, STDDEV_POP, STDDEV_SAMP, STREAM, STRING, STRUCTURE, STYLE, SUBCLASS_ORIGIN, SUBMULTISET, SUBSTITUTE, SUBSTRING, SUM, SYMMETRIC, SYSTEM, SYSTEM_USER, TABLE, TABLESAMPLE, TABLE_NAME, TEMPORARY, THEN, TIES, TIME, TIMESTAMP, TIMESTAMPADD, TIMESTAMPDIFF, TIMEZONE_HOUR, TIMEZONE_MINUTE, TINYINT, TO, TOP_LEVEL_COUNT, TRAILING, TRANSACTION, TRANSACTIONS_ACTIVE, TRANSACTIONS_COMMITTED, TRANSACTIONS_ROLLED_BACK, TRANSFORM, TRANSFORMS, TRANSLATE, TRANSLATION, TREAT, TRIGGER, TRIGGER_CATALOG, TRIGGER_NAME, TRIGGER_SCHEMA, TRIM, TRUE, TYPE, UESCAPE, UNBOUNDED, UNCOMMITTED, UNDER, UNION, UNIQUE, UNKNOWN, UNNAMED, UNNEST, UPDATE, UPPER, UPSERT, USAGE, USER, USER_DEFINED_TYPE_CATALOG, USER_DEFINED_TYPE_CODE, USER_DEFINED_TYPE_NAME, USER_DEFINED_TYPE_SCHEMA, USING, VALUE, VALUES, VARBINARY, VARCHAR, VARYING, VAR_POP, VAR_SAMP, VERSION, VIEW, WEEK, WHEN, WHENEVER, WHERE, WIDTH_BUCKET, WINDOW, WITH, WITHIN, WITHOUT, WORK, WRAPPER, WRITE, XML, YEAR, ZONE

              +
              +
              +
              +
              Parent topic: Constraints and Definitions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15005.html b/docs/dli/sqlreference/dli_08_15005.html new file mode 100644 index 000000000..798394c5f --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15005.html @@ -0,0 +1,23 @@ + + +

              Data Definition Language (DDL)

              +
              + + diff --git a/docs/dli/sqlreference/dli_08_15006.html b/docs/dli/sqlreference/dli_08_15006.html new file mode 100644 index 000000000..a3ba51a7e --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15006.html @@ -0,0 +1,86 @@ + + +

              CREATE TABLE

              +

              Function

              This statement creates a table using the specified table name. However, if a table with the same name already exists in the catalog, the registration process will fail.

              +
              +

              Syntax

              CREATE TABLE [IF NOT EXISTS] [catalog_name.][db_name.]table_name
+  (
+    { <physical_column_definition> | <metadata_column_definition> | <computed_column_definition> }[ , ...n]
+    [ <watermark_definition> ]
+    [ <table_constraint> ][ , ...n]
+  )
+  [COMMENT table_comment]
+  [PARTITIONED BY (partition_column_name1, partition_column_name2, ...)]
+  WITH (key1=val1, key2=val2, ...)
+  [ LIKE source_table [( <like_options> )] ]
+
+<physical_column_definition>:
+  column_name column_type [ <column_constraint> ] [COMMENT column_comment]
+
+<column_constraint>:
+  [CONSTRAINT constraint_name] PRIMARY KEY NOT ENFORCED
+
+<table_constraint>:
+  [CONSTRAINT constraint_name] PRIMARY KEY (column_name, ...) NOT ENFORCED
+
+<metadata_column_definition>:
+  column_name column_type METADATA [ FROM metadata_key ] [ VIRTUAL ]
+
+<computed_column_definition>:
+  column_name AS computed_column_expression [COMMENT column_comment]
+
+<watermark_definition>:
+  WATERMARK FOR rowtime_column_name AS watermark_strategy_expression
+
+<source_table>:
+  [catalog_name.][db_name.]table_name
+
+<like_options>:
+{
+   { INCLUDING | EXCLUDING } { ALL | CONSTRAINTS | PARTITIONS }
+ | { INCLUDING | EXCLUDING | OVERWRITING } { GENERATED | OPTIONS | WATERMARKS } 
+}[, ...]
              +
              +

              Description

              COMPUTED COLUMN

              +

              A computed column is a virtual column generated using column_name AS computed_column_expression. A computed column evaluates an expression that can reference other columns declared in the same table. The column itself is not physically stored within the table. A computed column could be defined using cost AS price * quantity. This expression can contain any combination of physical columns, constants, functions, or variables, but cannot contain any subquery.

              +

              In Flink, a computed column is used to define the time attribute in CREATE TABLE statements. A processing time attribute can be defined easily via proc AS PROCTIME() using the system's PROCTIME() function. The event time column may be obtained from an existing field. In this case, you can use the computed column to obtain event time. For example, if the original field is not of the TIMESTAMP(3) type or is nested in a JSON string, you can use computed columns.

              +

              Note:

              +
              • An expression that defines a computed column in a source table is calculated after data is read from the data source. The column can be used in the SELECT statement.
              • A computed column cannot be the target of an INSERT statement. In an INSERT statement, the schema of the SELECT statement must be the same as that of the target table that does not have a computed column.
              +
              +

              WATERMARK

              +

              The WATERMARK clause defines the event time attribute of a table and takes the form WATERMARK FOR rowtime_column_name AS watermark_strategy_expression.

              +

              rowtime_column_name defines an existing column that is marked as the event time attribute of the table. The column must be of the TIMESTAMP(3) type and must be the top-level column in the schema. It can also be a computed column.

              +

              watermark_strategy_expression defines the watermark generation strategy. It allows arbitrary non-query expressions, including computed columns, to calculate the watermark. The expression return type must be TIMESTAMP(3), which represents the timestamp since the Epoch. The returned watermark will be emitted only if it is non-null and its value is greater than the previously emitted local watermark (to preserve the contract of ascending watermarks). The watermark generation expression is evaluated by the framework for every record. The framework will periodically emit the largest generated watermark. If the current watermark is still identical to the previous one, or is null, or the value of the returned watermark is smaller than that of the last emitted one, then no new watermark will be emitted. A watermark is emitted in an interval defined by pipeline.auto-watermark-interval. If the watermark interval is 0 ms, a watermark will be emitted per record if it is not null and greater than the last emitted one.

              +

              When using event time semantics, tables must contain an event time attribute and watermark strategy.

              +

              Flink provides several commonly used watermark strategies.

              +
              • Strictly ascending timestamps: WATERMARK FOR rowtime_column AS rowtime_column

                Emits a watermark of the maximum observed timestamp so far. Rows that have a timestamp bigger than the maximum timestamp are not late.

                +
              • Ascending timestamps: WATERMARK FOR rowtime_column AS rowtime_column - INTERVAL '0.001' SECOND

                Emits a watermark of the maximum observed timestamp so far minus 1. Rows that have a timestamp bigger than or equal to the maximum timestamp are not late.

                +
              • Bounded out-of-order timestamps: WATERMARK FOR rowtime_column AS rowtime_column - INTERVAL 'string' timeUnit

                Emits a watermark, which is the maximum observed timestamp minus the specified delay, for example, WATERMARK FOR rowtime_column AS rowtime_column - INTERVAL '5' SECOND is a 5-second delayed watermark strategy.

                +
                CREATE TABLE Orders (
+    user BIGINT,
+    product STRING,
+    order_time TIMESTAMP(3),
+    WATERMARK FOR order_time AS order_time - INTERVAL '5' SECOND
+) WITH ( . . . );
                +
              +

              PRIMARY KEY

              +

              The primary key constraint is a hint for Flink to leverage for optimizations. It tells that a column or a set of columns of a table or a view are unique and they do not contain null. Neither of columns in a primary can be nullable. The primary key therefore uniquely identifies a row in a table.

              +

              The primary key constraint can be either declared along with a column definition (a column constraint) or as a single line (a table constraint). For both cases, it should only be declared as a singleton. If you define multiple primary key constraints at the same time, an exception would be thrown.

              +

              Validity Check

              +

              SQL standard specifies that a constraint can either be ENFORCED or NOT ENFORCED. This controls if the constraint checks are performed on the incoming/outgoing data. Flink does not own the data and therefore the only mode we want to support is the NOT ENFORCED mode. It is up to the user to ensure that the query enforces key integrity.

              +

              Flink will assume correctness of the primary key by assuming that the columns nullability is aligned with the columns in the primary key. Connectors should ensure those are aligned.

              +

              Note: In a CREATE TABLE statement, creating a primary key constraint will alter the columns nullability, which means, a column with a primary key constraint is not nullable.

              +

              PARTITIONED BY

              +

              Partition the created table by the specified columns. A directory is created for each partition if this table is used as a file system sink.

              +

              WITH OPTIONS

              +

              Table properties used to create a table source/sink. The properties are usually used to find and create the underlying connector.

              +

              The key and value of expression key1=val1 should both be string literal.

              +

              Note: The table name can be in any of the following formats: 1. catalog_name.db_name.table_name 2. db_name.table_name 3. table_name. Tables named in the catalog_name.db_name.table_name format are registered with metastore along with the catalog named catalog_name and the database named db_name. Tables named in the uses db_name.table_name format will be registered with the current table environment's catalog and the database will be named db_name. Tables named in the table_name format will be registered with the running catalog and database.

              +

              Note: Tables registered using the CREATE TABLE statement can be used as both the table source and table sink. We cannot decide if it is used as a source or sink until it is referenced in the DMLs.

              +
              +
              +
              +
              Parent topic: Data Definition Language (DDL)
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15007.html b/docs/dli/sqlreference/dli_08_15007.html new file mode 100644 index 000000000..fd9ee408d --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15007.html @@ -0,0 +1,19 @@ + + +

              CREATE CATALOG

              +

              Function

              This statement creates a catalog using specified attributes. If a catalog with the same name already exists, an exception is thrown.

              +
              +

              Syntax

              CREATE CATALOG catalog_name
+  WITH (key1=val1, key2=val2, ...)
              +
              +

              Description

              WITH OPTIONS

              +

              Catalog attributes typically store additional information about the catalog.

              +

              The key and value of the key1=val1 expression are string literals.

              +
              +
              +
              +
              +
              Parent topic: Data Definition Language (DDL)
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15008.html b/docs/dli/sqlreference/dli_08_15008.html new file mode 100644 index 000000000..332901d1c --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15008.html @@ -0,0 +1,22 @@ + + +

              CREATE DATABASE

              +

              Function

              This statement creates a database using specified table attributes. If a table with the same name already exists in the database, an exception is thrown.

              +
              +

              Syntax

              CREATE DATABASE [IF NOT EXISTS] [catalog_name.]db_name
+  [COMMENT database_comment]
+  WITH (key1=val1, key2=val2, ...)
              +
              +

              Description

              IF NOT EXISTS

              +

              If the database already exists, no operation is performed.

              +

              WITH OPTIONS

              +

              Database attributes typically store additional information about the database.

              +

              The key and value of the key1=val1 expression are string literals.

              +
              +
              +
              +
              +
              Parent topic: Data Definition Language (DDL)
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15009.html b/docs/dli/sqlreference/dli_08_15009.html new file mode 100644 index 000000000..d53724a41 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15009.html @@ -0,0 +1,24 @@ + + +

              CREATE VIEW

              +

              Function

              This statement creates a view based on the given query statement. If a view with the same name already exists in the database, an exception is thrown.

              +
              +

              Syntax

              CREATE [TEMPORARY] VIEW [IF NOT EXISTS] [catalog_name.][db_name.]view_name
+  [{columnName [, columnName ]* }] [COMMENT view_comment]
+  AS query_expression
              +
              +

              Description

              TEMPORARY

              +

              Create a temporary view with catalogs and database namespaces and overwrite the original view.

              +

              IF NOT EXISTS

              +

              If the view already exists, nothing happens.

              +
              +

              Example

              Create a view named viewName.

              +
              create view viewName as select * from dataSource
              +
              +
              +
              +
              +
              Parent topic: Data Definition Language (DDL)
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15010.html b/docs/dli/sqlreference/dli_08_15010.html new file mode 100644 index 000000000..d6842cec9 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15010.html @@ -0,0 +1,29 @@ + + +

              CREATE FUNCTION

              +

              Function

              To create a catalog function with a catalog and a database namespace, you will need to specify an identifier. You can specify a language tag. You cannot register the function if a function with the same name has already been registered in the catalog. If the language tag is JAVA or SCALA, the identifier is the fully qualified name of the UDF implementation class.

              +

              For details about how to create a UDF, see UDFs.

              +
              +

              Syntax

              CREATE [TEMPORARY|TEMPORARY SYSTEM] FUNCTION
+  [IF NOT EXISTS] [[catalog_name.]db_name.]function_name
+  AS identifier [LANGUAGE JAVA|SCALA]
              +
              +

              Description

              TEMPORARY

              +

              Create a temporary catalog function with catalogs and database namespaces and overwrite the original catalog function.

              +

              TEMPORARY SYSTEM

              +

              Create a temporary system catalog function without database namespaces and overwrite the system's built-in functions.

              +

              IF NOT EXISTS

              +

              If the function already exists, nothing happens.

              +

              LANGUAGE JAVA|SCALA

              +

              The language tag is used to instruct Flink runtime how to execute the function. Currently, only JAVA and SCALA language tags are supported, and the default language for a function is JAVA.

              +
              +

              Example

              Create a function named STRINGBACK.

              +
              create function STRINGBACK as 'com.dli.StringBack'
              +
              +
              +
              +
              +
              Parent topic: Data Definition Language (DDL)
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15011.html b/docs/dli/sqlreference/dli_08_15011.html new file mode 100644 index 000000000..ba52f3c14 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15011.html @@ -0,0 +1,136 @@ + + +

              Data Manipulation Language (DML)

              +

              Constraints and Limitations

              • Flink SQL uses a lexical policy for identifier (table, attribute, function names) similar to Java:
                • The case of identifiers is preserved whether they are quoted.
                • Identifiers are matched case-sensitively.
                • Unlike Java, back-ticks allow identifiers to contain non-alphanumeric characters (for example, SELECT a AS `my field` FROM t).
                +
              • String literals must be enclosed in single quotes (for example, SELECT'Hello World'). Duplicate a single quote for escaping (for example, SELECT 'It''s me.'). Unicode characters are supported in string literals. If explicit Unicode points are required, use the following syntax:
                • Use the backslash (\) as an escaping character (default): SELECT U&'\263A'
                • Use a custom escaping character: SELECT U&'#263A' UESCAPE '#'
                +
              +
              +

              Syntax

              INSERT INTO table_name [PARTITION part_spec] query
+
+part_spec:  (part_col_name1=val1 [, part_col_name2=val2, ...])
+
+query:
+  values
+  | {
+      select
+      | selectWithoutFrom
+      | query UNION [ ALL ] query
+      | query EXCEPT query
+      | query INTERSECT query
+    }
+    [ ORDER BY orderItem [, orderItem ]* ]
+    [ LIMIT { count | ALL } ]
+    [ OFFSET start { ROW | ROWS } ]
+    [ FETCH { FIRST | NEXT } [ count ] { ROW | ROWS } ONLY]
+
+orderItem:
+  expression [ ASC | DESC ]
+
+select:
+  SELECT [ ALL | DISTINCT ]
+  { * | projectItem [, projectItem ]* }
+  FROM tableExpression
+  [ WHERE booleanExpression ]
+  [ GROUP BY { groupItem [, groupItem ]* } ]
+  [ HAVING booleanExpression ]
+  [ WINDOW windowName AS windowSpec [, windowName AS windowSpec ]* ]
+
+selectWithoutFrom:
+  SELECT [ ALL | DISTINCT ]
+  { * | projectItem [, projectItem ]* }
+
+projectItem:
+  expression [ [ AS ] columnAlias ]
+  | tableAlias . *
+
+tableExpression:
+  tableReference [, tableReference ]*
+  | tableExpression [ NATURAL ] [ LEFT | RIGHT | FULL ] JOIN tableExpression [ joinCondition ]
+
+joinCondition:
+  ON booleanExpression
+  | USING '(' column [, column ]* ')'
+
+tableReference:
+  tablePrimary
+  [ matchRecognize ]
+  [ [ AS ] alias [ '(' columnAlias [, columnAlias ]* ')' ] ]
+
+tablePrimary:
+  [ TABLE ] [ [ catalogName . ] schemaName . ] tableName
+  | LATERAL TABLE '(' functionName '(' expression [, expression ]* ')' ')'
+  | UNNEST '(' expression ')'
+
+values:
+  VALUES expression [, expression ]*
+
+groupItem:
+  expression
+  | '(' ')'
+  | '(' expression [, expression ]* ')'
+  | CUBE '(' expression [, expression ]* ')'
+  | ROLLUP '(' expression [, expression ]* ')'
+  | GROUPING SETS '(' groupItem [, groupItem ]* ')'
+
+windowRef:
+    windowName
+  | windowSpec
+
+windowSpec:
+    [ windowName ]
+    '('
+    [ ORDER BY orderItem [, orderItem ]* ]
+    [ PARTITION BY expression [, expression ]* ]
+    [
+        RANGE numericOrIntervalExpression {PRECEDING}
+      | ROWS numericExpression {PRECEDING}
+    ]
+    ')'
+
+matchRecognize:
+      MATCH_RECOGNIZE '('
+      [ PARTITION BY expression [, expression ]* ]
+      [ ORDER BY orderItem [, orderItem ]* ]
+      [ MEASURES measureColumn [, measureColumn ]* ]
+      [ ONE ROW PER MATCH ]
+      [ AFTER MATCH
+            ( SKIP TO NEXT ROW
+            | SKIP PAST LAST ROW
+            | SKIP TO FIRST variable
+            | SKIP TO LAST variable
+            | SKIP TO variable )
+      ]
+      PATTERN '(' pattern ')'
+      [ WITHIN intervalLiteral ]
+      DEFINE variable AS condition [, variable AS condition ]*
+      ')'
+
+measureColumn:
+      expression AS alias
+
+pattern:
+      patternTerm [ '|' patternTerm ]*
+
+patternTerm:
+      patternFactor [ patternFactor ]*
+
+patternFactor:
+      variable [ patternQuantifier ]
+
+patternQuantifier:
+      '*'
+  |   '*?'
+  |   '+'
+  |   '+?'
+  |   '?'
+  |   '??'
+  |   '{' { [ minRepeat ], [ maxRepeat ] } '}' ['?']
+  |   '{' repeat '}'
              +
              +
              +
              +
              +
              Parent topic: Constraints and Definitions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15012.html b/docs/dli/sqlreference/dli_08_15012.html new file mode 100644 index 000000000..7ff915486 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15012.html @@ -0,0 +1,101 @@ + + +

              Overview

              +

              This section describes the Flink OpenSource SQL 1.15 syntax supported by DLI. For details about the parameters and examples, see the syntax description.

              +

              Creating Tables

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Syntax for creating tables

              Classification

              +

              Function

              +

              Format

              +

              Avro

              +

              Canal

              +

              Confluent Avro

              +

              CSV

              +

              Debezium

              +

              JSON

              +

              Maxwell

              +

              Ogg

              +

              Orc

              +

              Parquet

              +

              Raw

              +

              Connectors

              +

              BlackHole

              +

              ClickHouse

              +

              DataGen

              +

              Doris

              +

              DWS

              +

              Elasticsearch

              +

              FileSystem

              +

              Hbase

              +

              Hive

              +

              JDBC

              +

              Kafka

              +

              Print

              +

              Redis

              +

              Upsert Kafka

              +
              +
              +
              +
              + + diff --git a/docs/dli/sqlreference/dli_08_15014.html b/docs/dli/sqlreference/dli_08_15014.html new file mode 100644 index 000000000..898e907d5 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15014.html @@ -0,0 +1,37 @@ + + +

              Formats

              +
              +
              + + +
              +
              Parent topic: Flink OpenSource SQL 1.15 Syntax Reference
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15015.html b/docs/dli/sqlreference/dli_08_15015.html new file mode 100644 index 000000000..c1f87f6ad --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15015.html @@ -0,0 +1,77 @@ + + +

              Overview

              +

              Flink provides a set of table formats that can be used with table connectors.

              +

              A table format is a storage format that defines how to map binary data onto table columns.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Formats supported by Flink

              Formats

              +

              Supported Connectors

              +

              CSV

              +

              Kafka, Upsert Kafka, FileSystem

              +

              JSON

              +

              Kafka, Upsert Kafka, FileSystem, Elasticsearch

              +

              Avro

              +

              Kafka, Upsert Kafka, FileSystem

              +

              Confluent Avro

              +

              Kafka, Upsert Kafka

              +

              Debezium

              +

              Kafka, FileSystem

              +

              Canal

              +

              Kafka, FileSystem

              +

              Maxwell

              +

              Kafka, FileSystem

              +

              Ogg

              +

              Kafka, FileSystem

              +

              Orc

              +

              FileSystem

              +

              Parquet

              +

              FileSystem

              +

              Raw

              +

              Kafka, Upsert Kafka, FileSystem

              +
              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15016.html b/docs/dli/sqlreference/dli_08_15016.html new file mode 100644 index 000000000..83575acd7 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15016.html @@ -0,0 +1,231 @@ + + +

              Avro

              +

              Function

              Apache Avro is supported for you to read and write Avro data based on an Avro schema with Flink. The Avro schema is derived from the table schema.

              +

              For details, see Avro Format.

              +
              +

              Supported Connectors

              • Kafka
              • Upsert Kafka
              • FileSystem
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameters

              Parameter

              +

              Mandatory

              +

              Default value

              +

              Type

              +

              Description

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              Format to be used. Set the value to avro.

              +

              avro.codec

              +

              No

              +

              None

              +

              String

              +

              For Filesystem only, the compression codec for avro. Snappy compression as default. The valid enumerations are: null, deflate, snappy, bzip2, and xz.

              +
              +
              +
              +

              Data Type Mapping

              Currently, the Avro schema is derived from the table schema and cannot be explicitly defined. The following table lists mappings between Flink to Avro types.

              +

              In addition to the following types, Flink supports reading/writing nullable types. Flink maps nullable types to Avro union(something, null), where something is an Avro type converted from Flink type.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Data type mapping

              Flink SQL Type

              +

              Avro Type

              +

              Avro Logical Type

              +

              CHAR/VARCHAR/STRING

              +

              String

              +

              -

              +

              BOOLEAN

              +

              Boolean

              +

              -

              +

              BINARY/VARBINARY

              +

              bytes

              +

              -

              +

              DECIMAL

              +

              fixed

              +

              decimal

              +

              TINYINT

              +

              int

              +

              -

              +

              SMALLINT

              +

              int

              +

              -

              +

              INT

              +

              int

              +

              -

              +

              BIGINT

              +

              long

              +

              -

              +

              FLOAT

              +

              float

              +

              -

              +

              DOUBLE

              +

              double

              +

              -

              +

              DATE

              +

              int

              +

              date

              +

              TIME

              +

              int

              +

              time-millis

              +

              TIMESTAMP

              +

              long

              +

              timestamp-millis

              +

              ARRAY

              +

              array

              +

              -

              +

              MAP (keys must be of the string, char, or varchar type.)

              +

              map

              +

              -

              +

              MULTISET (elements must be of the string, char, or varchar type.)

              +

              map

              +

              -

              +

              ROW

              +

              record

              +

              -

              +
              +
              +
              +

              Example

              Read data from Kafka, deserialize the data to the Avro format, and outputs the data to Print.

              +
              1. Create a datasource connection for access to the VPC and subnet where Kafka locates and bind the connection to the queue. Set a security group and inbound rule to allow access of the queue and test the connectivity of the queue using the Kafka IP address. For example, locate a general-purpose queue where the job runs and choose More > Test Address Connectivity in the Operation column. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Create a Flink OpenSource SQL job and select Flink 1.15. Copy the following statement and submit the job:

                CREATE TABLE kafkaSource (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'avro'
+);
+
+
+CREATE TABLE printSink (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'print'
+);
+insert into printSink select * from kafkaSource;
                +

              3. Insert the following data to Kafka using Avro data serialization:

                {"order_id":"202103241000000001","order_channel":"webShop","order_time":"2021-03-24 10:00:00","pay_amount":100.0,"real_pay":100.0,"pay_time":"2021-03-24 10:02:03","user_id":"0001","user_name":"Alice","area_id":"330106"}
+
+{"order_id":"202103241606060001","order_channel":"appShop","order_time":"2021-03-24 16:06:06","pay_amount":200.0,"real_pay":180.0,"pay_time":"2021-03-24 16:10:06","user_id":"0001","user_name":"Alice","area_id":"330106"}
                +

              4. Perform the following operations to view the data result in the taskmanager.out file:

                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the .out file, and view result logs.
                +
                +I[202103241000000001, webShop, 2021-03-24 10:00:00, 100.0, 100.0, 2021-03-24 10:02:03, 0001, Alice, 330106]
++I[202103241606060001, appShop, 2021-03-24 16:06:06, 200.0, 180.0, 2021-03-24 16:10:06, 0001, Alice, 330106]
                +

              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15017.html b/docs/dli/sqlreference/dli_08_15017.html new file mode 100644 index 000000000..3231e6e51 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15017.html @@ -0,0 +1,262 @@ + + +

              Canal

              +

              Function

              Canal is a Changelog Data Capture (CDC) tool that can stream changes in real-time from MySQL into other systems. Canal provides a unified format schema for changelog and supports to serialize messages using JSON and protobuf (the default format for Canal).

              +

              Flink supports to interpret Canal JSON messages as INSERT, UPDATE, and DELETE messages into the Flink SQL system. This is useful in many cases to leverage this feature, such as:

              +
              • synchronizing incremental data from databases to other systems
              • Auditing logs
              • Real-time materialized view on databases
              • Temporal join changing history of a database table, etc.
              +

              Flink also supports to encode the INSERT, UPDATE, and DELETE messages in Flink SQL as Canal JSON messages, and emit to storage like Kafka. However, currently Flink cannot combine UPDATE_BEFORE and UPDATE_AFTER into a single UPDATE message. Therefore, Flink encodes UPDATE_BEFORE and UPDATE_AFTER as DELETE and INSERT Canal messages.

              +

              For details, see Canal Format.

              +
              +

              Supported Connectors

              • Kafka
              • FileSystem
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameter description

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Type

              +

              Description

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              Format to be used. In this example.Set this parameter to canal-json.

              +

              canal-json.ignore-parse-errors

              +

              No

              +

              false

              +

              Boolean

              +

              Whether fields and rows with parse errors will be skipped or failed. The default value is false, indicating that an error will be thrown. Fields are set to null in case of errors.

              +

              canal-json.timestamp-format.standard

              +

              No

              +

              'SQL'

              +

              String

              +

              Input and output timestamp formats. Currently supported values are SQL and ISO-8601:

              +
              • SQL will parse input timestamp in "yyyy-MM-dd HH:mm:ss.s{precision}" format, for example 2020-12-30 12:13:14.123 and output timestamp in the same format.
              • ISO-8601 will parse input timestamp in "yyyy-MM-ddTHH:mm:ss.s{precision}" format, for example 2020-12-30T12:13:14.123 and output timestamp in the same format.
              +

              canal-json.map-null-key.mode

              +

              No

              +

              'FALL'

              +

              String

              +

              Handling mode when serializing null keys for map data. Available values are as follows:

              +
              • FAIL will throw exception when encountering map value with null key.
              • DROP will drop null key entries for map data.
              • LITERAL replaces the empty key value in the map with a string constant. The string literal is defined by canal-json.map-null-key.literal option.
              +

              canal-json.map-null-key.literal

              +

              No

              +

              'null'

              +

              String

              +

              String literal to replace null key when canal-json.map-null-key.mode is LITERAL.

              +

              canal-json.encode.decimal-as-plain-number

              +

              No

              +

              false

              +

              Boolean

              +

              Encode all decimals as plain numbers instead of possible scientific notations. By default, decimals may be written using scientific notation. For example, 0.000000027 is encoded as 2.7E-8 by default, and will be written as 0.000000027 if set this parameter to true.

              +

              canal-json.database.include

              +

              No

              +

              None

              +

              +

              String

              +

              +

              An optional regular expression to only read the specific databases changelog rows by regular matching the "database" meta field in the Canal record. The pattern string is compatible with Java's Pattern.

              +

              canal-json.table.include

              +

              No

              +

              None

              +

              String

              +

              An optional regular expression to only read the specific tables changelog rows by regular matching the "table" meta field in the Canal record. The pattern string is compatible with Java's Pattern.

              +
              +
              +
              +

              Metadata

              The following format metadata can be exposed as read-only (VIRTUAL) columns in DDL.

              +

              Format metadata fields are only available if the corresponding connector forwards format metadata. Currently, only the Kafka connector is able to expose metadata fields for its value format.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Metadata

              Key

              +

              Data Type

              +

              Description

              +

              database

              +

              STRING NULL

              +

              The originating database. Corresponds to the database field in the Canal record if available.

              +

              table

              +

              STRING NULL

              +

              The originating database table. Corresponds to the table field in the Canal record if available.

              +

              sql-type

              +

              MAP<STRING, INT> NULL

              +

              Map of various sql types. Corresponds to the sqlType field in the Canal record if available.

              +

              pk-names

              +

              ARRAY<STRING> NULL

              +

              Array of primary key names. Corresponds to the pkNames field in the Canal record if available.

              +

              ingestion-timestamp

              +

              TIMESTAMP_LTZ(3) NULL

              +

              The timestamp at which the connector processed the event. Corresponds to the ts field in the Canal record.

              +
              +
              +

              The following example shows how to access Canal metadata fields in Kafka:

              +
              CREATE TABLE KafkaTable (
+  origin_database STRING METADATA FROM 'value.database' VIRTUAL,
+  origin_table STRING METADATA FROM 'value.table' VIRTUAL,
+  origin_sql_type MAP<STRING, INT> METADATA FROM 'value.sql-type' VIRTUAL,
+  origin_pk_names ARRAY<STRING> METADATA FROM 'value.pk-names' VIRTUAL,
+  origin_ts TIMESTAMP(3) METADATA FROM 'value.ingestion-timestamp' VIRTUAL,
+  user_id BIGINT,
+  item_id BIGINT,
+  behavior STRING
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'earliest-offset',
+  'value.format' = 'canal-json'
+);
              +
              +

              Example

              Use canal-json to read Canal records in Kafka and output them to Print.

              +
              1. Create a datasource connection for the communication with the VPC and subnet where Kafka locates and bind the connection to the queue. Set a security group and inbound rule to allow access of the queue and test the connectivity of the queue using the Kafka IP address. For example, locate a general-purpose queue where the job runs and choose More > Test Address Connectivity in the Operation column. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Create a Flink OpenSource SQL job and select Flink 1.15. Copy the following statement and submit the job:

                create table kafkaSource(
+  id bigint,
+  name string,
+  description string,
+  weight DECIMAL(10, 2)
+  ) with (
+    'connector' = 'kafka',
+    'topic' = '<yourTopic>',
+    'properties.group.id' = '<yourGroupId>',
+    'properties.bootstrap.servers' = '<yourKafkaAddress>:<yourKafkaPort>',
+    'scan.startup.mode' = 'latest-offset',
+    'format' = 'canal-json'
+);
+create table printSink(
+  id bigint,
+  name string,
+  description string,
+  weight DECIMAL(10, 2)
+   ) with (
+     'connector' = 'print'
+   );
+insert into printSink select * from kafkaSource;
                +

              3. Insert the data below into the appropriate Kafka topics. The data shows that the MySQL products table has four columns: id, name, description, and weight. This JSON message is an update event on the products table, indicating that the value of the weight field has changed from 5.15 to 5.18 for the row with id = 111.

                {
+  "data": [
+    {
+      "id": "111",
+      "name": "scooter",
+      "description": "Big 2-wheel scooter",
+      "weight": "5.18"
+    }
+  ],
+  "database": "inventory",
+  "es": 1589373560000,
+  "id": 9,
+  "isDdl": false,
+  "mysqlType": {
+    "id": "INTEGER",
+    "name": "VARCHAR(255)",
+    "description": "VARCHAR(512)",
+    "weight": "FLOAT"
+  },
+  "old": [
+    {
+      "weight": "5.15"
+    }
+  ],
+  "pkNames": [
+    "id"
+  ],
+  "sql": "",
+  "sqlType": {
+    "id": 4,
+    "name": 12,
+    "description": 12,
+    "weight": 7
+  },
+  "table": "products",
+  "ts": 1589373560798,
+  "type": "UPDATE"
+}
                +

              4. Perform the following operations to view the data result in the taskmanager.out file:

                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the .out file, and view result logs.
                +
                -U[111, scooter, Big 2-wheel scooter, 5.15]
++U[111, scooter, Big 2-wheel scooter, 5.18]
                +

              +

              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15018.html b/docs/dli/sqlreference/dli_08_15018.html new file mode 100644 index 000000000..6db46fd5c --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15018.html @@ -0,0 +1,219 @@ + + +

              Confluent Avro

              +

              Function

              The Avro Schema Registry (avro-confluent) format allows you to read records that were serialized by the io.confluent.kafka.serializers.KafkaAvroSerializer and to write records that can in turn be read by the io.confluent.kafka.serializers.KafkaAvroDeserializer.

              +

              When reading (deserializing) a record with this format the Avro writer schema is fetched from the configured Confluent Schema Registry based on the schema version ID encoded in the record while the reader schema is inferred from table schema.

              +

              When writing (serializing) a record with this format the Avro schema is inferred from the table schema and used to retrieve a schema ID to be encoded with the data The lookup is performed with in the configured Confluent Schema Registry under the subject. The subject is specified by the avro-confluent.subject parameter.

              +
              +

              Supported Connectors

              • kafka
              • upsert kafka
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameter description

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Type

              +

              Description

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              Format to be used. Set this parameter to 'avro-confluent'.

              +

              avro-confluent.basic-auth.credentials-source

              +

              No

              +

              None

              +

              String

              +

              Basic auth credentials source for Schema Registry

              +

              avro-confluent.basic-auth.user-info

              +

              No

              +

              None

              +

              String

              +

              Basic auth user info for schema registry

              +

              avro-confluent.bearer-auth.credentials-source

              +

              No

              +

              None

              +

              String

              +

              Bearer auth credentials source for Schema Registry

              +

              avro-confluent.bearer-auth.token

              +

              No

              +

              None

              +

              String

              +

              Bearer auth token for Schema Registry

              +

              avro-confluent.properties

              +

              No

              +

              None

              +

              Map

              +

              Properties map that is forwarded to the underlying Schema Registry. This is useful for options that are not officially exposed via Flink config options. However, note that Flink options have higher precedence.

              +

              avro-confluent.ssl.keystore.location

              +

              No

              +

              None

              +

              String

              +

              Location/File of SSL keystore

              +

              avro-confluent.ssl.keystore.password

              +

              No

              +

              None

              +

              String

              +

              Password for SSL keystore

              +

              avro-confluent.ssl.truststore.location

              +

              No

              +

              None

              +

              String

              +

              Location/File of SSL truststore

              +

              avro-confluent.ssl.truststore.password

              +

              No

              +

              None

              +

              String

              +

              Password for SSL truststore

              +

              avro-confluent.subject

              +

              No

              +

              None

              +

              String

              +

              The Confluent Schema Registry subject under which to register the schema used by this format during serialization. By default, 'kafka' and 'upsert-kafka' connectors use '<topic_name>-value' or '<topic_name>-key' as the default subject name if this format is used as the value or key format. But for other connectors (e.g. 'filesystem'), the subject option is required when used as sink.

              +

              avro-confluent.url

              +

              No

              +

              None

              +

              String

              +

              The URL of the Confluent Schema Registry to fetch/register schemas.

              +
              +
              +
              +

              Data Type Mapping

              Currently, Apache Flink always uses the table schema to derive the Avro reader schema during deserialization and Avro writer schema during serialization. Explicitly defining an Avro schema is not supported yet. See Avro for the mapping between Avro and Flink DataTypes.

              +

              In addition to the types listed there, Flink supports reading/writing nullable types. Flink maps nullable types to Avro union(something, null), where something is the Avro type converted from Flink type.

              +
              +

              Example

              Read JSON data from the source topic in Kafka and write the data in Confluent Avro format to the sink topic.

              +
              1. Create a datasource connection for the communication with the VPC and subnet where Kafka and ECS locate and bind the connection to the queue. Set a security group and inbound rule to allow access of the queue and test the connectivity of the queue using the Kafka and ECS IP addresses. For example, locate a general-purpose queue where the job runs and choose More > Test Address Connectivity in the Operation column. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Purchase an ECS cluster, download Confluent 5.5.2 and jdk1.8.0_232, and upload them to the ECS cluster. Run the following command to decompress the packages (assume that the decompression directories are confluent-5.5.2 and jdk1.8.0_232):
                tar zxvf confluent-5.5.2-2.11.tar.gz
+tar zxvf jdk1.8.0_232.tar.gz
                +
              3. Run the following commands to install jdk1.8.0_232 in the current ECS cluster. You can run the pwd command in the jdk1.8.0_232 folder to view the value of yourJdkPath.
                export JAVA_HOME=<yourJdkPath>
+export PATH=$JAVA_HOME/bin:$PATH
+export CLASSPATH=.:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
                +
              4. Go to the confluent-5.5.2/etc/schema-registry/ directory and modify the following configuration items in the schema-registry.properties file:
                listeners=http://<yourEcsIp>:8081  
+kafkastore.bootstrap.servers=<yourKafkaAddress1>:<yourKafkaPort>,<yourKafkaAddress2>:<yourKafkaPort>
                +
              5. Switch to the confluent-5.5.2 directory and run the following command to start Confluent:
                bin/schema-registry-start etc/schema-registry/schema-registry.properties
                +
              6. Create a Flink OpenSource SQL job, select the Flink 1.15 version, and allow DLI to save job logs in OBS. Add the following statement to the job and submit it:
                CREATE TABLE kafkaSource (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,    
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaSourceTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE kafkaSink (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,    
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaSinkTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'format' = 'avro-confluent',
+  'avro-confluent.url' = 'http://EcsIp:8081'
+);
+insert into kafkaSink select * from kafkaSource;
                +
              7. Insert the following data into Kafka:
                {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
                +
              8. Read the data of the sink Kafka topic. You will find that the data has been written and the schema has been saved to the _schema topic of Kafka.
              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15019.html b/docs/dli/sqlreference/dli_08_15019.html new file mode 100644 index 000000000..cdece2ca2 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15019.html @@ -0,0 +1,268 @@ + + +

              CSV

              +

              Function

              The CSV format allows you to read and write CSV data based on a CSV schema. Currently, the CSV schema is derived from table schema. For details, see CSV Format.

              +
              +

              Supported Connectors

              • Kafka
              • Upsert Kafka
              • FileSystem
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Description

              Parameter

              +

              Mandatory

              +

              Default value

              +

              Type

              +

              Description

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              Format to be used. Set the value to csv.

              +

              csv.field-delimiter

              +

              No

              +

              ,

              +

              String

              +

              Field delimiter character, which must be a single character. You can use backslash to specify special characters, for example, \t represents the tab character. You can also use unicode to specify them in plain SQL, for example, 'csv.field-delimiter' = U&'\0001' represents the 0x01 character.

              +

              csv.disable-quote-character

              +

              No

              +

              false

              +

              Boolean

              +

              Disabled quote character for enclosing field values. If you set this parameter to true, csv.quote-character cannot be set.

              +

              csv.quote-character

              +

              No

              +

              ''

              +

              String

              +

              Quote character for enclosing field values.

              +

              csv.allow-comments

              +

              No

              +

              false

              +

              Boolean

              +

              Ignore comment lines that start with #. If you set this parameter to true, make sure to also ignore parse errors to allow empty rows.

              +

              csv.ignore-parse-errors

              +

              No

              +

              false

              +

              Boolean

              +

              Whether fields and rows with parse errors will be skipped or failed. The default value is false, indicating that an error will be thrown. Fields are set to null in case of errors.

              +

              csv.array-element-delimiter

              +

              No

              +

              ;

              +

              String

              +

              Array element delimiter string for separating array and row element values.

              +

              csv.escape-character

              +

              No

              +

              +

              None

              +

              String

              +

              +

              Escape character for escaping values

              +

              +

              csv.null-literal

              +

              No

              +

              None

              +

              String

              +

              Null literal string that is interpreted as a null value.

              +
              +
              +
              +

              Data Type Mapping

              Currently, the CSV schema is always derived from table schema. Explicitly defining a CSV schema is not supported yet. Flink CSV format uses jackson databind API to parse and generate CSV string.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Data type mapping

              Flink SQL Type

              +

              CSV Type

              +

              CHAR/VARCHAR/STRING

              +

              String

              +

              BOOLEAN

              +

              Boolean

              +

              BINARY/VARBINARY

              +

              string with encoding: base64

              +

              DECIMAL

              +

              Number

              +

              TINYINT

              +

              Number

              +

              SMALLINT

              +

              Number

              +

              INT

              +

              Number

              +

              BIGINT

              +

              Number

              +

              FLOAT

              +

              Number

              +

              DOUBLE

              +

              Number

              +

              DATE

              +

              string with format: date

              +

              TIME

              +

              string with format: time

              +

              TIMESTAMP

              +

              string with format: date-time

              +

              INTERVAL

              +

              Number

              +

              ARRAY

              +

              array

              +

              ROW

              +

              object

              +
              +
              +
              +

              Example

              Use Kafka to send data and output the data to Print.

              +
              1. Create a datasource connection for the communication with the VPC and subnet where Kafka locates and bind the connection to the queue. Set a security group and inbound rule to allow access of the queue and test the connectivity of the queue using the Kafka IP address. For example, locate a general-purpose queue where the job runs and choose More > Test Address Connectivity in the Operation column. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Create a Flink OpenSource SQL job. Copy the following statement and submit the job:

                CREATE TABLE kafkaSource (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'csv'
+);
+
+
+CREATE TABLE printSink (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'print'
+);
+insert into printSink select * from kafkaSource;
                +

              3. Insert the following data into the source Kafka topic:

                202103251505050001,appShop,2021-03-25 15:05:05,500.00,400.00,2021-03-25 15:10:00,0003,Cindy,330108
+
+202103241606060001,appShop,2021-03-24 16:06:06,200.00,180.00,2021-03-24 16:10:06,0001,Alice,330106
                +

              4. Perform the following operations to view the data result in the taskmanager.out file:

                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the .out file, and view result logs.
                +
                +I[202103251505050001, appShop, 2021-03-25 15:05:05, 500.0, 400.0, 2021-03-25 15:10:00, 0003, Cindy, 330108]
++I[202103241606060001, appShop, 2021-03-24 16:06:06, 200.0, 180.0, 2021-03-24 16:10:06, 0001, Alice, 330106]
                +

              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15020.html b/docs/dli/sqlreference/dli_08_15020.html new file mode 100644 index 000000000..bbf2bdfb3 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15020.html @@ -0,0 +1,414 @@ + + +

              Debezium

              +

              Function

              Debezium is a Changelog Data Capture (CDC) tool that can stream changes in real-time from MySQL, PostgreSQL, Oracle, Microsoft SQL Server and many other databases into Kafka. Debezium provides a unified format schema for changelog and supports to serialize messages using JSON and Apache Avro.

              +

              Flink supports to interpret Debezium JSON and Avro messages as INSERT/UPDATE/DELETE messages into Flink SQL system. This is useful in many cases to leverage this feature, such as

              +
              • Synchronizing incremental data from databases to other systems
              • Auditing logs
              • Real-time materialized views on databases
              • Temporal join changing history of a database table
              +

              Flink also supports to encode the INSERT/UPDATE/DELETE messages in Flink SQL as Debezium JSON or Avro messages, and emit to external systems like Kafka. However, currently Flink cannot combine UPDATE_BEFORE and UPDATE_AFTER into a single UPDATE message. Therefore, Flink encodes UPDATE_BEFORE and UDPATE_AFTER as DELETE and INSERT Debezium messages.

              +

              For details, see Debezium Format.

              +
              +

              Supported Connectors

              • Kafka
              • FileSystem
              +
              +

              Caveats

              • Duplicate change events

                Under normal operating scenarios, the Debezium application delivers every change event exactly-once. Flink works pretty well when consuming Debezium produced events in this situation. However, Debezium application works in at-least-once delivery if any failover happens. That means, in the abnormal situations, Debezium may deliver duplicate change events to Kafka and Flink will get the duplicate events. This may cause Flink query to get wrong results or unexpected exceptions.

                +

                Solution: Set table.exec.source.cdc-events-duplicate to true and define a primary key on this source.

                +

                Framework will generate an additional stateful operator, and use the primary key to deduplicate the change events and produce a normalized changelog stream.

                +

                For more information, see Debezium documentation.

                +
              • Consuming data produced by Debezium Postgres Connector

                If you are using Debezium Connector for PostgreSQL to capture the changes to Kafka, please make sure the REPLICA IDENTITY configuration of the monitored PostgreSQL table has been set to FULL which is by default DEFAULT. Otherwise, Flink SQL currently will fail to interpret the Debezium data.

                +

                In FULL strategy, the UPDATE and DELETE events will contain the previous values of all the table's columns.

                +

                In other strategies, the before field of UPDATE and DELETE events will only contain primary key columns or null if no primary key.

                +

                You can change the REPLICA IDENTITY by running ALTER TABLE <your-table-name> REPLICA IDENTITY FULL.

                +
              +
              +

              Parameter Description

              Flink provides debezium-avro-confluent and debezium-json formats to interpret Avro or Json messages produced by Debezium. Use format debezium-avro-confluent to interpret Debezium Avro messages and format debezium-json to interpret Debezium Json messages.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Debezium Avro parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              Format to be used. Set this parameter to 'debezium-avro-confluent'.

              +

              debezium-avro-confluent.basic-auth.credentials-source

              +

              No

              +

              None

              +

              String

              +

              Basic auth credentials source for Schema Registry

              +

              debezium-avro-confluent.basic-auth.user-info

              +

              No

              +

              None

              +

              String

              +

              Basic auth user info for schema registry

              +

              debezium-avro-confluent.bearer-auth.credentials-source

              +

              No

              +

              None

              +

              String

              +

              Bearer auth credentials source for Schema Registry

              +

              debezium-avro-confluent.bearer-auth.token

              +

              No

              +

              None

              +

              String

              +

              Bearer auth token for Schema Registry

              +

              debezium-avro-confluent.properties

              +

              No

              +

              None

              +

              Map

              +

              Properties map that is forwarded to the underlying Schema Registry. This is useful for options that are not officially exposed via Flink config options. However, note that Flink options have higher precedence.

              +

              debezium-avro-confluent.ssl.keystore.location

              +

              No

              +

              None

              +

              String

              +

              Location/File of SSL keystore

              +

              debezium-avro-confluent.ssl.keystore.password

              +

              No

              +

              None

              +

              String

              +

              Password for SSL keystore

              +

              debezium-avro-confluent.ssl.truststore.location

              +

              No

              +

              None

              +

              String

              +

              Location/File of SSL truststore

              +

              debezium-avro-confluent.ssl.truststore.password

              +

              No

              +

              None

              +

              String

              +

              Password for SSL truststore

              +

              debezium-avro-confluent.subject

              +

              No

              +

              None

              +

              String

              +

              The Confluent Schema Registry subject under which to register the schema used by this format during serialization. By default, 'kafka' and 'upsert-kafka' connectors use '<topic_name>-value' or '<topic_name>-key' as the default subject name if this format is used as the value or key format. But for other connectors (e.g. 'filesystem'), the subject option is required when used as sink.

              +

              debezium-avro-confluent.url

              +

              No

              +

              None

              +

              String

              +

              The URL of the Confluent Schema Registry to fetch/register schemas.

              +
              +
              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Debezium JSON parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Mandatory

              +

              Description

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              Format to be used. In this example.Set this parameter to debezium-json.

              +

              debezium-json.schema-include

              +

              No

              +

              false

              +

              Boolean

              +

              Whether the Debezium JSON messages contain the schema. When setting up Debezium Kafka Connect, enable the Kafka configuration value.converter.schemas.enable to include the schema in the message.

              +

              debezium-json.ignore-parse-errors

              +

              No

              +

              false

              +

              Boolean

              +

              Whether fields and rows with parse errors will be skipped or failed. The default value is false, indicating that an error will be thrown. Fields are set to null in case of errors.

              +

              debezium-json.timestamp-format.standard

              +

              No

              +

              'SQL'

              +

              String

              +

              Input and output timestamp formats. Currently supported values are SQL and ISO-8601.

              +
              • SQL will parse input timestamp in "yyyy-MM-dd HH:mm:ss.s{precision}" format, for example 2020-12-30 12:13:14.123 and output timestamp in the same format.
              • ISO-8601 will parse input timestamp in "yyyy-MM-ddTHH:mm:ss.s{precision}" format, for example 2020-12-30T12:13:14.123 and output timestamp in the same format.
              +

              debezium-json.map-null-key.mode

              +

              No

              +

              'FAIL'

              +

              String

              +

              Handling mode when serializing null keys for map data. Available values are as follows:

              +
              • FAIL will throw exception when encountering map value with null key.
              • DROP will drop null key entries for map data.
              • LITERAL replaces the empty key value in the map with a string constant. The string literal is defined by debezium-json.map-null-key.literal option.
              +

              debezium-json.map-null-key.literal

              +

              No

              +

              'null'

              +

              String

              +

              String literal to replace null key when debezium-json.map-null-key.mode is LITERAL.

              +

              debezium-json.encode.decimal-as-plain-number

              +

              No

              +

              false

              +

              Boolean

              +

              Encode all decimals as plain numbers instead of possible scientific notations. For example, 0.000000027 is encoded as 2.7E-8 by default, and will be written as 0.000000027 if set this parameter to true.

              +
              +
              +
              +

              Metadata

              The following format metadata can be exposed as read-only (VIRTUAL) columns in DDL.

              +

              Format metadata fields are only available if the corresponding connector forwards format metadata. Currently, only the Kafka connector is able to expose metadata fields for its value format.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 3 Metadata

              Key

              +

              Data Type

              +

              Description

              +

              schema

              +

              STRING NULL

              +

              JSON string describing the schema of the payload. Null if the schema is not included in the Debezium record.

              +

              ingestion-timestamp

              +

              TIMESTAMP_LTZ(3) NULL

              +

              The timestamp at which the connector processed the event. Corresponds to the ts_ms field in the Debezium record.

              +

              source.timestamp

              +

              TIMESTAMP_LTZ(3) NULL

              +

              The timestamp at which the source system created the event. Corresponds to the source.ts_ms field in the Debezium record.

              +

              source.database

              +

              STRING NULL

              +

              The originating database. Corresponds to the source.db field in the Debezium record if available.

              +

              source.schema

              +

              STRING NULL

              +

              The originating database schema. Corresponds to the source.schema field in the Debezium record if available.

              +

              source.table

              +

              STRING NULL

              +

              The originating database table. Corresponds to the source.table or source.collection field in the Debezium record if available.

              +

              source.properties

              +

              MAP<STRING, STRING> NULL

              +

              Map of various source properties. Corresponds to the source field in the Debezium record.

              +
              +
              +

              The following example shows how to access Canal metadata fields in Kafka:

              +
              CREATE TABLE KafkaTable (
+  origin_ts TIMESTAMP(3) METADATA FROM 'value.ingestion-timestamp' VIRTUAL,
+  event_time TIMESTAMP(3) METADATA FROM 'value.source.timestamp' VIRTUAL,
+  origin_database STRING METADATA FROM 'value.source.database' VIRTUAL,
+  origin_schema STRING METADATA FROM 'value.source.schema' VIRTUAL,
+  origin_table STRING METADATA FROM 'value.source.table' VIRTUAL,
+  origin_properties MAP<STRING, STRING> METADATA FROM 'value.source.properties' VIRTUAL,
+  user_id BIGINT,
+  item_id BIGINT,
+  behavior STRING
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'earliest-offset',
+  'value.format' = 'debezium-json'
+);
              +
              +

              Example

              Use Kafka to parse Debezium JSON data and output the result to Print.

              +
              1. Create a datasource connection for the communication with the VPC and subnet where Kafka locates and bind the connection to the queue. Set a security group and inbound rule to allow access of the queue and test the connectivity of the queue using the Kafka IP address. For example, locate a general-purpose queue where the job runs and choose More > Test Address Connectivity in the Operation column. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Create a Flink OpenSource SQL job and select Flink 1.15. Copy the following statement and submit the job:

                CREATE TABLE kafkaSource (
+  id bigint,
+  name string,
+  description string,  
+  weight DECIMAL(10, 2)
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'debezium-json'
+);
+CREATE TABLE printSink (
+  id bigint,
+  name string,
+  description string,  
+  weight DECIMAL(10, 2)
+) WITH (
+  'connector' = 'print'
+);
+insert into printSink select * from kafkaSource;
                +

              3. Insert the data below into the appropriate Kafka topics. The data shows that the MySQL products table has four columns: id, name, description, and weight. This JSON message represents an update event on the products table, where the weight value of the row with id = 111 has been changed from 5.18 to 5.15.

                {
+  "before": {
+    "id": 111,
+    "name": "scooter",
+    "description": "Big 2-wheel scooter",
+    "weight": 5.18
+  },
+  "after": {
+    "id": 111,
+    "name": "scooter",
+    "description": "Big 2-wheel scooter",
+    "weight": 5.15
+  },
+  "source": {
+    "version": "0.9.5.Final",
+	"connector": "mysql",
+	"name": "fullfillment",
+	"server_id" :1,
+	"ts_sec": 1629607909,
+	"gtid": "mysql-bin.000001",
+	"pos": 2238,"row": 0,
+	"snapshot": false,
+	"thread": 7,
+	"db": "inventory",
+	"table": "test",
+	"query": null},
+  "op": "u",
+  "ts_ms": 1589362330904,
+  "transaction": null
+}
                +

              4. Perform the following operations to view the data result in the taskmanager.out file:

                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the .out file, and view result logs.
                +
                -U[111, scooter, Big 2-wheel scooter, 5.18]
++U[111, scooter, Big 2-wheel scooter, 5.15]
                +

              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15021.html b/docs/dli/sqlreference/dli_08_15021.html new file mode 100644 index 000000000..956108f08 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15021.html @@ -0,0 +1,256 @@ + + +

              JSON

              +

              Function

              The JSON format allows you to read and write JSON data based on a JSON schema. Currently, the JSON schema is derived from table schema. For details, see JSON Format.

              +
              +

              Supported Connectors

              • Kafka
              • Upsert Kafka
              • Elasticsearch
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Type

              +

              Description

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              Format to be used. Set this parameter to json.

              +

              json.fail-on-missing-field

              +

              No

              +

              false

              +

              Boolean

              +

              Whether missing fields and rows will be skipped or failed. The default value is false, indicating that an error will be thrown.

              +

              json.ignore-parse-errors

              +

              No

              +

              false

              +

              Boolean

              +

              Whether fields and rows with parse errors will be skipped or failed. The default value is false, indicating that an error will be thrown. Fields are set to null in case of errors.

              +

              json.timestamp-format.standard

              +

              No

              +

              'SQL'

              +

              String

              +
              Specify the input and output timestamp format for TIMESTAMP and TIMESTAMP_LTZ type. Currently supported values are SQL and ISO-8601:
              • Option SQL will parse input TIMESTAMP values in "yyyy-MM-dd HH:mm:ss.s{precision}" format, e.g "2020-12-30 12:13:14.123", parse input TIMESTAMP_LTZ values in "yyyy-MM-dd HH:mm:ss.s{precision}'Z'" format, e.g "2020-12-30 12:13:14.123Z", and output timestamp in the same format.
              • Option ISO-8601 will parse input TIMESTAMP in "yyyy-MM-ddTHH:mm:ss.s{precision}" format, e.g "2020-12-30T12:13:14.123", parse input TIMESTAMP_LTZ in "yyyy-MM-ddTHH:mm:ss.s{precision}'Z'" format, e.g "2020-12-30T12:13:14.123Z", and output timestamp in the same format.
              +
              +

              json.map-null-key.mode

              +

              No

              +

              'FALL'

              +

              String

              +

              Handling mode when serializing null keys for map data. Available values are as follows:

              +
              • FAIL will throw exception when encountering map value with null key.
              • DROP will drop null key entries for map data.
              • LITERAL replaces the empty key value in the map with a string constant. The string literal is defined by json.map-null-key.literal option.
              +

              json.map-null-key.literal

              +

              No

              +

              'null'

              +

              String

              +

              String literal to replace null key when json.map-null-key.mode is LITERAL.

              +

              json.encode.decimal-as-plain-number

              +

              No

              +

              false

              +

              Boolean

              +

              Encode all decimals as plain numbers instead of possible scientific notations. For example, 0.000000027 is encoded as 2.7E-8 by default, and will be written as 0.000000027 if set this parameter to true.

              +
              +
              +
              +

              Data Type Mapping

              Currently, the JSON schema is always derived from table schema. Explicitly defining a JSON schema is not supported yet.

              +

              Flink JSON format uses jackson databind API to parse and generate JSON string.

              +

              The following table lists the type mapping from Flink type to JSON type.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Data type mapping

              Flink SQL Type

              +

              JSON Type

              +

              CHAR/VARCHAR/STRING

              +

              String

              +

              BOOLEAN

              +

              Boolean

              +

              BINARY/VARBINARY

              +

              string with encoding: base64

              +

              DECIMAL

              +

              Number

              +

              TINYINT

              +

              Number

              +

              SMALLINT

              +

              Number

              +

              INT

              +

              Number

              +

              BIGINT

              +

              Number

              +

              FLOAT

              +

              Number

              +

              DOUBLE

              +

              Number

              +

              DATE

              +

              string with format: date

              +

              TIME

              +

              string with format: time

              +

              TIMESTAMP

              +

              string with format: date-time

              +

              TIMESTAMP_WITH_LOCAL_TIME_ZONE

              +

              string with format: date-time (with UTC time zone)

              +

              INTERVAL

              +

              Number

              +

              ARRAY

              +

              array

              +

              MAP / MULTISET

              +

              object

              +

              ROW

              +

              object

              +
              +
              +
              +

              Example

              In this example, data is read from a topic and written to another using a Kafka sink.

              +
              1. Create a datasource connection for the communication with the VPC and subnet where Kafka locates and bind the connection to the queue. Set an inbound rule for the security group to allow access of the queue and test the connectivity using the Kafka address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Create a Flink OpenSource SQL job, select Flink 1.15, and allow DLI to save job logs in OBS. Use the following statement in the job and submit it:

                CREATE TABLE kafkaSource (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE printSink (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'print'
+);
+insert into printSink select * from kafkaSource;
                +

              3. Insert the following data into the source Kafka topic:

                {"order_id":"202103241000000001","order_channel":"webShop","order_time":"2021-03-24 10:00:00","pay_amount":100.0,"real_pay":100.0,"pay_time":"2021-03-24 10:02:03","user_id":"0001","user_name":"Alice","area_id":"330106"}
+
+{"order_id":"202103241606060001","order_channel":"appShop","order_time":"2021-03-24 16:06:06","pay_amount":200.0,"real_pay":180.0,"pay_time":"2021-03-24 16:10:06","user_id":"0001","user_name":"Alice","area_id":"330106"}
                +

              4. Perform the following operations to view the data result in the taskmanager.out file:

                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the .out file, and view result logs.
                  +I[202103241000000001, webShop, 2021-03-24 10:00:00, 100.0, 100.0, 2021-03-24 10:02:03, 0001, Alice, 330106]
++I[202103241606060001, appShop11, 2021-03-24 16:06:06, 200.0, 180.0, 2021-03-24 16:10:06, 0001, Alice, 330106]
                  +
                +

              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15022.html b/docs/dli/sqlreference/dli_08_15022.html new file mode 100644 index 000000000..039efedb1 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15022.html @@ -0,0 +1,216 @@ + + +

              Maxwell

              +

              Function

              Maxwell is a Changelog Data Capture (CDC) tool that can stream changes in real-time from MySQL into Kafka and other streaming connectors. Maxwell provides a unified format schema for changelog and supports to serialize messages using JSON.

              +

              Flink supports to interpret Maxwell JSON messages as INSERT/UPDATE/DELETE messages into Flink SQL system. This is useful in many cases to leverage this feature,

              +
              +

              such as:

              +
              • Synchronizing incremental data from databases to other systems
              • Auditing logs
              • Real-time materialized views on databases
              • Temporal join changing history of a database table and so on
              +

              Flink also supports to encode the INSERT/UPDATE/DELETE messages in Flink SQL as Maxwell JSON messages, and emit to external systems like Kafka. However, currently Flink cannot combine UPDATE_BEFORE and UPDATE_AFTER into a single UPDATE message. Therefore, Flink encodes UPDATE_BEFORE and UDPATE_AFTER as DELETE and INSERT Maxwell messages.

              +

              For details, see Maxwell Format.

              +

              Supported Connectors

              • Kafka
              • FileSystem
              +
              +

              Caveats

              The Maxwell application allows to deliver every change event exactly-once. Flink works pretty well when consuming Maxwell produced events in this situation. If Maxwell application works in at-least-once delivery, it may deliver duplicate change events to Kafka and Flink will get the duplicate events. This may cause Flink query to get wrong results or unexpected exceptions. Thus, it is recommended setting job configuration table.exec.source.cdc-events-duplicate to true and define PRIMARY KEY on the source in this situation. Framework will generate an additional stateful operator, and use the primary key to deduplicate the change events and produce a normalized changelog stream.

              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Type

              +

              Description

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              Format to be used. Set this parameter to 'maxwell-json'.

              +

              maxwell-json.ignore-parse-errors

              +

              No

              +

              false

              +

              Boolean

              +

              Whether fields and rows with parse errors will be skipped or failed. Fields are set to null in case of errors.

              +

              maxwell-json.timestamp-format.standard

              +

              No

              +

              'SQL'

              +

              String

              +

              Specify the input and output timestamp format. Currently supported values are SQL and ISO-8601:

              +
              • SQL will parse input timestamp in "yyyy-MM-dd HH:mm:ss.s{precision}" format, e.g '2020-12-30 12:13:14.123' and output timestamp in the same format.
              • ISO-8601 will parse input timestamp in "yyyy-MM-ddTHH:mm:ss.s{precision}" format, e.g '2020-12-30T12:13:14.123' and output timestamp in the same format.
              +

              maxwell-json.map-null-key.mode

              +

              No

              +

              'FAIL'

              +

              String

              +

              Specify the handling mode when serializing null keys for map data. Currently supported values are FAIL, DROP, and LITERAL:

              +
              • FAIL will throw exception when encountering map with null key.
              • DROP will drop null key entries for map data.
              • LITERAL will replace null key with string literal. The string literal is defined by maxwell-json.map-null-key.literal.
              +

              maxwell-json.map-null-key.literal

              +

              No

              +

              'null'

              +

              String

              +

              Specify string literal to replace null key when maxwell-json.map-null-key.mode is LITERAL.

              +

              maxwell-json.encode.decimal-as-plain-number

              +

              No

              +

              false

              +

              Boolean

              +

              Encode all decimals as plain numbers instead of possible scientific notations. By default, decimals may be written using scientific notation. For example, 0.000000027 is encoded as 2.7E-8 by default, and will be written as 0.000000027 if set this parameter to true.

              +
              +
              +
              +

              Metadata

              The following format metadata can be exposed as read-only (VIRTUAL) columns in DDL.

              + +
              + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Metadata

              Key

              +

              Data Type

              +

              Description

              +

              database

              +

              STRING NULL

              +

              The originating database. Corresponds to the database field in the Maxwell record if available.

              +

              table

              +

              STRING NULL

              +

              The originating database table. Corresponds to the table field in the Maxwell record if available.

              +

              primary-key-columns

              +

              ARRAY<STRING> NULL

              +

              Array of primary key names. Corresponds to the primary_key_columns field in the Maxwell record if available.

              +

              ingestion-timestamp

              +

              TIMESTAMP_LTZ(3) NULL

              +

              The timestamp at which the connector processed the event. Corresponds to the ts field in the Maxwell record.

              +
              +
              +

              The following is an example of using metadata:

              +
              CREATE TABLE KafkaTable (
+  origin_database STRING METADATA FROM 'value.database' VIRTUAL,
+  origin_table STRING METADATA FROM 'value.table' VIRTUAL,
+  origin_primary_key_columns ARRAY<STRING> METADATA FROM 'value.primary-key-columns' VIRTUAL,
+  origin_ts TIMESTAMP(3) METADATA FROM 'value.ingestion-timestamp' VIRTUAL,
+  user_id BIGINT,
+  item_id BIGINT,
+  behavior STRING
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'earliest-offset',
+  'value.format' = 'maxwell-json'
+);
              +
              +

              Example

              Use Kafka to send data and output the data to Print.

              +
              1. Create a datasource connection for the communication with the VPC and subnet where Kafka locates and bind the connection to the queue. Set a security group and inbound rule to allow access of the queue and test the connectivity of the queue using the Kafka IP address. For example, locate a general-purpose queue where the job runs and choose More > Test Address Connectivity in the Operation column. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Create a Flink OpenSource SQL job and select Flink 1.15. Copy the following statement and submit the job:

                CREATE TABLE kafkaSource (
+  id bigint,
+  name string,
+  description string,  
+  weight DECIMAL(10, 2)
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'maxwell-json'
+);
+
+
+CREATE TABLE printSink (
+  id bigint,
+  name string,
+  description string,  
+  weight DECIMAL(10, 2)
+) WITH (
+  'connector' = 'print'
+);
+insert into printSink select * from kafkaSource;
                +

              3. Insert the data below into the appropriate Kafka topics (for details about the meaning of each field, see Maxwell documentation):

                {
+   "database":"test",
+   "table":"e",
+   "type":"insert",
+   "ts":1477053217,
+   "xid":23396,
+   "commit":true,
+   "position":"master.000006:800911",
+   "server_id":23042,
+   "thread_id":108,
+   "primary_key": [1, "2016-10-21 05:33:37.523000"],
+   "primary_key_columns": ["id", "c"],
+   "data":{
+     "id":111,
+     "name":"scooter",
+     "description":"Big 2-wheel scooter",
+     "weight":5.15
+   },
+   "old":{
+     "weight":5.18
+   }
+}
                +

              4. Perform the following operations to view the data result in the taskmanager.out file:

                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the .out file, and view result logs.
                +
                +I[111, scooter, Big 2-wheel scooter, 5.15]
                +

              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15023.html b/docs/dli/sqlreference/dli_08_15023.html new file mode 100644 index 000000000..7b70345d8 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15023.html @@ -0,0 +1,202 @@ + + +

              Ogg

              +

              Function

              Oracle GoldenGate (a.k.a ogg) is a comprehensive software package for real-time data capture and replication in heterogeneous IT environments. The product set enables high availability solutions, real-time data integration, transactional change data capture, data replication, transformations, and verification between operational and analytical enterprise systems. Ogg provides a format schema for changelog and supports to serialize messages using JSON.

              +

              Flink supports to interpret Ogg JSON as INSERT/UPDATE/DELETE messages into Flink SQL system. This is useful in many cases to leverage this feature, such as:

              +
              • Synchronizing incremental data from databases to other systems
              • Auditing logs
              • Real-time materialized views on databases
              • Temporal join changing history of a database table and so on.
              +

              Flink also supports to encode the INSERT/UPDATE/DELETE messages in Flink SQL as Ogg JSON, and emit to external systems like Kafka. However, currently Flink cannot combine UPDATE_BEFORE and UPDATE_AFTER into a single UPDATE message. Therefore, Flink encodes UPDATE_BEFORE and UPDATE_AFTER as DELETE and INSERT Ogg messages.

              +
              +

              Supported Connectors

              • Kafka
              • FileSystem
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              format

              +

              Yes

              +

              (none)

              +

              String

              +

              Specify what format to use, here should be ogg-json.

              +

              ogg-json.ignore-parse-errors

              +

              No

              +

              false

              +

              Boolean

              +

              Whether fields and rows with parse errors will be skipped or failed. The default value is false, indicating that an error will be thrown. Fields are set to null in case of errors.

              +

              debezium-json.timestamp-format.standard

              +

              No

              +

              'SQL'

              +

              String

              +

              Input and output timestamp formats. Currently supported values are SQL and ISO-8601:

              +
              • SQL will parse input timestamp in "yyyy-MM-dd HH:mm:ss.s{precision}" format, for example 2020-12-30 12:13:14.123 and output timestamp in the same format.
              • ISO-8601 will parse input timestamp in "yyyy-MM-ddTHH:mm:ss.s{precision}" format, for example 2020-12-30T12:13:14.123 and output timestamp in the same format.
              +

              ogg-json.map-null-key.mode

              +

              No

              +

              'FAIL'

              +

              String

              +

              Handling mode when serializing null keys for map data. Currently supported values are FAIL, DROP, and LITERAL:

              +
              • Option FAIL will throw exception when encountering map with null key.
              • Option DROP will drop null key entries for map data.
              • Option LITERAL will replace null key with string literal. The string literal is defined by ogg-json.map-null-key.literal.
              +

              ogg-json.map-null-key.literal

              +

              No

              +

              'null'

              +

              String

              +

              Specify string literal to replace null key when ogg-json.map-null-key.mode is LITERAL.

              +
              +
              +
              +

              Metadata

              +
              + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Metadata

              Key

              +

              Data Type

              +

              Description

              +

              table

              +

              STRING NULL

              +

              Contains fully qualified table name. The format of the fully qualified table name is CATALOG NAME.SCHEMA NAME.TABLE NAME.

              +

              primary-keys

              +

              ARRAY<STRING> NULL

              +

              An array variable holding the column names of the primary keys of the source table.

              +

              The primary-keys field is only included in the JSON output if the includePrimaryKeys configuration property is set to true.

              +

              ingestion-timestamp

              +

              TIMESTAMP_LTZ(6) NULL

              +

              The timestamp at which the connector processed the event. Corresponds to the current_ts field in the Ogg record.

              +

              event-timestamp

              +

              TIMESTAMP_LTZ(6) NULL

              +

              The timestamp at which the source system created the event. Corresponds to the op_ts field in the Ogg record.

              +
              +
              +

              The following example shows how to access Canal metadata fields in Kafka:

              +
              CREATE TABLE KafkaTable (
+  origin_ts TIMESTAMP(3) METADATA FROM 'value.ingestion-timestamp' VIRTUAL,
+  event_time TIMESTAMP(3) METADATA FROM 'value.event-timestamp' VIRTUAL,
+  origin_table STRING METADATA FROM 'value.table' VIRTUAL,
+  primary_keys ARRAY<STRING> METADATA FROM 'value.primary_keys' VIRTUAL,
+  user_id BIGINT,
+  item_id BIGINT,
+  behavior STRING
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'earliest-offset',
+  'value.format' = 'ogg-json'
+);
              +
              +

              Example

              Use ogg-json to read Ogg records in Kafka and output them to Print.

              +
              1. Create a datasource connection for the communication with the VPC and subnet where Kafka locates and bind the connection to the queue. Set a security group and inbound rule to allow access of the queue and test the connectivity of the queue using the Kafka IP address. For example, locate a general-purpose queue where the job runs and choose More > Test Address Connectivity in the Operation column. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Create a Flink OpenSource SQL job and select Flink 1.15. Copy the following statement and submit the job:

                CREATE TABLE kafkaSource (
+  id bigint,
+  name string,
+  description string,  
+  weight DECIMAL(10, 2)
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'ogg-json'
+);
+
+CREATE TABLE printSink (
+  id bigint,
+  name string,
+  description string,  
+  weight DECIMAL(10, 2)
+) WITH (
+  'connector' = 'print'
+);
+insert into printSink select * from kafkaSource;    
+
                +

              3. Insert the data below into the appropriate Kafka topics. The data shows that the Oracle PRODUCTS table has four columns: id, name, description, and weight. This JSON message represents an update event on the PRODUCTS table, where the weight value of the row with id = 111 has been changed from 5.18 to 5.15.

                {
+  "before": {
+    "id": 111,
+    "name": "scooter",
+    "description": "Big 2-wheel scooter",
+    "weight": 5.18
+  },
+  "after": {
+    "id": 111,
+    "name": "scooter",
+    "description": "Big 2-wheel scooter",
+    "weight": 5.15
+  },
+  "op_type": "U",
+  "op_ts": "2020-05-13 15:40:06.000000",
+  "current_ts": "2020-05-13 15:40:07.000000",
+  "primary_keys": [
+    "id"
+  ],
+  "pos": "00000000000000000000143",
+  "table": "PRODUCTS"
+}
                +

              4. Perform the following operations to view the data result in the taskmanager.out file:

                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the .out file, and view result logs.
                +
                -U[111, scooter, Big 2-wheel scooter, 5.18]
++U[111, scooter, Big 2-wheel scooter, 5.15]
                +

              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15024.html b/docs/dli/sqlreference/dli_08_15024.html new file mode 100644 index 000000000..181e9c2cb --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15024.html @@ -0,0 +1,221 @@ + + +

              ORC

              +

              Function

              The Apache ORC format allows to read and write ORC data. For details, see ORC Format.

              +
              +

              Supported Connectors

              • FileSystem
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + +
              Table 1 Parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              Specify what format to use, here should be orc.

              +
              +
              +

              ORC format also supports table properties from Table properties. For example, you can configure orc.compress=SNAPPY to enable snappy compression.

              +
              +

              Data Type Mapping

              ORC format type mapping is compatible with Apache Hive. The following table lists the type mapping from Flink type to ORC type.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Data type mapping

              Flink SQL Type

              +

              ORC Physical Type

              +

              ORC Logical Type

              +

              CHAR

              +

              bytes

              +

              CHAR

              +

              VARCHAR

              +

              bytes

              +

              VARCHAR

              +

              STRING

              +

              bytes

              +

              STRING

              +

              BOOLEAN

              +

              long

              +

              BOOLEAN

              +

              BYTES

              +

              bytes

              +

              BINARY

              +

              DECIMAL

              +

              decimal

              +

              DECIMAL

              +

              TINYINT

              +

              long

              +

              BYTE

              +

              SMALLINT

              +

              long

              +

              SHORT

              +

              INT

              +

              long

              +

              INT

              +

              BIGINT

              +

              long

              +

              LONG

              +

              FLOAT

              +

              double

              +

              FLOAT

              +

              DOUBLE

              +

              double

              +

              DOUBLE

              +

              DATE

              +

              long

              +

              DATE

              +

              TIMESTAMP

              +

              timestamp

              +

              TIMESTAMP

              +

              ARRAY

              +

              -

              +

              LIST

              +

              MAP

              +

              -

              +

              MAP

              +

              ROW

              +

              -

              +

              STRUCT

              +
              +
              +
              +

              Example

              Use Kafka to send data and output the data to Print.

              +
              1. Create a datasource connection for the communication with the VPC and subnet where Kafka locates and bind the connection to the queue. Set a security group and inbound rule to allow access of the queue and test the connectivity of the queue using the Kafka IP address. For example, locate a general-purpose queue where the job runs and choose More > Test Address Connectivity in the Operation column. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Create a Flink OpenSource SQL job and enable checkpointing. Copy the following statement and submit the job:

                CREATE TABLE kafkaSource (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic-pattern' = kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId'',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'csv'
+);
+
+
+CREATE TABLE sink (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'filesystem',
+  'format' = 'orc',
+  'path' = 'obs://xx'
+);
+insert into sink select * from kafkaSource;
                +

              3. Insert the following data into the source Kafka topic:

                202103251505050001,appshop,2021-03-25 15:05:05,500.00,400.00,2021-03-25 15:10:00,0003,Cindy,330108
+
+202103241606060001,appShop,2021-03-24 16:06:06,200.00,180.00,2021-03-24 16:10:06,0001,Alice,330106
                +

              4. Read the ORC file in the OBS path configured in the sink table. The data results are as follows:

                202103251202020001, miniAppShop, 2021-03-25 12:02:02, 60.0, 60.0, 2021-03-25 12:03:00, 0002, Bob, 330110
+
+202103241606060001, appShop, 2021-03-24 16:06:06, 200.0, 180.0, 2021-03-24 16:10:06, 0001, Alice, 330106
                +

              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15025.html b/docs/dli/sqlreference/dli_08_15025.html new file mode 100644 index 000000000..4db00b408 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15025.html @@ -0,0 +1,227 @@ + + +

              Parquet

              +

              Function

              The Apache Parquet format allows to read and write Parquet data. For details, see Parquet Format.

              +
              +

              Supported Connectors

              • FileSystem
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              Specify what format to use, here should be parquet.

              +

              parquet.utc-timezone

              +

              No

              +

              false

              +

              Boolean

              +

              Use UTC timezone or local timezone to the conversion between epoch time and LocalDateTime. Hive 0.x/1.x/2.x use local timezone. But Hive 3.x use UTC timezone.

              +
              +
              +
              +

              Data Type Mapping

              Currently, Parquet format type mapping is compatible with Apache Hive, but different with Apache Spark:

              +
              • Timestamp: mapping timestamp type to int96 whatever the precision is.
              • Decimal: mapping decimal type to fixed length byte array according to the precision.
              +

              The following table lists the type mapping from Flink type to Parquet type.

              +

              Note that currently only writing is supported for composite data types (Array, Map, and Row), while reading is not supported.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Data type mapping

              Flink SQL Type

              +

              Parquet Type

              +

              Parquet Logical Type

              +

              CHAR/VARCHAR/STRING

              +

              BINARY

              +

              UTF8

              +

              BOOLEAN

              +

              BOOLEAN

              +

              -

              +

              BINARY/VARBINARY

              +

              BINARY

              +

              -

              +

              DECIMAL

              +

              FIXED_LEN_BYTE_ARRAY

              +

              DECIMAL

              +

              TINYINT

              +

              INT32

              +

              INT_8

              +

              SMALLINT

              +

              INT32

              +

              INT_16

              +

              INT

              +

              INT32

              +

              -

              +

              BIGINT

              +

              INT64

              +

              -

              +

              FLOAT

              +

              FLOAT

              +

              -

              +

              DOUBLE

              +

              DOUBLE

              +

              -

              +

              DATE

              +

              INT32

              +

              DATE

              +

              TIME

              +

              INT32

              +

              TIME_MILLIS

              +

              TIMESTAMP

              +

              INT96

              +

              -

              +

              ARRAY

              +

              -

              +

              LIST

              +

              MAP

              +

              -

              +

              MAP

              +

              ROW

              +

              -

              +

              STRUCT

              +
              +
              +
              +

              Example

              Use Kafka to send data and output the data to Print.

              +
              1. Create a datasource connection for the communication with the VPC and subnet where Kafka locates and bind the connection to the queue. Set a security group and inbound rule to allow access of the queue and test the connectivity of the queue using the Kafka IP address. For example, locate a general-purpose queue where the job runs and choose More > Test Address Connectivity in the Operation column. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Create a Flink OpenSource SQL job and enable checkpointing. Copy the following statement and submit the job:

                CREATE TABLE kafkaSource (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic-pattern' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+
+CREATE TABLE sink (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'filesystem',
+  'format' = 'parquet',
+  'path' = 'obs://xx'
+);
+insert into sink select * from kafkaSource;
                +

              3. Insert the following data into the source Kafka topic:

                202103251505050001,appShop,2021-03-25 15:05:05,500.00,400.00,2021-03-25 15:10:00,0003,Cindy,330108
+
+202103241606060001,appShop,2021-03-24 16:06:06,200.00,180.00,2021-03-24 16:10:06,0001,Alice,330106
                +

              4. Read the Parquet file in the OBS path configured in the sink table. The data results are as follows:

                202103251202020001, miniAppShop, 2021-03-25 12:02:02, 60.0, 60.0, 2021-03-25 12:03:00, 0002, Bob, 330110
+
+202103241606060001, appShop, 2021-03-24 16:06:06, 200.0, 180.0, 2021-03-24 16:10:06, 0001, Alice, 330106
                +

              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15026.html b/docs/dli/sqlreference/dli_08_15026.html new file mode 100644 index 000000000..e1156e9ec --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15026.html @@ -0,0 +1,151 @@ + + +

              Raw

              +

              Function

              The Raw format allows to read and write raw (byte based) values as a single column.

              +
              • This format encodes null values as null of byte[] type. This may have limitation when used in upsert-kafka, because upsert-kafka treats null values as a tombstone message (DELETE on the key). Therefore, we recommend avoiding using upsert-kafka connector and the raw format as a value.format if the field can have a null value.
              • The raw format connector is built-in, no additional dependencies are required. For details, see Raw Format.
              +
              +
              +

              Supported Connectors

              • Kafka
              • Upsert Kafka
              • FileSystem
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Type

              +

              Description

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              Format to be used. Set this parameter to raw.

              +

              raw.charset

              +

              No

              +

              UTF-8

              +

              String

              +

              Charset to encode the text string.

              +

              raw.endianness

              +

              No

              +

              big-endian

              +

              String

              +

              Endianness to encode the bytes of numeric value. Valid values are big-endian and little-endian. You can search for endianness for more details.

              +
              +
              +
              +

              Data Type Mapping

              The table below details the SQL types the format supports, including details of the serializer and deserializer class for encoding and decoding.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Data type mapping

              Flink SQL Type

              +

              Value

              +

              CHAR/VARCHAR/STRING

              +

              A UTF-8 (by default) encoded text string. The encoding charset can be configured by raw.charse.

              +

              BINARY / VARBINARY / BYTES

              +

              The sequence of bytes itself.

              +

              BOOLEAN

              +

              A single byte to indicate boolean value, 0 means false, 1 means true.

              +

              TINYINT

              +

              A single byte of the signed number value.

              +

              SMALLINT

              +

              Two bytes with big-endian (by default) encoding. The endianness can be configured by raw.endianness.

              +

              INT

              +

              Four bytes with big-endian (by default) encoding. The endianness can be configured by raw.endianness.

              +

              BIGINT

              +

              Eight bytes with big-endian (by default) encoding. The endianness can be configured by raw.endianness.

              +

              FLOAT

              +

              Four bytes with IEEE 754 format and big-endian (by default) encoding. The endianness can be configured by raw.endianness.

              +

              DOUBLE

              +

              Eight bytes with IEEE 754 format and big-endian (by default) encoding. The endianness can be configured by raw.endianness.

              +

              RAW

              +

              The sequence of bytes serialized by the underlying TypeSerializer of the RAW type.

              +
              +
              +
              +

              Example

              Use Kafka to send data and output the data to Print.

              +
              1. Create a datasource connection for the communication with the VPC and subnet where Kafka locates and bind the connection to the queue. Set a security group and inbound rule to allow access of the queue and test the connectivity of the queue using the Kafka IP address. For example, locate a general-purpose queue where the job runs and choose More > Test Address Connectivity in the Operation column. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Create a Flink OpenSource SQL job and select Flink 1.15. Copy the following statement and submit the job:

                CREATE TABLE kafkaSource (
+  log string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'raw'
+);
+
+CREATE TABLE printSink (
+  log string
+) WITH (
+  'connector' = 'print'
+);
+insert into printSink select * from kafkaSource;
                +

              3. Insert the following data to the corresponding topic in Kafka:

                47.29.201.179 - - [28/Feb/2019:13:17:10 +0000] "GET /?p=1 HTTP/2.0" 200 5316 "https://domain.com/?p=1" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36" "2.75"
                +

              4. Perform the following operations to view the data result in the taskmanager.out file:

                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the .out file, and view result logs.
                +
                +I[47.29.201.179 - - [28/Feb/2019:13:17:10 +0000] "GET /?p=1 HTTP/2.0" 200 5316 "https://domain.com/?p=1" "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36" "2.75"]
                +

              +
              +
              +
              +
              +
              Parent topic: Formats
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15027.html b/docs/dli/sqlreference/dli_08_15027.html new file mode 100644 index 000000000..2fb40b7a2 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15027.html @@ -0,0 +1,46 @@ + + +

              Connectors

              +

              +
              + + diff --git a/docs/dli/sqlreference/dli_08_15028.html b/docs/dli/sqlreference/dli_08_15028.html new file mode 100644 index 000000000..9e783da99 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15028.html @@ -0,0 +1,159 @@ + + +

              Overview

              +

              Table Type

              • Source table: A source table is the data input table for Flink jobs, such as real-time streaming data input from Kafka, etc.
              • Dimension Table: An auxiliary table for the data source table, used to enrich and expand the data in the source table. In Flink jobs, because the data collected by the data collection end is often limited, the required dimension information needs to be completed before data analysis can be performed, and the dimension table represents the data source that stores dimension information. Common user dimension tables include MySQL, Redis, etc.
              • Result table: The result data table output by the Flink job, which writes each real-time processed data into the target storage, such as MySQL, HBase, and other databases.
              +
              +

              Example:

              +

              Flink real-time consumes user order data from the Kafka source table, associates the product ID with the dimension table through Redis to obtain the product category, calculates the sales amount of different categories of products, and writes the calculation results into the RDS (Relational Database Service, such as MySQL) result table.

              +

              Table information is as follows:

              +
              • Source table: Order data table, including user ID, product ID, order ID, order amount, and other information.
              • Dimension table: User information table, including product ID and product category information.
              • Result table: Statistics of order sales amount data by product category.
              +

              The job first reads real-time order data from the order data source table, associates the order data stream with the product category information dimension table, then aggregates and calculates the total order amount, and finally writes the statistical results into the result table.

              +

              In this example, the order table serves as the driving source table input, the product category information table serves as the static dimension table, and the statistical result table serves as the final output of the job.

              +

              Supported Connectors

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Supported connectors

              Connector

              +

              Source Table

              +

              Dimension Table

              +

              Result Table

              +

              BlackHole

              +

              Not supported

              +

              Not supported

              +

              Supported

              +

              ClickHouse

              +

              Not supported

              +

              Not supported

              +

              Supported

              +

              DataGen

              +

              Supported

              +

              Not supported

              +

              Not supported

              +

              Doris

              +

              Supported

              +

              Supported

              +

              Supported

              +

              GaussDB(DWS)

              +

              Supported

              +

              Supported

              +

              Supported

              +

              Elasticsearch

              +

              Not supported

              +

              Not supported

              +

              Supported

              +

              FileSystem

              +

              Supported

              +

              Not supported

              +

              Supported

              +

              HBase

              +

              Supported

              +

              Supported

              +

              Supported

              +

              Hive

              +

              Supported

              +

              Supported

              +

              Supported

              +

              JDBC

              +

              Supported

              +

              Supported

              +

              Supported

              +

              Kafka

              +

              Supported

              +

              Not supported

              +

              Supported

              +

              Print

              +

              Not supported

              +

              Not supported

              +

              Supported

              +

              Redis

              +

              Supported

              +

              Supported

              +

              Supported

              +

              Upsert Kafka

              +

              Supported

              +

              Not supported

              +

              Supported

              +
              +
              +
              +
              +
              +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15029.html b/docs/dli/sqlreference/dli_08_15029.html new file mode 100644 index 000000000..5d972c674 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15029.html @@ -0,0 +1,82 @@ + + +

              BlackHole

              +

              Function

              The BlackHole connector allows for swallowing all input records. It is designed for high-performance testing and UDF output. It is not a substantive sink. The BlackHole result table is a built-in connector.

              +

              For example, if an error is reported when you register a result table of another type, but you are not sure whether it is caused by a system fault or an invalid setting of the WITH parameter for the result table, you can change the value of connector to blackhole and click Run. If no error is reported, the system is normal. You must check the settings of the WITH parameter.

              + +
              + + + + + + + +
              Table 1 Supported types

              Type

              +

              Description

              +

              Supported Table Types

              +

              Result table

              +
              +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              +
              +

              Syntax

              create table blackhole_table (
+ attr_name attr_type (',' attr_name attr_type) *
+) with (
+ 'connector' = 'blackhole'
+);
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + +
              Table 2 Parameter

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector to be used. Set this parameter to blackhole.

              +
              +
              +
              +

              Example

              The DataGen source table generates data, and the BlackHole result table receives the data.

              +
              create table datagenSource (
+ user_id string,
+ user_name string,
+ user_age int
+) with (
+ 'connector' = 'datagen',
+ 'rows-per-second'='1'
+);
+create table blackholeSink (
+ user_id string,
+ user_name string,
+ user_age int
+) with (
+ 'connector' = 'blackhole'
+);
+insert into blackholeSink select * from datagenSource;
              +
              +
              +
              +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15030.html b/docs/dli/sqlreference/dli_08_15030.html new file mode 100644 index 000000000..ba4f548e4 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15030.html @@ -0,0 +1,305 @@ + + +

              ClickHouse

              +

              Function

              DLI has the capability to export data from Flink jobs to the ClickHouse database. However, it only supports exporting data to result tables.

              +

              ClickHouse is a column-based database oriented to online analysis and processing. It supports SQL query and provides good query performance. The aggregation analysis and query performance based on large and wide tables is excellent, which is one order of magnitude faster than other analytical databases.

              + +
              + + + + + + + +
              Table 1 Supported types

              Type

              +

              Description

              +

              Supported Table Types

              +

              Result table

              +
              +
              +
              +

              Prerequisites

              • Your jobs are running on a dedicated queue of DLI.
              • You have established an enhanced datasource connection to ClickHouse and set the port in the security group rule of the ClickHouse cluster as needed.
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • When you create a ClickHouse cluster for MRS, set the cluster version to MRS 3.1.0 or later.
              • The ClickHouse result table does not support table data deletion.
              • Flink supports the following data types: string, tinyint, smallint, int, bigint, float, double, date, timestamp, decimal, and array.

                The array supports only the int, bigint, string, float, and double data types.

                +
              +
              +

              Syntax

              1
+2
+3
+4
+5
+6
+7
+8
+9
              create table clickhouseSink (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+)
+with (
+  'type' = 'clickhouse',
+  'url' = '',
+  'table-name' = ''
+);
+
              +
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Result table type. Set this parameter to clickhouse.

              +

              url

              +

              Yes

              +

              None

              +

              String

              +

              ClickHouse URL

              +

              It is in the format of jdbc:clickhouse://ClickHouseBalancer instance service IP address 1:ClickHouseBalancer port,ClickHouseBalancer instance service IP address 2:ClickHouseBalancer port/Database name.

              +
              • IP address of a ClickHouseBalancer instance:

                Log in to the MRS console and choose Clusters > Active Clusters in the navigation pane. Click a cluster name, and choose Components > ClickHouse > Instances to obtain the business IP address of the ClickHouseBalancer instance.

                +
              • ClickHouseBalancer port number:

                Log in to the MRS console and choose Clusters > Active Clusters in the navigation pane. Click a cluster name, and choose Components > ClickHouse > Service Configuration. On the Service Configuration page, select ClickHouseBalancer from the All Roles drop-down list. If the MRS cluster does not have Kerberos authentication enabled, search for lb_http_port and set it (defaults to 21425). If Kerberos authentication is enabled, search for lb_https_port and set it (defaults to 21426).

                +
              • The database name is the name of the database created for the ClickHouse cluster. If the database name does not exist, there is no need to specify it.
              • You can configure multiple IP addresses for ClickHouseBalancer instances to avoid single points of failure (SPOFs) of the instances.
              • If the MRS cluster has Kerberos authentication enabled, you also need to add the ssl and sslmode request parameters to the URL, setting ssl to true and sslmode to none. Refer to Example 2 for an example.
              +

              table-name

              +

              Yes

              +

              None

              +

              String

              +

              ClickHouse table name.

              +

              driver

              +

              No

              +

              ru.yandex.clickhouse.ClickHouseDriver

              +

              String

              +

              Driver required for connecting to the database. If you do not set this parameter, the automatically extracted driver will be used, which defaults to ru.yandex.clickhouse.ClickHouseDriver.

              +

              username

              +

              No

              +

              None

              +

              String

              +

              Username for accessing the ClickHouse database. This parameter is mandatory when Kerberos authentication is enabled for the MRS cluster.

              +

              password

              +

              No

              +

              None

              +

              String

              +

              Password for accessing the ClickHouse database. This parameter is mandatory when Kerberos authentication is enabled for the MRS cluster.

              +

              sink.buffer-flush.max-rows

              +

              No

              +

              100

              +

              Integer

              +

              Maximum number of rows to be updated when data is written. The default value is 100.

              +

              sink.buffer-flush.interval

              +

              No

              +

              1s

              +

              Duration

              +

              Interval for data update. The unit can be ms, milli, millisecond/s, sec, second/min, or minute. The default value is 1s. Value 0 indicates that data is not updated.

              +

              sink.max-retries

              +

              No

              +

              3

              +

              Integer

              +

              Maximum number of retries for writing data to the result table. The default value is 3.

              +
              +
              +
              +

              Example

              • Example 1: Read data from Kafka and insert the data into ClickHouse. (The ClickHouse version is 21.3.4.25 of MRS, and Kerberos authentication is not enabled for the MRS cluster):
                1. Create an enhanced datasource connection in the VPC and subnet where ClickHouse and Kafka clusters locate, and bind the connection to the required Flink elastic resource pool.
                2. Set ClickHouse and Kafka cluster security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the ClickHouse address. If the connection passes the test, it is bound to the queue.
                3. Use the ClickHouse client to connect to the ClickHouse server and run the following command to query other environment parameters such as the cluster identifier:
                  select cluster,shard_num,replica_num,host_name from system.clusters;
                  +
                  The following information is displayed:
                  ┌─cluster────┬────┬─shard_num─┐
+│ default_cluster │    1   │           1 │
+│ default_cluster │    1   │           2 │
+└──────── ┴────┴────── ┘
                  +
                  +

                  Run the following command to create database flink on a node of the ClickHouse cluster based on the obtained cluster ID, for example, default_cluster:

                  +
                  CREATE DATABASE flink ON CLUSTER default_cluster;
                  +
                4. Run the following command to create the ReplicatedMergeTree table named order on the node of cluster default_cluster and on database flink:
                  CREATE TABLE flink.order ON CLUSTER default_cluster(order_id String,order_channel String,order_time String,pay_amount Float64,real_pay Float64,pay_time String,user_id String,user_name String,area_id String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/flink/order', '{replica}')ORDER BY order_id;
                  +
                5. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. The job script uses the DMS Kafka data source and the ClickHouse result table.

                  Change the values of the parameters in bold as needed in the following script.

                  +
                  create table orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+create table clickhouseSink(
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) with (
+  'connector' = 'clickhouse',
+  'url' = 'jdbc:clickhouse://ClickhouseAddress1:ClickhousePort,ClickhouseAddress2:ClickhousePort/flink',
+  'username' = 'username',
+  'password' = 'password',
+  'table-name' = 'order',
+  'sink.buffer-flush.max-rows' = '10',
+  'sink.buffer-flush.interval' = '3s'
+);
+
+insert into clickhouseSink select * from orders;
                  +
                6. Connect to the Kafka cluster and insert the following test data into DMS Kafka:
                  {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103251202020001", "order_channel":"miniAppShop", "order_time":"2021-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2021-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
                  +
                7. Use the ClickHouse client to connect to the ClickHouse and run the following command to query the data written to table order in database flink:
                  select * from flink.order;
                  +

                  The query result is as follows:

                  +
                  202103241000000001 webShop 2021-03-24 10:00:00 100 100 2021-03-24 10:02:03 0001 Alice 330106
+
+202103241606060001 appShop 2021-03-24 16:06:06 200 180 2021-03-24 16:10:06 0001 Alice 330106 
+
+202103251202020001 miniAppShop 2021-03-25 12:02:02 60 60 2021-03-25 12:03:00 0002 Bob 330110
                  +
                +
              • Example 2: Read data from Kafka and insert the data into ClickHouse. The procedure is as follows (The ClickHouse version is 21.3.4.25 of MRS, and Kerberos authentication is enabled for the MRS cluster):
                1. Create an enhanced datasource connection in the VPC and subnet where ClickHouse and Kafka clusters locate, and bind the connection to the required Flink elastic resource pool.
                2. Set ClickHouse and Kafka cluster security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the ClickHouse address. If the connection passes the test, it is bound to the queue.
                3. Use the ClickHouse client to connect to the ClickHouse server and run the following command to query other environment parameters such as the cluster identifier:
                  select cluster,shard_num,replica_num,host_name from system.clusters;
                  +
                  The following information is displayed:
                  ┌─cluster────┬────┬─shard_num─┐
+│ default_cluster │    1   │           1 │
+│ default_cluster │    1   │           2 │
+└──────── ┴────┴────── ┘
                  +
                  +

                  Run the following command to create database flink on a node of the ClickHouse cluster based on the obtained cluster ID, for example, default_cluster:

                  +
                  CREATE DATABASE flink ON CLUSTER default_cluster;
                  +
                4. Run the following command to create the ReplicatedMergeTree table named order on the node of cluster default_cluster and on database flink:
                  CREATE TABLE flink.order ON CLUSTER default_cluster(order_id String,order_channel String,order_time String,pay_amount Float64,real_pay Float64,pay_time String,user_id String,user_name String,area_id String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/flink/order', '{replica}')ORDER BY order_id;
                  +
                5. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. The job script uses the Kafka data source and the ClickHouse result table.

                  Change the values of the parameters in bold as needed in the following script.

                  +
                  create table orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+create table clickhouseSink(
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) with (
+  'connector' = 'clickhouse',
+  'url' = 'jdbc:clickhouse://ClickhouseAddress1:ClickhousePort,ClickhouseAddress2:ClickhousePort/flink?ssl=true&sslmode=none',
+  'table-name' = 'order',
+  'username' = 'username',
+  'password' = 'password', --Key in the DEW secret
+  'sink.buffer-flush.max-rows' = '10',
+  'sink.buffer-flush.interval' = '3s',
+  'dew.endpoint'='kms.xx.xx.com', --Endpoint information for the DEW service being used
+  'dew.csms.secretName'='xx', --Name of the DEW shared secret
+  'dew.csms.decrypt.fields'='password', --The password field value must be decrypted and replaced using DEW secret management.
+  'dew.csms.version'='v1'
+);
+
+insert into clickhouseSink select * from orders;
                  +
                6. Connect to the Kafka cluster and insert the following test data into Kafka:
                  {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103251202020001", "order_channel":"miniAppShop", "order_time":"2021-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2021-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
                  +
                7. Use the ClickHouse client to connect to the ClickHouse and run the following command to query the data written to table order in database flink:
                  select * from flink.order;
                  +

                  The query result is as follows:

                  +
                  202103241000000001 webShop 2021-03-24 10:00:00 100 100 2021-03-24 10:02:03 0001 Alice 330106
+
+202103241606060001 appShop 2021-03-24 16:06:06 200 180 2021-03-24 16:10:06 0001 Alice 330106 
+
+202103251202020001 miniAppShop 2021-03-25 12:02:02 60 60 2021-03-25 12:03:00 0002 Bob 330110
                  +
                +
              +
              +
              +
              +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15031.html b/docs/dli/sqlreference/dli_08_15031.html new file mode 100644 index 000000000..4104c1aa3 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15031.html @@ -0,0 +1,200 @@ + + +

              DataGen

              +

              Function

              DataGen is used to generate random data for debugging and testing.

              + +
              + + + + + + + +
              Table 1 Supported types

              Type

              +

              Description

              +

              Supported Table Types

              +

              Source table

              +
              +
              +
              +

              Caveats

              • When you create a DataGen table, the table field type cannot be Array, Map, or Row. You can use COMPUTED COLUMN in CREATE TABLE to construct similar functions.
              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              +
              +

              Syntax

              create table dataGenSource(
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (',' WATERMARK FOR rowtime_column_name AS watermark-strategy_expression)
+)
+with (
+  'connector' = 'datagen'
+);
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector to be used. Set this parameter to datagen.

              +

              rows-per-second

              +

              No

              +

              10000

              +

              Long

              +

              Rows per second to control the emit rate.

              +

              number-of-rows

              +

              No

              +

              None

              +

              Long

              +

              The total number of rows to emit. By default, the total number of rows of generated data is not limited. If the generator type is a sequence generator, data generation will stop when either the maximum number of rows has been reached or the sequence number has reached its end value.

              +

              fields.#.kind

              +

              No

              +

              random

              +

              String

              +

              Generator of the # field. The # field must be an actual field in the DataGen table. Replace # with the corresponding field name. The meanings of the # field for other parameters are the same.

              +

              The value can be sequence or random.

              +
              • random is the default value, indicating an unbounded random generator. You can use the fields.#.max and fields.#.min parameters to specify the maximum and minimum values that are randomly generated. If the specified field type is char, varchar, or string, you can also use the fields.#.length parameter to specify the length. If the specified field type is a timestamp, you can use the fields.#.max-past parameter to specify the maximum offset from the current time towards the past.
              • sequence represents a bounded sequence generator. You can specify the start and end values of the sequence using fields.#.start and fields.#.end. Once the sequence number reaches the end value, no more data will be generated.
              +

              fields.#.min

              +

              No

              +

              Minimum value of the field type specified by #

              +

              Field type specified by #

              +

              This parameter is valid only when fields.#.kind is set to random.

              +

              Minimum value of the random generator. It applies only to numeric field types specified by #.

              +

              fields.#.max

              +

              No

              +

              Maximum value of the field type specified by #

              +

              Field type specified by #

              +

              This parameter is valid only when fields.#.kind is set to random.

              +

              Maximum value of the random number. It applies only to numeric field types specified by #.

              +

              fields.#.max-past

              +

              No

              +

              0

              +

              Duration

              +

              This parameter is valid only when fields.#.kind is set to random.

              +

              The random generator generates a maximum offset from the current time towards the past. The # specified field is only applicable to timestamp types.

              +

              fields.#.length

              +

              No

              +

              100

              +

              Integer

              +

              This parameter is valid only when fields.#.kind is set to random.

              +

              Length of the characters generated by the random generator. It applies only to char, varchar, and string types specified by #.

              +

              fields.#.start

              +

              No

              +

              None

              +

              Field type specified by #

              +

              This parameter is valid only when fields.#.kind is set to sequence.

              +

              Start value of a sequence generator.

              +

              fields.#.end

              +

              No

              +

              None

              +

              Field type specified by #

              +

              This parameter is valid only when fields.#.kind is set to sequence.

              +

              End value of a sequence generator.

              +
              +
              +
              +

              Example

              Create a Flink OpenSource SQL job. Run the following script to generate random data through the DataGen table and output the data to the Print result table.

              +
              create table dataGenSource(
+  user_id string,
+  amount int
+) with (
+  'connector' = 'datagen',
+  'rows-per-second' = '1', --Generates a piece of data per second.
+  'fields.user_id.kind' = 'random', --Specifies a random generator for the user_id field.
+  'fields.user_id.length' = '3' --Limits the length of the user_id field to 3.
+  'fields.amount.kind' = 'sequence', --Specify a sequence generator for the amount field.
+  'fields.amount.start' = '1', --Start value of the amount field
+  'fields.amount.end' = '1000' --End value of the amount field
+);
+
+create table printSink(
+  user_id string,
+  amount int
+) with (
+  'connector' = 'print'
+);
+
+insert into printSink select * from dataGenSource;
              +

              After the job is submitted, the job status changes to Running. You can perform the following operations of either method to view the output result:

              +
              • Method 1:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Locate the row that contains the target Flink job, and choose More > FlinkUI in the Operation column.
                3. On the Flink UI, choose Task Managers, click the task name, and select Stdout to view job logs.
                +
              • Method 2: If you select Save Job Log on the Running Parameters tab before submitting the job, perform the following operations:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the file whose name contains taskmanager.out, and view result logs.
                +
              +
              +
              +
              +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15032.html b/docs/dli/sqlreference/dli_08_15032.html new file mode 100644 index 000000000..03bace117 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15032.html @@ -0,0 +1,21 @@ + + +

              Doris

              +
              +
              + + +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15033.html b/docs/dli/sqlreference/dli_08_15033.html new file mode 100644 index 000000000..95c9b4f43 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15033.html @@ -0,0 +1,28 @@ + + +

              Overview

              +

              The Flink Doris Connector can support operations (read, insert, modify, delete) data stored in Doris through Flink.

              +

              Only tables in the Unique Key model can be modified or deleted.

              +
              + +
              + + + + + + + +
              Table 1 Supported types

              Type

              +

              Description

              +

              Supported Table Types

              +

              Source table, dimension table, and result table

              +
              +
              +
              +
              +
              +
              Parent topic: Doris
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15034.html b/docs/dli/sqlreference/dli_08_15034.html new file mode 100644 index 000000000..98bf92d5d --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15034.html @@ -0,0 +1,241 @@ + + +

              Source Table

              +

              Function

              Flink SQL jobs read from the Doris source table.

              +
              +

              Prerequisites

              • An enhanced datasource connection has been created for DLI to connect to Doris, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required. +
              • If MRS Doris is used, IP addresses of all hosts in the MRS cluster have been added to host information of the enhanced datasource connection.

                +

                For details, see "Modifying Host Information" in Data Lake Insight User Guide.

                +
              • Kerberos authentication is disabled for the cluster (the cluster is in normal mode)

                After connecting to Doris as user admin, create a role with administrator permissions, and bind the role to the user.

                +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • Kerberos authentication is disabled for the cluster (the cluster is in normal mode)
              • Doris table names are case sensitive.
              • When Doris of CloudTable is used, set the port number in the fenodes field to 8030, for example, xx:8030. In addition, enable ports 8030, 8040, and 9030 in the security group.
              • After HTTPS is enabled, add the following configuration parameters to the with clause for creating a table:
                • 'doris.enable.https' = 'true'
                • 'doris.ignore.https.ca' = 'true'
                +
              +
              +

              Syntax

              create table dorisSource (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+ )
+with (
+  'connector' = 'doris',
+  'fenodes' = 'FE_IP:PORT,FE_IP:PORT,FE_IP:PORT',
+  'table.identifier' = 'database.table',
+  'username' = 'dorisUsername',
+  'password' = 'dorisPassword'
+);
              +
              +

              Parameter Description

              Shared configuration

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

              Parameter

              +

              Default Value

              +

              Mandatory

              +

              Parameter Type Description

              +

              fenodes

              +

              --

              +

              Yes

              +

              IP address and port number of the Doris FE. Use commas (,) to separate them for multiple instances. To obtain the port number, log in to MRS Manager, choose Cluster > Services > Doris > Configurations, and search for http. Search for https instead if HTTPS is enabled.

              +

              table.identifier

              +

              --

              +

              Yes

              +

              Doris table name, for example, db.tbl.

              +

              username

              +

              --

              +

              Yes

              +

              User name for accessing Doris.

              +

              password

              +

              --

              +

              Yes

              +

              Password for accessing Doris.

              +

              doris.request.retries

              +

              3

              +

              No

              +

              Number of retry times for sending requests to Doris.

              +

              doris.request.connect.timeout.ms

              +

              30000

              +

              No

              +

              Connection timeout interval for sending requests to Doris.

              +

              doris.request.read.timeout.ms

              +

              30000

              +

              No

              +

              Read timeout interval for sending requests to Doris.

              +

              doris.request.query.timeout.s

              +

              3600

              +

              No

              +

              Timeout interval for querying Doris. The default value is 1 hour. The value -1 indicates that there is no timeout limit.

              +

              doris.request.tablet.size

              +

              Integer. MAX_VALUE

              +

              No

              +

              Number of Doris Tablets corresponding to a partition. The smaller the value set, the more partitions will be generated. This increases the degree of parallelism in Flink, but puts more pressure on Doris.

              +

              doris.batch.size

              +

              1024

              +

              No

              +

              Maximum number of rows to read from BE at a time. Increasing this value reduces the number of times Flink needs to establish a connection with Doris when reading data from BE, thereby reducing the additional time overhead caused by network latency.

              +

              doris.exec.mem.limit

              +

              2147483648

              +

              No

              +

              Memory limit for a single query, with a default value of 2 GB, in bytes.

              +

              doris.deserialize.arrow.async

              +

              FALSE

              +

              No

              +

              Whether to support asynchronous conversion of Arrow format to RowBatch required for flink-doris-connector iteration.

              +

              doris.deserialize.queue.size

              +

              64

              +

              No

              +

              The internal processing queue for asynchronous conversion of Arrow format is effective when doris.deserialize.arrow.async is set to true.

              +

              doris.read.field

              +

              --

              +

              No

              +

              The column name list for reading from Doris tables, with multiple columns separated by commas.

              +

              doris.filter.query

              +

              --

              +

              No

              +

              The expression used to filter the data to be read, which is passed through to Doris. Doris uses this expression to filter the source data.

              +
              +
              +
              +

              Example

              This example reads data from a Doris source table and inputs it into the Print connector.

              +
              1. Create an enhanced datasource connection in the VPC and subnet where Doris locates, and bind the connection to the required Flink elastic resource pool. Add MRS host information for the enhanced datasource connection.
              2. Set Doris security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Doris address. If the connection passes the test, it is bound to the queue.
              3. Create a Doris table and insert 10 data records into the table. The creation statement is as follows:
                CREATE TABLE IF NOT EXISTS dorisdemo
+(
+  `user_id` varchar(10) NOT NULL,
+  `city` varchar(10),
+  `age` int,
+  `gender` int
+)
+DISTRIBUTED BY HASH(`user_id`) BUCKETS 10;
+
+INSERT INTO dorisdemo VALUES ('user1', 'city1', 20, 1);
+INSERT INTO dorisdemo VALUES ('user2', 'city2', 21, 0);
+INSERT INTO dorisdemo VALUES ('user3', 'city3', 22, 1);
+INSERT INTO dorisdemo VALUES ('user4', 'city4', 23, 0);
+INSERT INTO dorisdemo VALUES ('user5', 'city5', 24, 1);
+INSERT INTO dorisdemo VALUES ('user6', 'city6', 25, 0);
+INSERT INTO dorisdemo VALUES ('user7', 'city7', 26, 1);
+INSERT INTO dorisdemo VALUES ('user8', 'city8', 27, 0);
+INSERT INTO dorisdemo VALUES ('user9', 'city9', 28, 1);
+INSERT INTO dorisdemo VALUES ('user10', 'city10', 29, 0);
                +
              4. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. The job script reads from the Doris table and prints the output.
                CREATE TABLE dorisDemo (
+  `user_id` String NOT NULL,
+  `city` String,
+  `age` int,
+  `gender` int
+) with (
+  'connector' = 'doris',
+  'fenodes' = 'FE_IP:PORT,FE_IP:PORT,FE_IP:PORT',
+  'table.identifier' = 'demo.dorisdemo',
+  'username' = 'dorisUser',
+  'password' = 'dorisPassword',
+  'doris.request.retries'='3',
+  'doris.batch.size' = '100'
+);
+
+CREATE TABLE print (
+  `user_id` String NOT NULL,
+  `city` String,
+  `age` int,
+  `gender` int
+) with (
+  'connector' = 'print'
+);
+
+insert into print select * from dorisDemo;
                +
              5. View the data in the Print result table.
                +I[user5, city5, 24, 1]
++I[user4, city4, 23, 0]
++I[user3, city3, 22, 1]
++I[user10, city10, 29, 0]
++I[user6, city6, 25, 0]
++I[user1, city1, 20, 1]
++I[user9, city9, 28, 1]
++I[user7, city7, 26, 1]
++I[user8, city8, 27, 0]
++I[user2, city2, 21, 0]
                +
              +
              +
              +
              +
              +
              Parent topic: Doris
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15035.html b/docs/dli/sqlreference/dli_08_15035.html new file mode 100644 index 000000000..4b49fa734 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15035.html @@ -0,0 +1,246 @@ + + +

              Result Table

              +

              Function

              Flink SQL jobs write to the Doris result table.

              +
              +

              Prerequisites

              • An enhanced datasource connection has been created for DLI to connect to Doris, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required. +
              • If MRS Doris is used, IP addresses of all hosts in the MRS cluster have been added to host information of the enhanced datasource connection.

                +

                For details, see "Modifying Host Information" in Data Lake Insight User Guide.

                +
              • Kerberos authentication is disabled for the cluster (the cluster is in normal mode)

                After connecting to Doris as user admin, create a role with administrator permissions, and bind the role to the user.

                +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • Kerberos authentication is disabled for the cluster (the cluster is in normal mode)
              • Doris table names are case sensitive.
              • When Doris of CloudTable is used, set the port number in the fenodes field to 8030, for example, xx:8030. In addition, enable ports 8030, 8040, and 9030 in the security group.
              • After HTTPS is enabled, add the following configuration parameters to the with clause for creating a table:
                • 'doris.enable.https' = 'true'
                • 'doris.ignore.https.ca' = 'true'
                +
              • On the Running Parameters tab of the Flink job editing page, check Enable Checkpointing. Otherwise, data can be written to the Doris result table, and the delay in writing to Doris depends on the value set for Checkpoint Interval.
              +
              +

              Syntax

              create table dorisSource (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+ )
+with (
+  'connector' = 'doris',
+  'fenodes' = 'FE_IP:PORT,FE_IP:PORT,FE_IP:PORT',
+  'table.identifier' = 'database.table',
+  'username' = 'dorisUsername',
+  'password' = 'dorisPassword'
+);
              +
              +

              Parameter Description

              Shared configuration

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

              Parameter

              +

              Default Value

              +

              Mandatory

              +

              Parameter Type Description

              +

              fenodes

              +

              --

              +

              Yes

              +

              IP address and port number of the Doris FE. Use commas (,) to separate them for multiple instances. To obtain the port number, log in to MRS Manager, choose Cluster > Services > Doris > Configurations, and search for http. Search for https instead if HTTPS is enabled.

              +

              table.identifier

              +

              --

              +

              Yes

              +

              Doris table name, for example, db.tbl.

              +

              username

              +

              --

              +

              Yes

              +

              User name for accessing Doris.

              +

              password

              +

              --

              +

              Yes

              +

              Password for accessing Doris.

              +

              sink.label-prefix

              +

              ""

              +

              Yes

              +

              Label prefix used for Stream load import. It must be globally unique in two-phase commit (2pc) scenarios to ensure Flink's EOS semantics.

              +

              sink.enable-2pc

              +

              TRUE

              +

              No

              +

              Whether to enable 2pc for ensuring Exactly-Once semantics. The default value is true.

              +

              sink.check-interval

              +

              10000

              +

              No

              +

              Interval for checking exceptions during loading.

              +

              sink.max-retries

              +

              3

              +

              No

              +

              Maximum number of retries when writing records to the database fails.

              +

              sink.buffer-size

              +

              256 * 1024

              +

              No

              +

              Buffer size for caching data during Stream load.

              +

              sink.buffer-count

              +

              3

              +

              No

              +

              Buffer count for caching data during Stream load.

              +

              sink.enable-delete

              +

              TRUE

              +

              No

              +

              Whether to enable deletion. This option requires batch deletion to be enabled for the Doris table (default in Doris 0.15 or later for Unique model only).

              +

              sink.properties.*

              +

              --

              +

              No

              +

              Import parameters for Stream load.

              +

              For example, 'sink.properties.column_separator' = ',' defines the column separator, and 'sink.properties.escape_delimiters' = 'true' treats special characters as separators, where '\x01' is converted to binary 0x01.

              +

              JSON format import

              +

              'sink.properties.format' = 'json' 'sink.properties.read_json_by_line' = 'true'

              +
              +
              +
              +

              Example

              In this example, data is read from the DataGen data source and written to the Doris result table.

              +
              1. Create an enhanced datasource connection in the VPC and subnet where Doris locates, and bind the connection to the required Flink elastic resource pool. Add MRS host information for the enhanced datasource connection.
              2. Set Doris security groups and add inbound rules to allow access from the Flink queue. Test the queue connectivity based on the Doris address. If the connection passes the test, it is bound to the queue.

                +
              3. Create a Doris table by referring to MRS Doris Usage Guide. The creation statement is as follows:
                CREATE TABLE IF NOT EXISTS dorisdemo
+(
+  `user_id` varchar(10) NOT NULL,
+  `city` varchar(10),
+  `age` int,
+  `gender` int
+)
+DISTRIBUTED BY HASH(`user_id`) BUCKETS 10
+
                +
              4. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. The job script uses DataGen as the data source and writes data to as a Doris result table.
                create table student_datagen_source(
+  `user_id` String NOT NULL,
+  `city` String,
+  `age` int,
+  `gender` int
+) with (
+  'connector' = 'datagen',
+  'rows-per-second' = '1',
+  'fields.user_id.kind' = 'random',
+  'fields.user_id.length' = '7', 
+  'fields.city.kind' = 'random',
+  'fields.city.length' = '7'
+);
+
+
+CREATE TABLE dorisDemo (
+  `user_id` String NOT NULL,
+  `city` String,
+  `age` int,
+  `gender` int
+) with (
+  'connector' = 'doris',
+  'fenodes' = 'FE_IP:PORT',
+  'table.identifier' = 'demo.dorisdemo',
+  'username' = 'dorisUser',
+  'password' = 'dorisPassword',
+  'sink.label-prefix' = 'demo',
+  'sink.enable-2pc' = 'true',
+  'sink.buffer-count' = '10'
+);
+
+insert into dorisDemo select * from student_datagen_source
                +
              1. Check whether data is successfully written to the Doris result table. +
                + + + + + + + + + + + + + + + + + + + + + +

                user_id

                +

                city

                +

                age

                +

                gender

                +

                50aff04

                +

                93406c5

                +

                12

                +

                1

                +

                681a230

                +

                1f27d06

                +

                16

                +

                1

                +

                006eff4

                +

                3521ded

                +

                18

                +

                0

                +
                +
                +
              +
              +
              +
              +
              +
              Parent topic: Doris
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15036.html b/docs/dli/sqlreference/dli_08_15036.html new file mode 100644 index 000000000..6dd24ab42 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15036.html @@ -0,0 +1,186 @@ + + +

              Dimension Table

              +

              Function

              Create a Doris dimension table to connect to the source streams for wide table generation.

              +
              +

              Prerequisites

              • An enhanced datasource connection has been created for DLI to connect to HBase, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required. +
              • If MRS Doris is used, IP addresses of all hosts in the MRS cluster have been added to host information of the enhanced datasource connection.

                +

                For details, see "Modifying Host Information" in Data Lake Insight User Guide.

                +
              • Kerberos authentication is disabled for the cluster (the cluster is in normal mode).

                After connecting to Doris as user admin, create a role with administrator permissions, and bind the role to the user.

                +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • Kerberos authentication is disabled for the cluster (the cluster is in normal mode).
              • Doris table names are case sensitive.
              • When Doris of CloudTable is used, set the port number in the fenodes field to 8030, for example, xx:8030. In addition, enable ports 8030, 8040, and 9030 in the security group.
              • After HTTPS is enabled, add the following configuration parameters to the with clause for creating a table:
                • 'doris.enable.https' = 'true'
                • 'doris.ignore.https.ca' = 'true'
                +
              +
              +

              Syntax

              create table hbaseSource (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+ )
+with (
+  'connector' = 'doris',
+  'fenodes' = 'FE_IP:PORT,FE_IP:PORT,FE_IP:PORT',
+  'table.identifier' = 'database.table',
+  'username' = 'dorisUsername',
+  'password' = 'dorisPassword'
+);
              +
              +

              Parameter Description

              Shared configuration

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

              Parameter

              +

              Default Value

              +

              Mandatory

              +

              Parameter Type Description

              +

              fenodes

              +

              --

              +

              Y

              +

              IP address and port number of the Doris FE. Use commas (,) to separate them for multiple instances. To obtain the port number, log in to MRS Manager, choose Cluster > Services > Doris > Configurations, and search for http. Search for https instead if HTTPS is enabled.

              +

              table.identifier

              +

              --

              +

              Y

              +

              Doris table name, for example, db.tbl.

              +

              username

              +

              --

              +

              Y

              +

              User name for accessing Doris.

              +

              password

              +

              --

              +

              Y

              +

              Password for accessing Doris.

              +

              lookup.cache.max-rows

              +

              -1L

              +

              N

              +

              Maximum number of rows to search in the cache, where the oldest row will be deleted if this value is exceeded.

              +

              To enable cache configuration, both the cache.max-rows and cache.ttl options must be specified.

              +

              lookup.cache.ttl

              +

              10s

              +

              N

              +

              Cache lifespan.

              +

              lookup.max-retries

              +

              3

              +

              N

              +

              Maximum number of retry attempts when a database lookup fails.

              +
              +
              +
              +

              Example

              This example reads data from a Doris source table and inputs it into the Print connector.

              +
              1. Create an enhanced datasource connection in the VPC and subnet where Doris locates, and bind the connection to the required Flink elastic resource pool. Add MRS host information for the enhanced datasource connection.
              2. Set Doris and Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Doris and Kafka addresses. If the connection passes the test, it is bound to the queue.
              3. Create a Doris table and insert 10 data records. The creation statement is as follows:
                CREATE TABLE IF NOT EXISTS dorisdemo
+(
+  `user_id` varchar(10) NOT NULL,
+  `city` varchar(10),
+  `age` int,
+  `gender` int
+)
+DISTRIBUTED BY HASH(`user_id`) BUCKETS 10;
+
+INSERT INTO dorisdemo VALUES ('user1', 'city1', 20, 1);
+INSERT INTO dorisdemo VALUES ('user2', 'city2', 21, 0);
+INSERT INTO dorisdemo VALUES ('user3', 'city3', 22, 1);
+INSERT INTO dorisdemo VALUES ('user4', 'city4', 23, 0);
+INSERT INTO dorisdemo VALUES ('user5', 'city5', 24, 1);
+INSERT INTO dorisdemo VALUES ('user6', 'city6', 25, 0);
+INSERT INTO dorisdemo VALUES ('user7', 'city7', 26, 1);
+INSERT INTO dorisdemo VALUES ('user8', 'city8', 27, 0);
+INSERT INTO dorisdemo VALUES ('user9', 'city9', 28, 1);
+INSERT INTO dorisdemo VALUES ('user10', 'city10', 29, 0);
                +
              4. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. This job simulates reading data from Kafka, performs a join with a Doris dimension table to denormalize the data, and outputs it to Print.
                CREATE TABLE ordersSource (
+  user_id string,
+  user_name string,
+  proctime as Proctime()
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafka-topic',
+  'properties.bootstrap.servers' = 'kafkaIp:port,kafkaIp:port,kafkaIp:port',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE dorisDemo (
+  `user_id` String NOT NULL,
+  `city` String,
+  `age` int,
+  `gender` int
+) with (
+  'connector' = 'doris',
+  'fenodes' = 'IP address of the FE instance:Port number',
+  'table.identifier' = 'demo.dorisdemo',
+  'username' = 'dorisUsername',
+  'password' = 'dorisPassword',
+  'lookup.cache.ttl'='10 m',
+  'lookup.cache.max-rows' = '100'
+);
+
+CREATE TABLE print (
+  user_id string,
+  user_name string,
+  `city` String,
+  `age` int,
+  `gender` int
+) WITH (
+  'connector' = 'print'
+);
+
+insert into print 
+select 
+  orders.user_id,
+  orders.user_name,
+  dim.city,
+  dim.age,
+  dim.sex
+from ordersSource orders
+left join dorisDemo for system_time as of orders.proctime as dim on orders.user_id = dim.user_id;
                +
              5. Write two data records to the Kafka data source.
                {"user_id": "user1", "user_name": "name1"}
+{"user_id": "user2", "user_name": "name2"}
                +
              6. View the data in the Print result table.
                +I[user1, name1, city1, 20, 1]
++I[user2, name2, city2, 21, 0]
                +
              +
              +
              +
              +
              +
              Parent topic: Doris
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15037.html b/docs/dli/sqlreference/dli_08_15037.html new file mode 100644 index 000000000..df4d2ac0a --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15037.html @@ -0,0 +1,41 @@ + + +

              Overview

              +

              GaussDB(DWS) is an online data processing database based on the cloud infrastructure and platform and helps you mine and analyze massive sets of data. DLI reads data of Flink jobs from GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex types and delivers space information services, multi-version concurrent control (MVCC), and high concurrency. It applies to location applications, financial insurance, and e-commerce.

              +

              +
              DLI Flink 1.15 now offers two GaussDB(DWS) connector options for accessing GaussDB data:
              • GaussDB(DWS)'s self-developed GaussDB(DWS) connector (recommended): This option focuses on the performance of and direct interaction with GaussDB(DWS), allowing users to easily and flexibly read and write data.

                You can use GaussDB(DWS)'s self-developed GaussDB(DWS) connector by creating UDFs. For details about how to create a UDF, see UDFs.

                +
              • DLI's GaussDB(DWS) connector (discarded and not recommended): This option allows users to customize sink and source functions to meet specific data read and write needs.
                For how to use DLI's GaussDB(DWS) connector, see Table 1. +
                + + + + + + + + + + + + + +
                Table 1 Supported GaussDB(DWS) connector types

                Type

                +

                Instruction

                +

                Source table

                +

                GaussDB(DWS) Source Table (Not Recommended)

                +

                Result table

                +

                GaussDB(DWS) Result Table (Not Recommended)

                +

                Dimension table

                +

                GaussDB(DWS) Dimension Table (Not Recommended)

                +
                +
                +
                +
              +
              +
              +
              +
              +
              Parent topic: GaussDB(DWS)
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15038.html b/docs/dli/sqlreference/dli_08_15038.html new file mode 100644 index 000000000..708a55c04 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15038.html @@ -0,0 +1,444 @@ + + +

              Elasticsearch

              +

              Function

              DLI outputs the output data of the Flink job to an index in the Elasticsearch engine of the Cloud Search Service (CSS).

              +

              Elasticsearch is a popular enterprise-class Lucene-powered search server and provides the distributed multi-user capabilities. It delivers multiple functions, including full-text retrieval, structured search, analytics, aggregation, and highlighting. With Elasticsearch, you can achieve stable, reliable, real-time search. Elasticsearch applies to diversified scenarios, such as log analysis and site search.

              +

              CSS is a fully managed, distributed search service. It is fully compatible with open-source Elasticsearch and provides DLI with structured and unstructured data search, statistics, and report capabilities.

              +

              For details, see Elasticsearch SQL Connector.

              + +
              + + + + + + + + + + +
              Table 1 Supported types

              Type

              +

              Description

              +

              Supported Table Types

              +

              Result table

              +

              Supported Data Formats

              +

              JSON

              +
              +
              +
              +

              Prerequisites

              • Ensure that you have created a cluster on CSS using your account.
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • Fields in the with parameter can only be enclosed in single quotes.
              • Only CSS 7.X or later clusters are currently supported.
              • If security mode is enabled and HTTPS is enabled, you need to configure the username, password, and certificate location. Note that the hosts field value in this scenario starts with https.
              • ICMP must be enabled for the security group inbound rule of the CSS cluster.
              • Fields in the with parameter can only be enclosed in single quotes.
              • For details about how to use data types, see section Format.
              +
              +

              Syntax

              create table esSink (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (','PRIMARY KEY (attr_name, ...) NOT ENFORCED)
+)
+with (
+  'connector' = 'elasticsearch-7',
+  'hosts' = '',
+  'index' = ''
+);
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Elasticsearch result table parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Specify what connector to use. Set this parameter to elasticsearch-7. This indicates connecting to Elasticsearch 7.x cluster.

              +

              hosts

              +

              Yes

              +

              None

              +

              String

              +

              Host name of the cluster where Elasticsearch is located. Use semicolons (;) to separate multiple host names.

              +

              index

              +

              Yes

              +

              None

              +

              String

              +

              Elasticsearch index for every record. The index can be a static index (for example, 'myIndex') or a dynamic index (for example, 'index-{log_ts|yyyy-MM-dd}'). See the following Dynamic Index section for more details.

              +

              username

              +

              No

              +

              None

              +

              String

              +

              Username of the cluster where Elasticsearch locates. This parameter must be configured in pair with password.

              +

              password

              +

              No

              +

              None

              +

              String

              +

              Password of the cluster where Elasticsearch locates. This parameter must be configured in pair with username.

              +

              document-id.key-delimiter

              +

              No

              +

              _

              +

              String

              +

              Delimiter for composite keys ("_" by default), e.g., $ would result in IDs KEY1$KEY2$KEY3.

              +

              failure-handler

              +

              No

              +

              fail

              +

              String

              +

              Failure handling strategy in case a request to Elasticsearch fails. Valid strategies are:

              +
              • fail: throws an exception if a request fails and causes a job failure.
              • ignore: ignores failures and drops the request.
              • retry-rejected: Requests that failed due to saturated queue are re-added.
              • custom class name: The subclass of ActionRequestFailureHandler is used to handle failures.
              +

              sink.flush-on-checkpoint

              +

              No

              +

              true

              +

              Boolean

              +

              Flush on checkpoint or not.

              +

              When disabled, a sink will not wait for all pending action requests to be acknowledged by Elasticsearch on checkpoints. Thus, a sink does not provide any strong guarantees for at-least-once delivery of action requests.

              +

              sink.bulk-flush.max-actions

              +

              No

              +

              1000

              +

              Interger

              +

              Maximum number of buffered actions per bulk request. You can set this parameter to 0 to disable it.

              +

              sink.bulk-flush.max-size

              +

              No

              +

              2mb

              +

              MemorySize

              +

              Maximum size in memory of buffered actions per bulk request. Must be in MB granularity. Can be set to 0 to disable it.

              +

              sink.bulk-flush.interval

              +

              No

              +

              1s

              +

              Duration

              +

              The interval to flush buffered actions. Can be set to 0 to disable it.

              +

              Note, both sink.bulk-flush.max-size and sink.bulk-flush.max-actions can be set to 0 with the flush interval set allowing for complete async processing of buffered actions.

              +

              sink.bulk-flush.backoff.strategy

              +

              No

              +

              DISABLED

              +

              String

              +

              Specify how to perform retries if any flush actions failed due to a temporary request error. Valid strategies are:

              +
              • DISABLED: no retry performed, i.e. fail after the first request error.
              • CONSTANT: wait for backoff delay between retries.
              • EXPONENTIAL: initially wait for backoff delay and increase exponentially between retries.
              +

              sink.bulk-flush.backoff.max-retries

              +

              No

              +

              None

              +

              Integer

              +

              Maximum number of rollback retries.

              +

              sink.bulk-flush.backoff.delay

              +

              No

              +

              None

              +

              Duration

              +

              Delay between each backoff attempt.

              +

              For CONSTANT backoff, this is simply the delay between each retry. For EXPONENTIAL backoff, this is the initial base delay.

              +

              connection.path-prefix

              +

              No

              +

              None

              +

              String

              +

              Prefix string added to each REST communication, for example, '/v1'.

              +

              connection.request-timeout

              +

              No

              +

              None

              +

              Duration

              +

              The timeout in milliseconds for requesting a connection from the connection manager. The timeout must be larger than or equal to 0. A timeout value of zero is interpreted as an infinite timeout.

              +

              connection.timeout

              +

              No

              +

              None

              +

              Duration

              +

              The timeout in milliseconds for establishing a connection.

              +

              The timeout must be larger than or equal to 0. A timeout value of zero is interpreted as an infinite timeout.

              +

              socket.timeout

              +

              No

              +

              None

              +

              Duration

              +

              The socket timeout (SO_TIMEOUT) for waiting for data. The timeout must be larger than or equal to 0. A timeout value of zero is interpreted as an infinite timeout.

              +

              format

              +

              No

              +

              json

              +

              String

              +

              Elasticsearch connector supports to specify a format. The format must produce a valid JSON document. By default, the built-in JSON format is used.

              +

              Refer to Format for more details and format parameters.

              +

              certificate

              +

              No

              +

              None

              +

              String

              +

              Location of the Elasticsearch cluster certificate in OBS.

              +

              This parameter is required only when the security mode and HTTPS are enabled.

              +

              Download the certificate from the CSS management console and upload the certificate to OBS. This parameter specifies the OBS address.

              +

              Example: obs://bucket/path/CloudSearchService.cer

              +
              +
              +
              +

              Key Handling

              The Elasticsearch sink can work in either upsert mode or append mode, depending on whether a primary key is defined.

              +
              • If a primary key is defined, the Elasticsearch sink works in upsert mode which can consume queries containing UPDATE/DELETE messages.
              • If a primary key is not defined, the Elasticsearch sink works in append mode which can only consume queries containing INSERT only messages.
              +

              In the Elasticsearch connector, the primary key is used to calculate the Elasticsearch document ID, which is a string of up to 512 bytes. It cannot have whitespaces.

              +

              The Elasticsearch connector generates a document ID string for every row by concatenating all primary key fields in the order defined in the DDL using a key delimiter specified by document-id.key-delimiter. Certain types are not allowed as a primary key field as they do not have a good string representation, e.g. BYTES, ROW, ARRAY, MAP, etc.

              +

              If no primary key is specified, Elasticsearch will generate a document ID automatically.

              +
              +

              Dynamic Index

              The Elasticsearch sink supports both static index and dynamic index.

              +
              • If you want to have a static index, the index option value should be a plain string, e.g. myusers, all the records will be consistently written into myusers index.
              • If you want to have a dynamic index, you can use {field_name} to reference a field value in the record to dynamically generate a target index.
                • You can use {field_name|date_format_string} to convert a field value of TIMESTAMP/DATE/TIME type into the format specified by the date_format_string. The date_format_string is compatible with Java's DateTimeFormatter. For example, if the option value is myusers-{log_ts|yyyy-MM-dd}, then a record with log_ts field value 2020-03-27 12:25:55 will be written into myusers-2020-03-27 index.
                • You can use {now()|date_format_string} to convert the current system time to the format specified by date_format_string. The corresponding time type of now() is TIMESTAMP_WITH_LTZ. When formatting the system time as a string, the time zone configured in the session through table.local-time-zone will be used. You can use NOW(), now(), CURRENT_TIMESTAMP, or current_timestamp.

                  When using the dynamic index generated by the current system time, for changelog stream, there is no guarantee that the records with the same primary key can generate the same index name. Therefore, the dynamic index based on the system time can only support append only stream.

                  +
                  +
                +
              +
              +

              Example

              In this example, data is read from the Kafka data source and written to the Elasticsearch result table (Elasticsearch 7.10.2). The procedure is as follows:

              +
              1. Create an enhanced datasource connection in the VPC and subnet where Elasticsearch and Kafka locate, and bind the connection to the required Flink elastic resource pool.
              2. Set Elasticsearch and Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Elasticsearch and Kafka addresses. If the connection passes the test, it is bound to the queue.
              3. Log in to Kibana of the Elasticsearch cluster, select Dev Tools, enter and execute the following statement to create an index whose value is orders:
                PUT /orders
+{
+  "settings": {
+    "number_of_shards": 1
+  },
+	"mappings": {
+	  "properties": {
+	    "order_id": {
+	      "type": "text"
+	    },
+	    "order_channel": {
+	      "type": "text"
+	    },
+	    "order_time": {
+	      "type": "text"
+	    },
+	    "pay_amount": {
+	      "type": "double"
+	    },
+	    "real_pay": {
+	      "type": "double"
+	    },
+	    "pay_time": {
+	      "type": "text"
+	    },
+	    "user_id": {
+	      "type": "text"
+	    },
+	    "user_name": {
+	      "type": "text"
+	    },
+	    "area_id": {
+	      "type": "text"
+	    }
+	  }
+	}
+}
                +
              4. Create a Flink OpenSource SQL job. Enter the following job script and submit the job.
                Change the values of the parameters in bold as needed in the following script.
                CREATE TABLE kafkaSource (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE elasticsearchSink (
+  order_id string,
+  order_channel string,
+  order_time string, 
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'elasticsearch-7',
+  'hosts' = 'ElasticsearchAddress:ElasticsearchPort',
+  'index' = 'orders'
+);
+insert into elasticsearchSink select * from kafkaSource;
                +
                +
              5. Connect to the Kafka cluster and insert the following test data into Kafka:
                {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
                +
              6. Enter the following statement in Kibana of the Elasticsearch cluster and view the result:
                GET orders/_search
                +

                +
                {
+  "took" : 201,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 1,
+    "successful" : 1,
+    "skipped" : 0,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : {
+      "value" : 2,
+      "relation" : "eq"
+    },
+    "max_score" : 1.0,
+    "hits" : [
+      {
+        "_index" : "orders",
+        "_type" : "_doc",
+        "_id" : "fopyx4sBUuT2wThgYGcp",
+        "_score" : 1.0,
+        "_source" : {
+          "order_id" : "202103241606060001",
+          "order_channel" : "appShop",
+          "order_time" : "2021-03-24 16:06:06",
+          "pay_amount" : 200.0,
+          "real_pay" : 180.0,
+          "pay_time" : "2021-03-24 16:10:06",
+          "user_id" : "0001",
+          "user_name" : "Alice",
+          "area_id" : "330106"
+        }
+      },
+      {
+        "_index" : "orders",
+        "_type" : "_doc",
+        "_id" : "f4pyx4sBUuT2wThgYGcr",
+        "_score" : 1.0,
+        "_source" : {
+          "order_id" : "202103241000000001",
+          "order_channel" : "webShop",
+          "order_time" : "2021-03-24 10:00:00",
+          "pay_amount" : 100.0,
+          "real_pay" : 100.0,
+          "pay_time" : "2021-03-24 10:02:03",
+          "user_id" : "0001",
+          "user_name" : "Alice",
+          "area_id" : "330106"
+        }
+      }
+    ]
+  }
+}
+
                +
              +
              +
              +
              +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15039.html b/docs/dli/sqlreference/dli_08_15039.html new file mode 100644 index 000000000..f6d94e731 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15039.html @@ -0,0 +1,17 @@ + + +

              OBS

              +
              +
              + + +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15040.html b/docs/dli/sqlreference/dli_08_15040.html new file mode 100644 index 000000000..1a29faa75 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15040.html @@ -0,0 +1,142 @@ + + +

              OBS Source Table

              +

              Function

              The file system connector can be used to read single files or entire directories into a single table.

              +

              When using a directory as the source path, there is no defined order of ingestion for the files inside the directory. For more information, see FileSystem SQL Connector.

              +
              +

              Syntax

               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
              CREATE TABLE sink_table (
+   name string,
+   num INT,
+   p_day string,
+   p_hour string
+) partitioned by (p_day, p_hour) WITH (
+   'connector' = 'filesystem',
+   'path' = 'obs://*** ',
+   'format' = 'parquet',
+   'source.monitor-interval'=''
+);
+
              +
              +
              +

              Parameter Description

              • Directory watching

                By default, the file system connector is bounded, that is it will scan the configured path once and then close itself.

                +

                You can enable continuous directory watching by configuring the source.monitor-interval parameter:

                + +
                + + + + + + + + + + + +

                Key

                +

                Default Value

                +

                Data Type

                +

                Description

                +

                source.monitor-interval

                +

                None

                +

                Duration

                +

                The interval in which the source checks for new files. The interval must be greater than 0.

                +

                Each file is uniquely identified by its path, and will be processed once, as soon as it is discovered.

                +

                The set of files already processed is kept in state during the whole lifecycle of the source, so it's persisted in checkpoints and savepoints together with the source state.

                +

                Shorter intervals mean that files are discovered more quickly, but also imply more frequent listing or directory traversal of the file system/object store.

                +

                If this config option is not set, the provided path will be scanned once, hence the source will be bounded.

                +
                +
                +
              +
              +
              • Available Metadata

                The following connector metadata can be accessed as metadata columns in a table definition. All the metadata are read only.

                + +
                + + + + + + + + + + + + + + + + + + + + + +

                Key

                +

                Data Type

                +

                Description

                +

                file.path

                +

                STRING NOT NULL

                +

                Full path of the input file

                +

                file.name

                +

                STRING NOT NULL

                +

                Name of the file, that is the farthest element from the root of the filepath

                +

                file.size

                +

                STRING NOT NULL

                +

                Byte count of the file

                +

                file.modification-time

                +

                TIMESTAMP_LTZ(3) NOT NULL

                +

                Modification time of the file

                +
                +
                +
              +

              Example

              Read data from the OBS table as the data source and output it to the Print connector.

              +
              CREATE TABLE obs_source(
+   name string,
+   num INT,
+   `file.path` STRING NOT NULL METADATA
+) WITH (
+   'connector' = 'filesystem',
+   'path' = 'obs://demo/sink_parquent_obs',
+   'format' = 'parquet',
+   'source.monitor-interval'='1 h'
+);
+
+
+CREATE TABLE print (
+   name string,
+   num INT,
+   path  STRING
+) WITH (
+   'connector' = 'print'
+);
+
+insert into print
+select * from obs_source;
+
              +
              +
              Print result:
              +I[0e72e, 841255524, /spark.db/sink_parquent_obs/compacted-part-fd4d4cc8-8b18-42d5-b522-9b524500fa23-0-0]
++I[53524, -2032270969, /spark.db/sink_parquent_obs/compacted-part-fd4d4cc8-8b18-42d5-b522-9b524500fa23-0-0]
++I[77225, 245599258, /spark.db/sink_parquent_obs/compacted-part-fd4d4cc8-8b18-42d5-b522-9b524500fa23-0-0]
++I[fc202, -545621464, /spark.db/sink_parquent_obs/compacted-part-fd4d4cc8-8b18-42d5-b522-9b524500fa23-0-0]
++I[07e9d, 1511139764, /spark.db/sink_parquent_obs/compacted-part-fd4d4cc8-8b18-42d5-b522-9b524500fa23-0-0]
++I[4e48b, 278014413, /spark.db/sink_parquent_obs/compacted-part-fd4d4cc8-8b18-42d5-b522-9b524500fa23-0-0]
              +
              +
              +
              +
              +
              Parent topic: OBS
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15041.html b/docs/dli/sqlreference/dli_08_15041.html new file mode 100644 index 000000000..56ce886fa --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15041.html @@ -0,0 +1,232 @@ + + +

              OBS Result Table

              +

              Function

              The FileSystem result (sink) table is used to export data to the HDFS or OBS file system. It is applicable to scenarios such as data dumping, big data analysis, data backup, and active, deep, or cold archiving.

              +

              Considering that the input stream can be unbounded, you can put the data in each bucket into part files of a limited size. Data can be written into a bucket based on time. For example, you can write data into a bucket every hour. This bucket contains the records received within one hour, and

              +

              data in the bucket directory is split into multiple part files. Each sink bucket that receives data contains at least one part file for each subtask. Other part files are created based on the configured rolling policy. For Row Formats, the default rolling policy is based on the part file size. You need to specify the maximum timeout period for opening a file and the timeout period for the inactive state after closing a file. Bulk Formats are rolled each time a checkpoint is created. You can add other rolling conditions based on size or time. For more information, see FileSystem SQL Connector.

              +
              • To use FileSink in STREAMING mode, you need to enable the checkpoint function. Part files are generated only when the checkpoint is successful. If the checkpoint function is not enabled, the files remain in the in-progress or pending state, and downstream systems cannot securely read the file data.
              • The number recorded by the sink end operator is the number of checkpoints, not the actual volume of the sent data. For the actual volume, see the number recorded by the streaming-writer or StreamingFileWriter operator.
              +
              +
              +

              Caveats

              On the Flink job's editing page, select Enable Checkpointing on the Running Parameters tab. Otherwise, data cannot be written to the FileSystem result table.

              +
              +

              Syntax

               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
              CREATE TABLE sink_table (
+   name string,
+   num INT,
+   p_day string,
+   p_hour string
+) partitioned by (p_day, p_hour) WITH (
+   'connector' = 'filesystem',
+   'path' = 'obs://*** ',
+   'format' = 'parquet',
+   'auto-compaction' = 'true'
+);
+
              +
              +
              +

              Usage

              • Rolling Policy

                The Rolling Policy determines when to close the current in-progress part file and transition it from the in-progress state to the pending state, and then to the finished state. Part files in the "finished" state are the ones that are ready for viewing and are guaranteed to contain valid data that will not be reverted in case of failure.

                +

                In STREAMING mode, the Rolling Policy in combination with the checkpointing interval (pending files become finished on the next checkpoint) control how quickly part files become available for downstream readers and also the size and number of these parts. For details, see Parameter Description.

                +
              • Part File Lifecycle

                To use the output of the FileSink in downstream systems, we need to understand the naming and lifecycle of the output files produced.

                +

                Part files can be in one of three states:

                +
                • In-progress: The part file that is currently being written to is in-progress.
                • Pending: Closed (due to the specified rolling policy) in-progress files that are waiting to be committed.
                • Finished: On successful checkpoints (STREAMING) or at the end of input (BATCH) pending files transition to Finished
                +

                Only finished files are safe to read by downstream systems as those are guaranteed to not be modified later.

                +

                By default, the file naming strategy is as follows:

                +
                • In-progress / Pending: part-<uid>-<partFileIndex>.inprogress.uid
                • Finished: part-<uid>-<partFileIndex>
                +

                uid is a random ID assigned to a subtask of the sink when the subtask is instantiated. This uid is not fault-tolerant so it is regenerated when the subtask recovers from a failure.

                +
              • Compaction

                FileSink supports compaction of the pending files, which allows the application to have smaller checkpoint interval without generating a lot of small files.

                +

                Once enabled, the compaction happens between the files become pending and get committed. The pending files will be first committed to temporary files whose path starts with a dot (.). Then these files will be compacted according to the strategy by the compactor specified by the users, and the new compacted pending files will be generated. Then these pending files will be emitted to the committer to be committed to the formal files. After that, the source files will be removed.

                +
              • Partitions

                Filesystem sink supports the partitioning function. Partitions are generated based on the selected fields by using the partitioned by syntax. The following is an example:

                +
                path
+└── datetime=2022-06-25
+    └── hour=10
+        ├── part-0.parquet
+        ├── part-1.parquet
+└── datetime=2022-06-26
+    └── hour=16
+        ├── part-0.parquet
+    └── hour=17
+        ├── part-0.parquet
                +

                Similar to files, partitions also need to be submitted to notify downstream applications that files in the partitions can be securely read. Filesystem sink provides multiple configuration submission policies.

                +
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              The value is fixed at filesystem.

              +

              path

              +

              Yes

              +

              None

              +

              String

              +

              OBS path

              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              File format

              +

              Available values are: csv and parquet

              +

              sink.rolling-policy.file-size

              +

              No

              +

              128MB

              +

              MemorySize

              +

              Maximum size of a part file. If the size of a part file exceeds this value, a new file will be generated.

              +
              NOTE:

              The Rolling Policy determines when to close the current in-progress part file and transition it from the in-progress state to the pending state, and then to the finished state. Part files in the "finished" state are the ones that are ready for viewing and are guaranteed to contain valid data that will not be reverted in case of failure. In STREAMING mode, the Rolling Policy in combination with the checkpointing interval (pending files become finished on the next checkpoint) control how quickly part files become available for downstream readers and also the size and number of these parts.

              +
              +

              sink.rolling-policy.rollover-interval

              +

              No

              +

              30 min

              +

              Duration

              +

              Maximum duration that a part file can be opened. If a part file is opened longer than the maximum duration, a new file will be generated in rolling mode. The default value is 30 minutes so that there will not be a large number of small files. The check frequency is specified by sink.rolling-policy.check-interval.

              +
              NOTE:

              There must be a space between the number and the unit.

              +

              The supported time units include d, h, min, s, and ms.

              +

              For bulk files (parquet, orc, and avro), the checkpoint interval also controls the maximum open duration of a part file.

              +
              +

              sink.rolling-policy.check-interval

              +

              No

              +

              1 min

              +

              Duration

              +

              Check interval of the time-based rolling policy

              +

              This parameter controls the frequency of checking whether a file should be rolled based on sink.rolling-policy.rollover-interval.

              +

              auto-compaction

              +

              No

              +

              false

              +

              Boolean

              +

              Whether automatic compaction is enabled for the streaming sink. Data is first written to temporary files. After the checkpoint is complete, the temporary files generated by the checkpoint are compacted.

              +

              compaction.file-size

              +

              No

              +

              Size of sink.rolling-policy.file-size

              +

              MemorySize

              +

              Size of the files that will be compacted. The default value is the size of the files that will be rolled.

              +
              NOTE:
              • Only files in the same checkpoint are compacted. The final files must be more than or equal to the number of checkpoints.
              • If the compaction takes a long time, back pressure may occur and the checkpointing may be prolonged.
              • After this function is enabled, final files are generated during checkpoint and a new file is opened to receive the data generated at the next checkpoint.
              +
              +
              +
              +
              +

              Example 1

              Use datagen to randomly generate data and write the data into the fileName directory in the OBS bucket bucketName. The generation time of a file is related to the checkpoint. If the file is open for 30 minutes or more, or if it exceeds 128 MB in size, a new file will be created.
              create table orders(
+  name string,
+  num INT
+) with (
+  'connector' = 'datagen',
+  'rows-per-second' = '100', 
+  'fields.name.kind' = 'random', 
+  'fields.name.length' = '5' 
+);
+
+CREATE TABLE sink_table (
+   name string,
+   num INT
+) WITH (
+   'connector' = 'filesystem',
+   'path' = 'obs://bucketName/fileName',
+   'format' = 'csv',
+   'sink.rolling-policy.file-size'='128m',
+   'sink.rolling-policy.rollover-interval'='30 min'
+);
+INSERT into sink_table SELECT * from orders;
+
              +
              +
              +

              Example 2

              Use datagen to randomly generate data and write the data into the fileName directory in the OBS bucket bucketName. The file generation time is relevant to the checkpoint. When the checkpoint interval is reached or the file size reaches 100 MB, a new file is generated.
              create table orders(
+  name string,
+  num INT
+) with (
+  'connector' = 'datagen',
+  'rows-per-second' = '100', 
+  'fields.name.kind' = 'random', 
+  'fields.name.length' = '5' 
+);
+
+CREATE TABLE sink_table (
+   name string,
+   num INT
+) WITH (
+   'connector' = 'filesystem',
+   'path' = 'obs://bucketName/fileName',
+   'format' = 'parquet',
+   'sink.rolling-policy.file-size'='128m',
+   'sink.rolling-policy.rollover-interval'='30 min',
+   'auto-compaction'='true',
+   'compaction.file-size'='100m'
+
+);
+INSERT into sink_table SELECT * from orders;
              +
              +
              +
              +
              +
              +
              Parent topic: OBS
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15042.html b/docs/dli/sqlreference/dli_08_15042.html new file mode 100644 index 000000000..55c6957c9 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15042.html @@ -0,0 +1,21 @@ + + +

              HBase

              +

              The HBase connector allows for reading from and writing to an HBase cluster. This section describes how to set up the HBase Connector to run SQL queries against HBase.

              +

              HBase always works in upsert mode for exchange changelog messages with the external system using a primary key defined on the DDL. The primary key must be defined on the HBase rowkey field (rowkey field must be declared). If the PRIMARY KEY clause is not declared, the HBase connector will take rowkey as the primary key by default. For details, see HBase SQL Connector.

              +
              +
              + + +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15043.html b/docs/dli/sqlreference/dli_08_15043.html new file mode 100644 index 000000000..f7f9c8c5d --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15043.html @@ -0,0 +1,267 @@ + + +

              Source Table

              +

              Function

              Create a source stream to obtain data from HBase as input for jobs. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excellent performance, and elastic scalability. It applies to the storage of massive amounts of data and distributed computing. You can use HBase to build a storage system capable of storing TB- or even PB-level data. With HBase, you can filter and analyze data with ease and get responses in milliseconds, rapidly mining data value. DLI can read data from HBase for filtering, analysis, and data dumping.

              +
              +

              Prerequisites

              • An enhanced datasource connection has been created for DLI to connect to HBase, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required. +
              • If MRS HBase is used, IP addresses of all hosts in the MRS cluster have been added to host information of the enhanced datasource connection.

                +

                For details, see "Modifying Host Information" in Data Lake Insight User Guide.

                +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • The column families in created HBase source table must be declared as the ROW type, the field names map the column family names, and the nested field names map the column qualifier names.

                There is no need to declare all the families and qualifiers in the schema. Users can declare what is used in the query. Except the ROW type fields, the single atomic type field (for example, STRING or BIGINT) will be recognized as the HBase rowkey. The rowkey field can be an arbitrary name, but should be quoted using backticks if it is a reserved keyword.

                +
              +
              +

              Syntax

              create table hbaseSource (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (',' watermark for rowtime_column_name as watermark-strategy_expression)
+  ','PRIMARY KEY (attr_name, ...) NOT ENFORCED)
+)
+with (
+  'connector' = 'hbase-2.2',
+  'table-name' = '',
+  'zookeeper.quorum' = ''
+);
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameter description

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector to be used. Set this parameter to hbase-2.2.

              +

              table-name

              +

              Yes

              +

              None

              +

              String

              +

              Name of the HBase table to connect.

              +

              zookeeper.quorum

              +

              Yes

              +

              None

              +

              String

              +

              HBase ZooKeeper quorum, in the format of "ZookeeperAddress:ZookeeperPort".

              +

              The following uses an MRS HBase cluster as an example to describe how to obtain the IP address and port number of ZooKeeper used by this parameter:

              +
              • On MRS Manager, choose Cluster and click the name of the desired cluster. Choose Services > ZooKeeper > Instance, and obtain the IP address of the ZooKeeper instance.
              • On MRS Manager, choose Cluster and click the name of the desired cluster. Choose Services > ZooKeeper > Configurations > All Configurations, search for the clientPort parameter, and obtain its value, that is, the ZooKeeper port number.
              +

              zookeeper.znode.parent

              +

              No

              +

              /hbase

              +

              String

              +

              Root directory in ZooKeeper. The default value is /hbase.

              +

              null-string-literal

              +

              No

              +

              None

              +

              String

              +

              Representation for null values for string fields.

              +

              HBase source encodes/decodes empty bytes as null values for all types except the string type.

              +
              +
              +
              +

              Data Type Mapping

              HBase stores all data as byte arrays. The data needs to be serialized and deserialized during read and write operations.

              +

              When serializing and de-serializing, Flink HBase connector uses utility class org.apache.hadoop.hbase.util.Bytes provided by HBase (Hadoop) to convert Flink data types to and from byte arrays.

              +

              Flink HBase connector encodes null values to empty bytes, and decodes empty bytes to null values for all data types except the string type. For string type, the null literal is determined by the null-string-literal option.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Data type mapping

              Flink SQL Type

              +

              HBase Conversion

              +

              CHAR/VARCHAR/STRING

              +

              byte[] toBytes(String s)

              +

              String toString(byte[] b)

              +

              BOOLEAN

              +

              byte[] toBytes(boolean b)

              +

              boolean toBoolean(byte[] b)

              +

              BINARY/VARBINARY

              +

              Returns byte[] as is.

              +

              DECIMAL

              +

              byte[] toBytes(BigDecimal v)

              +

              BigDecimal toBigDecimal(byte[] b)

              +

              TINYINT

              +

              new byte[] { val }

              +

              bytes[0] // returns first and only byte from bytes

              +

              SMALLINT

              +

              byte[] toBytes(short val)

              +

              short toShort(byte[] bytes)

              +

              INT

              +

              byte[] toBytes(int val)

              +

              int toInt(byte[] bytes)

              +

              BIGINT

              +

              byte[] toBytes(long val)

              +

              long toLong(byte[] bytes)

              +

              FLOAT

              +

              byte[] toBytes(float val)

              +

              float toFloat(byte[] bytes)

              +

              DOUBLE

              +

              byte[] toBytes(double val)

              +

              double toDouble(byte[] bytes)

              +

              DATE

              +

              Stores the number of days since epoch as an int value.

              +

              TIME

              +

              Stores the number of milliseconds of the day as an int value.

              +

              TIMESTAMP

              +

              Stores the milliseconds since epoch as a long value.

              +

              ARRAY

              +

              Not supported

              +

              MAP/MULTISET

              +

              Not supported

              +

              ROW

              +

              Not supported

              +
              +
              +
              +

              Example

              In this example, data is read from the HBase data source and written to the Print result table. (The HBase version used in this example is 2.2.3.)

              +
              1. Create an enhanced datasource connection in the VPC and subnet where HBase locates, and bind the connection to the required Flink queue.
              2. Set HBase cluster security groups and add inbound rules to allow access from the Flink job queue. Test the connectivity using the HBase address. If the connection passes the test, it is bound to the queue.
              3. Use the HBase shell to create HBase table order that has only one column family detail. The creation statement is as follows:
                create 'order', {NAME => 'detail'}
                +
              4. Run the following command in the HBase shell to insert a data record:
                put 'order', '202103241000000001', 'detail:order_channel','webShop'
+put 'order', '202103241000000001', 'detail:order_time','2021-03-24 10:00:00'
+put 'order', '202103241000000001', 'detail:pay_amount','100.00'
+put 'order', '202103241000000001', 'detail:real_pay','100.00'
+put 'order', '202103241000000001', 'detail:pay_time','2021-03-24 10:02:03'
+put 'order', '202103241000000001', 'detail:user_id','0001'
+put 'order', '202103241000000001', 'detail:user_name','Alice'
+put 'order', '202103241000000001', 'detail:area_id','330106'
                +
              5. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. The job script uses the HBase data source and the Print result table.
                When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.
                create table hbaseSource (
+  order_id string,-- Indicates the unique rowkey.
+  detail Row( -- Indicates the column family.
+    order_channel string,
+    order_time string,
+    pay_amount string,
+    real_pay string,
+    pay_time string,
+    user_id string,
+    user_name string,
+    area_id string),
+  primary key (order_id) not enforced
+) with (
+  'connector' = 'hbase-2.2',
+   'table-name' = 'order',
+   'zookeeper.quorum' = 'ZookeeperAddress:ZookeeperPort'
+) ;
+
+create table printSink (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount string,
+  real_pay string,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) with (
+ 'connector' = 'print'
+);
+
+insert into printSink select order_id, detail.order_channel,detail.order_time,detail.pay_amount,detail.real_pay,
+detail.pay_time,detail.user_id,detail.user_name,detail.area_id from hbaseSource;
                +
                +
              6. Perform the following operations to view the data result in the taskmanager.out file:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the taskmanager.out file, and view result logs.
                +

                The data result is as follows:

                +
                +I(202103241000000001,webShop,2021-03-24 10:00:00,100.00,100.00,2021-03-24 10:02:03,0001,Alice,330106)
                +
              +
              +

              FAQ

              • Q: What should I do if the Flink job execution fails and the log contains the following error information?
                java.lang.IllegalArgumentException: offset (0) + length (8) exceed the capacity of the array: 6
                +

                A: If data in the HBase table is imported in other modes, the data is represented in the string format. Therefore, this error is reported when other data formats are used. Change the type of the non-string fields in the HBase source table created by Flink to the string format.

                +
              +
              +
              • Q: What should I do if the Flink job execution fails and the log contains the following error information?
                org.apache.zookeeper.ClientCnxn$SessionTimeoutException: Client session timed out, have not heard from server in 90069ms for connection id 0x0
                +

                A: The datasource connection is not bound, the binding fails, or the security group of the HBase cluster is not configured to allow access from the network segment of the DLI queue. Configure the datasource connection or configure the security group of the HBase cluster to allow access from the DLI queue.

                +
              +
              +
              +
              +
              Parent topic: HBase
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15044.html b/docs/dli/sqlreference/dli_08_15044.html new file mode 100644 index 000000000..ec46d1036 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15044.html @@ -0,0 +1,394 @@ + + +

              HBase Result Table

              +

              Function

              DLI outputs the job data to HBase. HBase is a column-oriented distributed cloud storage system that features enhanced reliability, excellent performance, and elastic scalability. It applies to the storage of massive amounts of data and distributed computing. You can use HBase to build a storage system capable of storing TB- or even PB-level data. With HBase, you can filter and analyze data with ease and get responses in milliseconds, rapidly mining data value. Structured and semi-structured key-value data can be stored, including messages, reports, recommendation data, risk control data, logs, and orders. With DLI, you can write massive volumes of data to HBase at a high speed and with low latency.

              +
              +

              Prerequisites

              • An enhanced datasource connection has been created for DLI to connect to HBase, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required. +
              • If MRS HBase is used, IP addresses of all hosts in the MRS cluster have been added to host information of the enhanced datasource connection.

                +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • The column families in created HBase result table must be declared as the ROW type, the field names map the column family names, and the nested field names map the column qualifier names. There is no need to declare all the families and qualifiers in the schema. Users can declare what is used in the query. Except the ROW type fields, the single atomic type field (for example, STRING or BIGINT) will be recognized as the HBase rowkey. The rowkey field can be an arbitrary name, but should be quoted using backticks if it is a reserved keyword.
              +
              +

              Syntax

              create table hbaseSink (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  ','PRIMARY KEY (attr_name, ...) NOT ENFORCED)
+) with (
+  'connector' = 'hbase-2.2',
+  'table-name' = '',
+  'zookeeper.quorum' = ''
+);
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameter description

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector to be used. Set this parameter to hbase-2.2.

              +

              table-name

              +

              Yes

              +

              None

              +

              String

              +

              Name of the HBase table to connect.

              +

              zookeeper.quorum

              +

              Yes

              +

              None

              +

              String

              +

              HBase ZooKeeper instance information, in the format of ZookeeperAddress:ZookeeperPort.

              +

              The following uses an MRS HBase cluster as an example to describe how to obtain the IP address and port number of ZooKeeper used by this parameter:

              +
              • On MRS Manager, choose Cluster and click the name of the desired cluster. Choose Services > ZooKeeper > Instance, and obtain the IP address of the ZooKeeper instance.
              • On MRS Manager, choose Cluster and click the name of the desired cluster. Choose Services > ZooKeeper > Configurations > All Configurations, search for the clientPort parameter, and obtain its value, that is, the ZooKeeper port number.
              +

              zookeeper.znode.parent

              +

              No

              +

              /hbase

              +

              String

              +

              Root directory in ZooKeeper. The default value is /hbase.

              +

              null-string-literal

              +

              No

              +

              null

              +

              String

              +

              Representation for null values for string fields.

              +

              The HBase sink encodes/decodes empty bytes as null values for all types except the string type.

              +

              sink.buffer-flush.max-size

              +

              No

              +

              2mb

              +

              MemorySize

              +

              Maximum size in memory of buffered rows for each write request.

              +

              This can improve performance for writing data to the HBase database, but may increase the latency.

              +

              You can set this parameter to 0 to disable it.

              +

              sink.buffer-flush.max-rows

              +

              No

              +

              1000

              +

              Integer

              +

              Maximum number of rows to buffer for each write request.

              +

              This can improve performance for writing data to the HBase database, but may increase the latency.

              +

              You can set this parameter to 0 to disable it.

              +

              sink.buffer-flush.interval

              +

              No

              +

              1s

              +

              Duration

              +

              Interval for refreshing the buffer, during which data is refreshed by asynchronous threads.

              +

              This can improve performance for writing data to the HBase database, but may increase the latency.

              +

              You can set this parameter to 0 to disable it.

              +

              Note: If sink.buffer-flush.max-size and sink.buffer-flush.max-rows are both set to 0 and the buffer refresh interval is configured, the buffer is asynchronously refreshed.

              +

              The format is {length value}{time unit label}, for example, 123ms, 321s. The supported time units include d, h, min, s, and ms (default unit).

              +

              sink.parallelism

              +

              No

              +

              None

              +

              Integer

              +

              Defines the parallelism of the HBase sink operator.

              +

              By default, the parallelism is determined by the framework: using the same parallelism as the upstream join operator.

              +

              properties.connector.kerberos.principal

              +

              No

              +

              None

              +

              String

              +

              Username for logging in to the security cluster. This parameter is mandatory if Kerberos authentication is enabled.

              +

              properties.connector.kerberos.keytab

              +

              No

              +

              None

              +

              String

              +

              OBS path to which the user.keytab file is uploaded. This parameter is mandatory if Kerberos authentication is enabled.

              +

              properties.connector.kerberos.krb5

              +

              No

              +

              None

              +

              String

              +

              OBS path to which the krb5.conf file is uploaded. This parameter is mandatory if Kerberos authentication is enabled.

              +

              Note: The renew_lifetime configuration item under [libdefaults] must be removed from krb5.conf. Otherwise, the "Message stream modified (41)" error may occur.

              +
              +
              +
              +

              Data Type Mapping

              HBase stores all data as byte arrays. The data needs to be serialized and deserialized during read and write operations.

              +

              When serializing and de-serializing, Flink HBase connector uses utility class org.apache.hadoop.hbase.util.Bytes provided by HBase (Hadoop) to convert Flink data types to and from byte arrays.

              +

              Flink HBase connector encodes null values to empty bytes, and decodes empty bytes to null values for all data types except the string type. For string type, the null literal is determined by the null-string-literal option.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Data type mapping

              Flink SQL Type

              +

              HBase Conversion

              +

              CHAR/VARCHAR/STRING

              +

              byte[] toBytes(String s)

              +

              String toString(byte[] b)

              +

              BOOLEAN

              +

              byte[] toBytes(boolean b)

              +

              boolean toBoolean(byte[] b)

              +

              BINARY/VARBINARY

              +

              Returns byte[] as is.

              +

              DECIMAL

              +

              byte[] toBytes(BigDecimal v)

              +

              BigDecimal toBigDecimal(byte[] b)

              +

              TINYINT

              +

              new byte[] { val }

              +

              bytes[0] // returns first and only byte from bytes

              +

              SMALLINT

              +

              byte[] toBytes(short val)

              +

              short toShort(byte[] bytes)

              +

              INT

              +

              byte[] toBytes(int val)

              +

              int toInt(byte[] bytes)

              +

              BIGINT

              +

              byte[] toBytes(long val)

              +

              long toLong(byte[] bytes)

              +

              FLOAT

              +

              byte[] toBytes(float val)

              +

              float toFloat(byte[] bytes)

              +

              DOUBLE

              +

              byte[] toBytes(double val)

              +

              double toDouble(byte[] bytes)

              +

              DATE

              +

              Stores the number of days since epoch as an int value.

              +

              TIME

              +

              Stores the number of milliseconds of the day as an int value.

              +

              TIMESTAMP

              +

              Stores the milliseconds since epoch as a long value.

              +

              ARRAY

              +

              Not supported

              +

              MAP / MULTISET

              +

              Not supported

              +

              ROW

              +

              Not supported

              +
              +
              +
              +

              Example

              In this example, data is read from the Kafka data source and written to the HBase result table. The procedure is as follows (the HBase version used in this example is 2.2.3):

              +
              1. Create an enhanced datasource connection in the VPC and subnet where HBase and Kafka locate, and bind the connection to the required Flink elastic resource pool.
              2. Set HBase and Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the HBase and Kafka addresses. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              3. Use the HBase shell to create HBase table order that has only one column family detail.
                create 'order', {NAME => 'detail'}
                +
              4. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. The job script uses Kafka as the data source and HBase as the result table (the Rowkey is order_id and the column family name is detail).
                When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.
                CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+create table hbaseSink(
+  order_id string,
+  detail Row(
+    order_channel string,
+    order_time string,
+    pay_amount double,
+    real_pay double,
+    pay_time string,
+    user_id string,
+    user_name string,
+    area_id string)
+) with (
+  'connector' = 'hbase-2.2',
+  'table-name' = 'order',
+  'zookeeper.quorum' = 'ZookeeperAddress:ZookeeperPort',
+  'sink.buffer-flush.max-rows' = '1'
+);
+
+insert into hbaseSink select order_id, Row(order_channel,order_time,pay_amount,real_pay,pay_time,user_id,user_name,area_id) from orders;
                +
                +
              5. Connect to the Kafka cluster and enter the following data to Kafka:
                {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103251202020001", "order_channel":"miniAppShop", "order_time":"2021-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2021-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
                +
              6. Run the following statement on the HBase shell to view the data result:
                 scan 'order'
                +
                The data result is as follows:
                202103241000000001   column=detail:area_id, timestamp=2021-12-16T21:30:37.954, value=330106
+
+202103241000000001   column=detail:order_channel, timestamp=2021-12-16T21:30:37.954, value=webShop
+
+202103241000000001   column=detail:order_time, timestamp=2021-12-16T21:30:37.954, value=2021-03-24 10:00:00
+
+202103241000000001   column=detail:pay_amount, timestamp=2021-12-16T21:30:37.954, value=@Y\x00\x00\x00\x00\x00\x00
+
+202103241000000001   column=detail:pay_time, timestamp=2021-12-16T21:30:37.954, value=2021-03-24 10:02:03
+
+202103241000000001   column=detail:real_pay, timestamp=2021-12-16T21:30:37.954, value=@Y\x00\x00\x00\x00\x00\x00
+
+202103241000000001   column=detail:user_id, timestamp=2021-12-16T21:30:37.954, value=0001
+
+202103241000000001   column=detail:user_name, timestamp=2021-12-16T21:30:37.954, value=Alice
+
+202103241606060001   column=detail:area_id, timestamp=2021-12-16T21:30:44.842, value=330106
+
+202103241606060001   column=detail:order_channel, timestamp=2021-12-16T21:30:44.842, value=appShop
+
+202103241606060001   column=detail:order_time, timestamp=2021-12-16T21:30:44.842, value=2021-03-24 16:06:06
+
+202103241606060001   column=detail:pay_amount, timestamp=2021-12-16T21:30:44.842, value=@i\x00\x00\x00\x00\x00\x00
+
+202103241606060001   column=detail:pay_time, timestamp=2021-12-16T21:30:44.842, value=2021-03-24 16:10:06
+
+202103241606060001   column=detail:real_pay, timestamp=2021-12-16T21:30:44.842, value=@f\x80\x00\x00\x00\x00\x00
+
+202103241606060001   column=detail:user_id, timestamp=2021-12-16T21:30:44.842, value=0001
+
+202103241606060001   column=detail:user_name, timestamp=2021-12-16T21:30:44.842, value=Alice
+
+202103251202020001   column=detail:area_id, timestamp=2021-12-16T21:30:52.181, value=330110
+
+202103251202020001   column=detail:order_channel, timestamp=2021-12-16T21:30:52.181, value=miniAppShop
+
+202103251202020001   column=detail:order_time, timestamp=2021-12-16T21:30:52.181, value=2021-03-25 12:02:02
+
+202103251202020001   column=detail:pay_amount, timestamp=2021-12-16T21:30:52.181, value=@N\x00\x00\x00\x00\x00\x00
+
+202103251202020001   column=detail:pay_time, timestamp=2021-12-16T21:30:52.181, value=2021-03-25 12:03:00
+
+202103251202020001   column=detail:real_pay, timestamp=2021-12-16T21:30:52.181, value=@N\x00\x00\x00\x00\x00\x00
+
+202103251202020001   column=detail:user_id, timestamp=2021-12-16T21:30:52.181, value=0002
+
+202103251202020001   column=detail:user_name, timestamp=2021-12-16T21:30:52.181, value=Bob
                +
                +
              +
              +

              FAQ

              Q: What should I do if the Flink job execution fails and the log contains the following error information?

              +
              org.apache.zookeeper.ClientCnxn$SessionTimeoutException: Client session timed out, have not heard from server in 90069ms for connection id 0x0
              +

              A: The datasource connection is not bound or the binding fails. Configure the datasource connection or configure the security group of the Kafka cluster to allow access from the DLI queue.

              +
              +
              +
              +
              +
              Parent topic: HBase
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15045.html b/docs/dli/sqlreference/dli_08_15045.html new file mode 100644 index 000000000..4171405c9 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15045.html @@ -0,0 +1,337 @@ + + +

              Dimension Table

              +

              Function

              Create an HBase dimension table to connect to the source streams for wide table generation.

              +
              +

              Prerequisites

              • An enhanced datasource connection has been created for DLI to connect to HBase, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required. +
              • If MRS HBase is used, IP addresses of all hosts in the MRS cluster have been added to host information of the enhanced datasource connection.

                +

                For details, see "Modifying Host Information" in Data Lake Insight User Guide.

                +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • All the column families in HBase table must be declared as ROW type, the field name maps to the column family name, and the nested field names map to the column qualifier names. There is no need to declare all the families and qualifiers in the schema, users can declare what is used in the query. Except the ROW type fields, the single atomic type field (for example, STRING, BIGINT) will be recognized as HBase rowkey. The rowkey field can be an arbitrary name, but should be quoted using backticks if it is a reserved keyword.
              +
              +

              Syntax

              create table hbaseSource (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+ )
+with (
+  'connector' = 'hbase-2.2',
+  'table-name' = '',
+  'zookeeper.quorum' = ''
+);
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameter description

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector type. Set this parameter to hbase-2.2.

              +

              table-name

              +

              Yes

              +

              None

              +

              String

              +

              Name of the HBase table

              +

              zookeeper.quorum

              +

              Yes

              +

              None

              +

              String

              +

              HBase Zookeeper quorum. The format is ZookeeperAddress:ZookeeperPort.

              +

              The following describes how to obtain the ZooKeeper IP address and port number:

              +
              • On the MRS Manager console, choose Cluster > Name of the desired cluster > Service > ZooKeeper > Instance. On the displayed page, obtain the IP address of the ZooKeeper instance.
              • On the MRS Manager console, choose Cluster > Name of the desired cluster > Service > ZooKeeper > Configuration, and click All Configurations. Search for the clientPort parameter, and obtain the ZooKeeper port number.
              +

              zookeeper.znode.parent

              +

              No

              +

              /hbase

              +

              String

              +

              Root directory in ZooKeeper for the HBase cluster.

              +

              lookup.async

              +

              No

              +

              false

              +

              Boolean

              +

              Whether async lookup is enabled.

              +

              lookup.cache.max-rows

              +

              No

              +

              -1

              +

              Long

              +

              Maximum number of cached rows in a dimension table. When the rows exceed this value, the first item added to the cache will be marked as expired.

              +

              Lookup cache is disabled by default.

              +

              lookup.cache.ttl

              +

              No

              +

              -1

              +

              Long

              +

              Maximum time to live (TTL) for each row in lookup cache. Caches exceeding the TTL will be expired. The format is {length value}{time unit label}, for example, 123ms, 321s. The supported time units include d, h, min, s, and ms (default unit).

              +

              Lookup cache is disabled by default.

              +

              lookup.max-retries

              +

              No

              +

              3

              +

              Integer

              +

              Maximum retry times if lookup database failed.

              +
              +
              +
              +

              Data Type Mapping

              HBase stores all data as byte arrays. The data needs to be serialized and deserialized during read and write operations.

              +

              When serializing and de-serializing, Flink HBase connector uses utility class org.apache.hadoop.hbase.util.Bytes provided by HBase (Hadoop) to convert Flink data types to and from byte arrays.

              +

              Flink HBase connector encodes null values to empty bytes, and decodes empty bytes to null values for all data types except the string type. For string type, the null literal is determined by the null-string-literal option.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Data type mapping

              Flink SQL Type

              +

              HBase Conversion

              +

              CHAR/VARCHAR/STRING

              +

              byte[] toBytes(String s)

              +

              String toString(byte[] b)

              +

              BOOLEAN

              +

              byte[] toBytes(boolean b)

              +

              boolean toBoolean(byte[] b)

              +

              BINARY/VARBINARY

              +

              Returns byte[] as is.

              +

              DECIMAL

              +

              byte[] toBytes(BigDecimal v)

              +

              BigDecimal toBigDecimal(byte[] b)

              +

              TINYINT

              +

              new byte[] { val }

              +

              bytes[0] // returns first and only byte from bytes

              +

              SMALLINT

              +

              byte[] toBytes(short val)

              +

              short toShort(byte[] bytes)

              +

              INT

              +

              byte[] toBytes(int val)

              +

              int toInt(byte[] bytes)

              +

              BIGINT

              +

              byte[] toBytes(long val)

              +

              long toLong(byte[] bytes)

              +

              FLOAT

              +

              byte[] toBytes(float val)

              +

              float toFloat(byte[] bytes)

              +

              DOUBLE

              +

              byte[] toBytes(double val)

              +

              double toDouble(byte[] bytes)

              +

              DATE

              +

              Number of days since 1970-01-01 00:00:00 UTC. The value is an integer.

              +

              TIME

              +

              Number of milliseconds since 1970-01-01 00:00:00 UTC. The value is an integer.

              +

              TIMESTAMP

              +

              Number of milliseconds since 1970-01-01 00:00:00 UTC. The value is of the long type.

              +

              ARRAY

              +

              Not supported

              +

              MAP / MULTISET

              +

              Not supported

              +

              ROW

              +

              Not supported

              +
              +
              +
              +

              Example

              In this example, data is read from a DMS Kafka data source, an HBase table is used as a dimension table to generate a wide table, and the result is written to a Kafka result table. The procedure is as follows (the HBase version in this example is 2.2.3):

              +
              1. Create an enhanced datasource connection in the VPC and subnet where HBase and Kafka locate, and bind the connection to the required Flink elastic resource pool. Add MRS host information for the enhanced datasource connection.
              2. Set HBase and Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the HBase and Kafka addresses. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              3. Create an HBase table and name it area_info using the HBase shell. The table has only one column family detail. The creation statement is as follows:
                create 'area_info', {NAME => 'detail'}
                +
              4. Run the following statement in the HBase shell to insert dimension table data:
                put 'area_info', '330106', 'detail:area_province_name', 'a1'
+put 'area_info', '330106', 'detail:area_city_name', 'b1'
+put 'area_info', '330106', 'detail:area_county_name', 'c2'
+put 'area_info', '330106', 'detail:area_street_name', 'd2'
+put 'area_info', '330106', 'detail:region_name', 'e1'
+
+put 'area_info', '330110', 'detail:area_province_name', 'a1'
+put 'area_info', '330110', 'detail:area_city_name', 'b1'
+put 'area_info', '330110', 'detail:area_county_name', 'c4'
+put 'area_info', '330110', 'detail:area_street_name', 'd4'
+put 'area_info', '330110', 'detail:region_name', 'e1'
                +
              5. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. The job script uses Kafka as the data source and an HBase table as the dimension table. Data is output to a Kafka result table.
                When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Set the values of the parameters in bold in the following script as needed.
                CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string,
+  proctime as Proctime()
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaSourceTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+-- Create an address dimension table
+create table area_info (
+  area_id string,   
+  detail row(
+    area_province_name string, 
+    area_city_name string, 
+    area_county_name string, 
+    area_street_name string, 
+    region_name string) 
+) WITH (
+  'connector' = 'hbase-2.2',
+  'table-name' = 'area_info',
+  'zookeeper.quorum' = 'ZookeeperAddress:ZookeeperPort',
+  'lookup.async' = 'true',
+  'lookup.cache.max-rows' = '10000',
+  'lookup.cache.ttl' = '2h'
+);
+
+-- Generate a wide table based on the address dimension table containing detailed order information.
+create table order_detail(
+    order_id string,
+    order_channel string,
+    order_time string,
+    pay_amount double,
+    real_pay double,
+    pay_time string,
+    user_id string,
+    user_name string,
+    area_id string,
+    area_province_name string,
+    area_city_name string,
+    area_county_name string,
+    area_street_name string,
+    region_name string
+) with (
+  'connector' = 'kafka',
+  'topic' = '<yourSinkTopic>',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'format' = 'json'
+);
+
+insert into order_detail
+    select orders.order_id, orders.order_channel, orders.order_time, orders.pay_amount, orders.real_pay, orders.pay_time, orders.user_id, orders.user_name,
+           area.area_id, area.area_province_name, area.area_city_name, area.area_county_name,
+           area.area_street_name, area.region_name  from orders
+    left join area_info for system_time as of orders.proctime as area on orders.area_id = area.area_id;
                +
                +
              6. Connect to the Kafka cluster and insert the following test data into the source topic in Kafka:
                {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103251202020001", "order_channel":"miniAppShop", "order_time":"2021-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2021-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
                +
              7. Connect to the Kafka cluster and read data from the sink topic of Kafka. The result data is as follows:
                {"order_id":"202103241000000001","order_channel":"webShop","order_time":"2021-03-24 10:00:00","pay_amount":100.0,"real_pay":100.0,"pay_time":"2021-03-24 10:02:03","user_id":"0001","user_name":"Alice","area_id":"330106","area_province_name":"a1","area_city_name":"b1","area_county_name":"c2","area_street_name":"d2","region_name":"e1"}
+
+{"order_id":"202103241606060001","order_channel":"appShop","order_time":"2021-03-24 16:06:06","pay_amount":200.0,"real_pay":180.0,"pay_time":"2021-03-24 16:10:06","user_id":"0001","user_name":"Alice","area_id":"330106","area_province_name":"a1","area_city_name":"b1","area_county_name":"c2","area_street_name":"d2","region_name":"e1"}
+
+{"order_id":"202103251202020001","order_channel":"miniAppShop","order_time":"2021-03-25 12:02:02","pay_amount":60.0,"real_pay":60.0,"pay_time":"2021-03-25 12:03:00","user_id":"0002","user_name":"Bob","area_id":"330110","area_province_name":"a1","area_city_name":"b1","area_county_name":"c4","area_street_name":"d4","region_name":"e1"}
                +
              +
              +

              FAQs

              Q: What should I do if Flink job logs contain the following error information?

              +
              org.apache.zookeeper.ClientCnxn$SessionTimeoutException: Client session timed out, have not heard from server in 90069ms for connection id 0x0
              +

              A: The datasource connection is not bound or the binding fails. Configure the datasource connection or configure the security group of the Kafka cluster to allow access from the DLI queue.

              +
              +
              +
              +
              +
              Parent topic: HBase
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15046.html b/docs/dli/sqlreference/dli_08_15046.html new file mode 100644 index 000000000..4420c4a3a --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15046.html @@ -0,0 +1,28 @@ + + +

              Hive

              +

              +
              + + diff --git a/docs/dli/sqlreference/dli_08_15047.html b/docs/dli/sqlreference/dli_08_15047.html new file mode 100644 index 000000000..cdfd612e9 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15047.html @@ -0,0 +1,218 @@ + + +

              Creating a Hive Catalog

              +

              Introduction

              Catalogs provide metadata, such as databases, tables, partitions, views, and functions and information needed to access data stored in a database or other external systems.

              +

              One of the most crucial aspects of data processing is managing metadata. It may be transient metadata like temporary tables, or UDFs registered against the table environment. Or permanent metadata, like that in a Hive Metastore. Catalogs provide a unified API for managing metadata and making it accessible from the Table API and SQL Queries. For details, see Apache Flink Catalogs.

              +
              +

              Function

              The HiveCatalog serves two purposes; as persistent storage for pure Flink metadata, and as an interface for reading and writing existing Hive metadata.

              +

              Flink's Hive documentation provides full details on setting up the catalog and interfacing with an existing Hive installation. For details, see Apache Flink Hive Catalog.

              +

              HiveCatalog can be used to handle two kinds of tables: Hive-compatible tables and generic tables.

              +
              • Hive-compatible tables are those stored in a Hive-compatible way, in terms of both metadata and data in the storage layer. Therefore, Hive-compatible tables created via Flink can be queried from Hive side.
              • Generic tables, on the other hand, are specific to Flink. When creating generic tables with HiveCatalog, we're just using HMS to persist the metadata. While these tables are visible to Hive, it is unlikely Hive is able to understand the metadata. And therefore using such tables in Hive leads to undefined behavior.
              +

              You are advised to switch to Hive dialect to create Hive-compatible tables. If you want to create Hive-compatible tables with default dialect, make sure to set 'connector'='hive' in your table properties, otherwise a table is considered generic by default in HiveCatalog. Note that the connector property is not required if you use Hive dialect. Refer to Hive Dialect.

              +
              +

              Caveats

              • Warning: The Hive Metastore stores all meta-object names in lower case.
              • If a directory with the same name already exists, an exception is thrown.
              • To use Hudi tables, you need to use the Hudi catalog, which is not compatible with the Hive catalog.
              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              +
              +

              Syntax

              CREATE CATALOG myhive
+WITH (
+    'type' = 'hive',
+    'default-database' = 'default',
+    'hive-conf-dir' = '/opt/flink/conf'
+);
+
+USE CATALOG myhive;
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              type

              +

              Yes

              +

              None

              +

              String

              +

              Catalog type. Set this parameter to hive when creating a HiveCatalog.

              +

              hive-conf-dir

              +

              Yes

              +

              None

              +

              String

              +

              This refers to the URI that points to the directory containing the hive-site.xml file.

              +

              The value is fixed at 'hive-conf-dir' = '/opt/flink/conf'.

              +

              default-database

              +

              No

              +

              default

              +

              String

              +

              When a catalog is set as the current catalog, the default current database is used.

              +
              +
              +
              +

              Supported Types

              The HiveCatalog supports universal tables for all Flink types.

              +

              For Hive-compatible tables, the HiveCatalog needs to map Flink data types to their corresponding Hive types.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Data type mapping

              Flink SQL Type

              +

              Hive Data Type

              +

              CHAR(p)

              +

              CHAR(p)

              +

              VARCHAR(p)

              +

              VARCHAR(p)

              +

              STRING

              +

              STRING

              +

              BOOLEAN

              +

              BOOLEAN

              +

              TINYINT

              +

              TINYINT

              +

              SMALLINT

              +

              SMALLINT

              +

              INT

              +

              INT

              +

              BIGINT

              +

              LONG

              +

              FLOAT

              +

              FLOAT

              +

              DOUBLE

              +

              DOUBLE

              +

              DECIMAL(p, s)

              +

              DECIMAL(p, s)

              +

              DATE

              +

              DATE

              +

              TIMESTAMP(9)

              +

              TIMESTAMP

              +

              BYTES

              +

              BINARY

              +

              ARRAY<T>

              +

              LIST<T>

              +

              MAP

              +

              MAP

              +

              ROW

              +

              STRUCT

              +
              +
              +
              • The maximum length for Hive's CHAR(p) is 255.
              • The maximum length for Hive's VARCHAR(p) is 65535.
              • Hive's MAP only supports primitive types as keys, while Flink's MAP can be any data type.
              • Hive does not support UNION types.
              • Hive's TIMESTAMP always has a precision of 9 and does not support other precisions. However, Hive UDFs can handle TIMESTAMP values with precision <= 9.
              • Hive does not support Flink's TIMESTAMP_WITH_TIME_ZONE, TIMESTAMP_WITH_LOCAL_TIME_ZONE, and MULTISET.
              • Flink's INTERVAL type cannot yet be mapped to Hive's INTERVAL type.
              +
              +
              +

              Example

              1. Create a catalog named myhive in the Flink OpenSource SQL job and use it to manage metadata.
                CREATE CATALOG myhive WITH (
+    'type' = 'hive'
+    ,'hive-conf-dir' = '/opt/flink/conf'
+);
+
+USE CATALOG myhive;
+
+create table dataGenSource(
+  user_id string,
+  amount int
+) with (
+  'connector' = 'datagen',
+  'rows-per-second' = '1', --Generates a piece of data per second.
+  'fields.user_id.kind' = 'random', --Specifies a random generator for the user_id field.
+  'fields.user_id.length' = '3' --Limits the length of user_id to 3.
+);
+
+create table printSink(
+  user_id string,
+  amount int
+) with (
+  'connector' = 'print'
+);
+
+insert into printSink select * from dataGenSource;
                +
              2. Check if the dataGenSource and printSink tables exist in the default database.

                The Hive Metastore stores all meta-object names in lower case.

                +
                +
              3. Create a new Flink OpenSource SQL job using the metadata in the catalog named myhive.
                CREATE CATALOG myhive WITH (
+    'type' = 'hive'
+    ,'hive-conf-dir' = '/opt/flink/conf'
+);
+
+USE CATALOG myhive;
+
+insert into printSink select * from dataGenSource;
                +
              +
              +
              +
              +
              +
              Parent topic: Hive
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15048.html b/docs/dli/sqlreference/dli_08_15048.html new file mode 100644 index 000000000..5a95d6dbc --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15048.html @@ -0,0 +1,24 @@ + + +

              Hive Dialect

              +

              Introduction

              Starting from 1.11.0, Flink allows users to write SQL statements in Hive syntax when Hive dialect is used. By providing compatibility with Hive syntax, we aim to improve the interoperability with Hive and reduce the scenarios when users need to switch between Flink and Hive in order to execute different statements. For details, see Apache Flink Hive Dialect.

              +
              +

              Function

              Flink currently supports two SQL dialects: default and hive. You need to switch to Hive dialect before you can write in Hive syntax. The following describes how to set dialect with SQL Client.

              +

              Also notice that you can dynamically switch dialect for each statement you execute. There's no need to restart a session to use a different dialect.

              +
              +

              Syntax

              SQL dialect can be specified via the table.sql-dialect property.

              +
              1
              set table.sql-dialect=hive;
+
              +
              +
              +

              Caveats

              • Hive dialect should only be used to process Hive meta objects, and requires the current catalog to be a HiveCatalog.
              • Hive dialect only supports 2-part identifiers, so you can not specify catalog for an identifier. Refer to Apache Flink Hive Read & Write for more details.

                While all Hive versions support the same syntax, whether a specific feature is available still depends on the Hive version you use.

                +

                For example, updating database location is only supported in Hive-2.4.0 or later.

                +
              • Use HiveModule to run DML and DQL.
              • Since Flink 1.15 you need to swap flink-table-planner-loader located in /lib with flink-table-planner_2.12 located in /opt to avoid the following exception. Please see FLINK-25128 for more details.
              +
              +
              +
              +
              +
              Parent topic: Hive
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15049.html b/docs/dli/sqlreference/dli_08_15049.html new file mode 100644 index 000000000..e51bd27c0 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15049.html @@ -0,0 +1,305 @@ + + +

              Hive Source Table

              +

              Introduction

              Apache Hive has established itself as a focal point of the data warehousing ecosystem. It serves as not only a SQL engine for big data analytics and ETL, but also a data management platform, where data is discovered, defined, and evolved.

              +

              Flink offers a two-fold integration with Hive. The first is to leverage Hive's Metastore as a persistent catalog. The second is to offer Flink as an alternative engine for reading and writing Hive tables. Overview | Apache Flink

              +

              Starting from 1.11.0, Flink allows users to write SQL statements in Hive syntax when Hive dialect is used. By providing compatibility with Hive syntax, we aim to improve the interoperability with Hive and reduce the scenarios when users need to switch between Flink and Hive in order to execute different statements. For details, see Apache Flink Hive Dialect.

              +

              Using the HiveCatalog, Apache Flink can be used for unified BATCH and STREAM processing of Apache Hive Tables. This means Flink can be used as a more performant alternative to Hive's batch engine, or to continuously read and write data into and out of Hive tables to power real-time data warehousing applications. Apache Flink Hive Read & Write

              +
              +

              Function

              This section describes how to use Flink to read and write Hive tables, the definition of the Hive source table, parameters used for creating the source table, and sample code. For details, see Apache Flink Hive Read & Write.

              +

              Flink supports reading data from Hive in both BATCH and STREAMING modes. When running as a BATCH application, Flink will execute its query over the state of the table at the point in time when the query is executed. STREAMING reads will continuously monitor the table and incrementally fetch new data as it is made available. Flink will read tables as bounded by default.

              +

              STREAMING reads support consuming both partitioned and non-partitioned tables. For partitioned tables, Flink will monitor the generation of new partitions, and read them incrementally when available. For non-partitioned tables, Flink will monitor the generation of new files in the folder and read new files incrementally.

              +
              +

              Prerequisites

              To create a FileSystem source table, an enhanced datasource connection is required. You can set security group rules as required when you configure the connection. +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • For details about how to use data types, see Format.
              • Flink 1.15 currently only supports creating OBS tables and DLI lakehouse tables using Hive syntax, which is supported by Hive dialect DDL statements.
                • To create an OBS table using Hive syntax:
                  • For the default dialect, set hive.is-external to true in the with properties.
                  • For the Hive dialect, use the EXTERNAL keyword in the create table statement.
                  +
                • To create a DLI lakehouse table using Hive syntax:
                  • For the Hive dialect, add 'is_lakehouse'='true' to the table properties.
                  +
                +
              • Enable checkpointing.
              • You are advised to switch to Hive dialect to create Hive-compatible tables. If you want to create Hive-compatible tables with default dialect, make sure to set 'connector'='hive' in your table properties, otherwise a table is considered generic by default in HiveCatalog. Note that the connector property is not required if you use Hive dialect.
              • Monitor strategy is to scan all directories/files currently in the location path. Many partitions may cause performance degradation.
              • Streaming reads for non-partitioned tables requires that each file be written atomically into the target directory.
              • Streaming reading for partitioned tables requires that each partition should be added atomically in the view of hive metastore. If not, new data added to an existing partition will be consumed.
              • Streaming reads do not support watermark grammar in Flink DDL. These tables cannot be used for window operators.
              +
              +

              Syntax

               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
              CREATE EXTERNAL TABLE [IF NOT EXISTS] table_name
+  [(col_name data_type [column_constraint] [COMMENT col_comment], ... [table_constraint])]
+  [COMMENT table_comment]
+  [PARTITIONED BY (col_name data_type [COMMENT col_comment], ...)]
+  [
+    [ROW FORMAT row_format]
+    [STORED AS file_format]
+  ]
+  [LOCATION obs_path]
+  [TBLPROPERTIES (property_name=property_value, ...)]
+
+row_format:
+  : DELIMITED [FIELDS TERMINATED BY char [ESCAPED BY char]] [COLLECTION ITEMS TERMINATED BY char]
+      [MAP KEYS TERMINATED BY char] [LINES TERMINATED BY char]
+      [NULL DEFINED AS char]
+  | SERDE serde_name [WITH SERDEPROPERTIES (property_name=property_value, ...)]
+
+file_format:
+  : SEQUENCEFILE
+  | TEXTFILE
+  | RCFILE
+  | ORC
+  | PARQUET
+  | AVRO
+  | INPUTFORMAT input_format_classname OUTPUTFORMAT output_format_classname
+
+column_constraint:
+  : NOT NULL [[ENABLE|DISABLE] [VALIDATE|NOVALIDATE] [RELY|NORELY]]
+
+table_constraint:
+  : [CONSTRAINT constraint_name] PRIMARY KEY (col_name, ...) [[ENABLE|DISABLE] [VALIDATE|NOVALIDATE] [RELY|NORELY]]
+
              +
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 TBLPROPERTIES parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              streaming-source.enable

              +

              No

              +

              false

              +

              Boolean

              +

              Enable streaming source or not. Note: Make sure that each partition/file should be written atomically, otherwise the reader may get incomplete data.

              +

              streaming-source.partition.include

              +

              No

              +

              all

              +

              String

              +

              Option to set the partitions to read, supported options are all and latest. By default, this parameter is set to all.

              +

              all means read all partitions.

              +

              latest only works when the streaming hive source table used as temporal table. latest means reading latest partition in order of streaming-source.partition.order.

              +

              Flink supports temporal join the latest hive partition by enabling streaming-source.enable and setting streaming-source.partition.include to latest. At the same time, user can assign the partition compare order and data update interval by configuring following partition-related options.

              +

              streaming-source.monitor-interval

              +

              No

              +

              None

              +

              Duration

              +

              Time interval for consecutively monitoring partition/file. Notes: The default interval for hive streaming reading is '1 m', the default interval for hive streaming temporal join is '60 m', this is because there's one framework limitation that every TM will visit the Hive metaStore in current hive streaming temporal join implementation which may produce pressure to metaStore, this will improve in the future.

              +

              streaming-source.partition-order

              +

              No

              +

              partition-name

              +

              String

              +

              The partition order of streaming source, supporting create-time, partition-time, and partition-name.

              +

              create-time compares partition/file creation time, this is not the partition create time in Hive metaStore, but the folder/file modification time in filesystem, if the partition folder somehow gets updated, e.g. add new file into folder, it can affect how the data is consumed.

              +

              partition-time compares the time extracted from partition name.

              +

              partition-name compares partition name's alphabetical order.

              +

              For a non-partition table, this value should always be create-time.

              +

              By default the value is partition-name. The option is equality with deprecated option streaming-source.consume-order.

              +

              streaming-source.consume-start-offset

              +

              No

              +

              None

              +

              String

              +

              Start offset for streaming consuming. How to parse and compare offsets depends on your order. For create-time and partition-time, should be a timestamp string (yyyy-[m]m-[d]d [hh:mm:ss]).

              +

              For partition-time, will use partition time extractor to extract time from partition. For partition-name, is the partition name string (e.g. pt_year=2020/pt_mon=10/pt_day=01).

              +

              is_lakehouse

              +

              No

              +

              None

              +

              Boolean

              +

              If DLI lakehouse tables using Hive syntax are used, set this parameter to true.

              +
              +
              +
              • Source Parallelism Inference

                By default, Flink infers the hive source parallelism based on the number of splits, and the number of splits is based on the number of files and the number of blocks in the files.

                +

                Flink allows you to flexibly configure the policy of parallelism inference. You can configure the following parameters in TableConfig (note that these parameters affect all sources of the job):

                + +
                + + + + + + + + + + + + + + + + +

                Key

                +

                Default

                +

                Type

                +

                Description

                +

                table.exec.hive.infer-source-parallelism

                +

                true

                +

                Boolean

                +

                If it is true, source parallelism is inferred according to splits number. If it is false, parallelism of source is set by config.

                +

                table.exec.hive.infer-source-parallelism.max

                +

                1000

                +

                Integer

                +

                Sets max infer parallelism for source operator.

                +
                +
                +
              +
              • Load Partition Splits

                Multi-thread is used to split hive's partitions. You can use table.exec.hive.load-partition-splits.thread-num to configure the thread number. The default value is 3 and the configured value should be greater than 0.

                + +
                + + + + + + + + + + + +

                Key

                +

                Default

                +

                Type

                +

                Description

                +

                table.exec.hive.load-partition-splits.thread-num

                +

                3

                +

                Integer

                +

                The configured value should be greater than 0.

                +
                +
                +

                SQL hints can be used to apply configurations to a Hive table without changing its definition in the Hive metastore. See Hints | Apache Flink.

                +
              • Vectorized Optimization upon Read

                Flink will automatically used vectorized reads of Hive tables when the following conditions are met:

                +
                • Format: ORC or Parquet.
                • Columns without complex data type, like hive types: List, Map, Struct, Union.
                  This feature is enabled by default. It may be disabled with the following configuration.
                  table.exec.hive.fallback-mapred-reader=true
                  +
                  +
                +
              +
              +
              • Reading Hive Views

                Flink is able to read from Hive defined views, but some limitations apply:

                +
                • The Hive catalog must be set as the current catalog before you can query the view. This can be done by either tableEnv.useCatalog(...) in Table API or USE CATALOG ... in SQL Client.
                • Hive and Flink SQL have different syntax, e.g. different reserved keywords and literals. Make sure the view's query is compatible with Flink grammar.
                +
              +

              Example

              1. Create an OBS table in Hive syntax using Spark SQL and insert 10 data records. Simulate the data source.
                CREATE TABLE IF NOT EXISTS demo.student(
+  name STRING, 
+ score DOUBLE) 
+PARTITIONED BY (classNo INT) 
+STORED AS PARQUET 
+LOCATION 'obs://demo/spark.db/student';
+
+INSERT INTO demo.student PARTITION(classNo=1) VALUES ('Alice', 90.0), ('Bob', 80.0), ('Charlie', 70.0), ('David', 60.0), ('Eve', 50.0), ('Frank', 40.0), ('Grace', 30.0), ('Hank', 20.0), ('Ivy', 10.0), ('Jack', 0.0);
                +
              2. Demonstrate batch processing using Flink SQL to read data from the Hive syntax OBS table demo.student in batch mode and print it out. Checkpointing is required.
                CREATE CATALOG myhive WITH (
+    'type' = 'hive',
+    'default-database' = 'demo',
+     'hive-conf-dir' = '/opt/flink/conf'
+);
+
+USE CATALOG myhive;
+
+create table if not exists print (
+    name STRING, 
+    score DOUBLE, 
+    classNo INT)
+with ('connector' = 'print');
+
+insert into print
+select * from student;
                +
                Result (out log of TaskManager):
                +I[Alice, 90.0, 1]
++I[Bob, 80.0, 1]
++I[Charlie, 70.0, 1]
++I[David, 60.0, 1]
++I[Eve, 50.0, 1]
++I[Frank, 40.0, 1]
++I[Grace, 30.0, 1]
++I[Hank, 20.0, 1]
++I[Ivy, 10.0, 1]
++I[Jack, 0.0, 1]
                +
                +
              3. Demonstrate stream processing by using Flink SQL to read data from the Hive syntax OBS table demo.student in stream mode and print it out.
                CREATE CATALOG myhive WITH (
+    'type' = 'hive' ,
+    'default-database' = 'demo',
+     'hive-conf-dir' = '/opt/flink/conf'
+);
+
+USE CATALOG myhive;
+
+create table if not exists print (
+    name STRING, 
+    score DOUBLE, 
+    classNo INT)
+with ('connector' = 'print');
+
+insert into print
+select * from student /*+ OPTIONS('streaming-source.enable' = 'true', 'streaming-source.monitor-interval' = '3 m') */;
                +
              +
              +

              The SQL hints function is used. SQL hints can be used to apply configurations to a Hive table without changing its definition in the Hive metastore. For details, see SQL Hints.

              +
              +
              +
              +
              Parent topic: Hive
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15050.html b/docs/dli/sqlreference/dli_08_15050.html new file mode 100644 index 000000000..a6b7b1fd3 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15050.html @@ -0,0 +1,129 @@ + + +

              Result Table

              +

              Function

              This section describes how to use Flink to write Hive tables, the definition of the Hive result table, parameters used for creating the result table, and sample code. For details, see Apache Flink Hive Read & Write.

              +

              Flink supports writing data to Hive in both BATCH and STREAMING modes.

              +
              • When run as a BATCH application, Flink will write to a Hive table only making those records visible when the Job finishes. BATCH writes support both appending to and overwriting existing tables.
              • STREAMING writes continuously adding new data to Hive, committing records - making them visible - incrementally. Users control when/how to trigger commits with several properties. Insert overwrite is not supported for streaming write. Please see the streaming sink for a full list of available configurations.
              +
              +

              Prerequisites

              To create a FileSystem source table, an enhanced datasource connection is required. You can set security group rules as required when you configure the connection. +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • For details about how to use data types, see Format.
              • Flink 1.15 currently only supports creating OBS tables and DLI lakehouse tables using Hive syntax, which is supported by Hive dialect DDL statements.
                • To create an OBS table using Hive syntax:
                  • For the default dialect, set hive.is-external to true in the with properties.
                  • For the Hive dialect, use the EXTERNAL keyword in the create table statement.
                  +
                • To create a DLI lakehouse table using Hive syntax:
                  • For the Hive dialect, add 'is_lakehouse'='true' to the table properties.
                  +
                +
              • When creating a Flink OpenSource SQL job, enable checkpointing in the job editing interface.
              +
              +

              Syntax

               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
              CREATE EXTERNAL TABLE [IF NOT EXISTS] table_name
+  [(col_name data_type [column_constraint] [COMMENT col_comment], ... [table_constraint])]
+  [COMMENT table_comment]
+  [PARTITIONED BY (col_name data_type [COMMENT col_comment], ...)]
+  [
+    [ROW FORMAT row_format]
+    [STORED AS file_format]
+  ]
+  [LOCATION obs_path]
+  [TBLPROPERTIES (property_name=property_value, ...)]
+
+row_format:
+  : DELIMITED [FIELDS TERMINATED BY char [ESCAPED BY char]] [COLLECTION ITEMS TERMINATED BY char]
+      [MAP KEYS TERMINATED BY char] [LINES TERMINATED BY char]
+      [NULL DEFINED AS char]
+  | SERDE serde_name [WITH SERDEPROPERTIES (property_name=property_value, ...)]
+
+file_format:
+  : SEQUENCEFILE
+  | TEXTFILE
+  | RCFILE
+  | ORC
+  | PARQUET
+  | AVRO
+  | INPUTFORMAT input_format_classname OUTPUTFORMAT output_format_classname
+
+column_constraint:
+  : NOT NULL [[ENABLE|DISABLE] [VALIDATE|NOVALIDATE] [RELY|NORELY]]
+
+table_constraint:
+  : [CONSTRAINT constraint_name] PRIMARY KEY (col_name, ...) [[ENABLE|DISABLE] [VALIDATE|NOVALIDATE] [RELY|NORELY]]
+
              +
              +
              +

              Parameter Description

              Please see the streaming sink for a full list of available configurations.

              +
              +

              Example

              The following example demonstrates how to use Datagen to write to a Hive table with partition submission functionality.
              CREATE CATALOG myhive WITH (
+    'type' = 'hive' ,
+    'default-database' = 'demo',
+    'hive-conf-dir' = '/opt/flink/conf'
+);
+
+USE CATALOG myhive;
+
+SET table.sql-dialect=hive;
+
+-- drop table demo.student_hive_sink;
+CREATE EXTERNAL TABLE IF NOT EXISTS demo.student_hive_sink(
+  name STRING, 
+ score DOUBLE) 
+PARTITIONED BY (classNo INT) 
+STORED AS PARQUET 
+LOCATION 'obs://demo/spark.db/student_hive_sink'
+TBLPROPERTIES (
+   'sink.partition-commit.policy.kind'='metastore,success-file'
+ );
+
+SET table.sql-dialect=default;
+create table if not exists student_datagen_source(
+  name STRING, 
+  score DOUBLE, 
+  classNo INT
+) with (
+  'connector' = 'datagen',
+  'rows-per-second' = '1', --Generates a piece of data per second.
+  'fields.name.kind' = 'random', --Specifies a random generator for the user_id field.
+  'fields.name.length' = '7', --Limits the user_id length to 7.
+  'fields.classNo.kind' ='random',
+  'fields.classNo.min' = '1',
+  'fields.classNo.max' = '10'
+);
+
+insert into student_hive_sink select * from student_datagen_source;
              +
              +

              Query the result table using Spark SQL.

              +
              select * from  demo.student_hive_sink where classNo > 0 limit 10
              +
              +
              +
              +
              +
              Parent topic: Hive
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15051.html b/docs/dli/sqlreference/dli_08_15051.html new file mode 100644 index 000000000..6fc096f97 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15051.html @@ -0,0 +1,21 @@ + + +

              Hive Dimension Table

              +

              Function

              You can use Hive tables as temporal tables and associate them through temporal joins. For more information on temporal joins, refer to temporal join.

              +

              Flink supports processing-time temporal joins with Hive tables, which always join the latest version of the temporal table. Flink supports temporary joins with both partitioned and non-partitioned Hive tables. For partitioned tables, Flink automatically tracks the latest partition of the Hive table. For details, see Apache Flink Hive Read & Write.

              +
              +

              Caveats

              • Currently, Flink does not support event-time temporal joins with Hive tables.
              • The "Temporal Join The Latest Partition" feature is only supported in Flink STREAMING mode.
              +
              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • For details about how to use data types, see Format.
              • Flink 1.15 currently only supports creating OBS tables and DLI lakehouse tables using Hive syntax, which is supported by Hive dialect DDL statements.
                • To create an OBS table using Hive syntax:
                  • For the default dialect, set hive.is-external to true in the with properties.
                  • For the Hive dialect, use the EXTERNAL keyword in the create table statement.
                  +
                • To create a DLI lakehouse table using Hive syntax:
                  • For the Hive dialect, add 'is_lakehouse'='true' to the table properties.
                  +
                +
              • When creating a Flink OpenSource SQL job, enable checkpointing in the job editing interface.
              +
              +

              Syntax Format and Parameter Description

              For details, see the syntax format and parameter description in Hive Source Table.

              +
              +
              +
              +
              +
              Parent topic: Hive
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15052.html b/docs/dli/sqlreference/dli_08_15052.html new file mode 100644 index 000000000..dcdd46fa2 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15052.html @@ -0,0 +1,116 @@ + + +

              Using Temporal Join to Associate the Latest Partition of a Dimension Table

              +

              Function

              For partitioned tables that change over time, we can read them as unbounded streams. If each partition contains a complete set of data for a certain version, the partition can be considered as a version of the temporal table, which retains the data of the partition. Flink supports automatically tracking the latest partition (version) of the temporal table in processing-time joins.

              +

              The latest partition (version) is defined by the streaming-source.partition-order parameter.

              +

              This is the most common use case for using Hive tables as dimension tables in Flink streaming applications.

              +
              +

              Caveats

              Using Temporal join to associate the latest partition of a dimension table is only supported in Flink STREAMING mode.

              +
              +

              Example

              The following example shows a classic business pipeline where the dimension table comes from Hive and is updated once a day through batch processing or Flink jobs. The Kafka stream comes from real-time online business data or logs and needs to be joined with the dimension table to expand the stream.

              +
              1. Create a Hive OBS external table using Spark SQL and insert data.
                CREATE TABLE if not exists dimension_hive_table (
+  product_id STRING,
+  product_name STRING,
+  unit_price DECIMAL(10, 4),
+  pv_count BIGINT,
+  like_count BIGINT,
+  comment_count BIGINT,
+  update_time TIMESTAMP,
+  update_user STRING
+) 
+STORED AS PARQUET 
+LOCATION 'obs://demo/spark.db/dimension_hive_table' 
+PARTITIONED BY (
+    create_time   STRING
+);
                +
                INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_11', 'product_name_11', 1.2345, 100, 50, 20, '2023-11-25 02:10:58', 'update_user_1');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_12', 'product_name_12', 2.3456, 200, 100, 40, '2023-11-25 02:10:58', 'update_user_2');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_13', 'product_name_13', 3.4567, 300, 150, 60, '2023-11-25 02:10:58', 'update_user_3');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_14', 'product_name_14', 4.5678, 400, 200, 80, '2023-11-25 02:10:58', 'update_user_4');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_15', 'product_name_15', 5.6789, 500, 250, 100, '2023-11-25 02:10:58', 'update_user_5');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_16', 'product_name_16', 6.7890, 600, 300, 120, '2023-11-25 02:10:58', 'update_user_6');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_17', 'product_name_17', 7.8901, 700, 350, 140, '2023-11-25 02:10:58', 'update_user_7');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_18', 'product_name_18', 8.9012, 800, 400, 160, '2023-11-25 02:10:58', 'update_user_8');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_19', 'product_name_19', 9.0123, 900, 450, 180, '2023-11-25 02:10:58', 'update_user_9');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_10', 'product_name_10', 10.1234, 1000, 500, 200, '2023-11-25 02:10:58', 'update_user_10');
                +
                +
              1. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. This job simulates reading data from Kafka, performs a join with a Hive dimension table to denormalize the data, and outputs it to Print.
                Change the values of the parameters in bold as needed in the following script.
                CREATE CATALOG myhive WITH (
+    'type' = 'hive' ,
+    'default-database' = 'demo',
+     'hive-conf-dir' = '/opt/flink/conf'
+);
+
+USE CATALOG myhive;
+
+CREATE TABLE if not exists ordersSource (
+  product_id STRING,
+  user_name string,
+  proctime as Proctime()
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'TOPIC',
+  'properties.bootstrap.servers' = 'KafkaIP:PROT,KafkaIP:PROT,KafkaIP:PROT',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+create table if not exists print (
+  product_id STRING,
+  user_name string,
+  product_name STRING,
+  unit_price DECIMAL(10, 4),
+  pv_count BIGINT,
+  like_count BIGINT,
+  comment_count BIGINT,
+  update_time TIMESTAMP,
+  update_user STRING,
+  create_time   STRING
+) with (
+  'connector' = 'print'
+);
+
+insert into print 
+select 
+  orders.product_id,
+  orders.user_name,
+  dim.product_name,
+  dim.unit_price,
+  dim.pv_count,
+  dim.like_count,
+  dim.comment_count,
+  dim.update_time,
+  dim.update_user,
+  dim.create_time
+from ordersSource orders
+left join dimension_hive_table /*+ OPTIONS('streaming-source.enable'='true',
+   'streaming-source.partition.include' = 'latest', 'streaming-source.monitor-interval' = '10 m') */
+   for system_time as of orders.proctime as dim on orders.product_id = dim.product_id;
                +
                +
              2. Connect to the Kafka cluster and insert the following test data into the source topic in Kafka:
                {"product_id": "product_id_11", "user_name": "name11"}
+{"product_id": "product_id_12", "user_name": "name12"}
                +
              3. View the data in the Print result table.
                +I[product_id_11, name11, product_name_11, 1.2345, 100, 50, 20, 2023-11-24T18:10:58, update_user_1, create_time_1]
++I[product_id_12, name12, product_name_12, 2.3456, 200, 100, 40, 2023-11-24T18:10:58, update_user_2, create_time_1]
                +
              4. Simulate inserting new partition data into the Hive dimension table.
                INSERT INTO dimension_hive_table PARTITION (create_time='create_time_2') VALUES ('product_id_21', 'product_name_21', 1.2345, 100, 50, 20, '2023-11-25 02:10:58', 'update_user_1');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_2') VALUES ('product_id_22', 'product_name_22', 2.3456, 200, 100, 40, '2023-11-25 02:10:58', 'update_user_2');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_2') VALUES ('product_id_23', 'product_name_23', 3.4567, 300, 150, 60, '2023-11-25 02:10:58', 'update_user_3');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_2') VALUES ('product_id_24', 'product_name_24', 4.5678, 400, 200, 80, '2023-11-25 02:10:58', 'update_user_4');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_2') VALUES ('product_id_25', 'product_name_25', 5.6789, 500, 250, 100, '2023-11-25 02:10:58', 'update_user_5');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_2') VALUES ('product_id_26', 'product_name_26', 6.7890, 600, 300, 120, '2023-11-25 02:10:58', 'update_user_6');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_2') VALUES ('product_id_27', 'product_name_27', 7.8901, 700, 350, 140, '2023-11-25 02:10:58', 'update_user_7');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_2') VALUES ('product_id_28', 'product_name_28', 8.9012, 800, 400, 160, '2023-11-25 02:10:58', 'update_user_8');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_2') VALUES ('product_id_29', 'product_name_29', 9.0123, 900, 450, 180, '2023-11-25 02:10:58', 'update_user_9');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_2') VALUES ('product_id_20', 'product_name_20', 10.1234, 1000, 500, 200, '2023-11-25 02:10:58', 'update_user_10');
                +
              5. Connect to the Kafka cluster and insert the following test data into the source topic in Kafka. Associate the data from the previous partition with create_time='create_time_1':
                 {"product_id": "product_id_13", "user_name": "name13"}
                +
              6. View the data in the Print result table. The data of the previous partition create_time='create_time_1' in the Hive dimension table has been deleted.
                  +I[product_id_13, name13, null, null, null, null, null, null, null, null]
                +
              7. Connect to the Kafka cluster and insert the following test data into the source topic in Kafka. Associate the latest partition data with create_time='create_time_2':
                 {"product_id": "product_id_21", "user_name": "name21"}
                +
              8. View the data in the Print result table. The Hive dimension table retains the data of the latest partition with create_time='create_time_2'.
                  +I[product_id_21, name21, product_name_21, 1.2345, 100, 50, 20, 2023-11-24T18:10:58, update_user_1, create_time_2]
                +
              +
              +
              +
              +
              +
              Parent topic: Hive
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15053.html b/docs/dli/sqlreference/dli_08_15053.html new file mode 100644 index 000000000..26cd68550 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15053.html @@ -0,0 +1,125 @@ + + +

              Using Temporal Join to Associate the Latest Version of a Dimension Table

              +

              Function

              For Hive tables, we can read them as bounded streams. In this case, the Hive table can only track its latest version when queried. The latest version of the table retains all the data of the Hive table.

              +
              +

              Caveats

              • Each joining subtask needs to keep its own cache of the Hive table. Make sure the Hive table can fit into the memory of a TM task slot.
              • It is encouraged to set a relatively large value both for streaming-source.monitor-interval (latest partition as temporal table) or lookup.join.cache.ttl (all partitions as temporal table). Otherwise, jobs are prone to performance issues as the table needs to be updated and reloaded too frequently.
              • Currently we simply load the whole Hive table whenever the cache needs refreshing. There's no way to differentiate new data from the old.
              +
              +

              Parameter Description

              When performing the temporal join the latest Hive table, the Hive table will be cached in Slot memory and each record from the stream is joined against the table by key to decide whether a match is found. Using the latest Hive table as a temporal table does not require any additional configuration. Optionally, you can configure the TTL of the Hive table cache with the following property. After the cache expires, the Hive table will be scanned again to load the latest data.

              + +
              + + + + + + + + + + + +

              Parameter

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              lookup.join.cache.ttl

              +

              60 min

              +

              Duration

              +

              The cache TTL (e.g. 10 min) for the build table in lookup join. By default the TTL is 60 minutes.

              +

              The option only works when looking up bounded hive table source, if you are using streaming hive source as temporal table, use streaming-source.monitor-interval to configure the interval of data update.

              +
              +
              +
              +

              Example

              The example shows a classic business pipeline where the dimension table comes from Hive and is updated once a day through batch processing or Flink jobs. The Kafka stream comes from real-time online business data or logs and needs to be joined with the dimension table to expand the stream.

              +
              1. Create a Hive OBS external table using Spark SQL and insert data.
                CREATE TABLE if not exists dimension_hive_table (
+  product_id STRING,
+  product_name STRING,
+  unit_price DECIMAL(10, 4),
+  pv_count BIGINT,
+  like_count BIGINT,
+  comment_count BIGINT,
+  update_time TIMESTAMP,
+  update_user STRING
+) 
+STORED AS PARQUET 
+LOCATION 'obs://demo/spark.db/dimension_hive_table' 
+PARTITIONED BY (
+    create_time   STRING
+);
                +
                INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_11', 'product_name_11', 1.2345, 100, 50, 20, '2023-11-25 02:10:58', 'update_user_1');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_12', 'product_name_12', 2.3456, 200, 100, 40, '2023-11-25 02:10:58', 'update_user_2');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_13', 'product_name_13', 3.4567, 300, 150, 60, '2023-11-25 02:10:58', 'update_user_3');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_14', 'product_name_14', 4.5678, 400, 200, 80, '2023-11-25 02:10:58', 'update_user_4');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_15', 'product_name_15', 5.6789, 500, 250, 100, '2023-11-25 02:10:58', 'update_user_5');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_16', 'product_name_16', 6.7890, 600, 300, 120, '2023-11-25 02:10:58', 'update_user_6');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_17', 'product_name_17', 7.8901, 700, 350, 140, '2023-11-25 02:10:58', 'update_user_7');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_18', 'product_name_18', 8.9012, 800, 400, 160, '2023-11-25 02:10:58', 'update_user_8');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_19', 'product_name_19', 9.0123, 900, 450, 180, '2023-11-25 02:10:58', 'update_user_9');
+INSERT INTO dimension_hive_table PARTITION (create_time='create_time_1') VALUES ('product_id_10', 'product_name_10', 10.1234, 1000, 500, 200, '2023-11-25 02:10:58', 'update_user_10');
                +
                +
              1. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. This job simulates reading data from Kafka, performs a join with a Hive dimension table to denormalize the data, and outputs it to Print.
                Change the values of the parameters in bold as needed in the following script.
                CREATE CATALOG myhive WITH (
+    'type' = 'hive' ,
+    'default-database' = 'demo',
+     'hive-conf-dir' = '/opt/flink/conf'
+);
+
+USE CATALOG myhive;
+
+CREATE TABLE if not exists ordersSource (
+  product_id STRING,
+  user_name string,
+  proctime as Proctime()
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'TOPIC',
+  'properties.bootstrap.servers' = 'KafkaIP:PROT,KafkaIP:PROT,KafkaIP:PROT',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+create table if not exists print (
+  product_id STRING,
+  user_name string,
+  product_name STRING,
+  unit_price DECIMAL(10, 4),
+  pv_count BIGINT,
+  like_count BIGINT,
+  comment_count BIGINT,
+  update_time TIMESTAMP,
+  update_user STRING,
+  create_time   STRING
+) with (
+  'connector' = 'print'
+);
+
+insert into print 
+select 
+  orders.product_id,
+  orders.user_name,
+  dim.product_name,
+  dim.unit_price,
+  dim.pv_count,
+  dim.like_count,
+  dim.comment_count,
+  dim.update_time,
+  dim.update_user,
+  dim.create_time
+from ordersSource orders
+left join dimension_hive_table /*+ OPTIONS('lookup.join.cache.ttl'='60 m') */
+   for system_time as of orders.proctime as dim on orders.product_id = dim.product_id;
                +
                +
              2. Connect to the Kafka cluster and insert the following test data into the source topic in Kafka:
                {"product_id": "product_id_11", "user_name": "name11"}
+{"product_id": "product_id_12", "user_name": "name12"}
                +
              3. View the data in the Print result table.
                +I[product_id_11, name11, product_name_11, 1.2345, 100, 50, 20, 2023-11-24T18:10:58, update_user_1, create_time_1]
++I[product_id_12, name12, product_name_12, 2.3456, 200, 100, 40, 2023-11-24T18:10:58, update_user_2, create_time_1]
                +
              +
              +
              +
              +
              +
              Parent topic: Hive
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15057.html b/docs/dli/sqlreference/dli_08_15057.html new file mode 100644 index 000000000..23407843a --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15057.html @@ -0,0 +1,596 @@ + + +

              JDBC

              +

              Function

              The JDBC connector is provided by Apache Flink and can be used to read data from and write data to common databases, such as MySQL and PostgreSQL. Source tables, result tables, and dimension tables are supported.

              + +
              + + + + + + + +
              Table 1 Supported types

              Type

              +

              Description

              +

              Supported Table Types

              +

              Source table, dimension table, and result table

              +
              +
              +
              +

              Prerequisites

              • An enhanced datasource connection with the database has been established, so that you can configure security group rules as required.
              +
              +

              Caveats

              • The JDBC sink operates in upsert mode for exchanging UPDATE/DELETE messages with the external system if a primary key is defined on the DDL, otherwise, it operates in append mode and does not support to consume UPDATE/DELETE messages.
              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              +
              +

              Syntax

              create table jbdcTable (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (','PRIMARY KEY (attr_name, ...) NOT ENFORCED)
+  (',' watermark for rowtime_column_name as watermark-strategy_expression)
+) with (
+  'connector' = 'jdbc',
+  'url' = '',
+  'table-name' = '',
+  'username' = '',
+  'password' = ''
+);
              +
              +

              Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector to be used. Set this parameter to jdbc.

              +

              url

              +

              Yes

              +

              None

              +

              String

              +

              Database URL

              +
              • To connect to a MySQL database, the format is jdbc:mysql://MySQL address:MySQL port/Database name.
              • To connect to a PostgreSQL database, the format is jdbc:postgresql://PostgreSQL address:PostgreSQL port/Database name.
              +

              table-name

              +

              Yes

              +

              None

              +

              String

              +

              Name of the table where the data will be read from the database

              +

              driver

              +

              No

              +

              None

              +

              String

              +

              Driver required for connecting to the database. If you do not set this parameter, the automatically extracted URL will be used.

              +
              • The default driver of the MySQL database is com.mysql.jdbc.Driver.
              • The default driver of the PostgreSQL database is org.postgresql.Driver.
              +

              username

              +

              No

              +

              None

              +

              String

              +

              Database authentication user name. This parameter must be configured in pair with password.

              +

              password

              +

              No

              +

              None

              +

              String

              +

              Database authentication password. This parameter must be configured in pair with username.

              +

              connection.max-retry-timeout

              +

              No

              +

              60s

              +

              Duration

              +

              Maximum timeout between retries. The timeout should be in second granularity and should not be smaller than 1 second.

              +

              scan.partition.column

              +

              No

              +

              None

              +

              String

              +

              Name of the column used to partition the input. For details, see Partitioned Scan.

              +

              scan.partition.num

              +

              No

              +

              None

              +

              Integer

              +

              Number of partitions to be created. For details, see Partitioned Scan.

              +

              scan.partition.lower-bound

              +

              No

              +

              None

              +

              Integer

              +

              Lower bound of values to be fetched for the first partition. For details, see Partitioned Scan.

              +

              scan.partition.upper-bound

              +

              No

              +

              None

              +

              Integer

              +

              Upper bound of values to be fetched for the last partition. For details, see Partitioned Scan.

              +

              scan.fetch-size

              +

              No

              +

              0

              +

              Integer

              +

              Number of rows fetched from the database each time. If this parameter is set to 0, the SQL hint is ignored.

              +

              scan.auto-commit

              +

              No

              +

              true

              +

              Boolean

              +

              Whether each statement is committed in a transaction automatically.

              +

              lookup.cache.max-rows

              +

              No

              +

              None

              +

              Integer

              +

              Maximum number of rows in the lookup cache. When the rows exceed this value, the first item added to the cache will be marked as expired. By default, the lookup cache is not enabled. For details, see Lookup Cache Functions.

              +

              lookup.cache.ttl

              +

              No

              +

              None

              +

              Duration

              +

              Maximum survival time of each record in the lookup cache. When the rows exceed this value, the first item added to the cache will be marked as expired. By default, the lookup cache is not enabled. For details, see Lookup Cache Functions.

              +

              lookup.cache.caching-missing-key

              +

              No

              +

              true

              +

              Boolean

              +

              Whether to cache empty query results. The default value is true. For details, see Lookup Cache Functions.

              +

              lookup.max-retries

              +

              No

              +

              3

              +

              Integer

              +

              Maximum number of retry attempts when a database query fails.

              +

              sink.buffer-flush.max-rows

              +

              No

              +

              100

              +

              Integer

              +

              Maximum number of cached records before flushing, which can be set to 0 to disable it.

              +

              sink.buffer-flush.interval

              +

              No

              +

              1s

              +

              Duration

              +

              The interval for flushing, after which the asynchronous thread will flush the data. Can be set to 0 to disable it. To fully handle the flush events of the cache asynchronously, sink.buffer-flush.max-rows can be set to 0 and an appropriate flush time interval can be configured.

              +

              sink.max-retries

              +

              No

              +

              3

              +

              Integer

              +

              Maximum number of retries after a failed attempt to write records to the database.

              +

              sink.parallelism

              +

              No

              +

              None

              +

              Integer

              +

              Defines the parallelism of the JDBC sink operator. By default, the parallelism is determined by the framework: using the same parallelism as the upstream chained operator.

              +
              +
              +
              +

              Partitioned Scan

              To accelerate reading data in parallel Source task instances, Flink provides the partitioned scan feature for the JDBC table. The following parameters describe how to partition the table when reading in parallel from multiple tasks.

              +
              • scan.partition.column: name of the column used to partition the input. The data type of the column must be number, date, or timestamp.
              • scan.partition.num: number of partitions.
              • scan.partition.lower-bound: minimum value of the first partition.
              • scan.partition.upper-bound: maximum value of the last partition.
              +
              • When a table is created, the preceding partitioned scan parameters must all be specified if any of them is specified.
              • The scan.partition.lower-bound and scan.partition.upper-bound parameters are used to decide the partition stride instead of filtering rows in the table. All rows in the table are partitioned and returned.
              +
              +
              +

              Lookup Cache Functions

              The JDBC connector can be used as a lookup dimension table in temporal table joins, and currently only supports synchronous lookup mode.

              +

              By default, lookup cache is disabled. Therefore, all requests are sent to the external database. You can set lookup.cache.max-rows and lookup.cache.ttl to enable this feature. The main purpose of the lookup cache is to improve the performance of the JDBC connector in temporal table joins.

              +

              When the lookup cache is enabled, each process (i.e. TaskManager) will maintain a cache. Flink will first look up the cache, and only when the cache is not found will it send a request to the external database and update the cache with the returned data. When the cache hits the maximum cache rows lookup.cache.max-rows or when the rows exceed the maximum survival time lookup.cache.ttl, the first item added to the cache will be marked as expired. The records in the cache may not be the latest, and users can set lookup.cache.ttl to a smaller value to get better data refresh, but this may increase the number of requests sent to the database. Therefore, a balance between throughput and correctness should be maintained.

              +

              By default, Flink caches empty query results for primary keys, but you can switch this behavior by setting lookup.cache.caching-missing-key to false.

              +
              +

              Data Type Mapping

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 3 Data type mapping

              MySQL Type

              +

              PostgreSQL Type

              +

              Flink SQL Type

              +

              TINYINT

              +

              -

              +

              TINYINT

              +

              SMALLINT

              +

              TINYINT UNSIGNED

              +

              SMALLINT

              +

              INT2

              +

              SMALLSERIAL

              +

              SERIAL2

              +

              SMALLINT

              +

              INT

              +

              MEDIUMINT

              +

              SMALLINT UNSIGNED

              +

              INTEGER

              +

              SERIAL

              +

              INT

              +

              BIGINT

              +

              INT UNSIGNED

              +

              BIGINT

              +

              BIGSERIAL

              +

              BIGINT

              +

              BIGINT UNSIGNED

              +

              -

              +

              DECIMAL(20, 0)

              +

              BIGINT

              +

              BIGINT

              +

              BIGINT

              +

              FLOAT

              +

              REAL

              +

              FLOAT4

              +

              FLOAT

              +

              DOUBLE

              +

              DOUBLE PRECISION

              +

              FLOAT8

              +

              DOUBLE PRECISION

              +

              DOUBLE

              +

              NUMERIC(p, s)

              +

              DECIMAL(p, s)

              +

              NUMERIC(p, s)

              +

              DECIMAL(p, s)

              +

              DECIMAL(p, s)

              +

              BOOLEAN

              +

              TINYINT(1)

              +

              BOOLEAN

              +

              BOOLEAN

              +

              DATE

              +

              DATE

              +

              DATE

              +

              TIME [(p)]

              +

              TIME [(p)] [WITHOUT TIMEZONE]

              +

              TIME [(p)] [WITHOUT TIMEZONE]

              +

              DATETIME [(p)]

              +

              TIMESTAMP [(p)] [WITHOUT TIMEZONE]

              +

              TIMESTAMP [(p)] [WITHOUT TIMEZONE]

              +

              CHAR(n)

              +

              VARCHAR(n)

              +

              TEXT

              +

              CHAR(n)

              +

              CHARACTER(n)

              +

              VARCHAR(n)

              +

              CHARACTER

              +

              VARYING(n)

              +

              TEXT

              +

              STRING

              +

              BINARY

              +

              VARBINARY

              +

              BLOB

              +

              BYTEA

              +

              BYTES

              +

              -

              +

              ARRAY

              +

              ARRAY

              +
              +
              +
              +

              Example

              • Example 1: Use JDBC as the data source and Print as the result table to read data from an RDS MySQL database and write it into the Print result table.
                1. Create an enhanced datasource connection in the VPC and subnet where RDS MySQL locates, and bind the connection to the required Flink elastic resource pool.
                2. Set RDS MySQL security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the RDS address. If the connection passes the test, it is bound to the queue.
                3. Log in to the RDS MySQL database, create table orders in the Flink database, and insert data.

                  Create table orders in the Flink database.

                  +
                  CREATE TABLE `flink`.`orders` (
+	`order_id` VARCHAR(32) NOT NULL,
+	`order_channel` VARCHAR(32) NULL,
+	PRIMARY KEY (`order_id`)
+)	ENGINE = InnoDB
+	DEFAULT CHARACTER SET = utf8mb4
+	COLLATE = utf8mb4_general_ci;
                  +

                  Insert data into the table.

                  +
                  insert into orders(
+  order_id,
+  order_channel
+) values
+  ('1', 'webShop'),  
+  ('2', 'miniAppShop');
                  +
                4. Create a Flink OpenSource SQL job. Enter the following job script and submit the job.

                  When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.

                  +

                  Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .

                  +
                  CREATE TABLE jdbcSource (
+  order_id string,
+  order_channel string
+) WITH (
+  'connector' = 'jdbc',
+  'url' = 'jdbc:mysql://MySQLAddress:MySQLPort/flink',--flink is the database name created in RDS MySQL.
+  'table-name' = 'orders',
+  'username' = 'MySQLUsername',
+  'password' = 'MySQLPassword',
+  'scan.fetch-size' = '10',
+  'scan.auto-commit' = 'true'
+);
+
+CREATE TABLE printSink (
+  order_id string,
+  order_channel string
+) WITH (
+  'connector' = 'print'
+);
+
+insert into printSink select * from jdbcSource;
                  +
                5. View the data result in the taskmanager.out file. The data result is as follows:
                  +I(1,webShop)
++I(2,miniAppShop)
                  +
                +
              • Example 2: Send data using the DataGen source table and output data to a MySQL database through the JDBC result table.
                1. Create an enhanced datasource connection in the VPC and subnet where RDS MySQL locates, and bind the connection to the required Flink elastic resource pool.
                2. Set RDS MySQL security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the RDS address. If the connection passes the test, it is bound to the queue.
                3. Log in to the RDS MySQL database, create table orders in the Flink database, and insert data.
                  Create table orders in the Flink database.
                  CREATE TABLE `flink`.`orders` (
+	`order_id` VARCHAR(32) NOT NULL,
+	`order_channel` VARCHAR(32) NULL,
+	PRIMARY KEY (`order_id`)
+)	ENGINE = InnoDB
+	DEFAULT CHARACTER SET = utf8mb4
+	COLLATE = utf8mb4_general_ci;
                  +
                  +
                4. Create a Flink OpenSource SQL job. Enter the following job script and submit the job.

                  When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.

                  +
                  CREATE TABLE dataGenSource (
+  order_id string,
+  order_channel string
+) WITH (
+  'connector' = 'datagen',
+  'fields.order_id.kind' = 'sequence',
+  'fields.order_id.start' = '1',
+  'fields.order_id.end' = '1000',
+  'fields.order_channel.kind' = 'random',
+  'fields.order_channel.length' = '5'
+);
+
+CREATE TABLE jdbcSink (
+  order_id string,
+  order_channel string,
+  PRIMARY KEY(order_id) NOT ENFORCED
+) WITH (
+  'connector' = 'jdbc',
+  'url? = 'jdbc:mysql://MySQLAddress:MySQLPort/flink',-- flink is the MySQL database where the orders table locates.
+  'table-name' = 'orders',
+  'username' = 'MySQLUsername',
+  'password' = 'MySQLPassword',
+  'sink.buffer-flush.max-rows' = '1'
+);
+
+insert into jdbcSink select * from dataGenSource;
                  +
                5. Run the SQL statement in the MySQL database to view data in the table:
                  select * from orders;
                  +
                +
              • Example 3: Read data from the DataGen source table, use the JDBC table as the dimension table, and write the table information generated by both into the Print result table.
                1. Create an enhanced datasource connection in the VPC and subnet where RDS MySQL locates, and bind the connection to the required Flink elastic resource pool.
                2. Set RDS MySQL security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the RDS address. If the connection passes the test, it is bound to the queue.
                3. Log in to the RDS MySQL database, create table orders in the Flink database, and insert data.

                  Create table orders in the Flink database.

                  +
                  CREATE TABLE `flink`.`orders` (
+	`order_id` VARCHAR(32) NOT NULL,
+	`order_channel` VARCHAR(32) NULL,
+	PRIMARY KEY (`order_id`)
+)	ENGINE = InnoDB
+	DEFAULT CHARACTER SET = utf8mb4
+	COLLATE = utf8mb4_general_ci;
                  +

                  Insert data into the table.

                  +
                  insert into orders(
+  order_id,
+  order_channel
+) values
+  ('1', 'webShop'),  
+  ('2', 'miniAppShop');
                  +
                4. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. This job script uses DataGen as the data source and JDBC as the dimension table to write data into the Print result table.

                  When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.

                  +
                  CREATE TABLE dataGenSource (
+  order_id string,
+  order_time timestamp,
+  proctime as Proctime()
+) WITH (
+  'connector' = 'datagen',
+  'fields.order_id.kind' = 'sequence',
+  'fields.order_id.start' = '1',
+  'fields.order_id.end' = '2'
+);
+
+--Creating a dimension table
+CREATE TABLE jdbcTable (
+  order_id string,
+  order_channel string
+) WITH (
+  'connector' = 'jdbc',
+  'url' = 'jdbc:mysql://JDBC address:JDBC port/flink',--flink is the name of the database where the orders table of RDS for MySQL is located.
+  'table-name' = 'orders',
+  'username' = 'JDBCUserName',
+  'password' = 'JDBCPassWord',
+  'lookup.cache.max-rows' = '100',
+  'lookup.cache.ttl' = '1000',
+  'lookup.cache.caching-missing-key' = 'false',
+  'lookup.max-retries' = '5'
+);
+
+CREATE TABLE printSink (
+  order_id string,
+  order_time timestamp,
+  order_channel string
+) WITH (
+  'connector' = 'print'
+);
+
+insert into
+  printSink
+SELECT
+  dataGenSource.order_id, dataGenSource.order_time, jdbcTable.order_channel
+from
+  dataGenSource
+  left join jdbcTable for system_time as of dataGenSource.proctime on dataGenSource.order_id = jdbcTable.order_id;
                  +
                5. View the data result in the taskmanager.out file. The data result is as follows:
                  +I(1, xxx, webShop)
++I(2, xxx, miniAppShop)
                  +
                +
              +
              +

              FAQ

              None

              +
              +
              +
              +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15058.html b/docs/dli/sqlreference/dli_08_15058.html new file mode 100644 index 000000000..747410f2b --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15058.html @@ -0,0 +1,899 @@ + + +

              Kafka

              +

              Function

              The Kafka connector allows for reading data from and writing data into Kafka topics.

              +

              Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It delivers high throughput and built-in partitions and provides data replicas and fault tolerance. Apache Kafka is applicable to scenarios of handling massive messages.

              + +
              + + + + + + + + + + +
              Table 1 Supported types

              Type

              +

              Description

              +

              Supported Table Types

              +

              Source table and result table

              +

              Supported Data Formats

              +

              CSV

              +

              JSON

              +

              Apache Avro

              +

              Confluent Avro

              +

              Debezium CDC

              +

              Canal CDC

              +

              Maxwell CDC

              +

              OGG CDC

              +

              Raw

              +
              +
              +
              +

              Prerequisites

              • You have created a Kafka cluster.
              • An enhanced datasource connection has been created for DLI to connect to Kafka clusters, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required. +
              +
              +

              Caveats

              • For details, see Apache Kafka SQL Connector.
              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • Fields in the with parameter can only be enclosed in single quotes.
              • For details about how to use data types when creating tables, see Format.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              +
              +

              Syntax

               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
              create table kafkaSource(
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (','PRIMARY KEY (attr_name, ...) NOT ENFORCED)
+  (',' WATERMARK FOR rowtime_column_name AS watermark-strategy_expression)
+)
+with (
+  'connector' = 'kafka',
+  'topic' = '',
+  'properties.bootstrap.servers' = '',
+  'properties.group.id' = '',
+  'scan.startup.mode' = '',
+  'format' = ''
+);
+
              +
              +
              +

              Source Table Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Source table parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Specify what connector to use, for Kafka use kafka.

              +

              topic

              +

              No

              +

              None

              +

              String

              +

              Topic name(s) to read data from when the table is used as source. It also supports topic list for source by separating topic by semicolon like topic-1;topic-2.

              +

              Note, only one of topic-pattern and topic can be specified for sources.

              +

              When the table is used as sink, the topic name is the topic to write data to. Note topic list is not supported for sinks.

              +

              topic-pattern

              +

              No

              +

              None

              +

              String

              +

              The regular expression for a pattern of topic names to read from.

              +

              All topics with names that match the specified regular expression will be subscribed by the consumer when the job starts running.

              +

              Note, only one of topic-pattern and topic can be specified for sources.

              +

              For more information, see Topic and Partition Discovery.

              +

              properties.bootstrap.servers

              +

              Yes

              +

              None

              +

              String

              +

              Comma separated list of Kafka brokers.

              +

              properties.group.id

              +

              optional for source, not applicable for sink

              +

              None

              +

              String

              +

              The ID of the consumer group for Kafka source. If group ID is not specified, an automatically generated ID KafkaSource-{tableIdentifier} will be used.

              +

              properties.*

              +

              No

              +

              None

              +

              String

              +

              This can set and pass arbitrary Kafka configurations.

              +
              • Suffix names must match the configuration key defined in Apache Kafka.

                Flink will remove the properties. key prefix and pass the transformed key and values to the underlying KafkaClient. For example, you can disable automatic topic creation via 'properties.allow.auto.create.topics' = 'false'.

                +
              • But there are some configurations that do not support to set, because Flink will override them, e.g. key.deserializer and value.deserializer.
              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              The format used to deserialize and serialize the value part of Kafka messages.

              +

              Either this parameter or the value.format parameter is required.

              +
              • For details about the message key and body of Kafka messages, see Key and Value Formats.
              • Refer to Format for more details and format parameters.
              +

              key.format

              +

              No

              +

              None

              +

              String

              +

              The format used to deserialize and serialize the key part of Kafka messages.

              +
              • If a key format is defined, the key.fields parameter is required as well. Otherwise the Kafka records will have an empty key.
              • Refer to Format for more details and format parameters.
              +

              key.fields

              +

              No

              +

              []

              +

              List<String>

              +

              Defines an explicit list of physical columns from the table schema that configure the data type for the key format.

              +

              By default, this list is empty and thus a key is undefined. The list should look like field1;field2.

              +

              key.fields-prefix

              +

              No

              +

              None

              +

              String

              +

              Defines a custom prefix for all fields of the key format to avoid name clashes with fields of the value format. By default, the prefix is empty.

              +

              If a custom prefix is defined, both the table schema and key.fields will work with prefixed names.

              +

              When constructing the data type of the key format, the prefix will be removed and the non-prefixed names will be used within the key format.

              +

              Note that this parameter requires that value.fields-include must be set to EXCEPT_KEY.

              +

              value.format

              +

              No

              +

              None

              +

              String

              +

              The format used to deserialize and serialize the value part of Kafka messages.

              +
              • Either this parameter or the format parameter is required. If two parameters are configured, a conflict occurs.
              • Refer to Format for more details and format parameters.
              +

              value.fields-include

              +

              No

              +

              ALL

              +

              Enum

              +

              Possible values: [ALL, EXCEPT_KEY]

              +

              Defines a strategy how to deal with key columns in the data type of the value format.

              +

              By default, ALL physical columns of the table schema will be included in the value format which means that key columns appear in the data type for both the key and value format.

              +

              scan.startup.mode

              +

              No

              +

              group-offsets

              +

              String

              +

              Startup mode for Kafka consumer.

              +

              Valid values are:

              +
              • earliest-offset: start from the earliest offset possible.
              • latest-offset: start from the latest offset.
              • group-offsets: start from committed offsets in ZooKeeper/Kafka brokers of a specific consumer group.
              • timestamp: start from user-supplied timestamp for each partition.
              • specific-offsets: start from user-supplied specific offsets for each partition, and the position is specified by scan.startup.specific-offsets.
              +

              scan.startup.specific-offsets

              +

              No

              +

              None

              +

              String

              +

              Specify offsets for each partition in case of specific-offsets startup mode, e.g. partition:0,offset:42;partition:1,offset:300.

              +

              scan.startup.timestamp-millis

              +

              No

              +

              None

              +

              Long

              +

              Start from the specified epoch timestamp (milliseconds) used in case of timestamp startup mode.

              +

              scan.topic-partition-discovery.interval

              +

              No

              +

              None

              +

              Duration

              +

              Interval for consumer to discover dynamically created Kafka topics and partitions periodically.

              +
              +
              +
              +

              Result Table Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 3 Result table parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Specify what connector to use, for Kafka use kafka.

              +

              topic

              +

              No

              +

              None

              +

              String

              +

              Topic name(s) to read data from when the table is used as source. It also supports topic list for source by separating topic by semicolon like topic-1;topic-2.

              +

              Note, only one of topic-pattern and topic can be specified for sources.

              +

              When the table is used as sink, the topic name is the topic to write data to. Note topic list is not supported for sinks.

              +

              properties.bootstrap.servers

              +

              Yes

              +

              None

              +

              String

              +

              Comma separated list of Kafka brokers.

              +

              properties.*

              +

              No

              +

              None

              +

              String

              +

              This can set and pass arbitrary Kafka configurations.

              +
              • Suffix names must match the configuration key defined in Apache Kafka.

                Flink will remove the properties. key prefix and pass the transformed key and values to the underlying KafkaClient. For example, you can disable automatic topic creation via 'properties.allow.auto.create.topics' = 'false'.

                +
              • But there are some configurations that do not support to set, because Flink will override them, e.g. key.deserializer and value.deserializer.
              +

              format

              +

              Yes

              +

              None

              +

              String

              +

              The format used to deserialize and serialize the value part of Kafka messages. Note, either this parameter or the value.format parameter is required.

              +
              • For details about the message key and body of Kafka messages, see Key and Value Formats.
              • Refer to Format for more details and format parameters.
              +

              key.format

              +

              No

              +

              None

              +

              String

              +

              The format used to deserialize and serialize the key part of Kafka messages.

              +
              • If a key format is defined, the key.fields parameter is required as well. Otherwise the Kafka records will have an empty key.
              • Refer to Format for more details and format parameters.
              +

              key.fields

              +

              No

              +

              []

              +

              List<String>

              +

              Defines an explicit list of physical columns from the table schema that configure the data type for the key format.

              +

              By default, this list is empty and thus a key is undefined. The list should look like field1;field2.

              +

              key.fields-prefix

              +

              No

              +

              None

              +

              String

              +

              Defines a custom prefix for all fields of the key format to avoid name clashes with fields of the value format. By default, the prefix is empty.

              +

              If a custom prefix is defined, both the table schema and key.fields will work with prefixed names.

              +

              When constructing the data type of the key format, the prefix will be removed and the non-prefixed names will be used within the key format. Note that this parameter requires that value.fields-include must be set to EXCEPT_KEY.

              +

              value.format

              +

              No

              +

              None

              +

              String

              +

              The format used to deserialize and serialize the value part of Kafka messages.

              +
              • Either this parameter or the format parameter is required. If two parameters are configured, a conflict occurs.
              • Refer to Format for more details and format parameters.
              +

              value.fields-include

              +

              No

              +

              ALL

              +

              Enum

              +

              Possible values: [ALL, EXCEPT_KEY]

              +

              Defines a strategy how to deal with key columns in the data type of the value format.

              +

              By default, ALL physical columns of the table schema will be included in the value format which means that key columns appear in the data type for both the key and value format.

              +

              sink.partitioner

              +

              No

              +

              'default'

              +

              String

              +

              Output partitioning from Flink's partitions into Kafka's partitions. Valid values are:

              +
              • default: use the kafka default partitioner to partition records.
              • fixed: each Flink partition ends up in at most one Kafka partition.
              • round-robin: a Flink partition is distributed to Kafka partitions sticky round-robin. It only works when record's keys are not specified.
              • Custom FlinkKafkaPartitioner subclass: e.g. org.mycompany.MyPartitioner.
              +

              sink.semantic

              +

              No

              +

              at-least-once

              +

              String

              +

              Defines the delivery semantic for the Kafka sink. Valid enumerationns are at-least-once, exactly-once, and none.

              +

              sink.parallelism

              +

              No

              +

              None

              +

              Integer

              +

              Defines the parallelism of the Kafka sink operator. By default, the parallelism is determined by the framework: using the same parallelism as the upstream chained operator.

              +
              +
              +
              +

              Metadata

              You can define metadata in the source table to obtain the metadata of Kafka messages.

              +

              For example, if multiple topics are defined in the WITH parameter and metadata is defined in the Kafka source table, the data read by Flink is labeled with the topic from which the data is read.

              +
              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 4 Metadata

              Key

              +

              Data Type

              +

              R/W

              +

              Description

              +

              topic

              +

              STRING NOT NULL

              +

              R

              +

              Topic name of the Kafka record.

              +

              partition

              +

              INT NOT NULL

              +

              R

              +

              Partition ID of the Kafka record.

              +

              headers

              +

              MAP<STRING, BYTES> NOT NULL

              +

              R/W

              +

              Headers of the Kafka record as a map of raw bytes.

              +

              leader-epoch

              +

              INT NULL

              +

              R

              +

              Leader epoch of the Kafka record if available.

              +

              offset

              +

              BIGINT NOT NULL

              +

              R

              +

              Offset of the Kafka record in the partition.

              +

              timestamp

              +

              TIMESTAMP(3) WITH LOCAL TIME ZONE NOT NULL

              +

              R/W

              +

              Timestamp of the Kafka record.

              +

              timestamp-type

              +

              STRING NOT NULL

              +

              R

              +

              Timestamp type of the Kafka record.

              +
              • NoTimestampType: No timestamp is defined in the message.
              • CreateTime: time when the message is generated.
              • LogAppendTime: time when the message is added to the Kafka broker.
              +
              +
              +

              Key and Value Formats

              Both the key and value part of a Kafka record can be serialized to and deserialized from raw bytes using one of the given formats.

              +
              • Value Format

                Since a key is optional in Kafka records, the following statement reads and writes records with a configured value format but without a key format. The format parameter is a synonym for value.format. All format options are prefixed with the format identifier.

                +
                CREATE TABLE KafkaTable (
+  `ts` TIMESTAMP(3) METADATA FROM 'timestamp',
+  `user_id` BIGINT,
+  `item_id` BIGINT,
+  `behavior` STRING
+) WITH (
+  'connector' = 'kafka',
+  ...
+
+  'format' = 'json',
+  'json.ignore-parse-errors' = 'true'
+)
                +

                The value format will be configured with the following data type:

                +
                ROW<`user_id` BIGINT, `item_id` BIGINT, `behavior` STRING>
                +
              • Key and Value Format

                The following example shows how to specify and configure key and value formats. The format options are prefixed with either the key or value plus format identifier.

                +
                CREATE TABLE KafkaTable (
+  `ts` TIMESTAMP(3) METADATA FROM 'timestamp',
+  `user_id` BIGINT,
+  `item_id` BIGINT,
+  `behavior` STRING
+) WITH (
+  'connector' = 'kafka',
+  ...
+
+  'key.format' = 'json',
+  'key.json.ignore-parse-errors' = 'true',
+  'key.fields' = 'user_id;item_id',
+
+  'value.format' = 'json',
+  'value.json.fail-on-missing-field' = 'false',
+  'value.fields-include' = 'ALL'
+)
                +

                The key format includes the fields listed in key.fields (using ; as the delimiter) in the same order. Thus, it will be configured with the following data type:

                +
                ROW<`user_id` BIGINT, `item_id` BIGINT>
                +

                Since the value format is configured with 'value.fields-include' = 'ALL', key fields will also end up in the value format's data type:

                +
                ROW<`user_id` BIGINT, `item_id` BIGINT, `behavior` STRING>
                +
              • Overlapping Format Fields

                The connector cannot split the table's columns into key and value fields based on schema information if both key and value formats contain fields of the same name. The key.fields-prefix parameter allows to give key columns a unique name in the table schema while keeping the original names when configuring the key format.

                +

                The following example shows a key and value format that both contain a version field:

                +
                CREATE TABLE KafkaTable (
+  `k_version` INT,
+  `k_user_id` BIGINT,
+  `k_item_id` BIGINT,
+  `version` INT,
+  `behavior` STRING
+) WITH (
+  'connector' = 'kafka',
+  ...
+
+  'key.format' = 'json',
+  'key.fields-prefix' = 'k_',
+  'key.fields' = 'k_version;k_user_id;k_item_id',
+
+  'value.format' = 'json',
+  'value.fields-include' = 'EXCEPT_KEY'
+)
                +

                The value format must be configured in EXCEPT_KEY mode. The formats will be configured with the following data types:

                +
                Key format:
+ROW<`version` INT, `user_id` BIGINT, `item_id` BIGINT>
+
+Value format:
+ROW<`version` INT, `behavior` STRING>
                +
              +
              +

              Topic and Partition Discovery

              The config parameters topic and topic-pattern specify the topics or topic pattern to consume for source. The config parameter topic can accept topic list using semicolon separator like topic-1;topic-2. The config parameter topic-pattern will use regular expression to discover the matched topic. For example, if the topic-pattern is test-topic-[0-9], then all topics with names that match the specified regular expression (starting with test-topic- and ending with a single digit)) will be subscribed by the consumer when the job starts running.

              +

              To allow the consumer to discover dynamically created topics after the job started running, set a non-negative value for scan.topic-partition-discovery.interval. This allows the consumer to discover partitions of new topics with names that also match the specified pattern.

              +

              Note that topic list and topic pattern only work in sources. In sinks, Flink currently only supports a single topic.

              +
              +
              +

              Example 1: Reading DMS Kafka Metadata in CSV Format and Outputting It to a Kafka Sink (Applicable for Kafka Clusters Without SASL_SSL Enabled)

              1. Create an enhanced datasource connection in the VPC and subnet where Kafka locates, and bind the connection to the required Flink elastic resource pool.
              2. Set Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Kafka address. If the connection passes the test, it is bound to the queue.
              3. Create a Flink OpenSource SQL job. Enter the following job script and submit the job.
                When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.
                CREATE TABLE kafkaSource(
+  `topic` String metadata virtual,
+  `partition` int metadata virtual,
+  `headers` MAP<STRING, BYTES> metadata virtual,
+  `leader-epoch` INT metadata virtual,
+  `offset` bigint metadata virtual,
+  `timestamp-type` string metadata virtual,
+  `event_time` TIMESTAMP(3) metadata FROM 'timestamp',
+  `message` string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'SourceKafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'csv',
+  'csv.field-delimiter' = '\u0001',  
+  'csv.quote-character' = ''''
+);
+
+CREATE TABLE kafkaSink (
+  `topic` String,
+  `partition` int,
+  `headers` MAP<STRING, BYTES>,
+  `leader-epoch` INT,
+  `offset` bigint,
+  `timestampType` string,
+  `event_time` TIMESTAMP(3),
+  `message` string -- Indicates that data written by users is read from Kafka.
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'SinkKafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'format' = 'json'
+);
+insert into kafkaSink select * from kafkaSource;
                +
                +
              4. Send the following data to the topic of the source table in Kafka. The Kafka topic is kafkaSource.

                +
                {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103251202020001", "order_channel":"miniAppShop", "order_time":"2021-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2021-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
                +
              5. Read the topic of the Kafka result table. The Kafka topic is kafkaSink.
                {"topic":"kafkaSource","partition":1,"headers":{},"leader-epoch":0,"offset":4,"timestampType":"LogAppendTime","event_time":"2023-11-16 11:16:30.369","message":"{\"order_id\":\"202103251202020001\", \"order_channel\":\"miniAppShop\", \"order_time\":\"2021-03-25 12:02:02\", \"pay_amount\":\"60.00\", \"real_pay\":\"60.00\", \"pay_time\":\"2021-03-25 12:03:00\", \"user_id\":\"0002\", \"user_name\":\"Bob\", \"area_id\":\"330110\"}"}
+
+{"topic":"kafkaSource","partition":0,"headers":{},"leader-epoch":0,"offset":6,"timestampType":"LogAppendTime","event_time":"2023-11-16 11:16:30.367","message":"{\"order_id\":\"202103241000000001\",\"order_channel\":\"webShop\",\"order_time\":\"2021-03-24 10:00:00\",\"pay_amount\":100.0,\"real_pay\":100.0,\"pay_time\":\"2021-03-24 10:02:03\",\"user_id\":\"0001\",\"user_name\":\"Alice\",\"area_id\":\"330106\"}"}
+
+{"topic":"kafkaSource","partition":2,"headers":{},"leader-epoch":0,"offset":5,"timestampType":"LogAppendTime","event_time":"2023-11-16 11:16:30.368","message":"{\"order_id\":\"202103241606060001\",\"order_channel\":\"appShop\",\"order_time\":\"2021-03-24 16:06:06\",\"pay_amount\":200.0,\"real_pay\":180.0,\"pay_time\":\"2021-03-24 16:10:06\",\"user_id\":\"0001\",\"user_name\":\"Alice\",\"area_id\":\"330106\"}"}
                +
              +
              +

              Example 2: Using DMS Kafka in JSON Format as the Source Table and Outputting It to a Kafka Sink (Applicable for Kafka Clusters Without SASL_SSL Enabled)

              Use the Kafka source table and Kafka result table to read JSON data from Kafka and output it to the log file.
              1. Create an enhanced datasource connection in the VPC and subnet where Kafka locates, and bind the connection to the required Flink elastic resource pool.
              2. Set Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Kafka address. If the connection passes the test, it is bound to the queue.
              3. Create a Flink OpenSource SQL job. Enter the following job script and submit the job.
                When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.
                CREATE TABLE kafkaSource(
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,  
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaSourceTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE kafkaSink (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,  
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaSinkTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'format' = 'json'
+);
+insert into kafkaSink select * from kafkaSource;
                +
                +
              4. Send the following data to the topic of the source table in Kafka:
                {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"} 
+
+{"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103251202020001", "order_channel":"miniAppShop", "order_time":"2021-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2021-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
                +
              5. Read the topic of the Kafka result table. The data results are as follows:
                {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"} 
+
+{"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103251202020001", "order_channel":"miniAppShop", "order_time":"2021-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2021-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
                +
              +
              +
              +

              Example 3: Using DMS Kafka as the Source Table and Print as the Result Table (Applicable for Kafka Clusters with SASL_SSL Enabled)

              Create a Kafka cluster for DMS, enable SASL_SSL, download the SSL certificate, and upload the downloaded certificate client.jks to an OBS bucket.

              +

              The properties.sasl.jaas.config field contains account passwords encrypted using DEW.

              +
              CREATE TABLE ordersSource (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:9093,KafkaAddress2:9093',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'properties.connector.auth.open' = 'true',
+  'properties.ssl.truststore.location' = 'obs://xx/client.jks',  -- Location where the user uploads the certificate to
+  'properties.sasl.mechanism' = 'PLAIN', 
+  'properties.security.protocol' = 'SASL_SSL',
+  'properties.sasl.jaas.config' = 'xx',  -- Key in DEW secret management, whose value is like org.apache.kafka.common.security.plain.PlainLoginModule required username=xx password=xx;
+  'format' = 'json',
+  'dew.endpoint' = 'kms.xx.com', --Endpoint information for the DEW service being used
+  'dew.csms.secretName' = 'xx', --Name of the DEW shared secret
+  'dew.csms.decrypt.fields' = 'properties.sasl.jaas.config', --The properties.sasl.jaas.config field value must be decrypted and replaced using DEW secret management.
+  'dew.csms.version' = 'v1'
+);
+
+CREATE TABLE ordersSink (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'print'
+);
+ insert into ordersSink select * from ordersSource;
              + +
              +

              Example 4: Using Kafka (MRS Cluster) as the Source Table and Print as the Result Table (Applicable for Kafka with SASL_SSL Enabled and MRS Using Kerberos Authentication)

              • Enable Kerberos authentication for the MRS cluster.
              • Click the Components tab and click Kafka. On the displayed page, click the Service Configuration tab, locate the ssl.mode.enable, set it to true, and restart Kafka.
              • Download the user credential. Log in to the MRS Manager of the MRS cluster and choose System > Permission > User. Locate the row that contains the target user, click More, and select Download Authentication Credential.

                Obtain the truststore.jks file using the authentication credential and store the credential and truststore.jks file in OBS.

                +
              • If "Message stream modified (41)" is displayed, the JDK version may be incorrect. Change the JDK version in the sample code to a version earlier than 8u_242 or delete the renew_lifetime = 0m configuration item from the krb5.conf configuration file.
              • Set the port to the sasl_ssl.port configured in the Kafka service configuration. The default value is 21009.
              • In the following statements, set security.protocol to SASL_SSL.
              • The properties.ssl.truststore.password field in the with parameter is encrypted using DEW.
              +
              CREATE TABLE ordersSource (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'properties.sasl.kerberos.service.name' = 'kafka', -- Value configured in the MRS cluster
+  'properties.connector.auth.open' = 'true',
+  'properties.connector.kerberos.principal' = 'xx', --Username
+  'properties.connector.kerberos.krb5' = 'obs://xx/krb5.conf',
+  'properties.connector.kerberos.keytab' = 'obs://xx/user.keytab',
+  'properties.security.protocol' = 'SASL_SSL',
+  'properties.ssl.truststore.location' = 'obs://xx/truststore.jks',
+  'properties.ssl.truststore.password' = 'xx',  -- Key in the DEW secret
+  'properties.sasl.mechanism' = 'GSSAPI',
+  'format' = 'json',
+  'dew.endpoint'='kms.xx.xx.com', --Endpoint information for the DEW service being used
+  'dew.csms.secretName'='xx', --Name of the DEW shared secret
+  'dew.csms.decrypt.fields'='properties.ssl.truststore.password', --The properties.ssl.truststore.password field value must be decrypted and replaced using DEW secret management.
+  'dew.csms.version'='v1'
+);
+
+CREATE TABLE ordersSink (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'print'
+);
+ insert into ordersSink select * from ordersSource;
              +
              +

              Example 5: Using Kafka (MRS Cluster) as the Source Table and Print as the Result Table (Applicable for Kafka with SASL_SSL Enabled and MRS Using SASL_PLAINTEXT with Kerberos Authentication)

              • Enable Kerberos authentication for the MRS cluster.
              • Click the Components tab and click Kafka. On the displayed page, click the Service Configuration tab, locate the ssl.mode.enable, set it to true, and restart Kafka.
              • Log in to the MRS Manager of the MRS cluster and download the user credential. Choose System > Permission > User. Locate the row that contains the target user, choose More > Download Authentication Credential. Upload the credential to OBS.
              • If error message "Message stream modified (41)" is displayed, the JDK version may be incorrect. Change the JDK version in the sample code to a version earlier than 8u_242 or delete the renew_lifetime = 0m configuration item from the krb5.conf configuration file.
              • Set the port to the sasl.port configured in the Kafka service configuration. The default value is 21007.
              • In the following statements, set security.protocol to SASL_PLAINTEXT.
              +
              CREATE TABLE ordersSource (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'properties.sasl.kerberos.service.name' = 'kafka', -- Configured in the MRS cluster
+  'properties.connector.auth.open' = 'true',
+  'properties.connector.kerberos.principal' = 'xx',
+  'properties.connector.kerberos.krb5' = 'obs://xx/krb5.conf',
+  'properties.connector.kerberos.keytab' = 'obs://xx/user.keytab',
+  'properties.security.protocol' = 'SASL_PLAINTEXT',
+  'properties.sasl.mechanism' = 'GSSAPI',
+  'format' = 'json'
+);
+
+CREATE TABLE ordersSink (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'print'
+);
+ insert into ordersSink select * from ordersSource;
              +
              +

              Example 6: Using Kafka (MRS Cluster) as the Source Table and Print as the Result Table (Applicable for Kafka with SSL Enabled and MRS Without Kerberos Authentication Enabled)

              • Do not enable Kerberos authentication for the MRS cluster.
              • Download the user credential. Log in to the MRS Manager of the MRS cluster and choose System > Permission > User. Locate the row that contains the target user, click More, and select Download Authentication Credential.

                Obtain the truststore.jks file using the authentication credential and store the credential and truststore.jks file in OBS.

                +
              • Set the port to the ssl.port configured in the Kafka service configuration. The default value is 9093.
              • Set security.protocol in the with parameter to SSL.
              • In the Kafka configuration of the MRS cluster, set ssl.mode.enable to true and restart Kafka.
              • The properties.ssl.truststore.password field in the with parameter is encrypted using DEW.
                CREATE TABLE ordersSource (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'properties.connector.auth.open' = 'true',
+  'properties.ssl.truststore.location' = 'obs://xx/truststore.jks',
+  'properties.ssl.truststore.password' = 'xx',  -- Key for DEW secret management, whose value is the password set when generating truststore.jks
+  'properties.security.protocol' = 'SSL',
+  'format' = 'json',
+  'dew.endpoint' = 'kms.xx.com', --Endpoint information for the DEW service being used
+  'dew.csms.secretName' = 'xx', --Name of the DEW shared secret
+  'dew.csms.decrypt.fields' = 'properties.ssl.truststore.password', --The properties.ssl.truststore.password field value must be decrypted and replaced using DEW secret management.
+  'dew.csms.version' = 'v1'
+);
+
+CREATE TABLE ordersSink (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'print'
+);
+ insert into ordersSink select * from ordersSource;
                +
              +
              +

              FAQ

              • Q: What should I do if the Flink job execution fails and the log contains the following error information?
                org.apache.kafka.common.errors.TimeoutException: Timeout expired while fetching topic metadata
                +

                A: The datasource connection is not bound, the binding fails, or the security group of the Kafka cluster is not configured to allow access from the network segment of the DLI queue. Reconfigure the datasource connection or configure the security group of the Kafka cluster to allow access from the DLI queue.

                +
              • Q: What should I do if the Flink job execution fails and the log contains the following error information?
                Caused by: java.lang.RuntimeException: RealLine:45;Table 'default_catalog.default_database.printSink' declares persistable metadata columns, but the underlying DynamicTableSink doesn't implement the SupportsWritingMetadata interface. If the column should not be persisted, it can be declared with the VIRTUAL keyword.
                +

                A: The metadata type is defined in the sink table, but the Print connector does not support deletion of matadata from the sink table.

                +
              +
              +
              +
              +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15059.html b/docs/dli/sqlreference/dli_08_15059.html new file mode 100644 index 000000000..b2f00e536 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15059.html @@ -0,0 +1,729 @@ + + +

              MySQL CDC

              +

              Function

              The MySQL CDC source table, that is, the MySQL streaming source table, reads all historical data in the database first and then smoothly switches data read to the Binlog to ensure data integrity.

              + +
              + + + + + + + +
              Table 1 Supported types

              Type

              +

              Description

              +

              Supported Table Types

              +

              Source table

              +
              +
              +
              +

              Prerequisites

              • MySQL CDC requires MySQL 5.6, 5.7, or 8.0.x.
              • Fields in the with parameter can only be enclosed in single quotes.
              • An enhanced datasource connection has been created for DLI to connect to the MySQL database, so that you can configure security group rules as required. +
              • Binlog is enabled for MySQL, and binlog_row_image is set to FULL.
              • A MySQL user has been created and granted the SELECT, SHOW DATABASES, REPLICATION SLAVE, and REPLICATION CLIENT permissions. Note: When the scan.incremental.snapshot.enabled parameter is enabled (enabled by default), there is no need to grant the reload permission.
                GRANT SELECT, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'user' IDENTIFIED BY 'password';
                +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • Set a different SERVER ID for each reader.
                • Each MySQL client used for reading Binlog should have a unique server ID to ensure that the MySQL server can distinguish between different clients and maintain their respective Binlog reading positions.
                • Sharing the same server ID among different jobs may lead to reading data from incorrect Binlog positions, resulting in data inconsistency.
                • You can assign a unique server ID to each source reader through SQL hints, such as using SELECT * FROM source_table /*+ OPTIONS('server-id'='5401-5404') */ ; to allocate unique server IDs for four source readers.
                +
              +
              • Set up MySQL session timeouts.
                When an initial consistent snapshot is made for large databases, your established connection could time out while the tables are being read. You can prevent this behavior by configuring interactive_timeout and wait_timeout in your MySQL configuration file.
                • interactive_timeout: The number of seconds the server waits for activity on an interactive connection before closing it. See MySQL documentations.
                • wait_timeout: The number of seconds the server waits for activity on a noninteractive connection before closing it. See MySQL documentations.
                +
                +
              • Precautions when using tables without primary keys:
                • To use a table without primary keys, you must configure the scan.incremental.snapshot.chunk.key-column parameter and specify one non-null field.
                • If there is an index in the table, use a column which is contained in the index in scan.incremental.snapshot.chunk.key-column. This will increase the speed of select statement.
                  The processing semantics of a MySQL CDC table without primary keys is determined based on the behavior of the column that is specified by the scan.incremental.snapshot.chunk.key-column.
                  • If no update operation is performed on the specified column, the exactly-once semantics is ensured.
                  • If the update operation is performed on the specified column, only the at-least-once semantics is ensured. However, you can specify primary keys at downstream and perform the idempotence operation to ensure data correctness.
                  +
                  +
                +
              • Watermarks cannot be defined for MySQL CDC source tables. For details about window aggregation, see FAQ.
              • If you connect to a sink source that supports upsert, such as GaussDB(DWS) and MySQL, you need to define the primary key in the statement for creating the sink table. For details, see the printSink table creation statement in Example.
              +
              +

              Features

              • Incremental snapshot reading

                Incremental snapshot reading is a new mechanism to read snapshot of a table. Compared to the old snapshot mechanism, the incremental snapshot has many advantages, including:

                +
                • MySQL CDC Source can be parallel during snapshot reading
                • MySQL CDC Source can perform checkpoints in the chunk granularity during snapshot reading
                • MySQL CDC Source does not need to acquire global read lock (FLUSH TABLES WITH READ LOCK) before snapshot reading
                +

                If you would like the source run in parallel, each parallel reader should have a unique server ID, so the server-id must be a range like 5400-6400, and the range must be larger than the parallelism. During the incremental snapshot reading, the MySQL CDC Source firstly splits snapshot chunks (splits) by primary key of table, and then MySQL CDC Source assigns the chunks to multiple readers to read the data of snapshot chunk.

                +
              • Lock-free

                The MySQL CDC source uses incremental snapshot algorithm, which avoids acquiring global read lock (FLUSH TABLES WITH READ LOCK) and thus does not need RELOAD permission.

                +
              • Controlling parallelism

                Incremental snapshot reading provides the ability to read snapshot data parallelly.

                +
              • Checkpoint

                Incremental snapshot reading provides the ability to perform checkpoint in chunk level. It resolves the checkpoint timeout problem in previous version with old snapshot reading mechanism.

                +
              +
              +

              Syntax

              create table mySqlCdcSource (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (','PRIMARY KEY (attr_name, ...) NOT ENFORCED)
+)
+with (
+  'connector' = 'mysql-cdc',   
+  'hostname' = 'mysqlHostname',
+  'username' = 'mysqlUsername',
+  'password' = 'mysqlPassword',
+  'database-name' = 'mysqlDatabaseName',
+  'table-name' = 'mysqlTableName'
+);
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Source table parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Specify what connector to use, here should be mysql-cdc.

              +

              hostname

              +

              Yes

              +

              None

              +

              String

              +

              IP address or hostname of the MySQL database server.

              +

              username

              +

              Yes

              +

              None

              +

              String

              +

              Name of the MySQL database to use when connecting to the MySQL database server.

              +

              password

              +

              Yes

              +

              None

              +

              String

              +

              Password to use when connecting to the MySQL database server.

              +

              database-name

              +

              Yes

              +

              None

              +

              String

              +

              Database name of the MySQL server to monitor.

              +

              The database-name also supports regular expressions to monitor multiple tables match the regular expression.

              +
              • Prefix matching: ^(test).* matches database names with the prefix test, for example, test1 and test2.
              • Suffix matching: .*[p$] matches database names with the suffix p, for example, cdcp and edcp.
              • Specific matching: txc matches a specific database name.
              +

              table-name

              +

              Yes

              +

              None

              +

              String

              +

              Table name of the MySQL database to monitor. The table-name also supports regular expressions to monitor multiple tables that satisfy the regular expressions.

              +
              NOTE:

              When the MySQL CDC connector regularly matches the table name, it will concat the database-name and table-name filled in by the user through the string `\\.` to form a full-path regular expression, and then use the regular expression to match the fully qualified name of the table in the MySQL database.

              +
              • Prefix matching: ^(test).* matches table names with the prefix test, for example, test1 and test2.
              • Suffix matching: .*[p$] matches table names with the suffix p, for example, cdcp and edcp.
              • Specific matching: txc matches a specific table name.
              +
              +

              port

              +

              No

              +

              3306

              +

              Integer

              +

              Integer port number of the MySQL database server.

              +

              server-id

              +

              No

              +

              None

              +

              String

              +

              A numeric ID or a numeric ID range of this database client. The numeric ID syntax is like 5400, the numeric ID range syntax is like 5400-5408.

              +

              The numeric ID range syntax is recommended when scan.incremental.snapshot.enabled enabled.

              +

              Every ID must be unique across all currently-running database processes in the MySQL cluster. This connector joins the MySQL cluster as another server (with this unique ID) so it can read the binlog. By default, a random number is generated between 5400 and 6400, though we recommend setting an explicit value.

              +

              scan.incremental.snapshot.enabled

              +

              No

              +

              true

              +

              Boolean

              +

              Incremental snapshot is a new mechanism to read snapshot of a table. Compared to the old snapshot mechanism, the incremental snapshot has many advantages, including:

              +
              • MySQL CDC Source can be parallel during snapshot reading
              • MySQL CDC Source can perform checkpoints in the chunk granularity during snapshot reading
              • MySQL CDC Source does not need to acquire global read lock (FLUSH TABLES WITH READ LOCK) before snapshot reading

                If you would like the source run in parallel, each parallel reader should have a unique server ID, so the server-id must be a range like 5400-6400, and the range must be larger than the parallelism.

                +
              +

              scan.incremental.snapshot.chunk.size

              +

              No

              +

              8096

              +

              Integer

              +

              The chunk size (number of rows) of table snapshot, captured tables are split into multiple chunks when reading the snapshot of table.

              +

              scan.startup.mode

              +

              No

              +

              initial

              +

              String

              +

              Optional startup mode for MySQL CDC consumer, valid enumerations are initial, earliest-offset, latest-offset, specific-offset, and timestamp.

              +
              • initial (default): Perform an initial snapshot on the monitored database tables upon first startup, and continue to read the latest binlog.
              • earliest-offset: Skip snapshot phase and start reading binlog events from the earliest accessible binlog offset.
              • latest-offset: Never to perform snapshot on the monitored database tables upon first startup, just read from the end of the binlog which means only have the changes since the connector was started.
              • specific-offset: Skip snapshot phase and start reading binlog events from a specific offset. The offset could be specified with binlog filename and position, or a GTID set if GTID is enabled on server.
              • timestamp: Skip snapshot phase and start reading binlog events from a specific timestamp.
              +

              scan.startup.specific-offset.file

              +

              No

              +

              None

              +

              String

              +

              Optional binlog file name used in case of specific-offset startup mode

              +

              scan.startup.specific-offset.pos

              +

              No

              +

              None

              +

              Long

              +

              Optional binlog file position used in case of specific-offset startup mode

              +

              scan.startup.specific-offset.gtid-set

              +

              No

              +

              None

              +

              String

              +

              Optional GTID set used in case of specific-offset startup mode

              +

              scan.startup.specific-offset.skip-events

              +

              No

              +

              None

              +

              Long

              +

              Optional number of events to skip after the specific starting offset

              +

              scan.startup.specific-offset.skip-rows

              +

              No

              +

              None

              +

              Long

              +

              Optional number of rows to skip after the specific starting offset

              +

              server-time-zone

              +

              No

              +

              None

              +

              String

              +

              Session time zone on the database server

              +

              It controls how the TIMESTAMP type in MYSQL converted to STRING. If not set, then ZoneId.systemDefault() is used to determine the server time zone.

              +

              debezium.min.row. count.to.stream.result

              +

              No

              +

              1000

              +

              Integer

              +

              During a snapshot operation, the connector will query each included table to produce a read event for all rows in that table.

              +

              This parameter determines whether the MySQL connection will pull all results for a table into memory (which is fast but requires large amounts of memory), or whether the results will instead be streamed (can be slower, but will work for very large tables). The value specifies the minimum number of rows a table must contain before the connector will stream results, and defaults to 1000.

              +

              Set this parameter to 0 to skip all table size checks and always stream all results during a snapshot.

              +

              connect.timeout

              +

              No

              +

              30s

              +

              Duration

              +

              The maximum time that the connector should wait after trying to connect to the MySQL database server before timing out.

              +

              connect.max-retries

              +

              No

              +

              3

              +

              Integer

              +

              The max retry times that the connector should retry to build MySQL database server connection.

              +

              connection.pool.size

              +

              No

              +

              20

              +

              Integer

              +

              The connection pool size.

              +

              jdbc.properties.*

              +

              No

              +

              None

              +

              String

              +

              Option to pass custom JDBC URL properties.

              +

              User can pass custom properties like 'jdbc.properties.useSSL' = 'false'.

              +

              heartbeat.interval

              +

              No

              +

              30s

              +

              Duration

              +

              The interval of sending heartbeat event for tracing the latest available binlog offsets.

              +

              debezium.*

              +

              No

              +

              None

              +

              String

              +

              Pass-through Debezium's properties to Debezium Embedded Engine which is used to capture data changes from MySQL server.

              +

              For example: 'debezium.snapshot.mode' = 'never'. See more about the Debezium's MySQL Connector properties.

              +

              scan.incremental.close-idle-reader.enabled

              +

              No

              +

              false

              +

              Boolean

              +

              Whether to close idle readers at the end of the snapshot phase. This feature requires that execution.checkpointing.checkpoints-after-tasks-finish.enabled be set to true.

              +
              +
              +
              +

              Metadata

              The following format metadata can be exposed as read-only (VIRTUAL) columns in DDL.

              + +
              + + + + + + + + + + + + + + + + + +
              Table 3 Metadata

              Key

              +

              Data Type

              +

              Description

              +

              table_name

              +

              STRING NOT NULL

              +

              Name of the table that contains the row.

              +

              database_name

              +

              STRING NOT NULL

              +

              Name of the database that contains the row.

              +

              op_ts

              +

              TIMESTAMP_LTZ(3) NOT NULL

              +

              It indicates the time that the change was made in the database.

              +

              If the record is read from snapshot of the table instead of the binlog, the value is always 0.

              +
              +
              +
              +

              Data Type Mapping

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 4 Data type mapping

              MySQL Type

              +

              Flink SQL Type

              +

              Remarks

              +

              TINYINT

              +

              TINYINT

              +

              -

              +

              SMALLINT

              +

              TINYINT UNSIGNED

              +

              TINYINT UNSIGNED ZEROFILL

              +

              SMALLINT

              +

              -

              +

              INT

              +

              MEDIUMINT

              +

              SMALLINT UNSIGNED

              +

              SMALLINT UNSIGNED ZEROFILL

              +

              INT

              +

              -

              +

              BIGINT

              +

              INT UNSIGNED

              +

              INT UNSIGNED ZEROFILL

              +

              MEDIUMINT UNSIGNED

              +

              MEDIUMINT UNSIGNED ZEROFILL

              +

              BIGINT

              +

              -

              +

              BIGINT UNSIGNED

              +

              BIGINT UNSIGNED ZEROFILL

              +

              SERIAL

              +

              DECIMAL(20, 0)

              +

              -

              +

              FLOAT

              +

              FLOAT UNSIGNED

              +

              FLOAT UNSIGNED ZEROFILL

              +

              FLOAT

              +

              -

              +

              REAL

              +

              REAL UNSIGNED

              +

              REAL UNSIGNED ZEROFILL

              +

              DOUBLE

              +

              DOUBLE UNSIGNED

              +

              DOUBLE UNSIGNED ZEROFILL

              +

              DOUBLE PRECISION

              +

              DOUBLE PRECISION UNSIGNED

              +

              DOUBLE PRECISION UNSIGNED ZEROFILL

              +

              DOUBLE

              +

              -

              +

              NUMERIC(p, s)

              +

              NUMERIC(p, s) UNSIGNED

              +

              NUMERIC(p, s) UNSIGNED ZEROFILL

              +

              DECIMAL(p, s)

              +

              DECIMAL(p, s) UNSIGNE

              +

              DDECIMAL(p, s) UNSIGNED ZEROFILL

              +

              FIXED(p, s)

              +

              FIXED(p, s) UNSIGNED

              +

              FIXED(p, s) UNSIGNED ZEROFILL

              +

              where p <= 38

              +

              DECIMAL(p, s)

              +

              -

              +

              NUMERIC(p, s)

              +

              NUMERIC(p, s) UNSIGNED

              +

              NUMERIC(p, s) UNSIGNED ZEROFILL

              +

              DECIMAL(p, s)

              +

              DECIMAL(p, s) UNSIGNED

              +

              DECIMAL(p, s) UNSIGNED ZEROFILL

              +

              FIXED(p, s)

              +

              FIXED(p, s) UNSIGNED

              +

              FIXED(p, s) UNSIGNED ZEROFILL

              +

              where 38 < p <= 65

              +

              STRING

              +

              The precision for DECIMAL data type is up to 65 in MySQL, but the precision for DECIMAL is limited to 38 in Flink.

              +

              So if you define a decimal column whose precision is greater than 38, you should map it to STRING to avoid precision loss.

              +

              BOOLEAN

              +

              TINYINT(1)

              +

              BIT(1)

              +

              BOOLEAN

              +

              -

              +

              DATE

              +

              DATE

              +

              -

              +

              TIME [(p)]

              +

              TIME [(p)]

              +

              -

              +

              TIMESTAMP [(p)]

              +

              DATETIME [(p)]

              +

              TIMESTAMP [(p)]

              +

              -

              +

              CHAR(n)

              +

              CHAR(n)

              +

              -

              +

              VARCHAR(n)

              +

              VARCHAR(n)

              +

              -

              +

              BIT(n)

              +

              BINARY(⌈n/8⌉)

              +

              -

              +

              BINARY(n)

              +

              BINARY(n)

              +

              -

              +

              VARBINARY(N)

              +

              VARBINARY(N)

              +

              -

              +

              TINYTEXT

              +

              TEXT

              +

              MEDIUMTEXT

              +

              LONGTEXT

              +

              STRING

              +

              -

              +

              TINYBLOB

              +

              BLOB

              +

              MEDIUMBLOB

              +

              LONGBLOB

              +

              BYTES

              +

              Currently, for BLOB data type in MySQL, only the blob whose length is not greater than 2,147,483,647(2 ** 31 – 1) is supported.

              +

              YEAR

              +

              INT

              +

              -

              +

              ENUM

              +

              STRING

              +

              -

              +

              JSON

              +

              STRING

              +

              The JSON data type will be converted into STRING with JSON format in Flink.

              +

              SET

              +

              ARRAY<STRING>

              +

              As the SET data type in MySQL is a string object that can have zero or more values, it should always be mapped to an array of string.

              +

              GEOMETRY

              +

              POINT

              +

              LINESTRING

              +

              POLYGON

              +

              MULTIPOINT

              +

              MULTILINESTRING

              +

              MULTIPOLYGON

              +

              GEOMETRYCOLLECTION

              +

              STRING

              +

              The spatial data types in MySQL will be converted into STRING with a fixed Json format.

              +
              +
              +
              +

              Example

              This example demonstrates the use of MySQL-CDC to read data and metadata from an RDS for MySQL database in real-time and write it to a Print result table.

              +

              In this example, the RDS for MySQL database engine version is MySQL 5.7.33.

              +
              1. Create an enhanced datasource connection in the VPC and subnet where MySQL locates, and bind the connection to the required Flink elastic resource pool.
              2. Set MySQL security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the MySQL address. If the connection passes the test, it is bound to the queue.
              3. Create the test user in MySQL and grant them permissions. The SQL statements are as follows:
                CREATE USER 'test'@'%' IDENTIFIED BY 'xxx';
+GRANT SELECT, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'test';
+FLUSH PRIVILEGES;
                +
              4. Create a table named cdc_order in the Flink database of MySQL. The SQL statement is as follows (this statement requires the user to have the CREATE permission):
                CREATE TABLE `flink`.`cdc_order` (
+	`order_id` VARCHAR(32) NOT NULL,
+	`order_channel` VARCHAR(32) NULL,
+	`order_time` VARCHAR(32) NULL,
+	`pay_amount` DOUBLE  NULL,
+	`real_pay` DOUBLE  NULL,
+	`pay_time` VARCHAR(32) NULL,
+	`user_id` VARCHAR(32) NULL,
+	`user_name` VARCHAR(32) NULL,
+	`area_id` VARCHAR(32) NULL,
+	PRIMARY KEY (`order_id`)
+)	ENGINE = InnoDB
+	DEFAULT CHARACTER SET = utf8mb4
+	COLLATE = utf8mb4_general_ci;
                +
              5. Create a Flink OpenSource SQL job. Enter the following job script and submit the job.
                When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.
                create table mysqlCdcSource(
+  database_name STRING METADATA VIRTUAL,
+  table_name STRING METADATA VIRTUAL,
+  operation_ts TIMESTAMP_LTZ(3) METADATA FROM 'op_ts' VIRTUAL,
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id STRING,
+  primary key(order_id) not enforced
+) with (
+  'connector' = 'mysql-cdc',
+  'hostname' = 'mysqlHostname',
+  'username' = 'mysqlUsername',
+  'password' = 'mysqlPassword',
+  'database-name' = 'mysqlDatabaseName',
+  'table-name' = 'mysqlTableName'
+);
+
+create table printSink(
+  database_name string,
+  table_name string,
+  operation_ts TIMESTAMP_LTZ(3),
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id STRING,
+  primary key(order_id) not enforced
+) with (
+  'connector' = 'print'
+);
+insert into printSink select * from mysqlCdcSource;
                +
                +
              6. Run the following commands in MySQL to insert test data (this statement requires the user to have the corresponding permission):
                insert into flink.cdc_order values
+('202103241000000001','webShop','2021-03-24 10:00:00','100.00','100.00','2021-03-24 10:02:03','0001','Alice','330106'),
+('202103241606060001','appShop','2021-03-24 16:06:06','200.00','180.00','2021-03-24 16:10:06','0001','Alice','330106');
+
+delete from flink.cdc_order  where order_channel = 'webShop';
+insert into flink.cdc_order values('202103251202020001','miniAppShop','2021-03-25 12:02:02','60.00','60.00','2021-03-25 12:03:00','0002','Bob','330110');
                +
              7. Perform the following operations to view the data result in the taskmanager.out file:
                1. Log in to the DLI management console. In the navigation pane on the left, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the taskmanager.out file, and view result logs.
                +

                The data result is as follows:

                +
                 +I[flink, cdc_order, 2023-11-10T07:41:12Z, 202103241000000001, webShop, 2021-03-24 10:00:00, 100.0, 100.0, 2021-03-24 10:02:03, 0001, Alice, 330106]
++I[flink, cdc_order, 2023-11-10T07:41:12Z, 202103241606060001, appShop, 2021-03-24 16:06:06, 200.0, 180.0, 2021-03-24 16:10:06, 0001, Alice, 330106]
+-D[flink, cdc_order, 2023-11-10T07:41:59Z, 202103241000000001, webShop, 2021-03-24 10:00:00, 100.0, 100.0, 2021-03-24 10:02:03, 0001, Alice, 330106]
++I[flink, cdc_order, 2023-11-10T07:42:00Z, 202103251202020001, miniAppShop, 2021-03-25 12:02:02, 60.0, 60.0, 2021-03-25 12:03:00, 0002, Bob, 330110]
                +
              +
              +

              FAQ

              Q: How do I perform window aggregation if the MySQL CDC source table does not support definition of watermarks?

              +

              A: You can use the non-window aggregation method. That is, convert the time field into a window value, and then use GROUP BY to perform aggregation based on the window value.

              +

              For example, you can use the following script to collect statistics on the number of orders per minute (order_time indicates the order time, in the string format):

              +
              insert into printSink select DATE_FORMAT(order_time, 'yyyy-MM-dd HH:mm'), count(*) from mysqlCdcSource group by DATE_FORMAT(order_time, 'yyyy-MM-dd HH:mm');
              +
              +
              +
              +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15060.html b/docs/dli/sqlreference/dli_08_15060.html new file mode 100644 index 000000000..71ea998a5 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15060.html @@ -0,0 +1,167 @@ + + +

              Print

              +

              Function

              The Print connector is used to print output data to the error file or out file in the TaskManager, making it easier for you to view the result in code debugging.

              +
              +

              Prerequisites

              None

              +
              +

              Caveats

              • The Print result table supports the following output formats: +
                + + + + + + + + + + + + + + + + + + + + + +

                Print

                +

                Condition 1

                +

                Condition 2

                +

                Identifier:Task ID> Output data

                +

                A print identifier prefix must be provided. That is, you must specify print-identifier in the WITH parameter when creating the Print result table.

                +

                parallelism > 1

                +

                Identifier> Output data

                +

                A print identifier prefix must be provided. That is, you must specify print-identifier in the WITH parameter when creating the Print result table.

                +

                parallelism == 1

                +

                Task ID> Output data

                +

                A print identifier prefix is not needed. That is, you do not specify print-identifier in the WITH parameter when creating the Print result table.

                +

                parallelism > 1

                +

                Output data

                +

                A print identifier prefix is not needed. That is, you do not specify print-identifier in the WITH parameter when creating the Print result table.

                +

                parallelism == 1

                +
                +
                +
              +
              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              +
              +

              Syntax

              1
+2
+3
+4
+5
+6
+7
+8
+9
              create table printSink (
+  attr_name attr_type 
+  (',' attr_name attr_type) * 
+  (',' PRIMARY KEY (attr_name,...) NOT ENFORCED)
+) with (
+  'connector' = 'print',
+  'print-identifier' = '',
+  'standard-error' = ''
+);
+
              +
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector to be used. Set this parameter to print.

              +

              print-identifier

              +

              No

              +

              None

              +

              String

              +

              Message that identifies print and is prefixed to the output of the value.

              +

              standard-error

              +

              No

              +

              false

              +

              Boolean

              +

              The value can be only true or false. The default value is false.

              +
              • If the value is true, data is output to the error file of the TaskManager.
              • If the value is false, data is output to the out file of the TaskManager.
              +

              sink.parallelism

              +

              No

              +

              None

              +

              Integer

              +

              Defines the parallelism of the Print result table. By default, the parallelism is determined by the framework: using the same parallelism as the upstream chained operator.

              +
              +
              +
              +

              Example

              Create a Flink OpenSource SQL job. Run the following script to generate random data through the DataGen table and output the data to the Print result table.

              +
              create table dataGenSource(
+  user_id string,
+  amount int
+) with (
+  'connector' = 'datagen',
+  'rows-per-second' = '1', --Generates a piece of data per second.
+  'fields.user_id.kind' = 'random', --Specifies a random generator for the user_id field.
+  'fields.user_id.length' = '3' --Limits the length of user_id to 3.
+);
+
+create table printSink(
+  user_id string,
+  amount int
+) with (
+  'connector' = 'print',
+  'print-identifier' = '', --Configure the data prefix.
+  'standard-error' = 'false', --Output data to the out file of TaskManager.
+  'sink.parallelism' = '2' --Configure the parallelism.
+);
+
+insert into printSink select * from dataGenSource;
              +

              After the job is submitted, the job status changes to Running. You can perform the following operations of either method to view the output result:

              +
              • Method 1:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Locate the row that contains the target Flink job, and choose More > FlinkUI in the Operation column.
                3. On the Flink UI, choose Task Managers, click the task name, and select Stdout to view job logs.
                +
              • Method 2: If you select Save Job Log on the Running Parameters tab before submitting the job, perform the following operations:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the taskmanager.out file, and view result logs.
                +
              • Method 3: If the queue is a new version, perform the following operations:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. In the job list, click the name of your desired Flink job. On the displayed page, click the Logs tab.
                3. Select the corresponding TaskManager name from the drop-down list in the upper left corner and click the taskmanager.out file to view its result log.
                +
              +
              +
              +
              +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15061.html b/docs/dli/sqlreference/dli_08_15061.html new file mode 100644 index 000000000..79fd21837 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15061.html @@ -0,0 +1,19 @@ + + +

              Redis

              +
              +
              + + +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15062.html b/docs/dli/sqlreference/dli_08_15062.html new file mode 100644 index 000000000..0818c39da --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15062.html @@ -0,0 +1,451 @@ + + +

              Source Table

              +

              Function

              Create a source stream to obtain data from Redis as input for jobs.

              +
              +

              Prerequisites

              An enhanced datasource connection has been created for DLI to connect to the Redis database, so that you can configure security group rules as required.

              + +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • To obtain the key values, you can set the primary key in Flink. The primary key maps to the Redis key.
              • The primary key cannot be a composite primary key, and only can be one field.
              • Constraints on schema-syntax:
                • If schema-syntax is map or array, there can be only one non-primary key and it must be of the same map or array type.
                • If schema-syntax is fields-scores, the number of non-primary keys must be an even number, and the second key of every two keys except the primary key must be of the double type. The double value is the score of the previous key. The following is an example:
                  CREATE TABLE redisSource (
+  redisKey string,
+  order_id string,
+  score1 double,
+  order_channel string,
+  score2 double,
+  order_time string,
+  score3 double,
+  pay_amount double,
+  score4 double,
+  real_pay double,
+  score5 double,
+  pay_time string,
+  score6 double,
+  user_id string,
+  score7 double,
+  user_name string,
+  score8 double,
+  area_id string,
+  score9 double,
+  primary key (redisKey) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'sorted-set',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'fields-scores'
+);
                  +
                +
              • Restrictions on data-type:
                • When data-type is set, the types of non-primary keys defined in Flink must be the same.
                • If data-type is sorted-set and schema-syntax is fields or array, only sorted-set values can be read from Redis, and the score value cannot be read.
                • If data-type is string, only one non-primary key field is allowed.
                • If data-type is sorted-set and schema-syntax is map, only one non-primary key field is allowed besides the primary key field.

                  This non-primary key field must be of the map type. The map value of the field must be of the double type, indicating the score. The map key of the field indicates the value in the Redis set.

                  +
                • If data-type is sorted-set and schema-syntax is array-scores, only two non-primary keys are allowed and must be of the array type.
                  The first key indicates values in the Redis set. The second key is of the array<double> type, indicating index scores. The following is an example:
                  CREATE TABLE redisSink (
+  order_id string,
+  arrayField Array<String>,
+  arrayScore array<double>,
+  primary key (order_id) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'sorted-set',
+  "default-score" = '3',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'array-scores'
+);
                  +
                  +
                +
              +
              +

              Syntax

               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
              create table dwsSource (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (',' watermark for rowtime_column_name as watermark-strategy_expression)
+  ,PRIMARY KEY (attr_name, ...) NOT ENFORCED
+)
+with (
+  'connector' = 'redis',
+  'host' = ''
+);
+
              +
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameter description

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector to be used. Set this parameter to redis.

              +

              host

              +

              Yes

              +

              None

              +

              String

              +

              Redis connector address.

              +

              port

              +

              No

              +

              6379

              +

              Integer

              +

              Redis connector port.

              +

              password

              +

              No

              +

              None

              +

              String

              +

              Redis authentication password.

              +

              namespace

              +

              No

              +

              None

              +

              String

              +

              Redis key namespace.

              +

              delimiter

              +

              No

              +

              :

              +

              String

              +

              Delimiter between the Redis key and namespace.

              +

              data-type

              +

              No

              +

              hash

              +

              String

              +

              Redis data type. Available values are as follows:

              +
              • hash
              • list
              • set
              • sorted-set
              • string
              +

              For details about the constraints, see Constraints on data-type.

              +

              schema-syntax

              +

              No

              +

              fields

              +

              String

              +

              Redis schema semantics. Available values are as follows (for details, see Caveats and FAQ):

              +
              • fields: applicable to all data types
              • fields-scores: applicable to sorted-set data
              • array: applicable to list, set, and sorted-set data
              • array-scores: applicable to sorted-set data
              • map: applicable to hash and sorted-set data
              +

              For details about the constraints, see Constraints on schema-syntax.

              +

              deploy-mode

              +

              No

              +

              standalone

              +

              String

              +

              Deployment mode of the Redis cluster. The value can be standalone, master-replica, or cluster. The default value is standalone.

              +

              The deployment mode varies depending on the Redis instance type.

              +

              Select standalone for single-node, master/standby, and Proxy Cluster instances.

              +

              For a cluster instance, select cluster.

              +

              retry-count

              +

              No

              +

              5

              +

              Integer

              +

              Number of attempts to connect to the Redis cluster.

              +

              connection-timeout-millis

              +

              No

              +

              10000

              +

              Integer

              +

              Maximum timeout for connecting to the Redis cluster.

              +

              commands-timeout-millis

              +

              No

              +

              2000

              +

              Integer

              +

              Maximum time for waiting for a completion response.

              +

              rebalancing-timeout-millis

              +

              No

              +

              15000

              +

              Integer

              +

              Sleep time when the Redis cluster fails.

              +

              scan-keys-count

              +

              No

              +

              1000

              +

              Integer

              +

              Number of data records read in each scan.

              +

              default-score

              +

              No

              +

              0

              +

              Double

              +

              Default score when data-type is sorted-set.

              +

              deserialize-error-policy

              +

              No

              +

              fail-job

              +

              Enum

              +

              Policy of how to process a data parsing failure. Available values are as follows:

              +
              • fail-job: Fail the job.
              • skip-row: Skip the current data.
              • null-field: Set the current data to null.
              +

              skip-null-values

              +

              No

              +

              true

              +

              Boolean

              +

              Whether null values will be skipped.

              +

              ignore-retractions

              +

              No

              +

              false

              +

              Boolean

              +

              The connector should ignore retraction messages in the update insert/withdraw flow mode.

              +

              key-column

              +

              No

              +

              None

              +

              String

              +

              Schema key of the Redis table.

              +

              source.parallelism

              +

              No

              +

              None

              +

              int

              +

              Defines the custom parallelism of the source. By default, if this option is not defined, the parallelism from the global configuration is used.

              +
              +
              +
              +

              Example

              In this example, data is read from the DCS Redis data source and written to the Print result table. The procedure is as follows:

              +
              1. Create an enhanced datasource connection in the VPC and subnet where Redis locates, and bind the connection to the required Flink elastic resource pool.
              2. Set Redis security groups and add inbound rules to allow access from the Flink queue.

                Test the connectivity using the Redis address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.

                +
              3. Run the following commands on the Redis client to insert data into different keys and store the data in hash format:
                HMSET redisSource order_id 202103241000000001 order_channel webShop order_time "2021-03-24 10:00:00" pay_amount 100.00 real_pay 100.00 pay_time "2021-03-24 10:02:03" user_id 0001 user_name Alice area_id 330106
+
+HMSET redisSource1 order_id 202103241606060001 order_channel appShop order_time "2021-03-24 16:06:06" pay_amount 200.00 real_pay 180.00 pay_time "2021-03-24 16:10:06" user_id 0001 user_name Alice area_id 330106
+
+HMSET redisSource2 order_id 202103251202020001 order_channel miniAppShop order_time "2021-03-25 12:02:02" pay_amount 60.00 real_pay 60.00 pay_time "2021-03-25 12:03:00" user_id 0002 user_name Bob area_id 330110
                +
              4. Create a Flink OpenSource SQL job. Enter the following job script to read data in hash format from Redis.
                Change the values of the parameters in bold as needed in the following script.
                CREATE TABLE redisSource (
+  redisKey string,
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string,
+  primary key (redisKey) not enforced  --Obtains the key value from Redis.
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'hash',
+  'deploy-mode' = 'master-replica'
+);
+
+CREATE TABLE printSink (
+  redisKey string,
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'print'
+);
+
+insert into printSink select * from redisSource;
                +
                +
              5. Perform the following operations to view the data result in the taskmanager.out file:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the taskmanager.out file, and view result logs.
                +

                The data result is as follows:

                +
                +I(redisSource1,202103241606060001,appShop,2021-03-24 16:06:06,200.0,180.0,2021-03-24 16:10:06,0001,Alice,330106)
++I(redisSource,202103241000000001,webShop,2021-03-24 10:00:00,100.0,100.0,2021-03-24 10:02:03,0001,Alice,330106)
++I(redisSource2,202103251202020001,miniAppShop,2021-03-25 12:02:02,60.0,60.0,2021-03-25 12:03:00,0002,Bob,330110)
                +
              +
              +

              FAQ

              • Q: What should I do if the Flink job execution fails and the log contains the following error information?
                Caused by: org.apache.flink.client.program.ProgramInvocationException: The main method caused an error: RealLine:36;Usage of 'set' data-type and 'fields' schema syntax in source Redis connector with multiple non-key column types. As 'set' in Redis is not sorted, it's not possible to map 'set's values to table schema with different types.
                +

                A: If data-type is set, the data types of non-primary key fields in Flink are different. As a result, this error is reported. When data-type is set, the types of non-primary keys defined in Flink must be the same.

                +
              • Q: If data-type is hash, what are the differences between schema-syntax set to fields and that to map?

                A: When schema-syntax is set to fields, the hash value in the Redis key is assigned to the field with the same name in Flink. When schema-syntax is set to map, the hash key and hash value of each hash in Redis are put into a map, which represents the value of the corresponding Flink field. Specifically, this map contains all hash keys and hash values of a key in Redis.

                +
                • For fields:
                  1. Insert the following data into Redis:
                    HMSET redisSource order_id 202103241000000001 order_channel webShop order_time "2021-03-24 10:00:00" pay_amount 100.00 real_pay 100.00 pay_time "2021-03-24 10:02:03" user_id 0001 user_name Alice area_id 330106
                    +
                  2. When schema-syntax is set to fields, use the following job script:
                    CREATE TABLE redisSource (
+  redisKey string,
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string,
+  primary key (redisKey) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'hash',
+  'deploy-mode' = 'master-replica'
+);
+
+CREATE TABLE printSink (
+  redisKey string,
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'print'
+);
+
+insert into printSink select * from redisSource;
                    +
                  3. The job execution result is as follows:
                    +I(redisSource,202103241000000001,webShop,2021-03-24 10:00:00,100.0,100.0,2021-03-24 10:02:03,0001,Alice,330106)
                    +
                  +
                • For map:
                  1. Insert the following data into Redis:
                    HMSET redisSource order_id 202103241000000001 order_channel webShop order_time "2021-03-24 10:00:00" pay_amount 100.00 real_pay 100.00 pay_time "2021-03-24 10:02:03" user_id 0001 user_name Alice area_id 330106
                    +
                  2. When schema-syntax is set to map, use the following job script:
                    CREATE TABLE redisSource (
+  redisKey string,
+  order_result map<string, string>,
+  primary key (redisKey) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'hash',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'map'
+);
+
+CREATE TABLE printSink (
+  redisKey string,
+  order_result map<string, string>
+) WITH (
+  'connector' = 'print'
+);
+
+insert into printSink select * from redisSource;
                    +
                  3. The job execution result is as follows:
                    +I(redisSource,{user_id=0001, user_name=Alice, pay_amount=100.00, real_pay=100.00, order_time=2021-03-24 10:00:00, area_id=330106, order_id=202103241000000001, order_channel=webShop, pay_time=2021-03-24 10:02:03})
                    +
                  +
                +
              +
              +
              +
              +
              +
              Parent topic: Redis
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15063.html b/docs/dli/sqlreference/dli_08_15063.html new file mode 100644 index 000000000..2bb4464f5 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15063.html @@ -0,0 +1,610 @@ + + +

              Result Table

              +

              Function

              DLI outputs the Flink job output data to Redis. Redis is a key-value storage system that supports multiple types of data structures. It can be used in scenarios such as caching, event publish/subscribe, and high-speed queuing. Redis supports direct read/write of strings, hashes, lists, queues, and sets. Redis works with in-memory datasets and provides persistence. For more information about Redis, visit https://redis.io/.

              +
              +

              Prerequisites

              An enhanced datasource connection with Redis has been established, so that you can configure security group rules as required. +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • If the Redis key field is not defined in the statement for creating the Redis result table, the generated UUID is used as the key.
              • To specify a key in Redis, you need to define a primary key in the Redis result table of Flink. The value of the primary key is the Redis key.
              • If the primary key defined for the Redis result table, it cannot be a composite primary key and only can be one field.
              • Constraints on schema-syntax:
                • If schema-syntax is map or array, there can be only one non-primary key and it must be of the same map or array type.
                • If schema-syntax is fields-scores, the number of non-primary keys must be an even number, and the second key of every two keys except the primary key must be of the double type. The double value is the score of the previous key. The following is an example:
                  CREATE TABLE redisSink (
+  order_id string,
+  order_channel string,
+  order_time double,
+  pay_amount STRING,
+  real_pay double,
+  pay_time string,
+  user_id double,
+  user_name string,
+  area_id double,
+  primary key (order_id) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'sorted-set',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'fields-scores'
+);
                  +
                +
              • Restrictions on data-type:
                • If data-type is string, only one non-primary key field is allowed.
                • If data-type is sorted-set and schema-syntax is fields or array, default-score is used as the score.
                • If data-type is sorted-set and schema-syntax is map, there can be only one non-primary key in addition to the primary key and the non-primary key must be of the map type. The map values of the non-primary key must be of the double type, indicating the score. The keys in the map are the values in the Redis set.
                • If data-type is sorted-set and schema-syntax is array-scores, only two non-primary keys are allowed and must be of the array type.
                  The first key indicates values in the Redis set. The second key is of the array<double> type, indicating index scores. The following is an example:
                  CREATE TABLE redisSink (
+  order_id string,
+  arrayField Array<String>,
+  arrayScore array<double>,
+  primary key (order_id) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'sorted-set',
+  "default-score" = '3',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'array-scores'
+);
                  +
                  +
                +
              +
              +

              Syntax

              1
+2
+3
+4
+5
+6
+7
+8
+9
              create table dwsSink (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (','PRIMARY KEY (attr_name, ...) NOT ENFORCED)
+)
+with (
+  'connector' = 'redis',
+  'host' = ''
+);
+
              +
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameter description

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector to be used. Set this parameter to redis.

              +

              host

              +

              Yes

              +

              None

              +

              String

              +

              Redis connector address.

              +

              port

              +

              No

              +

              6379

              +

              Integer

              +

              Redis connector port.

              +

              password

              +

              No

              +

              None

              +

              String

              +

              Redis authentication password.

              +

              namespace

              +

              No

              +

              None

              +

              String

              +

              Redis key namespace.

              +

              For example, if the value is set to "person" and the key is "jack", the value in the Redis is person:jack.

              +

              delimiter

              +

              No

              +

              :

              +

              String

              +

              Delimiter between the Redis key and namespace.

              +

              data-type

              +

              No

              +

              hash

              +

              String

              +

              Redis data type. Available values are as follows:

              +
              • hash
              • list
              • set
              • sorted-set
              • string
              +

              For details about the constraints, see Constraints on data-type.

              +

              schema-syntax

              +

              No

              +

              fields

              +

              String

              +

              Redis schema semantics. Available values are as follows:

              +
              • fields: applicable to all data types. This value indicates that multiple fields can be set and the value of each field is read when data is written.
              • fields-scores: applicable to sorted-set data, indicating that each field is read as an independent score.
              • array: applicable to list, set, and sorted-set data.
              • array-scores: applicable to sorted-set data.
              • map: applicable to hash and sorted-set data.
              +

              For details about the constraints, see Constraints on schema-syntax.

              +

              deploy-mode

              +

              No

              +

              standalone

              +

              String

              +

              Deployment mode of the Redis cluster. The value can be standalone, master-replica, or cluster. The default value is standalone.

              +

              For details about the setting, see the instance type description of the Redis cluster.

              +

              retry-count

              +

              No

              +

              5

              +

              Integer

              +

              Number of attempts to connect to the Redis cluster.

              +

              connection-timeout-millis

              +

              No

              +

              10000

              +

              Integer

              +

              Maximum timeout for connecting to the Redis cluster.

              +

              commands-timeout-millis

              +

              No

              +

              2000

              +

              Integer

              +

              Maximum time for waiting for a completion response.

              +

              rebalancing-timeout-millis

              +

              No

              +

              15000

              +

              Integer

              +

              Sleep time when the Redis cluster fails.

              +

              default-score

              +

              No

              +

              0

              +

              Double

              +

              Default score when data-type is sorted-set.

              +

              ignore-retraction

              +

              No

              +

              false

              +

              Boolean

              +

              Whether to ignore Retract messages.

              +

              skip-null-values

              +

              No

              +

              true

              +

              Boolean

              +

              Whether null values will be skipped. If this parameter is false, null will be assigned for null values.

              +

              ignore-retractions

              +

              No

              +

              false

              +

              Boolean

              +

              The connector should ignore retraction messages in the update insert/withdraw flow mode.

              +

              key-column

              +

              No

              +

              None

              +

              String

              +

              Schema key of the Redis table.

              +

              sink.delivery-guarantee

              +

              No

              +

              at-least-once

              +

              String

              +
              • exactly-once:

                Each record is delivered only once, even in the event of a failover. To create a complete exactly-once pipeline, both the source and the sink must support exactly-once and be properly configured.

                +
              • at-least-once:

                Records are definitely to be delivered, but may be delivered multiple times. This mode is typically faster than exactly-once.

                +
              • none:

                Records are delivered on a best-effort basis. This is often the fastest way to process records, but may result in lost or duplicate records.

                +
              +

              sink.parallelism

              +

              No

              +

              None

              +

              int

              +

              Defines the custom parallelism of the sink. If this parameter is not defined, the planner will derive the parallelism for each statement separately by considering the global configuration.

              +

              key-ttl-mode

              +

              No

              +

              no-ttl

              +

              String

              +

              Whether the Redis sink TTL function will be enabled. The value can be no-ttl, expire-msec, expire-at-date or expire-at-timestamp.

              +
              • no-ttl: No expiration time is set.
              • expire-msec: validity period of the key. The parameter is a long string, in milliseconds.
              • expire-at-date: Date and time when the key expires. The value is in UTC time format.
              • expire-at-timestamp: Timestamp when the key expires.
              +

              key-ttl

              +

              No

              +

              None

              +

              String

              +

              Supplementary parameter of key-ttl-mode. Available values are as follows:

              +
              • If key-ttl-mode is no-ttl, this parameter does not need to be configured.
              • If key-ttl-mode is expire-msec, set this parameter to a string that can be parsed into the Long type. For example, 5000 indicates that the key will expire in 5000 ms.
              • If key-ttl-mode is expire-at-date, set this parameter to a date.
              • If key-ttl-mode is expire-at-timestamp, set this parameter to a timestamp, in milliseconds. For example, 1679385600000 indicates that the expiration time is 2023-03-21 16:00:00.
              +
              +
              +
              +

              Example

              In this example, data is read from the Kafka data source and written to the Redis result table. The procedure is as follows:

              +
              1. Create an enhanced datasource connection in the VPC and subnet where Redis locates, and bind the connection to the required Flink elastic resource pool.
              2. Set Redis security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Redis address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              3. Create a Flink OpenSource SQL job. Enter the following job script and submit the job.
                Change the values of the parameters in bold as needed in the following script.
                CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+--In the following redisSink table, data-type is set to default value hash, schema-syntax is fields, and order_id is defined as the primary key. Therefore, the value of this field is used as the Redis key.
+CREATE TABLE redisSink (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string,
+  primary key (order_id) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = '<yourRedis>',
+  'password' = '<yourPassword>',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'fields'
+);
+
+insert into redisSink select * from orders;
                +
                +
              4. Connect to the Kafka cluster and insert the following test data into Kafka:
                {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
                +
              5. Run the following commands in Redis and view the result:
                • Obtain the result whose key is 202103241606060001.

                  Run following command:

                  +
                  HGETALL 202103241606060001
                  +
                  Command output:
                   1) "user_id"
+ 2) "0001"
+ 3) "user_name"
+ 4) "Alice"
+ 5) "pay_amount"
+ 6) "200.0"
+ 7) "real_pay"
+ 8) "180.0"
+ 9) "order_time"
+10) "2021-03-24 16:06:06"
+11) "area_id"
+12) "330106"
+13) "order_channel"
+14) "appShop"
+15) "pay_time"
+16) "2021-03-24 16:10:06"
                  +
                  +
                • Obtain the result whose key is 202103241000000001.

                  Run following command:

                  +
                  HGETALL 202103241000000001
                  +
                  Command output:
                   1) "user_id"
+ 2) "0001"
+ 3) "user_name"
+ 4) "Alice"
+ 5) "pay_amount"
+ 6) "100.0"
+ 7) "real_pay"
+ 8) "100.0"
+ 9) "order_time"
+10) "2021-03-24 10:00:00"
+11) "area_id"
+12) "330106"
+13) "order_channel"
+14) "webShop"
+15) "pay_time"
+16) "2021-03-24 10:02:03"
                  +
                  +
                +
              +
              +

              FAQ

              • Q: When data-type is set, why is the final result data less than the input data?

                A: This is because the input data contains duplicate data. Deduplication is performed in the Redis set, and the number of records in the result decreases.

                +
              • Q: What should I do if Flink job logs contain the following error information?
                org.apache.flink.table.api.ValidationException: SQL validation failed. From line 1, column 40 to line 1, column 105: Parameters must be of the same type
                +

                A: The array type is used. However, the types of fields in the array are different. You need to ensure that the types of fields in the array in Redis are the same.

                +
              • Q: What should I do if Flink job logs contain the following error information?
                org.apache.flink.addons.redis.core.exception.RedisConnectorException: Wrong Redis schema for 'map' syntax: There should be a key (possibly) and 1 MAP non-key column.
                +

                A: When schema-syntax is map, the table creation statement in Flink can contain only one non-primary key column, and the column type must be map.

                +
              • Q: What should I do if Flink job logs contain the following error information?
                org.apache.flink.addons.redis.core.exception.RedisConnectorException: Wrong Redis schema for 'array' syntax: There should be a key (possibly) and 1 ARRAY non-key column.
                +

                A: When schema-syntax is array, the table creation statement in Flink can contain only one non-primary key column, and the column type must be array.

                +
              • Q: What is the function of schema-syntax since data-type has been set?

                A: schema-syntax is used to process special types, such as map and array.

                +
                • If it is set to fields, the value of each field is processed. If it is set to array or map, each element in the field is processed. For fields, the field value of the map or array type is directly used as a value in Redis.
                • For array or map, each value in the array is used as a Redis value, and the field value of the map is used as the Redis value. array-scores is used to process the sorted-set data type. It indicates that two array fields are used, the first one is the value in the set, and the second one is the score. fields-scores is used to process the sorted-set data type, indicating that the score is derived from the defined field. The field of an odd number except the primary key indicates the value in the set, and its next field indicates its score. Therefore, its next field must be of the double type.
                +
              • Q: If data-type is hash, what are the differences between schema-syntax set to fields and that to map?

                A: When fields is used, the field name in Flink is used as the Redis field of the hash data type, and the value of that field is used as the value of the hash data type in Redis. When map is used, the field key in Flink is used as the Redis field of the hash data type, and the value of that field is used as the value of the hash data type in Redis. The following is an example:

                +
                • For fields:
                  1. The execution script of the Flink job is as follows:
                    CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE redisSink (
+  order_id string,
+  maptest Map<string, String>,
+  primary key (order_id) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'fields'
+);
+
+insert into redisSink select order_id, Map[user_id, area_id] from orders;
                    +
                  2. Connect to the Kafka cluster and insert the following test data into the Kafka topic:
                    {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
                    +
                  3. In the Redis, the result is as follows:
                    1) "maptest"
+2) "{0001=330106}"
                    +
                  +
                • For map:
                  1. The execution script of the Flink job is as follows:
                    CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE redisSink (
+  order_id string,
+  maptest Map<string, String>,
+  primary key (order_id) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'map'
+);
+
+insert into redisSink select order_id, Map[user_id, area_id] from orders;
                    +
                  2. Connect to the Kafka cluster and insert the following test data into the Kafka topic:
                    {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
                    +
                  3. In the Redis, the result is as follows:
                    1) "0001"
+2) "330106"
                    +
                  +
                +
              +
              +
              • Q: If data-type is list, what are the differences between schema-syntax set to fields and that to array?

                A: The setting to fields or array does not result in different results. The only difference is that in the Flink table creation statement. fields can be multiple fields. However, array requires that the field is of the array type and the data types in the array must be the same. Therefore, fields are more flexible.

                +
                • For fields:
                  1. The execution script of the Flink job is as follows:
                    CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE redisSink (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string, 
+  primary key (order_id) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'list',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'fields'
+);
+
+insert into redisSink select * from orders;
                    +
                  2. Connect to the Kafka cluster and insert the following test data into the Kafka topic:
                    {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
                    +
                  3. View the result.

                    Run the following command in Redis:

                    +
                    LRANGE 202103241000000001 0 8
                    +
                    The command output is as follows:
                    1) "webShop"
+2) "2021-03-24 10:00:00"
+3) "100.0"
+4) "100.0"
+5) "2021-03-24 10:02:03"
+6) "0001"
+7) "Alice"
+8) "330106"
                    +
                    +
                  +
                • For array:
                  1. The execution script of the Flink job is as follows:
                    CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE redisSink (
+  order_id string,
+  arraytest Array<String>,
+  primary key (order_id) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'list',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'array'
+);
+
+insert into redisSink select order_id, array[order_channel,order_time,pay_time,user_id,user_name,area_id] from orders;
                    +
                  2. Connect to the Kafka cluster and insert the following test data into the Kafka topic:
                    {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
                    +
                  3. In Redis, view the result. (The result is different from that of fields because data of the double type is not added to the table creation statement of the sink in Flink. Therefore, two values are missing. This is not caused by the difference between fields and array.)
                    1) "webShop"
+2) "2021-03-24 10:00:00"
+3) "2021-03-24 10:02:03"
+4) "0001"
+5) "Alice"
+6) "330106"
                    +
                  +
                +
              +
              +
              +
              +
              Parent topic: Redis
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15064.html b/docs/dli/sqlreference/dli_08_15064.html new file mode 100644 index 000000000..aa7d01628 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15064.html @@ -0,0 +1,437 @@ + + +

              Dimension Table

              +

              Function

              Create a Redis table to connect to source streams for wide table generation.

              +
              +

              Prerequisites

              • An enhanced datasource connection with Redis has been established, so that you can configure security group rules as required. +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • To obtain the key values, you can set the primary key in Flink. The primary key maps to the Redis key.
              • If the primary key cannot be a composite primary key, and only can be one field.
              • Constraints on schema-syntax:
                • If schema-syntax is map or array, there can be only one non-primary key and it must be of the same map or array type.
                • If schema-syntax is fields-scores, the number of non-primary keys must be an even number, and the second key of every two keys except the primary key must be of the double type. The double value is the score of the previous key. The following is an example:
                  CREATE TABLE redisSource (
+  redisKey string,
+  order_id string,
+  score1 double,
+  order_channel string,
+  score2 double,
+  order_time string,
+  score3 double,
+  pay_amount double,
+  score4 double,
+  real_pay double,
+  score5 double,
+  pay_time string,
+  score6 double,
+  user_id string,
+  score7 double,
+  user_name string,
+  score8 double,
+  area_id string,
+  score9 double,
+  primary key (redisKey) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'sorted-set',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'fields-scores'
+);
                  +
                +
              • Restrictions on data-type:
                • When data-type is set, the types of non-primary keys defined in Flink must be the same.
                • If data-type is sorted-set and schema-syntax is fields or array, only sorted set values can be read from Redis, and the score value cannot be read.
                • If data-type is string, only one non-primary key field is allowed.
                • If data-type is sorted-set and schema-syntax is map, there can be only one non-primary key in addition to the primary key and the non-primary key must be of the map type. The map values of the non-primary key must be of the double type, indicating the score. The keys in the map are the values in the Redis set.
                • If data-type is sorted-set and schema-syntax is array-scores, only two non-primary keys are allowed and must be of the array type.
                  The first key indicates values in the Redis set. The second key is of the array<double> type, indicating index scores. The following is an example:
                  CREATE TABLE redisSink (
+  order_id string,
+  arrayField Array<String>,
+  arrayScore array<double>,
+  primary key (order_id) not enforced
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'sorted-set',
+  "default-score" = '3',
+  'deploy-mode' = 'master-replica',
+  'schema-syntax' = 'array-scores'
+);
                  +
                  +
                +
              +
              +

              Syntax

              create table dwsSource (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (',' watermark for rowtime_column_name as watermark-strategy_expression)
+  ,PRIMARY KEY (attr_name, ...) NOT ENFORCED
+)
+with (
+  'connector' = 'redis',
+  'host' = ''
+);
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameter description

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Types

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector type. Set this parameter to redis.

              +

              host

              +

              Yes

              +

              None

              +

              String

              +

              Redis connector address

              +

              port

              +

              No

              +

              6379

              +

              Integer

              +

              Redis connector port

              +

              password

              +

              No

              +

              None

              +

              String

              +

              Redis authentication password

              +

              namespace

              +

              No

              +

              None

              +

              String

              +

              Redis key namespace

              +

              delimiter

              +

              No

              +

              :

              +

              String

              +

              Delimiter between the Redis key and namespace

              +

              data-type

              +

              No

              +

              hash

              +

              String

              +

              Redis data type. Available values are as follows:

              +
              • hash
              • list
              • set
              • sorted-set
              • string
              +

              For details about the constraints, see Constraints on data-type.

              +

              schema-syntax

              +

              No

              +

              fields

              +

              String

              +

              Redis schema semantics. Available values are as follows:

              +
              • fields: applicable to all data types
              • fields-scores: applicable to sorted set data
              • array: applicable to list, set, and sorted set data
              • array-scores: applicable to sorted set data
              • map: applicable to hash and sorted set data
              +

              For details about the constraints, see Constraints on schema-syntax.

              +

              deploy-mode

              +

              No

              +

              standalone

              +

              String

              +

              Deployment mode of the Redis cluster. The value can be standalone, master-replica, or cluster. The default value is standalone.

              +

              retry-count

              +

              Yes

              +

              5

              +

              Integer

              +

              Size of each connection request queue. If the number of connection requests in a queue exceeds the queue size, command calling will cause RedisException. Setting requestQueueSize to a small value will cause exceptions to occur earlier during overload or disconnection. A larger value indicates more time required to reach the boundary, but more requests may be queued and more heap space may be used. The default value is 2147483647.

              +

              connection-timeout-millis

              +

              No

              +

              10000

              +

              Integer

              +

              Maximum timeout for connecting to the Redis cluster

              +

              commands-timeout-millis

              +

              No

              +

              2000

              +

              Integer

              +

              Maximum time for waiting for a completion response

              +

              rebalancing-timeout-millis

              +

              No

              +

              15000

              +

              Integer

              +

              Sleep time when the Redis cluster fails

              +

              scan-keys-count

              +

              No

              +

              1000

              +

              Integer

              +

              Number of data records read in each scan

              +

              default-score

              +

              No

              +

              0

              +

              Double

              +

              Default score when data-type is sorted-set

              +

              deserialize-error-policy

              +

              No

              +

              fail-job

              +

              Enum

              +

              How to process a data parsing failure

              +

              Available values are as follows:

              +
              • fail-job: Fail the job
              • skip-row: Skip the current data.
              • null-field: Set the current data to null.
              +

              skip-null-values

              +

              No

              +

              true

              +

              Boolean

              +

              Whether null values will be skipped

              +

              lookup.async

              +

              No

              +

              false

              +

              Boolean

              +

              Whether asynchronous I/O will be used when this table is used as a dimension table

              +

              lookup.parallelism

              +

              No

              +

              None

              +

              int

              +

              Defines the custom parallelism of the lookup join operator. If this parameter is not defined, the planner will derive the parallelism by considering the global configuration (if the lookup.parallelism parameter is defined) or the parallelism of the input operator.

              +

              lookup.batch.interval

              +

              No

              +

              1s

              +

              Duration

              +

              Batch lookup join can buffer input records with a maximum delay. Batch lookup join can buffer input records with a maximum delay.

              +

              lookup.batch.size

              +

              No

              +

              100L

              +

              long

              +

              Maximum number of input records that can be buffered for batch lookup join.

              +

              lookup.batch

              +

              No

              +

              false

              +

              Boolean

              +

              Whether to enable batch lookup optimization. If enabled, the user must set both the lookup.batch.interval and lookup.batch.size parameters. Additionally, due to the implementation of the underlying batch processing interval interference mechanism, the user must explicitly enable the table.exec.batch-lookup.enabled parameter in the Flink configuration.

              +

              ignore-retractions

              +

              No

              +

              false

              +

              Boolean

              +

              The connector should ignore retraction messages in the update insert/withdraw flow mode.

              +
              +
              +
              +

              Example

              Read data from a Kafka source table, use a Redis table as the dimension table. Write wide table information generated by the source and dimension tables to a Kafka result table. The procedure is as follows:

              +
              1. Create an enhanced datasource connection in the VPC and subnet where Redis and Kafka locates, and bind the connection to the required Flink elastic resource pool.
              2. Set Redis and Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Redis address. If the connection passes the test, it is bound to the queue.
              3. Run the following commands on the Redis client to send data to Redis:
                HMSET 330102  area_province_name a1 area_province_name b1 area_county_name c1 area_street_name d1 region_name e1
+
+HMSET 330106  area_province_name a1 area_province_name b1 area_county_name c2 area_street_name d2 region_name e1
+
+HMSET 330108  area_province_name a1 area_province_name b1 area_county_name c3 area_street_name d3 region_name e1
+
+HMSET 330110  area_province_name a1 area_province_name b1 area_county_name c4 area_street_name d4 region_name e1
                +
              4. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. The job script uses Kafka as the data source and a Redis table as the dimension table. Data is output to a Kafka result table.
                Change the values of the parameters in bold as needed in the following script.
                CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string,
+  proctime as Proctime()
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaSourceTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+-- Create an address dimension table
+create table area_info (
+    area_id string, 
+    area_province_name string,
+    area_city_name string,
+    area_county_name string, 
+    area_street_name string, 
+    region_name string, 
+    primary key (area_id) not enforced -- Redis key
+) WITH (
+  'connector' = 'redis',
+  'host' = 'RedisIP',
+  'password' = 'RedisPassword',
+  'data-type' = 'hash',
+  'deploy-mode' = 'master-replica'
+);
+
+-- Generate a wide table based on the address dimension table containing detailed order information.
+create table order_detail(
+    order_id string,
+    order_channel string,
+    order_time string,
+    pay_amount double,
+    real_pay double,
+    pay_time string,
+    user_id string,
+    user_name string,
+    area_id string,
+    area_province_name string,
+    area_city_name string,
+    area_county_name string,
+    area_street_name string,
+    region_name string
+) with (
+  'connector' = 'kafka',
+  'topic' = 'kafkaSinkTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'format' = 'json'
+);
+
+insert into order_detail
+    select orders.order_id, orders.order_channel, orders.order_time, orders.pay_amount, orders.real_pay, orders.pay_time, orders.user_id, orders.user_name,
+           area.area_id, area.area_province_name, area.area_city_name, area.area_county_name,
+           area.area_street_name, area.region_name  from orders
+    left join area_info for system_time as of orders.proctime as area on orders.area_id = area.area_id;
                +
                +
              5. Connect to the Kafka cluster and insert the following test data into the source topic in Kafka:
                {"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103251202020001", "order_channel":"miniAppShop", "order_time":"2021-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2021-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
+
+{"order_id":"202103251505050001", "order_channel":"appShop", "order_time":"2021-03-25 15:05:05", "pay_amount":"500.00", "real_pay":"400.00", "pay_time":"2021-03-25 15:10:00", "user_id":"0003", "user_name":"Cindy", "area_id":"330108"}
                +
              6. Connect to the Kafka cluster and read data from the sink topic of Kafka. The result data is as follows:
                {"order_id":"202103241606060001","order_channel":"appShop","order_time":"2021-03-24 16:06:06","pay_amount":200.0,"real_pay":180.0,"pay_time":"2021-03-24 16:10:06","user_id":"0001","user_name":"Alice","area_id":"330106","area_province_name":"a1","area_city_name":"b1","area_county_name":"c2","area_street_name":"d2","region_name":"e1"}
+
+{"order_id":"202103251202020001","order_channel":"miniAppShop","order_time":"2021-03-25 12:02:02","pay_amount":60.0,"real_pay":60.0,"pay_time":"2021-03-25 12:03:00","user_id":"0002","user_name":"Bob","area_id":"330110","area_province_name":"a1","area_city_name":"b1","area_county_name":"c4","area_street_name":"d4","region_name":"e1"}
+
+{"order_id":"202103251505050001","order_channel":"appshop","order_time":"2021-03-25 15:05:05","pay_amount":500.0,"real_pay":400.0,"pay_time":"2021-03-25 15:10:00","user_id":"0003","user_name":"Cindy","area_id":"330108","area_province_name":"a1","area_city_name":"b1","area_county_name":"c3","area_street_name":"d3","region_name":"e1"}
+
                +
              +
              +
              +
              +
              +
              Parent topic: Redis
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15065.html b/docs/dli/sqlreference/dli_08_15065.html new file mode 100644 index 000000000..90565c4f4 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15065.html @@ -0,0 +1,367 @@ + + +

              Upsert Kafka

              +

              Function

              Apache Kafka is a fast, scalable, and fault-tolerant distributed message publishing and subscription system. It delivers high throughput and built-in partitions and provides data replicas and fault tolerance. Apache Kafka is applicable to scenarios of handling massive messages. The Upsert Kafka connector allows for reading data from and writing data into Kafka topics in the upsert fashion. Source tables and result tables are supported.

              +
              • As a source, the upsert-kafka connector produces a changelog stream, where each data record represents an update or delete event.

                The value in a data record is interpreted as an UPDATE of the last value for the same key, if any (if a corresponding key does not exist yet, the UPDATE will be considered an INSERT). Using the table analogy, a data record in a changelog stream is interpreted as an UPSERT, also known as INSERT/UPDATE, because any existing row with the same key is overwritten. Also, null values are interpreted in a special way: A record with a null value represents a DELETE.

                +
              • As a sink, the upsert-kafka connector can consume a changelog stream. It will write INSERT/UPDATE_AFTER data as normal Kafka messages value, and write DELETE data as Kafka messages with null values (indicate tombstone for the key). Flink will guarantee the message ordering on the primary key by partition data on the values of the primary key columns, so the UPDATE/DELETE messages on the same key will fall into the same partition.
              + +
              + + + + + + + +
              Table 1 Supported types

              Type

              +

              Description

              +

              Supported Table Types

              +

              Source table and result table

              +
              +
              +
              +

              Prerequisites

              An enhanced datasource connection has been created for DLI to connect to Kafka clusters, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required. +
              +
              +

              Caveats

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • The Upsert Kafka always works in the upsert fashion and requires to define the primary key in the DDL. With the assumption that records with the same key should be ordered in the same partition, the primary key semantic on the changelog source means the materialized changelog is unique on the primary keys. The primary key definition will also control which fields should end up in Kafka's key.
              • Because the connector is working in upsert mode, the last record on the same key will take effect when reading back as a source.
              • For details about how to use data types, see Format.
              +
              +

              Syntax

               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
              create table kafkaTable(
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (','PRIMARY KEY (attr_name, ...) NOT ENFORCED)
+)
+with (
+  'connector' = 'upsert-kafka',
+  'topic' = '',
+  'properties.bootstrap.servers' = '',
+  'key.format' = '',
+  'value.format' = ''
+);
+
              +
              +
              +

              Parameter Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Parameters

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector to be used. For the Upsert Kafka connector, set this parameter to upsert-kafka.

              +

              topic

              +

              Yes

              +

              None

              +

              String

              +

              Kafka topic name

              +

              properties.bootstrap.servers

              +

              Yes

              +

              None

              +

              String

              +

              Comma separated list of Kafka brokers

              +

              key.format

              +

              Yes

              +

              None

              +

              String

              +

              Format used to deserialize and serialize the key part of Kafka messages. The key fields are specified by the PRIMARY KEY syntax. The following formats are supported:

              +
              • csv
              • json
              • avro
              +

              Refer to Format for more details and format parameters.

              +

              key.fields-prefix

              +

              No

              +

              None

              +

              String

              +

              Defines a custom prefix for all fields of the key format to avoid name clashes with fields of the value format.

              +

              By default, the prefix is empty. If a custom prefix is defined, both the table schema and key.fields will work with prefixed names. When constructing the data type of the key format, the prefix will be removed and the non-prefixed names will be used within the key format. Note that this option requires that value.fields-include be set to EXCEPT_KEY.

              +

              value.format

              +

              Yes

              +

              None

              +

              String

              +

              Format used to deserialize and serialize the value part of Kafka messages. The following formats are supported:

              +
              • csv
              • json
              • avro
              +

              Refer to Format for more details and format parameters.

              +

              value.fields-include

              +

              Yes

              +

              ALL

              +

              String

              +

              Controls which fields should appear in the value part. Possible values are:

              +
              • ALL: All fields in the schema, including the primary key field, are included in the value part.
              • EXCEPT_KEY: All the fields of the table schema are included, except the primary key field.
              +

              properties.*

              +

              No

              +

              None

              +

              String

              +

              This option can set and pass arbitrary Kafka configurations.

              +

              The suffix to properties. must match the parameter defined in Kafka Configuration documentation. Flink will remove the properties. key prefix and pass the transformed key and value to the underlying KafkaClient.

              +

              For example, you can disable automatic topic creation via 'properties.allow.auto.create.topics' = 'false'.

              +

              But there are some configurations that do not support to set, because Flink will override them, for example, 'key.deserializer' and 'value.deserializer'.

              +

              sink.parallelism

              +

              No

              +

              None

              +

              Integer

              +

              Defines the parallelism of the Upsert Kafka sink operator. By default, the parallelism is determined by the framework: using the same parallelism as the upstream join operator.

              +

              sink.buffer-flush.max-rows

              +

              No

              +

              0

              +

              Integer

              +

              The max size of buffered records before flushing.

              +

              When the sink receives many updates on the same key, the buffer will retain the last record of the same key. This can help to reduce data shuffling and avoid possible tombstone messages to Kafka topic. Can be set to 0 to disable it.

              +

              By default, this is disabled. Note both sink.buffer-flush.max-rows and sink.buffer-flush.interval must be set to be greater than zero to enable sink buffer flushing.

              +

              sink.buffer-flush.interval

              +

              No

              +

              0

              +

              Duration

              +

              The flush interval mills, over this time, asynchronous threads will flush data. The unit can be millisecond (ms), second (s), minute (min), or hour (h). For example, 'sink.buffer-flush.interval'='10 ms'.

              +

              By default, this is disabled. Note both sink.buffer-flush.max-rows and sink.buffer-flush.interval must be set to be greater than zero to enable sink buffer flushing.

              +
              +
              +
              +

              Metadata

              For a list of available metadata fields, see Kafka Connector.

              +
              +

              Example

              • Example 1: This example reads data from a DMS Kafka data source and writes it to the Print result table.
                1. Create an enhanced datasource connection in the VPC and subnet where Kafka locates, and bind the connection to the required Flink elastic resource pool.
                2. Set Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Kafka address. If the connection passes the test, it is bound to the queue.
                3. Create a Flink OpenSource SQL job. Enter the following job script and submit the job.
                  When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.
                  CREATE TABLE upsertKafkaSource (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,  
+  area_id string,
+  PRIMARY KEY (order_id) NOT ENFORCED
+) WITH (
+  'connector' = 'upsert-kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' =  'KafkaAddress1:KafkaPort,KafkAddress2:KafkaPort',
+  'key.format' = 'csv',
+  'value.format' = 'json'
+);
+
+CREATE TABLE printSink (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,  
+  area_id string,
+  PRIMARY KEY (order_id) NOT ENFORCED
+) WITH (
+  'connector' = 'print'
+);
+
+INSERT INTO printSink SELECT * FROM upsertKafkaSource;
                  +
                  +
                4. Insert the following data to the specified topics in Kafka. (Note: Specify the key when inserting data to Kafka.)
                  {"order_id":"202303251202020001", "order_channel":"miniAppShop", "order_time":"2023-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2023-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
+
+{"order_id":"202303251505050001", "order_channel":"appshop", "order_time":"2023-03-25 15:05:05", "pay_amount":"500.00", "real_pay":"400.00", "pay_time":"2023-03-25 15:10:00", "user_id":"0003", "user_name":"Cindy", "area_id":"330108"}
+
+{"order_id":"202303251202020001", "order_channel":"miniAppShop", "order_time":"2023-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2023-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330111"}
                  +
                5. View the out file of the TaskManager. The data results are as follows:
                  +I(202303251202020001,miniAppShop,2023-03-2512:02:02,60.0,60.0,2023-03-2512:03:00,0002,Bob,330110)
++I(202303251505050001,appshop,2023-03-25 15:05:05,500.0,400.0,2023-03-2515:10:00,0003,Cindy,330108)
+-U(202303251202020001,miniAppShop,2023-03-2512:02:02,60.0,60.0,2023-03-2512:03:00,0002,Bob,330110)
++U(202303251202020001,miniAppShop,2023-03-2512:02:02,60.0,60.0,2023-03-2512:03:00,0002,Bob,330111)
                  +
                +
              +
              +
              • Example 2: This example retrieves DMS Kafka source topic data from a Kafka source table and writes it to a Kafka sink topic using Upsert Kafka result table.
                1. Create an enhanced datasource connection in the VPC and subnet where Kafka locates, and bind the connection to the required Flink elastic resource pool.
                2. Set Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Kafka address. If the connection passes the test, it is bound to the queue.
                3. Create a Flink OpenSource SQL job. Enter the following job script and submit the job.

                  When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.

                  +
                  CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE upsertKafkaSink (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string,
+  PRIMARY KEY(order_id) NOT ENFORCED
+) WITH (
+  'connector' = 'upsert-kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' =  'KafkaAddress1:KafkaPort,KafkAddress2:KafkaPort',
+  'key.format' = 'csv',
+  'value.format' = 'json'
+);
+
+insert into upsertKafkaSink select * from orders;
                  +
                4. Connect to the Kafka cluster and send the following test data to the Kafka source topic:
                  {"order_id":"202303251202020001", "order_channel":"miniAppShop", "order_time":"2023-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2023-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
+
+{"order_id":"202303251505050001", "order_channel":"appshop", "order_time":"2023-03-25 15:05:05", "pay_amount":"500.00", "real_pay":"400.00", "pay_time":"2023-03-25 15:10:00", "user_id":"0003", "user_name":"Cindy", "area_id":"330108"}
+
+{"order_id":"202303251202020001", "order_channel":"miniAppShop", "order_time":"2023-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2023-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330111"}
                  +
                5. Connect to the Kafka cluster and read data from the Kafka sink topic. The result is as follows:
                  {"order_id":"202303251202020001", "order_channel":"miniAppShop", "order_time":"2023-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2023-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
+
+{"order_id":"202303251505050001", "order_channel":"appshop", "order_time":"2023-03-25 15:05:05", "pay_amount":"500.00", "real_pay":"400.00", "pay_time":"2023-03-25 15:10:00", "user_id":"0003", "user_name":"Cindy", "area_id":"330108"}
+
+{"order_id":"202303251202020001", "order_channel":"miniAppShop", "order_time":"2023-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2023-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330111"}
                  +
                +
              • Example 3: In this scenario, the MRS cluster has enabled Kerberos authentication and Kafka is using the SASL_PLAINTEXT protocol. Data is retrieved from a Kafka source table and written to the Print result table.
                1. Create an enhanced datasource connection in the VPC and subnet where the MRS cluster locates, and bind the connection to the required Flink elastic resource pool.
                2. Set MRS cluster security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the Kafka address. If the connection passes the test, it is bound to the queue.
                3. Create a Flink OpenSource SQL job. Enter the following job script and submit the job.

                  When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.

                  +
                  CREATE TABLE upsertKafkaSource (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string,
+  PRIMARY KEY(order_id) NOT ENFORCED
+) WITH (
+  'connector' = 'upsert-kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' =  'KafkaAddress1:KafkaPort,KafkAddress2:KafkaPort',
+  'key.format' = 'csv',
+  'value.format' = 'json',
+  'properties.sasl.mechanism' = 'GSSAPI',
+  'properties.security.protocol' = 'SASL_PLAINTEXT',
+  'properties.sasl.kerberos.service.name' = 'kafka', -- Configured in MRS
+  'properties.connector.auth.open' = 'true',
+  'properties.connector.kerberos.principal' = 'username', --Username
+  'properties.connector.kerberos.krb5' = 'obs://xx/krb5.conf', --krb5_conf path
+  'properties.connector.kerberos.keytab' = 'obs://xx/user.keytab' --keytab path
+);
+
+CREATE TABLE printSink (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,  
+  area_id string,
+  PRIMARY KEY (order_id) NOT ENFORCED
+) WITH (
+  'connector' = 'print'
+);
+
+INSERT INTO printSink SELECT * FROM upsertKafkaSource;
                  +
                4. Insert the following data to the specified topics in Kafka. (Note: Specify the key when inserting data to Kafka.)
                  {"order_id":"202303251202020001", "order_channel":"miniAppShop", "order_time":"2023-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2023-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
+
+{"order_id":"202303251505050001", "order_channel":"appshop", "order_time":"2023-03-25 15:05:05", "pay_amount":"500.00", "real_pay":"400.00", "pay_time":"2023-03-25 15:10:00", "user_id":"0003", "user_name":"Cindy", "area_id":"330108"}
+
+{"order_id":"202303251202020001", "order_channel":"miniAppShop", "order_time":"2023-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2023-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330111"}
                  +
                5. View the out file of the TaskManager. The data results are as follows:
                  +I(202303251202020001,miniAppShop,2023-03-2512:02:02,60.0,60.0,2023-03-2512:03:00,0002,Bob,330110)
++I(202303251505050001,appshop,2023-03-2515:05:05,500.0,400.0,2023-03-2515:10:00,0003,Cindy,330108)
+-U(202303251202020001,miniAppShop,2023-03-2512:02:02,60.0,60.0,2023-03-2512:03:00,0002,Bob,330110)
++U(202303251202020001,miniAppShop,2023-03-2512:02:02,60.0,60.0,2023-03-2512:03:00,0002,Bob,330111)
                  +
                +
              +

              FAQ

              None

              +
              +
              +
              +
              +
              Parent topic: Connectors
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15066.html b/docs/dli/sqlreference/dli_08_15066.html new file mode 100644 index 000000000..4f33567c5 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15066.html @@ -0,0 +1,33 @@ + + +

              DML Snytax

              +
              + + diff --git a/docs/dli/sqlreference/dli_08_15067.html b/docs/dli/sqlreference/dli_08_15067.html new file mode 100644 index 000000000..69f8a04f7 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15067.html @@ -0,0 +1,184 @@ + + +

              SELECT

              +

              SELECT

              Syntax

              +
              1
+2
+3
+4
+5
+6
              SELECT [ ALL | DISTINCT ]
+  { * | projectItem [, projectItem ]* }
+  FROM tableExpression
+  [ WHERE booleanExpression ]
+  [ GROUP BY { groupItem [, groupItem ]* } ]
+  [ HAVING booleanExpression ]
+
              +
              +

              Description

              +

              SELECT is used to select data from a table.

              +

              ALL indicates that all results are returned.

              +

              DISTINCT indicates that the duplicated results are removed.

              +

              Precautions

              +
              • The to-be-queried table must exist. Otherwise, an error is reported.
              • WHERE is used to specify the search condition, which can be the arithmetic operator, relational operator, or logical operator.
              • GROUP BY is used to specify the grouping field, which can be one or more multiple fields.
              +

              Example

              +

              Select the order which contains more than 3 pieces of data.

              +
              1
              insert into temp SELECT  * FROM Orders WHERE units > 3; 
+
              +
              +

              Insert a group of constant data.

              +
              1
              insert into temp select 'Lily', 'male', 'student', 17;
+
              +
              +
              +

              WHERE

              Syntax

              +
              1
+2
+3
              SELECT   { * | projectItem [, projectItem ]* }
+  FROM tableExpression
+  [ WHERE booleanExpression ]
+
              +
              +

              Description

              +

              This clause is used to filter the query results using the WHERE clause.

              +

              Precautions

              +
              • The to-be-queried table must exist.
              • WHERE filters the records that do not meet the requirements.
              +

              Example

              +

              Search orders which contain more than 3 pieces and fewer than 10 pieces of data.

              +
              1
+2
              insert into temp SELECT  * FROM Orders
+  WHERE units > 3 and units < 10; 
+
              +
              +
              +

              HAVING

              Function

              +

              This clause is used to search for the query results that meet the search condition.

              +

              Syntax

              +
              1
+2
+3
+4
+5
              SELECT [ ALL | DISTINCT ]   { * | projectItem [, projectItem ]* }
+  FROM tableExpression
+  [ WHERE booleanExpression ]
+  [ GROUP BY { groupItem [, groupItem ]* } ]
+  [ HAVING booleanExpression ]
+
              +
              +

              Description

              +

              Generally, HAVING and GROUP BY are used together. You can use GROUP BY for grouping and then use HAVING for filtering. Arithmetic operations and aggregate functions are supported in the HAVING clause.

              +

              Precautions

              +

              If the filtering condition is subject to the results of GROUP BY, the HAVING clause, rather than the WHERE clause, must be used for search.

              +

              Example

              +

              Group the student table according to the name field and search for the records in which the maximum score is higher than 95 in the group.

              +
              1
+2
+3
              insert into temp SELECT name, max(score) FROM student
+  GROUP BY name
+  HAVING max(score) >95;
+
              +
              +
              +

              Column-Based GROUP BY

              Function

              +

              This clause is used to group a table based on columns.

              +

              Syntax

              +
              1
+2
+3
+4
              SELECT [ ALL | DISTINCT ]   { * | projectItem [, projectItem ]* }
+  FROM tableExpression
+  [ WHERE booleanExpression ]
+  [ GROUP BY { groupItem [, groupItem ]* } ]
+
              +
              +

              Description

              +

              Column-based GROUP BY can be categorized into single-column GROUP BY and multi-column GROUP BY.

              +
              • Single-column GROUP BY indicates that the GROUP BY clause contains only one column.
              • Multi-column GROUP BY indicates that the GROUP BY clause contains multiple columns. The table will be grouped according to all fields in the GROUP BY clause. The records whose fields are the same are grouped into one group.
              +

              Precautions

              +

              GroupBy generates update results in the stream processing table.

              +

              Example

              +

              Group the student table according to the score and name fields and return the grouping results.

              +
              1
+2
              insert into temp SELECT name,score, max(score) FROM student 
+  GROUP BY name,score;
+
              +
              +
              +

              Expression-Based GROUP BY

              Function

              +

              This clause is used to group streams according to expressions.

              +

              Syntax

              +
              1
+2
+3
+4
              SELECT [ ALL | DISTINCT ]   { * | projectItem [, projectItem ]* }
+  FROM tableExpression
+  [ WHERE booleanExpression ]
+  [ GROUP BY { groupItem [, groupItem ]* } ]
+
              +
              +

              Description

              +

              groupItem can have one or more fields. The fields can be called by string functions, but cannot be called by aggregate functions.

              +

              Precautions

              +

              None

              +

              Example

              +

              Use the substring function to obtain the character string from the name field, group the student table according to the obtained character string, and return each sub character string and the number of records.

              +
              1
+2
              insert into temp SELECT substring(name,6),count(name) FROM student
+  GROUP BY substring(name,6);
+
              +
              +
              +

              Grouping sets, Rollup, Cube

              Function

              +
              • The GROUP BY GROUPING SETS generates a result set equivalent to that generated by multiple simple GROUP BY UNION ALL statements. Using GROUPING SETS is more efficient.
              • The ROLLUP and CUBE generate multiple groups based on certain rules and then collect statistics by group.
              • The result set generated by CUBE contains all the combinations of values in the selected columns.
              • The result set generated by ROLLUP contains the combinations of a certain layer structure in the selected columns.
              +
              Syntax
              SELECT [ ALL | DISTINCT ]   { * | projectItem [, projectItem ]* }
+  FROM tableExpression
+  [ WHERE booleanExpression ]
+  [ GROUP BY groupingItem]
              +
              +

              Description

              +

              Values of groupingItem can be Grouping sets(columnName [, columnName]*), Rollup(columnName [, columnName]*), and Cube(columnName [, columnName]*).

              +

              Precautions

              +

              None

              +

              Example

              +

              Return the results generated based on user and product.

              +
              INSERT INTO temp SELECT SUM(amount)
+FROM Orders
+GROUP BY GROUPING SETS ((user), (product));
              +
              +

              GROUP BY Using HAVING

              Function

              +

              This clause filters a table after grouping it using the HAVING clause.

              +

              Syntax

              +
              1
+2
+3
+4
+5
              SELECT [ ALL | DISTINCT ]   { * | projectItem [, projectItem ]* }
+  FROM tableExpression
+  [ WHERE booleanExpression ]
+  [ GROUP BY { groupItem [, groupItem ]* } ]
+  [ HAVING booleanExpression ]
+
              +
              +

              Description

              +

              Generally, HAVING and GROUP BY are used together. You can use GROUP BY for grouping and the HAVING for filtering.

              +

              Precautions

              +
              • If the filtering condition is subject to the results of GROUP BY, the HAVING clause, rather than the WHERE clause, must be used for search. HAVING and GROUP BY are used together. Use GROUP BY for grouping and the HAVING for filtering.
              • Fields used in HAVING, except for those used for aggregate functions, must exist in GROUP BY.
              • The arithmetic operation and aggregate function are supported by the HAVING clause.
              +

              Example

              +

              Group the transactions by num, use the HAVING clause to search for the records in which the maximum value derived from multiplying price with amount is higher than 5000, and return the filtered results.

              +
              1
+2
+3
+4
              insert into temp SELECT num, max(price*amount) FROM transactions
+  WHERE time > '2016-06-01'
+  GROUP BY num
+  HAVING max(price*amount)>5000;
+
              +
              +
              +
              +
              +
              +
              Parent topic: DML Snytax
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15068.html b/docs/dli/sqlreference/dli_08_15068.html new file mode 100644 index 000000000..457e52dfc --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15068.html @@ -0,0 +1,52 @@ + + +

              Set Operations

              +

              Union/Union ALL/Intersect/Except

              Syntax

              +
              1
              query UNION [ ALL ] | Intersect | Except query
+
              +
              +

              Description

              +
              • UNION is used to return the union set of multiple query results.
              • INTERSECT is used to return the intersection of multiple query results.
              • EXCEPT is used to return the difference set of multiple query results.
              +

              Precautions

              +
              • Set operations join tables from head to tail under certain conditions. The quantity of columns returned by each SELECT statement must be the same. Column types must be the same. Column names can be different.
              • By default, UNION takes only distinct records while UNION ALL does not remove duplicates from the result.
              +

              Example

              +

              Output distinct records found in either Orders1 and Orders2 tables.

              +
              1
+2
              insert into temp SELECT  * FROM Orders1
+  UNION SELECT  * FROM Orders2;
+
              +
              +
              +

              IN

              Syntax

              +
              1
+2
+3
              SELECT [ ALL | DISTINCT ]   { * | projectItem [, projectItem ]* }
+  FROM tableExpression
+  WHERE column_name IN (value (, value)* ) | query
+
              +
              +

              Description

              +

              The IN operator allows multiple values to be specified in the WHERE clause. It returns true if the expression exists in the given table subquery.

              +

              Precautions

              +

              The subquery table must consist of a single column, and the data type of the column must be the same as that of the expression.

              +

              Example

              +

              Return user and amount information of the products in NewProducts of the Orders table.

              +
              1
+2
+3
+4
+5
              insert into temp SELECT user, amount
+FROM Orders
+WHERE product IN (
+    SELECT product FROM NewProducts
+);
+
              +
              +
              +
              +
              +
              +
              Parent topic: DML Snytax
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15069.html b/docs/dli/sqlreference/dli_08_15069.html new file mode 100644 index 000000000..a4fafb5ee --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15069.html @@ -0,0 +1,437 @@ + + +

              Window

              +

              GROUP WINDOW

              Description

              +

              Group Window is defined in GROUP BY. One record is generated from each group. Group Window involves the following functions:

              +
              • Grouping functions

                In streaming mode, the time_attr argument of the group window function must refer to a valid time attribute that specifies the processing time or event time of rows.

                +
                • event-time: The type is timestamp(3).
                • processing-time: No need to specify the type.
                +

                In batch mode, the time_attr argument of the group window function must be an attribute of type timestamp.

                +
                + +
                + + + + + + + + + + + + + +
                Table 1 Grouping functions

                Grouping Window Function

                +

                Description

                +

                TUMBLE(time_attr, interval)

                +

                Defines a tumbling time window.

                +

                A tumbling time window assigns rows to non-overlapping, continuous windows with a fixed duration (interval).

                +

                For example, a tumbling window of 5 minutes groups rows in 5 minutes intervals.

                +

                Tumbling windows can be defined on event-time (stream + batch) or processing-time (stream).

                +

                HOP(time_attr, interval, interval)

                +

                Defines a hopping time window (called sliding window in the Table API).

                +

                A hopping time window has a fixed duration (second interval parameter) and hops by a specified hop interval (first interval parameter).

                +

                If the hop interval is smaller than the window size, hopping windows are overlapping. Thus, rows can be assigned to multiple windows.

                +

                For example, a hopping window of 15 minutes size and 5 minute hop interval assigns each row to 3 different windows of 15 minute size, which are evaluated in an interval of 5 minutes. Hopping windows can be defined on event-time (stream + batch) or processing-time (stream).

                +

                SESSION(time_attr, interval)

                +

                Defines a session time window.

                +

                Session time windows do not have a fixed duration but their bounds are defined by a time interval of inactivity, i.e., a session window is closed if no event appears for a defined gap period.

                +

                For example a session window with a 30 minute gap starts when a row is observed after 30 minutes inactivity (otherwise the row would be added to an existing window) and is closed if no row is added within 30 minutes. Session windows can work on event-time (stream + batch) or processing-time (stream).

                +
                +
                +
              • Window helper functions
                You can use the following helper functions to select the start and end timestamps, as well as the time attribute, for grouping windows.

                When calling helper functions, it is important to use the same parameters as those used in the GROUP BY clause for grouping window functions.

                +
                + +
                + + + + + + + + + + + + + + + + +
                Table 2 Window helper functions

                Helper Function

                +

                Description

                +

                TUMBLE_START(time_attr, interval)

                +

                HOP_START(time_attr, interval, interval)

                +

                SESSION_START(time_attr, interval)

                +

                Returns the timestamp of the inclusive lower bound of the corresponding tumbling, hopping, or session window.

                +

                TUMBLE_END(time_attr, interval)

                +

                HOP_END(time_attr, interval, interval)

                +

                SESSION_END(time_attr, interval)

                +

                Returns the timestamp of the exclusive upper bound of the corresponding tumbling, hopping, or session window.

                +

                The exclusive upper bound timestamp cannot be used as a rowtime attribute in subsequent time-based operations, such as interval joins and group window or over window aggregations.

                +

                TUMBLE_ROWTIME(time_attr, interval)

                +

                HOP_ROWTIME(time_attr, interval, interval)

                +

                SESSION_ROWTIME(time_attr, interval)

                +

                Returns the timestamp of the inclusive upper bound of the corresponding tumbling, hopping, or session window. The resulting attribute is a rowtime attribute that can be used in subsequent time-based operations such as interval joins and group window or over window aggregations.

                +

                TUMBLE_PROCTIME(time_attr, interval)

                +

                HOP_PROCTIME(time_attr, interval, interval)

                +

                SESSION_PROCTIME(time_attr, interval)

                +

                Returns a processing time attribute that can be used in subsequent time-based operations such as interval joins and group window or over window aggregations.

                +
                +
                +
                +
              +

              Example

              +
               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
              // Calculate the SUM every day (event time).
+insert into temp SELECT name,
+    TUMBLE_START(ts, INTERVAL '1' DAY) as wStart,
+    SUM(amount)
+    FROM Orders
+    GROUP BY TUMBLE(ts, INTERVAL '1' DAY), name;
+
+// Calculate the SUM every day (processing time).
+insert into temp SELECT name, 
+    SUM(amount) 
+    FROM Orders 
+    GROUP BY TUMBLE(proctime, INTERVAL '1' DAY), name;
+
+// Calculate the SUM over the recent 24 hours every hour (event time).
+insert into temp SELECT product, 
+    SUM(amount) 
+    FROM Orders 
+    GROUP BY HOP(ts, INTERVAL '1' HOUR, INTERVAL '1' DAY), product;
+
+// Calculate the SUM of each session and an inactive interval every 12 hours (event time).
+insert into temp SELECT name, 
+    SESSION_START(ts, INTERVAL '12' HOUR) AS sStart,
+    SESSION_END(ts, INTERVAL '12' HOUR) AS sEnd,
+    SUM(amount)
+    FROM Orders
+    GROUP BY SESSION(ts, INTERVAL '12' HOUR), name;
+
              +
              +
              +

              TUMBLE WINDOW Extension

              Function

              +
              The extension functions of the DLI tumbling window are as follows:
              • A tumbling window is triggered periodically to reduce latency.

                Before the tumbling window ends, the window can be periodically triggered based on the configured frequency. The compute result from the start to the current time is output, which does not affect the final output. The latest result can be viewed in each period before the window ends.

                +
              • Custom latency for higher data accuracy

                You can set a latency for the end of the window. The output of the window is updated according to the configured latency each time a piece of late data reaches.

                +
              +
              +

              Caveats

              +
              • If you use the INSERT statement to write results to a sink, it must support the upsert mode. Ensure that the result table supports upsert operations and the primary key is defined.
              • Latency settings only take effect for event time and not for processing time.
              • Helper functions must be called with the same parameters as the grouping window functions in the GROUP BY clause.
              • If event time is used, watermark must be used. The code is as follows (order_time is identified as the event time column and watermark is set to 3 seconds):
                CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string,
+  watermark for order_time as order_time - INTERVAL '3' SECOND
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
                +
              • If the processing time is used, you need to use the computed column. The code is as follows (proc is the processing time column):
                CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string,
+  proc as proctime()
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
                +
              +

              Syntax

              +
              TUMBLE(time_attr, window_interval, period_interval, lateness_interval)
              +

              Example

              +
              The current time attribute column is testtime, the window interval is 10 seconds, and the latency is 10 seconds.
              TUMBLE(testtime, INTERVAL '10' SECOND, INTERVAL '10' SECOND, INTERVAL '10' SECOND)
              +
              +
              +

              Parameter description

              + +
              + + + + + + + + + + + + + + + + + + + +
              Table 3 Parameters

              Parameter

              +

              Description

              +

              Format

              +

              time_attr

              +

              Event time or processing time attribute column

              +
              • event-time: The type is timestamp(3).
              • processing-time: No need to specify the type.
              +

              -

              +

              window_interval

              +

              Duration of the window

              +
              • Format 1: INTERVAL '10' SECOND

                The window interval is 10 seconds. You can change the value as needed.

                +
              • Format 2: INTERVAL '10' MINUTE

                The window interval is 10 minutes. You can change the value as needed.

                +
              • Format 3: INTERVAL '10' DAY

                The window interval is 10 days. You can change the value as needed.

                +
              +

              period_interval

              +

              Frequency of periodic triggering within the window range. That is, before the window ends, the output result is updated at an interval specified by period_interval from the time when the window starts. If this parameter is not set, the periodic triggering policy is not used by default.

              +

              lateness_interval

              +

              Time to postpone the end of the window. The system continues to collect the data that reaches the window within lateness_interval after the window ends. The output is updated for each data that reaches the window within lateness_interval.

              +
              NOTE:

              If the time window is for processing time, lateness_interval does not take effect.

              +
              +
              +
              +
              Values of period_interval and lateness_interval cannot be negative numbers.
              • If period_interval is set to 0, periodic triggering is disabled for the window.
              • If lateness_interval is set to 0, the latency after the window ends is disabled.
              • If neither of the two parameters is set, both periodic triggering and latency are disabled and only the regular tumbling window functions are available .
              • If only the latency function needs to be used, set period_interval INTERVAL '0' SECOND.
              +
              +
              +

              Helper functions

              + +
              + + + + + + + + + + +
              Table 4 Helper functions

              Helper Function

              +

              Description

              +

              TUMBLE_START(time_attr, window_interval, period_interval, lateness_interval)

              +

              Returns the timestamp of the inclusive lower bound of the corresponding tumbling window.

              +

              TUMBLE_END(time_attr, window_interval, period_interval, lateness_interval)

              +

              Returns the timestamp of the exclusive upper bound of the corresponding tumbling window.

              +
              +
              +

              Example

              +

              1. The Kafka is used as the data source table containing the order information, and the JDBC is used as the data result table for statistics on the number of orders settled by a user within 30 seconds. The order ID and window opening time are used as primary keys to collect result statistics in real time to JDBC.

              +
              1. Create a datasource connection for the communication with the VPC and subnet where MySQL and Kafka locate and bind the connection to the queue. Set an inbound rule for the security group to allow access of the queue, and test the connectivity of the queue using the MySQL and Kafka addresses. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              2. Run the following statement to create the order_count table in the MySQL Flink database:

                CREATE TABLE `flink`.`order_count` (
+	`user_id` VARCHAR(32) NOT NULL,
+	`window_start` TIMESTAMP NOT NULL,
+	`window_end` TIMESTAMP NULL,
+	`total_num` BIGINT UNSIGNED NULL,
+	PRIMARY KEY (`user_id`, `window_start`)
+)	ENGINE = InnoDB
+	DEFAULT CHARACTER SET = utf8mb4
+	COLLATE = utf8mb4_general_ci;
                +

              3. Create a Flink OpenSource SQL job and submit the job. In this example, the window size is 30 seconds, the triggering period is 10 seconds, and the latency is 5 seconds. That is, if the result is updated before the window ends, the intermediate result will be output every 10 seconds. After the watermark is reached and the window ends, the data whose event time is within 5 seconds of the watermark will still be processed and counted in the current window. If the event time exceeds 5 seconds of the watermark, the data will be discarded.

                CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time timestamp(3),
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string,
+  watermark for order_time as order_time - INTERVAL '3' SECOND
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'kafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE jdbcSink (
+  user_id string,
+  window_start timestamp(3),
+  window_end timestamp(3),
+  total_num BIGINT,
+  primary key (user_id, window_start) not enforced
+) WITH (
+  'connector' = 'jdbc',
+  'url' = 'jdbc:mysql://<yourMySQL>:3306/flink',
+  'table-name' = 'order_count',
+  'username' = '<yourUserName>',
+  'password' = '<yourPassword>',
+  'sink.buffer-flush.max-rows' = '1'
+);
+
+insert into jdbcSink select 
+    order_id,
+    TUMBLE_START(order_time, INTERVAL '30' SECOND, INTERVAL '10' SECOND, INTERVAL '5' SECOND),
+    TUMBLE_END(order_time, INTERVAL '30' SECOND, INTERVAL '10' SECOND, INTERVAL '5' SECOND),
+    COUNT(*) from orders
+    GROUP BY user_id, TUMBLE(order_time, INTERVAL '30' SECOND, INTERVAL '10' SECOND, INTERVAL '5' SECOND);
                +

              4. Insert data to Kafka. Assume that orders are settled at different time and the order data at 10:00:13 arrives late.

                {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103241000000002", "order_channel":"webShop", "order_time":"2021-03-24 10:00:20", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103241000000003", "order_channel":"webShop", "order_time":"2021-03-24 10:00:33", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103241000000004", "order_channel":"webShop", "order_time":"2021-03-24 10:00:13", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
                +

              5. Run the following statement in the MySQL database to view the output result. The final result is displayed as follows because the periodic output result cannot be collected:

                select * from order_count
                +
                user_id      window_start         window_end        total_num
+0001      2021-03-24 10:00:00  2021-03-24 10:00:30    3
+0001      2021-03-24 10:00:30  2021-03-24 10:01:00    1
                +

              +

              OVER WINDOW

              The difference between Over Window and Group Window is that one record is generated from one row in Over Window.

              +

              Syntax

              +
               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
              SELECT agg1(attr1) OVER (
+  [PARTITION BY partition_name]
+  ORDER BY proctime|rowtime 
+  ROWS  
+ BETWEEN (UNBOUNDED|rowCOUNT) PRECEDING AND CURRENT ROW FROM TABLENAME
+
+SELECT agg1(attr1) OVER (
+  [PARTITION BY partition_name]
+  ORDER BY proctime|rowtime 
+  RANGE  
+  BETWEEN (UNBOUNDED|timeInterval) PRECEDING AND CURRENT ROW FROM TABLENAME
+
              +
              +

              Description

              + +
              + + + + + + + + + + + + + + + + +
              Table 5 Parameters

              Parameter

              +

              Description

              +

              PARTITION BY

              +

              Primary key of the specified group. Each group separately performs calculation.

              +

              ORDER BY

              +

              Processing time or event time as the timestamp for data.

              +

              ROWS

              +

              Count window.

              +

              RANGE

              +

              Time window.

              +
              +
              +

              Caveats

              +
              • All aggregates must be defined in the same window, that is, in the same partition, sort, and range.
              • Currently, only windows from PRECEDING (unbounded or bounded) to CURRENT ROW are supported. The range described by FOLLOWING is not supported.
              • ORDER BY must be specified for a single time attribute.
              +

              Example

              +
               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
              // Calculate the count and total number from syntax rules enabled to now (in proctime).
+insert into temp SELECT name,
+    count(amount) OVER (PARTITION BY name ORDER BY proctime RANGE UNBOUNDED preceding) as cnt1,
+    sum(amount) OVER (PARTITION BY name ORDER BY proctime RANGE UNBOUNDED preceding) as cnt2
+    FROM Orders;
+  
+// Calculate the count and total number of the recent four records (in proctime).
+insert into temp SELECT name,
+    count(amount) OVER (PARTITION BY name ORDER BY proctime ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) as cnt1,
+    sum(amount) OVER (PARTITION BY name ORDER BY proctime ROWS BETWEEN 4 PRECEDING AND CURRENT ROW) as cnt2
+    FROM Orders;
+
+// Calculate the count and total number last 60s (in eventtime). Process the events based on event time, which is the timeattr field in Orders.
+insert into temp SELECT name,
+    count(amount) OVER (PARTITION BY name ORDER BY timeattr RANGE BETWEEN INTERVAL '60' SECOND PRECEDING AND CURRENT ROW) as cnt1,
+    sum(amount) OVER (PARTITION BY name ORDER BY timeattr RANGE BETWEEN INTERVAL '60' SECOND PRECEDING AND CURRENT ROW) as cnt2
+    FROM Orders;
+
              +
              +
              +
              + + diff --git a/docs/dli/sqlreference/dli_08_15070.html b/docs/dli/sqlreference/dli_08_15070.html new file mode 100644 index 000000000..5f90a4845 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15070.html @@ -0,0 +1,367 @@ + + +

              Window Functions

              +

              Windowing Table-Valued Functions (Windowing TVFs)

              Windows are at the heart of processing infinite streams. Windows split the stream into "buckets" of finite size, over which we can apply computations.

              +

              Apache Flink provides several window table-valued functions (TVF) to divide the elements of your table into windows, including:

              +
              • Tumble Windows
              • Hop Windows
              • Cumulate Windows
              +

              Note that each element can logically belong to more than one window, depending on the windowing table-valued function you use. For example, HOP windowing creates overlapping windows wherein a single element can be assigned to multiple windows.

              +

              Windowing TVFs are Flink defined Polymorphic Table Functions (abbreviated PTF). PTF is part of the SQL 2016 standard, a special table-function, but can have a table as a parameter.

              +

              Windowing TVFs is a replacement of legacy Grouped Window Functions. Windowing TVFs is more SQL standard compliant and more powerful to support complex window-based computations, e.g. Window TopN, Window Join. However, Grouped Window Functions can only support Window Aggregation.

              +

              For more information, see Window Functions.

              +
              +

              Window Functions

              Apache Flink provides 3 built-in windowing TVFs: TUMBLE, HOP and CUMULATE.

              +

              The return value of windowing TVF is a new relation that includes all columns of original relation as well as additional 3 columns named "window_start", "window_end", "window_time" to indicate the assigned window.

              +

              In batch mode, the "window_time" field is an attribute of type TIMESTAMP or TIMESTAMP_LTZ based on input time field type. The "window_time" field can be used in subsequent time-based operations, e.g. another windowing TVF, or interval joins, over aggregations. The value of window_time always equal to window_end – 1 ms.

              +
              +

              TUMBLE

              • Function

                The TUMBLE function assigns each element to a window of specified window size. Tumbling windows have a fixed size and do not overlap.

                +

                For example, suppose you specify a tumbling window with a size of 5 minutes. In that case, Flink will evaluate the current window, and a new window started every five minutes.

                +
                Figure 1 Tumbling window
                +
              • Description

                The TUMBLE function assigns a window for each row of a relation based on a time attribute field. In streaming mode, the time attribute field must be either event or processing time attributes. In batch mode, the time attribute field of window table function must be an attribute of type TIMESTAMP or TIMESTAMP_LTZ.

                +

                The return value of TUMBLE is a new relation that includes all columns of original relation as well as additional 3 columns named "window_start", "window_end", "window_time" to indicate the assigned window. The original time attribute "timecol" will be a regular timestamp column after window TVF.

                +
                TUMBLE(TABLE data, DESCRIPTOR(timecol), size [, offset ])
                + +
                + + + + + + + + + + + + + + + + + + + + + +
                Table 1 TUMBLE function parameters

                Parameter

                +

                Mandatory

                +

                Description

                +

                data

                +

                Yes

                +

                A table parameter that can be any relation with a time attribute column.

                +

                timecol

                +

                Yes

                +

                A column descriptor indicating which time attributes column of data should be mapped to tumbling windows.

                +

                size

                +

                Yes

                +

                A duration specifying the width of the tumbling windows.

                +

                offset

                +

                No

                +

                Offset which window start would be shifted by.

                +
                +
                +
              • Example
                -- tables must have time attribute, e.g. `bidtime` in this table
+Flink SQL> desc Bid;
++-------------+------------------------+------+-----+--------+---------------------------------+
+|        name |                   type | null | key | extras |                       watermark |
++-------------+------------------------+------+-----+--------+---------------------------------+
+|     bidtime | TIMESTAMP(3) *ROWTIME* | true |     |        | `bidtime` - INTERVAL '1' SECOND |
+|       price |         DECIMAL(10, 2) | true |     |        |                                 |
+|        item |                 STRING | true |     |        |                                 |
++-------------+------------------------+------+-----+--------+---------------------------------+
+
+Flink SQL> SELECT * FROM Bid;
++------------------+-------+------+
+|          bidtime | price | item |
++------------------+-------+------+
+| 2020-04-15 08:05 |  4.00 | C    |
+| 2020-04-15 08:07 |  2.00 | A    |
+| 2020-04-15 08:09 |  5.00 | D    |
+| 2020-04-15 08:11 |  3.00 | B    |
+| 2020-04-15 08:13 |  1.00 | E    |
+| 2020-04-15 08:17 |  6.00 | F    |
++------------------+-------+------+
+
+Flink SQL> SELECT * FROM TABLE(
+   TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES));
+-- or with the named params
+-- note: the DATA param must be the first
+Flink SQL> SELECT * FROM TABLE(
+   TUMBLE(
+     DATA => TABLE Bid,
+     TIMECOL => DESCRIPTOR(bidtime),
+     SIZE => INTERVAL '10' MINUTES));
++------------------+-------+------+------------------+------------------+-------------------------+
+|          bidtime | price | item |     window_start |       window_end |            window_time  |
++------------------+-------+------+------------------+------------------+-------------------------+
+| 2020-04-15 08:05 |  4.00 | C    | 2020-04-15 08:00 | 2020-04-15 08:10 | 2020-04-15 08:09:59.999 |
+| 2020-04-15 08:07 |  2.00 | A    | 2020-04-15 08:00 | 2020-04-15 08:10 | 2020-04-15 08:09:59.999 |
+| 2020-04-15 08:09 |  5.00 | D    | 2020-04-15 08:00 | 2020-04-15 08:10 | 2020-04-15 08:09:59.999 |
+| 2020-04-15 08:11 |  3.00 | B    | 2020-04-15 08:10 | 2020-04-15 08:20 | 2020-04-15 08:19:59.999 |
+| 2020-04-15 08:13 |  1.00 | E    | 2020-04-15 08:10 | 2020-04-15 08:20 | 2020-04-15 08:19:59.999 |
+| 2020-04-15 08:17 |  6.00 | F    | 2020-04-15 08:10 | 2020-04-15 08:20 | 2020-04-15 08:19:59.999 |
++------------------+-------+------+------------------+------------------+-------------------------+
+
+-- apply aggregation on the tumbling windowed table
+Flink SQL> SELECT window_start, window_end, SUM(price)
+  FROM TABLE(
+    TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES))
+  GROUP BY window_start, window_end;
++------------------+------------------+-------+
+|     window_start |       window_end | price |
++------------------+------------------+-------+
+| 2020-04-15 08:00 | 2020-04-15 08:10 | 11.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 | 10.00 |
++------------------+------------------+-------+
                +
              +
              +

              HOP

              • Function

                The HOP function assigns elements to windows of fixed length. Like a TUMBLE windowing function, the size of the windows is configured by the window size parameter. An additional window slide parameter controls how frequently a hopping window is started. Hence, hopping windows can be overlapping if the slide is smaller than the window size. In this case, elements are assigned to multiple windows.

                +

                For example, you could have windows of size 10 minutes that slides by 5 minutes. With this, you get every 5 minutes a window that contains the events that arrived during the last 10 minutes, as depicted by the following figure.

                +
                Figure 2 Hopping window
                +
              +
              +
              • Description

                The HOP function assigns windows that cover rows within the interval of size and shifting every slide based on a time attribute field. In streaming mode, the time attribute field must be either event or processing time attributes. In batch mode, the time attribute field of window table function must be an attribute of type TIMESTAMP or TIMESTAMP_LTZ.

                +

                The return value of HOP is a new relation that includes all columns of original relation as well as additional 3 columns named "window_start", "window_end", "window_time" to indicate the assigned window. The original time attribute "timecol" will be a regular timestamp column after window TVF.

                +
                HOP(TABLE data, DESCRIPTOR(timecol), slide, size [, offset ])
                +

                + +
                + + + + + + + + + + + + + + + + + + + + + + + + + +
                Table 2 HOP function parameters

                Parameter

                +

                Mandatory

                +

                Description

                +

                data

                +

                Yes

                +

                A table parameter that can be any relation with a time attribute column.

                +

                timecol

                +

                Yes

                +

                A column descriptor indicating which time attributes column of data should be mapped to tumbling windows.

                +

                slide

                +

                Yes

                +

                A duration specifying the duration between the start of sequential hopping windows.

                +

                size

                +

                Yes

                +

                A duration specifying the width of the hopping windows.

                +

                offset

                +

                No

                +

                Offset which window start would be shifted by.

                +
                +
                +
              • Example
                > SELECT * FROM TABLE(
+    HOP(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '5' MINUTES, INTERVAL '10' MINUTES));
+-- or with the named params
+-- note: the DATA param must be the first
+> SELECT * FROM TABLE(
+    HOP(
+      DATA => TABLE Bid,
+      TIMECOL => DESCRIPTOR(bidtime),
+      SLIDE => INTERVAL '5' MINUTES,
+      SIZE => INTERVAL '10' MINUTES));
++------------------+-------+------+------------------+------------------+-------------------------+
+|          bidtime | price | item |     window_start |       window_end |           window_time   |
++------------------+-------+------+------------------+------------------+-------------------------+
+| 2020-04-15 08:05 |  4.00 | C    | 2020-04-15 08:00 | 2020-04-15 08:10 | 2020-04-15 08:09:59.999 |
+| 2020-04-15 08:05 |  4.00 | C    | 2020-04-15 08:05 | 2020-04-15 08:15 | 2020-04-15 08:14:59.999 |
+| 2020-04-15 08:07 |  2.00 | A    | 2020-04-15 08:00 | 2020-04-15 08:10 | 2020-04-15 08:09:59.999 |
+| 2020-04-15 08:07 |  2.00 | A    | 2020-04-15 08:05 | 2020-04-15 08:15 | 2020-04-15 08:14:59.999 |
+| 2020-04-15 08:09 |  5.00 | D    | 2020-04-15 08:00 | 2020-04-15 08:10 | 2020-04-15 08:09:59.999 |
+| 2020-04-15 08:09 |  5.00 | D    | 2020-04-15 08:05 | 2020-04-15 08:15 | 2020-04-15 08:14:59.999 |
+| 2020-04-15 08:11 |  3.00 | B    | 2020-04-15 08:05 | 2020-04-15 08:15 | 2020-04-15 08:14:59.999 |
+| 2020-04-15 08:11 |  3.00 | B    | 2020-04-15 08:10 | 2020-04-15 08:20 | 2020-04-15 08:19:59.999 |
+| 2020-04-15 08:13 |  1.00 | E    | 2020-04-15 08:05 | 2020-04-15 08:15 | 2020-04-15 08:14:59.999 |
+| 2020-04-15 08:13 |  1.00 | E    | 2020-04-15 08:10 | 2020-04-15 08:20 | 2020-04-15 08:19:59.999 |
+| 2020-04-15 08:17 |  6.00 | F    | 2020-04-15 08:10 | 2020-04-15 08:20 | 2020-04-15 08:19:59.999 |
+| 2020-04-15 08:17 |  6.00 | F    | 2020-04-15 08:15 | 2020-04-15 08:25 | 2020-04-15 08:24:59.999 |
++------------------+-------+------+------------------+------------------+-------------------------+
+
+-- apply aggregation on the hopping windowed table
+> SELECT window_start, window_end, SUM(price)
+  FROM TABLE(
+    HOP(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '5' MINUTES, INTERVAL '10' MINUTES))
+  GROUP BY window_start, window_end;
++------------------+------------------+-------+
+|     window_start |       window_end | price |
++------------------+------------------+-------+
+| 2020-04-15 08:00 | 2020-04-15 08:10 | 11.00 |
+| 2020-04-15 08:05 | 2020-04-15 08:15 | 15.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 | 10.00 |
+| 2020-04-15 08:15 | 2020-04-15 08:25 |  6.00 |
++------------------+------------------+-------+
                +
              +

              CUMULATE

              • Function

                Cumulating windows are very useful in some scenarios, such as tumbling windows with early firing in a fixed window interval. For example, a daily dashboard draws cumulative UVs from 00:00 to every minute, the UV at 10:00 represents the total number of UV from 00:00 to 10:00. This can be easily and efficiently implemented by CUMULATE windowing.

                +

                The CUMULATE function assigns elements to windows that cover rows within an initial interval of step size and expand to one more step size (keep window start fixed) every step until the max window size. You can think CUMULATE function as applying TUMBLE windowing with max window size first, and split each tumbling windows into several windows with same window start and window ends of step-size difference. So cumulating windows do overlap and do not have a fixed size.

                +

                For example, you could have a cumulating window for 1 hour step and 1 day max size, and you will get windows: [00:00, 01:00), [00:00, 02:00), [00:00, 03:00), …, [00:00, 24:00) for every day.

                +
                Figure 3 Cumulating window
                +
              +
              +
              • Description

                The CUMULATE functions assigns windows based on a time attribute column. In streaming mode, the time attribute field must be either event or processing time attributes. In batch mode, the time attribute field of window table function must be an attribute of type TIMESTAMP or TIMESTAMP_LTZ.

                +

                The return value of CUMULATE is a new relation that includes all columns of original relation as well as additional 3 columns named "window_start", "window_end", "window_time" to indicate the assigned window. The original time attribute "timecol" will be a regular timestamp column after window TVF.

                +
                CUMULATE(TABLE data, DESCRIPTOR(timecol), step, size)
                +

                + +
                + + + + + + + + + + + + + + + + + + + + + + + + + +
                Table 3 CUMULATE function parameters

                Parameter

                +

                Mandatory

                +

                Description

                +

                data

                +

                Yes

                +

                A table parameter that can be any relation with a time attribute column.

                +

                timecol

                +

                Yes

                +

                A column descriptor indicating which time attributes column of data should be mapped to cumulating windows.

                +

                step

                +

                Yes

                +

                A duration specifying the increased window size between the end of sequential cumulating windows.

                +

                size

                +

                Yes

                +

                A duration specifying the width of the cumulating windows.

                +

                offset

                +

                No

                +

                Offset which window start would be shifted by.

                +
                +
                +
              • Example
                > SELECT * FROM TABLE(
+    CUMULATE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '2' MINUTES, INTERVAL '10' MINUTES));
+-- or with the named params
+-- note: the DATA param must be the first
+> SELECT * FROM TABLE(
+    CUMULATE(
+      DATA => TABLE Bid,
+      TIMECOL => DESCRIPTOR(bidtime),
+      STEP => INTERVAL '2' MINUTES,
+      SIZE => INTERVAL '10' MINUTES));
++------------------+-------+------+------------------+------------------+-------------------------+
+|          bidtime | price | item |     window_start |       window_end |            window_time  |
++------------------+-------+------+------------------+------------------+-------------------------+
+| 2020-04-15 08:05 |  4.00 | C    | 2020-04-15 08:00 | 2020-04-15 08:06 | 2020-04-15 08:05:59.999 |
+| 2020-04-15 08:05 |  4.00 | C    | 2020-04-15 08:00 | 2020-04-15 08:08 | 2020-04-15 08:07:59.999 |
+| 2020-04-15 08:05 |  4.00 | C    | 2020-04-15 08:00 | 2020-04-15 08:10 | 2020-04-15 08:09:59.999 |
+| 2020-04-15 08:07 |  2.00 | A    | 2020-04-15 08:00 | 2020-04-15 08:08 | 2020-04-15 08:07:59.999 |
+| 2020-04-15 08:07 |  2.00 | A    | 2020-04-15 08:00 | 2020-04-15 08:10 | 2020-04-15 08:09:59.999 |
+| 2020-04-15 08:09 |  5.00 | D    | 2020-04-15 08:00 | 2020-04-15 08:10 | 2020-04-15 08:09:59.999 |
+| 2020-04-15 08:11 |  3.00 | B    | 2020-04-15 08:10 | 2020-04-15 08:12 | 2020-04-15 08:11:59.999 |
+| 2020-04-15 08:11 |  3.00 | B    | 2020-04-15 08:10 | 2020-04-15 08:14 | 2020-04-15 08:13:59.999 |
+| 2020-04-15 08:11 |  3.00 | B    | 2020-04-15 08:10 | 2020-04-15 08:16 | 2020-04-15 08:15:59.999 |
+| 2020-04-15 08:11 |  3.00 | B    | 2020-04-15 08:10 | 2020-04-15 08:18 | 2020-04-15 08:17:59.999 |
+| 2020-04-15 08:11 |  3.00 | B    | 2020-04-15 08:10 | 2020-04-15 08:20 | 2020-04-15 08:19:59.999 |
+| 2020-04-15 08:13 |  1.00 | E    | 2020-04-15 08:10 | 2020-04-15 08:14 | 2020-04-15 08:13:59.999 |
+| 2020-04-15 08:13 |  1.00 | E    | 2020-04-15 08:10 | 2020-04-15 08:16 | 2020-04-15 08:15:59.999 |
+| 2020-04-15 08:13 |  1.00 | E    | 2020-04-15 08:10 | 2020-04-15 08:18 | 2020-04-15 08:17:59.999 |
+| 2020-04-15 08:13 |  1.00 | E    | 2020-04-15 08:10 | 2020-04-15 08:20 | 2020-04-15 08:19:59.999 |
+| 2020-04-15 08:17 |  6.00 | F    | 2020-04-15 08:10 | 2020-04-15 08:18 | 2020-04-15 08:17:59.999 |
+| 2020-04-15 08:17 |  6.00 | F    | 2020-04-15 08:10 | 2020-04-15 08:20 | 2020-04-15 08:19:59.999 |
++------------------+-------+------+------------------+------------------+-------------------------+
+
+-- apply aggregation on the cumulating windowed table
+> SELECT window_start, window_end, SUM(price)
+  FROM TABLE(
+    CUMULATE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '2' MINUTES, INTERVAL '10' MINUTES))
+  GROUP BY window_start, window_end;
++------------------+------------------+-------+
+|     window_start |       window_end | price |
++------------------+------------------+-------+
+| 2020-04-15 08:00 | 2020-04-15 08:06 |  4.00 |
+| 2020-04-15 08:00 | 2020-04-15 08:08 |  6.00 |
+| 2020-04-15 08:00 | 2020-04-15 08:10 | 11.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:12 |  3.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:14 |  4.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:16 |  4.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:18 | 10.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 | 10.00 |
++------------------+------------------+-------+
                +
              +

              Window Offset

              Offset is an optional parameter which could be used to change the window assignment. It could be positive duration and negative duration. Default values for window offset is 0. The same record maybe assigned to the different window if set different offset value. For example, which window would be assigned to for a record with timestamp 2021-06-30 00:00:04 for a Tumble window with 10 MINUTE as size?

              +
              • If offset value is -16 MINUTE, the record assigns to window [2021-06-29 23:54:00, 2021-06-30 00:04:00).
              • If offset value is -6 MINUTE, the record assigns to window [2021-06-29 23:54:00, 2021-06-30 00:04:00).
              • If offset is -4 MINUTE, the record assigns to window [2021-06-29 23:56:00, 2021-06-30 00:06:00).
              • If offset is 0, the record assigns to window [2021-06-30 00:00:00, 2021-06-30 00:10:00).
              • If offset value is 4 MINUTE, the record assigns to window [2021-06-29 23:54:00, 2021-06-30 00:04:00).
              • If offset is 6 MINUTE, the record assigns to window [2021-06-29 23:56:00, 2021-06-30 00:06:00).
              • If offset is 16 MINUTE, the record assigns to window [2021-06-29 23:56:00, 2021-06-30 00:06:00). We could find that, some windows offset parameters may have same effect on the assignment of windows. In the above case, -16 MINUTE, -6 MINUTE and 4 MINUTE have same effect for a Tumble window with 10 MINUTE as size.
              +

              The effect of window offset is just for updating window assignment, it has no effect on Watermark.

              +
              +
              -- NOTE: Currently Flink doesn't support evaluating individual window table-valued function,
+--  window table-valued function should be used with aggregate operation,
+--  this example is just used for explaining the syntax and the data produced by table-valued function.
+Flink SQL> SELECT * FROM TABLE(
+   TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES, INTERVAL '1' MINUTES));
+-- or with the named params
+-- note: the DATA param must be the first
+Flink SQL> SELECT * FROM TABLE(
+   TUMBLE(
+     DATA => TABLE Bid,
+     TIMECOL => DESCRIPTOR(bidtime),
+     SIZE => INTERVAL '10' MINUTES,
+     OFFSET => INTERVAL '1' MINUTES));
++------------------+-------+------+------------------+------------------+-------------------------+
+|          bidtime | price | item |     window_start |       window_end |            window_time  |
++------------------+-------+------+------------------+------------------+-------------------------+
+| 2020-04-15 08:05 |  4.00 | C    | 2020-04-15 08:01 | 2020-04-15 08:11 | 2020-04-15 08:10:59.999 |
+| 2020-04-15 08:07 |  2.00 | A    | 2020-04-15 08:01 | 2020-04-15 08:11 | 2020-04-15 08:10:59.999 |
+| 2020-04-15 08:09 |  5.00 | D    | 2020-04-15 08:01 | 2020-04-15 08:11 | 2020-04-15 08:10:59.999 |
+| 2020-04-15 08:11 |  3.00 | B    | 2020-04-15 08:11 | 2020-04-15 08:21 | 2020-04-15 08:20:59.999 |
+| 2020-04-15 08:13 |  1.00 | E    | 2020-04-15 08:11 | 2020-04-15 08:21 | 2020-04-15 08:20:59.999 |
+| 2020-04-15 08:17 |  6.00 | F    | 2020-04-15 08:11 | 2020-04-15 08:21 | 2020-04-15 08:20:59.999 |
++------------------+-------+------+------------------+------------------+-------------------------+
+
+-- apply aggregation on the tumbling windowed table
+Flink SQL> SELECT window_start, window_end, SUM(price)
+  FROM TABLE(
+    TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES, INTERVAL '1' MINUTES))
+  GROUP BY window_start, window_end;
++------------------+------------------+-------+
+|     window_start |       window_end | price |
++------------------+------------------+-------+
+| 2020-04-15 08:01 | 2020-04-15 08:11 | 11.00 |
+| 2020-04-15 08:11 | 2020-04-15 08:21 | 10.00 |
++------------------+------------------+-------+
              +
              +
              +
              +
              +
              Parent topic: Window
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15071.html b/docs/dli/sqlreference/dli_08_15071.html new file mode 100644 index 000000000..2e77e5bc7 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15071.html @@ -0,0 +1,144 @@ + + +

              Window Aggregation

              +

              Window TVF Aggregation

              Window aggregations are defined in the GROUP BY clause contains "window_start" and "window_end" columns of the relation applied Windowing TVF. Just like queries with regular GROUP BY clauses, queries with a group by window aggregation will compute a single result row per group. Unlike other aggregations on continuous tables, window aggregation do not emit intermediate results but only a final result, the total aggregation at the end of the window. Moreover, window aggregations purge all intermediate state when no longer needed.

              +

              For more information, see Window Aggregation.

              +

              The start and end timestamps of group windows can be selected with the grouped window_start and window_end columns.

              +
              +
              • Windowing TVFs

                Flink supports TUMBLE, HOP and CUMULATE types of window aggregations.

                +
                • In streaming mode, the time attribute field of a window table-valued function must be on either event or processing time attributes. See Windowing TVF for more windowing functions information.
                • In batch mode, the time attribute field of a window table-valued function must be an attribute of type TIMESTAMP or TIMESTAMP_LTZ.
                +
                -- tables must have time attribute, e.g. `bidtime` in this table
+Flink SQL> desc Bid;
++-------------+------------------------+------+-----+--------+---------------------------------+
+|        name |                   type | null | key | extras |                       watermark |
++-------------+------------------------+------+-----+--------+---------------------------------+
+|     bidtime | TIMESTAMP(3) *ROWTIME* | true |     |        | `bidtime` - INTERVAL '1' SECOND |
+|       price |         DECIMAL(10, 2) | true |     |        |                                 |
+|        item |                 STRING | true |     |        |                                 |
+| supplier_id |                 STRING | true |     |        |                                 |
++-------------+------------------------+------+-----+--------+---------------------------------+
+
+Flink SQL> SELECT * FROM Bid;
++------------------+-------+------+-------------+
+|          bidtime | price | item | supplier_id |
++------------------+-------+------+-------------+
+| 2020-04-15 08:05 | 4.00  | C    | supplier1   |
+| 2020-04-15 08:07 | 2.00  | A    | supplier1   |
+| 2020-04-15 08:09 | 5.00  | D    | supplier2   |
+| 2020-04-15 08:11 | 3.00  | B    | supplier2   |
+| 2020-04-15 08:13 | 1.00  | E    | supplier1   |
+| 2020-04-15 08:17 | 6.00  | F    | supplier2   |
++------------------+-------+------+-------------+
+
+-- tumbling window aggregation
+Flink SQL> SELECT window_start, window_end, SUM(price)
+  FROM TABLE(
+    TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES))
+  GROUP BY window_start, window_end;
++------------------+------------------+-------+
+|     window_start |       window_end | price |
++------------------+------------------+-------+
+| 2020-04-15 08:00 | 2020-04-15 08:10 | 11.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 | 10.00 |
++------------------+------------------+-------+
+
+-- hopping window aggregation
+Flink SQL> SELECT window_start, window_end, SUM(price)
+  FROM TABLE(
+    HOP(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '5' MINUTES, INTERVAL '10' MINUTES))
+  GROUP BY window_start, window_end;
++------------------+------------------+-------+
+|     window_start |       window_end | price |
++------------------+------------------+-------+
+| 2020-04-15 08:00 | 2020-04-15 08:10 | 11.00 |
+| 2020-04-15 08:05 | 2020-04-15 08:15 | 15.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 | 10.00 |
+| 2020-04-15 08:15 | 2020-04-15 08:25 | 6.00  |
++------------------+------------------+-------+
+
+-- cumulative window aggregation
+Flink SQL> SELECT window_start, window_end, SUM(price)
+  FROM TABLE(
+    CUMULATE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '2' MINUTES, INTERVAL '10' MINUTES))
+  GROUP BY window_start, window_end;
++------------------+------------------+-------+
+|     window_start |       window_end | price |
++------------------+------------------+-------+
+| 2020-04-15 08:00 | 2020-04-15 08:06 | 4.00  |
+| 2020-04-15 08:00 | 2020-04-15 08:08 | 6.00  |
+| 2020-04-15 08:00 | 2020-04-15 08:10 | 11.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:12 | 3.00  |
+| 2020-04-15 08:10 | 2020-04-15 08:14 | 4.00  |
+| 2020-04-15 08:10 | 2020-04-15 08:16 | 4.00  |
+| 2020-04-15 08:10 | 2020-04-15 08:18 | 10.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 | 10.00 |
                +
              +
              +
              • GROUPING SETS

                Window aggregations also support GROUPING SETS syntax. Grouping sets allow for more complex grouping operations than those describable by a standard GROUP BY. Rows are grouped separately by each specified grouping set and aggregates are computed for each group just as for simple GROUP BY clauses.

                +

                Window aggregations with GROUPING SETS require both the window_start and window_end columns have to be in the GROUP BY clause, but not in the GROUPING SETS clause.

                +
                Flink SQL> SELECT window_start, window_end, supplier_id, SUM(price) as price
+  FROM TABLE(
+    TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES))
+  GROUP BY window_start, window_end, GROUPING SETS ((supplier_id), ());
++------------------+------------------+-------------+-------+
+|     window_start |       window_end | supplier_id | price |
++------------------+------------------+-------------+-------+
+| 2020-04-15 08:00 | 2020-04-15 08:10 |      (NULL) | 11.00 |
+| 2020-04-15 08:00 | 2020-04-15 08:10 |   supplier2 |  5.00 |
+| 2020-04-15 08:00 | 2020-04-15 08:10 |   supplier1 |  6.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 |      (NULL) | 10.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 |   supplier2 |  9.00 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 |   supplier1 |  1.00 |
++------------------+------------------+-------------+-------+
                +

                Each sublist of GROUPING SETS may specify zero or more columns or expressions and is interpreted the same way as though used directly in the GROUP BY clause. An empty grouping set means that all rows are aggregated down to a single group, which is output even if no input rows were present.

                +

                References to the grouping columns or expressions are replaced by null values in result rows for grouping sets in which those columns do not appear. For example, () in GROUPING SETS ((supplier_id), ()) in the preceding example is an empty sublist, and the supplier_id column in the corresponding result data is filled with NULL.

                +
              • ROLLUP

                ROLLUP is a shorthand notation for specifying a common type of grouping set. It represents the given list of expressions and all prefixes of the list, including the empty list.

                +

                For example, ROLLUP (one,two) is equivalent to GROUPING SET((one,two),(one),()).

                +

                Window aggregations with ROLLUP requires both the window_start and window_end columns have to be in the GROUP BY clause, but not in the ROLLUP clause.

                +

                For example, the following query is equivalent to the one above.

                +
                SELECT window_start, window_end, supplier_id, SUM(price) as price
+FROM TABLE(
+    TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES))
+GROUP BY window_start, window_end, ROLLUP (supplier_id);
                +
              • CUBE

                CUBE is a shorthand notation for specifying a common type of grouping set. It represents the given list and all of its possible subsets - the power set.

                +

                Window aggregations with CUBE requires both the window_start and window_end columns have to be in the GROUP BY clause, but not in the CUBE clause.

                +

                For example, the following two queries are equivalent.

                +
                SELECT window_start, window_end, item, supplier_id, SUM(price) as price
+  FROM TABLE(
+    TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES))
+  GROUP BY window_start, window_end, CUBE (supplier_id, item);
+
+SELECT window_start, window_end, item, supplier_id, SUM(price) as price
+  FROM TABLE(
+    TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES))
+  GROUP BY window_start, window_end, GROUPING SETS (
+      (supplier_id, item),
+      (supplier_id      ),
+      (             item),
+      (                 )
+)
                +
              +
              • Cascading Window Aggregation

                The window_start and window_end columns are regular timestamp columns, not time attributes. Thus they can not be used as time attributes in subsequent time-based operations.

                +

                To propagate time attributes, you need to additionally add window_time column into GROUP BY clause. The window_time is the third column produced by Windowing Table-Valued Functions (Windowing TVFs) which is a time attribute of the assigned window. Adding window_time into GROUP BY clause makes window_time also to be group key that can be selected. Then following queries can use this column for subsequent time-based operations, such as cascading window aggregations and Window TopN.

                +

                The following shows a cascading window aggregation where the first window aggregation propagates the time attribute for the second window aggregation.

                +
                -- tumbling 5 minutes for each supplier_id
+CREATE VIEW window1 AS
+-- Note: The window start and window end fields of inner Window TVF are optional in the select clause. However, if they appear in the clause, they need to be aliased to prevent name conflicting with the window start and window end of the outer Window TVF.
+SELECT window_start as window_5mintumble_start, window_end as window_5mintumble_end, window_time as rowtime, SUM(price) as partial_price
+  FROM TABLE(
+    TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '5' MINUTES))
+  GROUP BY supplier_id, window_start, window_end, window_time;
+
+-- tumbling 10 minutes on the first window
+SELECT window_start, window_end, SUM(partial_price) as total_price
+  FROM TABLE(
+      TUMBLE(TABLE window1, DESCRIPTOR(rowtime), INTERVAL '10' MINUTES))
+  GROUP BY window_start, window_end;
                +
              +
              +
              +
              +
              Parent topic: Window
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15072.html b/docs/dli/sqlreference/dli_08_15072.html new file mode 100644 index 000000000..49377a99b --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15072.html @@ -0,0 +1,93 @@ + + +

              Window Top-N

              +

              Function

              Window Top-N is a special Top-N which returns the N smallest or largest values for each window and other partitioned keys.

              +

              Unlike regular Top-N on continuous tables, window Top-N does not emit intermediate results but only a final result, the total top N records at the end of the window. Moreover, window Top-N purges all intermediate state when no longer needed.

              +

              Window Top-N queries have better performance if users do not need results updated per record. Usually, Window Top-N is used with Windowing Table-Valued Functions (Windowing TVFs) directly. Besides, Window Top-N could be used with other operations based on Windowing Table-Valued Functions (Windowing TVFs), such as Window Aggregation, Window TopN and Window Join.

              +

              Window Top-N can be defined in the same syntax as regular Top-N, see Top-N documentation for more information. Besides that, Window Top-N requires the PARTITION BY clause contains window_start and window_end columns of the relation applied Windowing TVF or Window Aggregation. Otherwise, the optimizer will not be able to translate the query.

              +

              For more information, see Window Top-N.

              +
              +

              Syntax

              SELECT [column_list]
+FROM (
+   SELECT [column_list],
+     ROW_NUMBER() OVER (PARTITION BY window_start, window_end [, col_key1...]
+       ORDER BY col1 [asc|desc][, col2 [asc|desc]...]) AS rownum
+   FROM table_name) -- relation applied windowing TVF
+WHERE rownum <= N [AND conditions]
              +
              +

              Caveats

              Flink only supports Window Top-N follows after Windowing TVF with Tumble Windows, Hop Windows and Cumulate Windows.

              +
              +

              Example

              Window Top-N follows after Window Aggregation

              +

              The following example shows how to calculate Top 3 suppliers who have the highest sales for every tumbling 10 minutes window.

              +
              -- tables must have time attribute, e.g. `bidtime` in this table
+Flink SQL> desc Bid;
++-------------+------------------------+------+-----+--------+---------------------------------+
+|        name |                   type | null | key | extras |                       watermark |
++-------------+------------------------+------+-----+--------+---------------------------------+
+|     bidtime | TIMESTAMP(3) *ROWTIME* | true |     |        | `bidtime` - INTERVAL '1' SECOND |
+|       price |         DECIMAL(10, 2) | true |     |        |                                 |
+|        item |                 STRING | true |     |        |                                 |
+| supplier_id |                 STRING | true |     |        |                                 |
++-------------+------------------------+------+-----+--------+---------------------------------+
+
+Flink SQL> SELECT * FROM Bid;
++------------------+-------+------+-------------+
+|          bidtime | price | item | supplier_id |
++------------------+-------+------+-------------+
+| 2020-04-15 08:05 |  4.00 |    A |   supplier1 |
+| 2020-04-15 08:06 |  4.00 |    C |   supplier2 |
+| 2020-04-15 08:07 |  2.00 |    G |   supplier1 |
+| 2020-04-15 08:08 |  2.00 |    B |   supplier3 |
+| 2020-04-15 08:09 |  5.00 |    D |   supplier4 |
+| 2020-04-15 08:11 |  2.00 |    B |   supplier3 |
+| 2020-04-15 08:13 |  1.00 |    E |   supplier1 |
+| 2020-04-15 08:15 |  3.00 |    H |   supplier2 |
+| 2020-04-15 08:17 |  6.00 |    F |   supplier5 |
++------------------+-------+------+-------------+
+
+Flink SQL> SELECT *
+  FROM (
+    SELECT *, ROW_NUMBER() OVER (PARTITION BY window_start, window_end ORDER BY price DESC) as rownum
+    FROM (
+      SELECT window_start, window_end, supplier_id, SUM(price) as price, COUNT(*) as cnt
+      FROM TABLE(
+        TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES))
+      GROUP BY window_start, window_end, supplier_id
+    )
+  ) WHERE rownum <= 3;
++------------------+------------------+-------------+-------+-----+--------+
+|     window_start |       window_end | supplier_id | price | cnt | rownum |
++------------------+------------------+-------------+-------+-----+--------+
+| 2020-04-15 08:00 | 2020-04-15 08:10 |   supplier1 |  6.00 |   2 |      1 |
+| 2020-04-15 08:00 | 2020-04-15 08:10 |   supplier4 |  5.00 |   1 |      2 |
+| 2020-04-15 08:00 | 2020-04-15 08:10 |   supplier2 |  4.00 |   1 |      3 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 |   supplier5 |  6.00 |   1 |      1 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 |   supplier2 |  3.00 |   1 |      2 |
+| 2020-04-15 08:10 | 2020-04-15 08:20 |   supplier3 |  2.00 |   1 |      3 |
++------------------+------------------+-------------+-------+-----+--------+
              +
              +

              Window Top-N follows after Windowing TVF

              +

              The following example shows how to calculate Top 3 items which have the highest price for every tumbling 10 minutes window.

              +
              Flink SQL> SELECT *
+  FROM (
+    SELECT *, ROW_NUMBER() OVER (PARTITION BY window_start, window_end ORDER BY price DESC) as rownum
+    FROM TABLE(
+               TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES))
+  ) WHERE rownum <= 3;
++------------------+-------+------+-------------+------------------+------------------+--------+
+|          bidtime | price | item | supplier_id |     window_start |       window_end | rownum |
++------------------+-------+------+-------------+------------------+------------------+--------+
+| 2020-04-15 08:05 |  4.00 |    A |   supplier1 | 2020-04-15 08:00 | 2020-04-15 08:10 |      2 |
+| 2020-04-15 08:06 |  4.00 |    C |   supplier2 | 2020-04-15 08:00 | 2020-04-15 08:10 |      3 |
+| 2020-04-15 08:09 |  5.00 |    D |   supplier4 | 2020-04-15 08:00 | 2020-04-15 08:10 |      1 |
+| 2020-04-15 08:11 |  2.00 |    B |   supplier3 | 2020-04-15 08:10 | 2020-04-15 08:20 |      3 |
+| 2020-04-15 08:15 |  3.00 |    H |   supplier2 | 2020-04-15 08:10 | 2020-04-15 08:20 |      2 |
+| 2020-04-15 08:17 |  6.00 |    F |   supplier5 | 2020-04-15 08:10 | 2020-04-15 08:20 |      1 |
++------------------+-------+------+-------------+------------------+------------------+--------+
              +
              +
              +
              +
              Parent topic: Window
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15073.html b/docs/dli/sqlreference/dli_08_15073.html new file mode 100644 index 000000000..0f26b73de --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15073.html @@ -0,0 +1,65 @@ + + +

              Window Deduplication

              +

              Function

              Window Deduplication is a special Deduplication which removes rows that duplicate over a set of columns, keeping the first one or the last one for each window and partitioned keys.

              +

              For streaming queries, unlike regular Deduplicate on continuous tables, Window Deduplication does not emit intermediate results but only a final result at the end of the window. Moreover, window Deduplication purges all intermediate state when no longer needed. Therefore, Window Deduplication queries have better performance if users do not need results updated per record. Usually, Window Deduplication is used with Windowing TVF directly. Besides, Window Deduplication could be used with other operations based on Windowing TVF, such as Window Aggregation, Window TopN and Window Join.

              +

              Window Top-N can be defined in the same syntax as regular Top-N, see Top-N documentation for more information. Besides that, Window Deduplication requires the PARTITION BY clause contains window_start and window_end columns of the relation. Otherwise, the optimizer will not be able to translate the query.

              +

              Flink uses ROW_NUMBER() to remove duplicates, just like the way of Window Top-N query. In theory, Window Deduplication is a special case of Window Top-N in which the N is one and order by the processing time or event time.

              +

              For more information, see Window Deduplication.

              +
              +

              Syntax

              SELECT [column_list]
+FROM (
+   SELECT [column_list],
+     ROW_NUMBER() OVER (PARTITION BY window_start, window_end [, col_key1...]
+       ORDER BY time_attr [asc|desc]) AS rownum
+   FROM table_name) -- relation applied windowing TVF
+WHERE (rownum = 1 | rownum <=1 | rownum < 2) [AND conditions]
              +
              +

              Parameter description:

              +
              • ROW_NUMBER(): Assigns an unique, sequential number to each row, starting with one.
              • PARTITION BY window_start, window_end [, col_key1...]: Specifies the partition columns which contain window_start, window_end and other partition keys.
              • ORDER BY time_attr [asc|desc]: Specifies the ordering column, it must be a time attribute. Currently Flink supports processing time attribute and event time attribute. Ordering by ASC means keeping the first row, ordering by DESC means keeping the last row.
              • WHERE (rownum = 1 | rownum <=1 | rownum < 2): The rownum = 1 | rownum <=1 | rownum < 2 is required for the optimizer to recognize the query could be translated to Window Deduplication.
              +

              Caveats

              • Flink can only perform window deduplication on window table value functions that are based on tumble, hop, or cumulate windows.
              • Window deduplication is only supported when sorting based on the event time attribute.
              +
              +

              Example

              The following example shows how to keep last record for every 10 minutes tumbling window.

              +
              -- tables must have time attribute, e.g. `bidtime` in this table
+Flink SQL> DESC Bid;
++-------------+------------------------+------+-----+--------+---------------------------------+
+|        name |                   type | null | key | extras |                       watermark |
++-------------+------------------------+------+-----+--------+---------------------------------+
+|     bidtime | TIMESTAMP(3) *ROWTIME* | true |     |        | `bidtime` - INTERVAL '1' SECOND |
+|       price |         DECIMAL(10, 2) | true |     |        |                                 |
+|        item |                 STRING | true |     |        |                                 |
++-------------+------------------------+------+-----+--------+---------------------------------+
+
+Flink SQL> SELECT * FROM Bid;
++------------------+-------+------+
+|          bidtime | price | item |
++------------------+-------+------+
+| 2020-04-15 08:05 |  4.00 | C    |
+| 2020-04-15 08:07 |  2.00 | A    |
+| 2020-04-15 08:09 |  5.00 | D    |
+| 2020-04-15 08:11 |  3.00 | B    |
+| 2020-04-15 08:13 |  1.00 | E    |
+| 2020-04-15 08:17 |  6.00 | F    |
++------------------+-------+------+
+
+Flink SQL> SELECT *
+  FROM (
+    SELECT bidtime, price, item, supplier_id, window_start, window_end, 
+      ROW_NUMBER() OVER (PARTITION BY window_start, window_end ORDER BY bidtime DESC) AS rownum
+    FROM TABLE(
+               TUMBLE(TABLE Bid, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES))
+  ) WHERE rownum <= 1;
++------------------+-------+------+-------------+------------------+------------------+--------+
+|          bidtime | price | item | supplier_id |     window_start |       window_end | rownum |
++------------------+-------+------+-------------+------------------+------------------+--------+
+| 2020-04-15 08:09 |  5.00 |    D |   supplier4 | 2020-04-15 08:00 | 2020-04-15 08:10 |      1 |
+| 2020-04-15 08:17 |  6.00 |    F |   supplier5 | 2020-04-15 08:10 | 2020-04-15 08:20 |      1 |
++------------------+-------+------+-------------+------------------+------------------+--------+
              +
              +
              +
              +
              +
              Parent topic: Window
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15074.html b/docs/dli/sqlreference/dli_08_15074.html new file mode 100644 index 000000000..62b2d891f --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15074.html @@ -0,0 +1,134 @@ + + +

              Window Join

              +

              A window join adds the dimension of time into the join criteria themselves. By doing so, the window join joins the elements of two streams that share a common key and are in the same window. The semantic of window join is same to the DataStream window join.

              +

              For streaming queries, unlike other joins on continuous tables, window join does not emit intermediate results but only emits final results at the end of the window. Moreover, window join purge all intermediate state when no longer needed. Usually, Window Join is used with Windowing TVF. Besides, Window Join could follow after other operations based on Windowing TVF, such as Window Aggregation, Window TopN and Window Join. Currently, Window Join requires the join on condition contains window starts equality of input tables and window ends equality of input tables. Window Join supports INNER/LEFT/RIGHT/FULL OUTER/ANTI/SEMI JOIN.

              +

              For more information, see Window Join.

              +

              Caveats

              • Currently, the window join requires the join on condition contains window starts equality of input tables and window ends equality of input tables.
              • Currently, the windowing TVFs must be the same of left and right inputs.
              • Currently, if Window Join follows after Windowing TVF, the Windowing TVF has to be with Tumble Windows, Hop Windows or Cumulate Windows instead of Session windows.
              +
              +

              INNER/LEFT/RIGHT/FULL OUTER

              The syntax of INNER/LEFT/RIGHT/FULL OUTER WINDOW JOIN are very similar with each other, we only give an example for FULL OUTER JOIN here. When performing a window join, all elements with a common key and a common tumbling window are joined together. We only give an example for a Window Join which works on a Tumble Window TVF. By scoping the region of time for the join into fixed five-minute intervals, we chopped our datasets into two distinct windows of time: [12:00, 12:05) and [12:05, 12:10). The L2 and R2 rows could not join together because they fell into separate windows.

              +

              Syntax

              +
              SELECT ...
+FROM L [LEFT|RIGHT|FULL OUTER] JOIN R -- L and R are relations applied windowing TVF
+ON L.window_start = R.window_start AND L.window_end = R.window_end AND ...
              +
              +

              Example

              +

              When performing a window join, all elements with a common key and a common tumbling window are joined together. We only give an example for a Window Join which works on a Tumble Window TVF. By scoping the region of time for the join into fixed five-minute intervals, we chopped our datasets into two distinct windows of time: [12:00, 12:05) and [12:05, 12:10). The L2 and R2 rows could not join together because they fell into separate windows.

              +
              Flink SQL> desc LeftTable;
++----------+------------------------+------+-----+--------+----------------------------------+
+|     name |                   type | null | key | extras |                        watermark |
++----------+------------------------+------+-----+--------+----------------------------------+
+| row_time | TIMESTAMP(3) *ROWTIME* | true |     |        | `row_time` - INTERVAL '1' SECOND |
+|      num |                    INT | true |     |        |                                  |
+|       id |                 STRING | true |     |        |                                  |
++----------+------------------------+------+-----+--------+----------------------------------+
+
+Flink SQL> SELECT * FROM LeftTable;
++------------------+-----+----+
+|         row_time | num | id |
++------------------+-----+----+
+| 2020-04-15 12:02 |   1 | L1 |
+| 2020-04-15 12:06 |   2 | L2 |
+| 2020-04-15 12:03 |   3 | L3 |
++------------------+-----+----+
+
+Flink SQL> desc RightTable;
++----------+------------------------+------+-----+--------+----------------------------------+
+|     name |                   type | null | key | extras |                        watermark |
++----------+------------------------+------+-----+--------+----------------------------------+
+| row_time | TIMESTAMP(3) *ROWTIME* | true |     |        | `row_time` - INTERVAL '1' SECOND |
+|      num |                    INT | true |     |        |                                  |
+|       id |                 STRING | true |     |        |                                  |
++----------+------------------------+------+-----+--------+----------------------------------+
+
+Flink SQL> SELECT * FROM RightTable;
++------------------+-----+----+
+|         row_time | num | id |
++------------------+-----+----+
+| 2020-04-15 12:01 |   2 | R2 |
+| 2020-04-15 12:04 |   3 | R3 |
+| 2020-04-15 12:05 |   4 | R4 |
++------------------+-----+----+
+
+Flink SQL> SELECT L.num as L_Num, L.id as L_Id, R.num as R_Num, R.id as R_Id,
+           COALESCE(L.window_start, R.window_start) as window_start,
+           COALESCE(L.window_end, R.window_end) as window_end
+           FROM (
+               SELECT * FROM TABLE(TUMBLE(TABLE LeftTable, DESCRIPTOR(row_time), INTERVAL '5' MINUTES))
+           ) L
+           FULL JOIN (
+               SELECT * FROM TABLE(TUMBLE(TABLE RightTable, DESCRIPTOR(row_time), INTERVAL '5' MINUTES))
+           ) R
+           ON L.num = R.num AND L.window_start = R.window_start AND L.window_end = R.window_end;
++-------+------+-------+------+------------------+------------------+
+| L_Num | L_Id | R_Num | R_Id |     window_start |       window_end |
++-------+------+-------+------+------------------+------------------+
+|     1 |   L1 |  null | null | 2020-04-15 12:00 | 2020-04-15 12:05 |
+|  null | null |     2 |   R2 | 2020-04-15 12:00 | 2020-04-15 12:05 |
+|     3 |   L3 |     3 |   R3 | 2020-04-15 12:00 | 2020-04-15 12:05 |
+|     2 |   L2 |  null | null | 2020-04-15 12:05 | 2020-04-15 12:10 |
+|  null | null |     4 |   R4 | 2020-04-15 12:05 | 2020-04-15 12:10 |
++-------+------+-------+------+------------------+------------------+
              +

              SEMI

              Semi Window Joins returns a row from one left record if there is at least one matching row on the right side within the common window.

              +
              Flink SQL> SELECT *
+           FROM (
+               SELECT * FROM TABLE(TUMBLE(TABLE LeftTable, DESCRIPTOR(row_time), INTERVAL '5' MINUTES))
+           ) L WHERE L.num IN (
+             SELECT num FROM (   
+               SELECT * FROM TABLE(TUMBLE(TABLE RightTable, DESCRIPTOR(row_time), INTERVAL '5' MINUTES))
+             ) R WHERE L.window_start = R.window_start AND L.window_end = R.window_end);
++------------------+-----+----+------------------+------------------+-------------------------+
+|         row_time | num | id |     window_start |       window_end |            window_time  |
++------------------+-----+----+------------------+------------------+-------------------------+
+| 2020-04-15 12:03 |   3 | L3 | 2020-04-15 12:00 | 2020-04-15 12:05 | 2020-04-15 12:04:59.999 |
++------------------+-----+----+------------------+------------------+-------------------------+
+
+Flink SQL> SELECT *
+           FROM (
+               SELECT * FROM TABLE(TUMBLE(TABLE LeftTable, DESCRIPTOR(row_time), INTERVAL '5' MINUTES))
+           ) L WHERE EXISTS (
+             SELECT * FROM (
+               SELECT * FROM TABLE(TUMBLE(TABLE RightTable, DESCRIPTOR(row_time), INTERVAL '5' MINUTES))
+             ) R WHERE L.num = R.num AND L.window_start = R.window_start AND L.window_end = R.window_end);
++------------------+-----+----+------------------+------------------+-------------------------+
+|         row_time | num | id |     window_start |       window_end |            window_time  |
++------------------+-----+----+------------------+------------------+-------------------------+
+| 2020-04-15 12:03 |   3 | L3 | 2020-04-15 12:00 | 2020-04-15 12:05 | 2020-04-15 12:04:59.999 |
++------------------+-----+----+------------------+------------------+-------------------------+
              +
              +

              ANTI

              Anti Window Joins are the obverse of the Inner Window Join: they contain all of the unjoined rows within each common window.

              +
              Flink SQL> SELECT *
+           FROM (
+               SELECT * FROM TABLE(TUMBLE(TABLE LeftTable, DESCRIPTOR(row_time), INTERVAL '5' MINUTES))
+           ) L WHERE L.num NOT IN (
+             SELECT num FROM (   
+               SELECT * FROM TABLE(TUMBLE(TABLE RightTable, DESCRIPTOR(row_time), INTERVAL '5' MINUTES))
+             ) R WHERE L.window_start = R.window_start AND L.window_end = R.window_end);
++------------------+-----+----+------------------+------------------+-------------------------+
+|         row_time | num | id |     window_start |       window_end |            window_time  |
++------------------+-----+----+------------------+------------------+-------------------------+
+| 2020-04-15 12:02 |   1 | L1 | 2020-04-15 12:00 | 2020-04-15 12:05 | 2020-04-15 12:04:59.999 |
+| 2020-04-15 12:06 |   2 | L2 | 2020-04-15 12:05 | 2020-04-15 12:10 | 2020-04-15 12:09:59.999 |
++------------------+-----+----+------------------+------------------+-------------------------+
+
+Flink SQL> SELECT *
+           FROM (
+               SELECT * FROM TABLE(TUMBLE(TABLE LeftTable, DESCRIPTOR(row_time), INTERVAL '5' MINUTES))
+           ) L WHERE NOT EXISTS (
+             SELECT * FROM (
+               SELECT * FROM TABLE(TUMBLE(TABLE RightTable, DESCRIPTOR(row_time), INTERVAL '5' MINUTES))
+             ) R WHERE L.num = R.num AND L.window_start = R.window_start AND L.window_end = R.window_end);
++------------------+-----+----+------------------+------------------+-------------------------+
+|         row_time | num | id |     window_start |       window_end |            window_time  |
++------------------+-----+----+------------------+------------------+-------------------------+
+| 2020-04-15 12:02 |   1 | L1 | 2020-04-15 12:00 | 2020-04-15 12:05 | 2020-04-15 12:04:59.999 |
+| 2020-04-15 12:06 |   2 | L2 | 2020-04-15 12:05 | 2020-04-15 12:10 | 2020-04-15 12:09:59.999 |
++------------------+-----+----+------------------+------------------+-------------------------+
              +
              +
              +
              +
              +
              Parent topic: Window
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15075.html b/docs/dli/sqlreference/dli_08_15075.html new file mode 100644 index 000000000..9154fde16 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15075.html @@ -0,0 +1,74 @@ + + +

              Group Aggregation

              +

              An aggregate function computes a single result from multiple input rows. For example, there are aggregates to compute the COUNT, SUM, AVG (average), MAX (maximum) and MIN (minimum) over a set of rows.

              +

              For streaming queries, the required state for computing the query result might grow infinitely. State size depends on the number of groups and the number and type of aggregation functions. For example MIN/MAX are heavy on state size while COUNT is cheap. You can provide a query configuration with an appropriate state time-to-live (TTL) to prevent excessive state size. Note that this might affect the correctness of the query result.

              +

              For more information, see Group Aggregation.

              +

              DISTINCT Aggregation

              Distinct aggregates remove duplicate values before applying an aggregation function. The following example counts the number of distinct order_ids instead of the total number of rows in the Orders table.

              +
              SELECT COUNT(DISTINCT order_id) FROM Orders
              +
              +

              GROUPING SETS

              Grouping sets allow for more complex grouping operations than those describable by a standard GROUP BY. Rows are grouped separately by each specified grouping set and aggregates are computed for each group just as for simple GROUP BY clauses.

              +

              Each sublist of GROUPING SETS may specify zero or more columns or expressions and is interpreted the same way as though used directly in the GROUP BY clause. An empty grouping set means that all rows are aggregated down to a single group, which is output even if no input rows were present.

              +

              References to the grouping columns or expressions are replaced by null values in result rows for grouping sets in which those columns do not appear.

              +
              SELECT supplier_id, rating, COUNT(*) AS total
+FROM (VALUES
+    ('supplier1', 'product1', 4),
+    ('supplier1', 'product2', 3),
+    ('supplier2', 'product3', 3),
+    ('supplier2', 'product4', 4))
+AS Products(supplier_id, product_id, rating)
+GROUP BY GROUPING SETS ((supplier_id, rating), (supplier_id), ())
              +
              +

              ROLLUP

              ROLLUP is a shorthand notation for specifying a common type of grouping set. It represents the given list of expressions and all prefixes of the list, including the empty list.

              +
              SELECT supplier_id, rating, COUNT(*)
+FROM (VALUES
+    ('supplier1', 'product1', 4),
+    ('supplier1', 'product2', 3),
+    ('supplier2', 'product3', 3),
+    ('supplier2', 'product4', 4))
+AS Products(supplier_id, product_id, rating)
+GROUP BY ROLLUP (supplier_id, rating)
              +
              +

              CUBE

              CUBE is a shorthand notation for specifying a common type of grouping set. It represents the given list and all of its possible subsets - the power set.

              +

              For example, the following two queries are equivalent.

              +
              SELECT supplier_id, rating, product_id, COUNT(*)
+FROM (VALUES
+    ('supplier1', 'product1', 4),
+    ('supplier1', 'product2', 3),
+    ('supplier2', 'product3', 3),
+    ('supplier2', 'product4', 4))
+AS Products(supplier_id, product_id, rating)
+GROUP BY CUBE (supplier_id, rating, product_id)
+
+SELECT supplier_id, rating, product_id, COUNT(*)
+FROM (VALUES
+    ('supplier1', 'product1', 4),
+    ('supplier1', 'product2', 3),
+    ('supplier2', 'product3', 3),
+    ('supplier2', 'product4', 4))
+AS Products(supplier_id, product_id, rating)
+GROUP BY GROUPING SET (
+    ( supplier_id, product_id, rating ),
+    ( supplier_id, product_id         ),
+    ( supplier_id,             rating ),
+    ( supplier_id                     ),
+    (              product_id, rating ),
+    (              product_id         ),
+    (                          rating ),
+    (                                 )
+)
              +
              +

              HAVING

              HAVING eliminates group rows that do not satisfy the condition. HAVING is different from WHERE: WHERE filters individual rows before the GROUP BY while HAVING filters group rows created by GROUP BY. Each column referenced in condition must unambiguously reference a grouping column unless it appears within an aggregate function.

              +

              The presence of HAVING turns a query into a grouped query even if there is no GROUP BY clause. It is the same as what happens when the query contains aggregate functions but no GROUP BY clause. The query considers all selected rows to form a single group, and the SELECT list and HAVING clause can only reference table columns from within aggregate functions. Such a query will emit a single row if the HAVING condition is true, zero rows if it is not true.

              +
              SELECT SUM(amount)
+FROM Orders
+GROUP BY users
+HAVING SUM(amount) > 50
              +
              +
              +
              +
              +
              Parent topic: DML Snytax
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15076.html b/docs/dli/sqlreference/dli_08_15076.html new file mode 100644 index 000000000..dc84a4ecd --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15076.html @@ -0,0 +1,62 @@ + + +

              Over Aggregation

              +

              OVER aggregates compute an aggregated value for every input row over a range of ordered rows. In contrast to GROUP BY aggregates, OVER aggregates do not reduce the number of result rows to a single row for every group. Instead OVER aggregates produce an aggregated value for every input row.

              +

              For more information, see Over Aggregation.

              +

              Syntax

              1
+2
+3
+4
+5
+6
+7
              SELECT
+  agg_func(agg_col) OVER (
+    [PARTITION BY col1[, col2, ...]]
+    ORDER BY time_col
+    range_definition),
+  ...
+FROM ...
+
              +
              +
              +

              Caveats

              • Currently, only windows from PRECEDING (unbounded or bounded) to CURRENT ROW are supported. The range described by FOLLOWING is not supported.
              • ORDER BY must be specified for a single time attribute.
              • You can define multiple OVER window aggregates in a SELECT clause. However, for streaming queries, the OVER windows for all aggregates must be identical due to current limitation.
              • OVER windows are defined on an ordered sequence of rows. Since tables do not have an inherent order, the ORDER BY clause is mandatory. For streaming queries, Flink currently only supports OVER windows that are defined with an ascending time attributes order. Additional orderings are not supported.
              +
              +

              Description

              SELECT order_id, order_time, amount,
+  SUM(amount) OVER w AS sum_amount,
+  AVG(amount) OVER w AS avg_amount
+FROM Orders
+WINDOW w AS (
+  PARTITION BY product
+  ORDER BY order_time
+  RANGE BETWEEN INTERVAL '1' HOUR PRECEDING AND CURRENT ROW)
              +
              • ORDER BY: OVER windows are defined on an ordered sequence of rows. Since tables do not have an inherent order, the ORDER BY clause is mandatory. For streaming queries, Flink currently only supports OVER windows that are defined with an ascending time attributes order. Additional orderings are not supported.
              • PARTITION BY: OVER windows can be defined on a partitioned table. In presence of a PARTITION BY clause, the aggregate is computed for each input row only over the rows of its partition.
              • Range Definitions: The range definition specifies how many rows are included in the aggregate. The range is defined with a BETWEEN clause that defines a lower and an upper boundary. All rows between these boundaries are included in the aggregate. Flink only supports CURRENT ROW as the upper boundary. There are two options to define the range, ROWS intervals and RANGE intervals.
                1. RANGE intervals

                  A RANGE interval is defined on the values of the ORDER BY column, which is in case of Flink always a time attribute. The following RANGE interval defines that all rows with a time attribute of at most 30 minutes less than the current row are included in the aggregate.

                  +
                  RANGE BETWEEN INTERVAL '30' MINUTE PRECEDING AND CURRENT ROW
                  +
                2. ROW intervals

                  A ROWS interval is a count-based interval. It defines exactly how many rows are included in the aggregate. The following ROWS interval defines that the 10 rows preceding the current row and the current row (so 11 rows in total) are included in the aggregate.

                  +
                  ROWS BETWEEN 10 PRECEDING AND CURRENT ROW
                  +
                +
              • WINDOW: The WINDOW clause can be used to define an OVER window outside of the SELECT clause. It can make queries more readable and also allows us to reuse the window definition for multiple aggregates.
              +
              +

              Example

              The following query computes for every order the sum of amounts of all orders for the same product that were received within one hour before the current order.

              +
              1
+2
+3
+4
+5
+6
+7
              SELECT order_id, order_time, amount,
+  SUM(amount) OVER (
+    PARTITION BY product
+    ORDER BY order_time
+    RANGE BETWEEN INTERVAL '1' HOUR PRECEDING AND CURRENT ROW
+  ) AS one_hour_prod_amount_sum
+FROM Orders
+
              +
              +
              +
              +
              +
              +
              Parent topic: DML Snytax
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15077.html b/docs/dli/sqlreference/dli_08_15077.html new file mode 100644 index 000000000..203fa893d --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15077.html @@ -0,0 +1,101 @@ + + +

              JOIN

              +

              Equi-join

              Syntax

              +
              1
+2
              FROM tableExpression INNER | LEFT | RIGHT | FULL JOIN tableExpression
+  ON value11 = value21 [ AND value12 = value22]
+
              +
              +
              +

              Precautions

              +
              • Currently, only equi-joins are supported, for example, joins that have at least one conjunctive condition with an equality predicate. Arbitrary cross or theta joins are not supported.
              • Tables are joined in the order in which they are specified in the FROM clause. Make sure to specify tables in an order that does not yield a cross join (Cartesian product), which are not supported and would cause a query to fail.
              • For streaming queries the required state to compute the query result might grow infinitely depending on the type of aggregation and the number of distinct grouping keys. Provide a query configuration with valid retention interval to prevent excessive state size.
              +

              Example

              +
              SELECT *
+FROM Orders INNER JOIN Product ON Orders.productId = Product.id;
+
+SELECT *
+FROM Orders LEFT JOIN Product ON Orders.productId = Product.id;
+
+SELECT *
+FROM Orders RIGHT JOIN Product ON Orders.productId = Product.id;
+
+SELECT *
+FROM Orders FULL OUTER JOIN Product ON Orders.productId = Product.id;
              +

              Time-windowed Join

              Function

              +

              Each piece of data in a stream is joined with data in different time zones in another stream.

              +

              Syntax

              +
              from t1 JOIN t2 ON t1.key = t2.key AND TIMEBOUND_EXPRESSIO
              +

              Description

              +

              TIMEBOUND_EXPRESSION can be in either of the following formats:

              +
              • L.time between LowerBound(R.time) and UpperBound(R.time)
              • R.time between LowerBound(L.time) and UpperBound(L.time)
              • Comparison expression with the time attributes (L.time/R.time)
              +

              Precautions

              +

              A time window join requires at least one equi join predicate and a join condition that limits the time of both streams.

              +

              For example, use two range predicates (<, <=, >=, or >), a BETWEEN predicate, or an equal predicate that compares the same type of time attributes (such as processing time and event time) in two input tables.

              +

              For example, the following predicate is a valid window join condition:

              +
              • ltime = rtime
              • ltime >= rtime AND ltime < rtime + INTERVAL '10' MINUTE
              • ltime BETWEEN rtime - INTERVAL '10' SECOND AND rtime + INTERVAL '5' SECOND
              +
              +

              Example

              +

              Join all orders shipped within 4 hours with their associated shipments.

              +
              SELECT *
+FROM Orders o, Shipments s
+WHERE o.id = s.orderId AND
+      o.ordertime BETWEEN s.shiptime - INTERVAL '4' HOUR AND s.shiptime;
              +

              Expanding arrays into a relation

              Precautions

              +

              This clause is used to return a new row for each element in the given array. Unnesting WITH ORDINALITY is not yet supported.

              +

              Example

              +
              SELECT users, tag
+FROM Orders CROSS JOIN UNNEST(tags) AS t (tag);
              +
              +

              User-Defined Table Functions

              Function

              +

              This clause is used to join a table with the results of a table function. ach row of the left (outer) table is joined with all rows produced by the corresponding call of the table function.

              +

              Precautions

              +

              A left outer join against a lateral table requires a TRUE literal in the ON clause.

              +

              Example

              +

              The row of the left (outer) table is dropped, if its table function call returns an empty result.

              +
              SELECT users, tag
+FROM Orders, LATERAL TABLE(unnest_udtf(tags)) t AS tag;
              +

              If a table function call returns an empty result, the corresponding outer row is preserved, and the result padded with null values.

              +
              SELECT users, tag
+FROM Orders LEFT JOIN LATERAL TABLE(unnest_udtf(tags)) t AS tag ON TRUE;
              +
              +

              Join Temporal Table Function

              Function

              +

              Precautions

              +

              Currently only inner join and left outer join with temporal tables are supported.

              +

              Example

              +

              Assuming Rates is a temporal table function, the join can be expressed in SQL as follows:

              +
              SELECT
+  o_amount, r_rate
+FROM
+  Orders,
+  LATERAL TABLE (Rates(o_proctime))
+WHERE
+  r_currency = o_currency;
              +
              +

              Join Temporal Tables

              Function

              +

              This clause is used to join the Temporal table.

              +

              Syntax

              +
              SELECT column-names
+FROM table1  [AS <alias1>]
+[LEFT] JOIN table2 FOR SYSTEM_TIME AS OF table1.proctime [AS <alias2>]
+ON table1.column-name1 = table2.key-name1
              +

              Description

              +
              • table1.proctime indicates the processing time attribute (computed column) of table1.
              • FOR SYSTEM_TIME AS OF table1.proctime indicates that when the records in the left table are joined with the dimension table on the right, only the snapshot data is used for matching the current processing time dimension table.
              +

              Precautions

              +

              Only inner and left joins are supported for temporal tables with processing time attributes.

              +

              Example

              +

              LatestRates is a dimension table (such as HBase table) that is materialized with the latest rate.

              +
              SELECT
+  o.amout, o.currency, r.rate, o.amount * r.rate
+FROM
+  Orders AS o
+  JOIN LatestRates FOR SYSTEM_TIME AS OF o.proctime AS r
+  ON r.currency = o.currency;
              +
              +
              +
              +
              +
              Parent topic: DML Snytax
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15078.html b/docs/dli/sqlreference/dli_08_15078.html new file mode 100644 index 000000000..0d7db846c --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15078.html @@ -0,0 +1,30 @@ + + +

              OrderBy & Limit

              +

              OrderBy

              Function

              +

              This clause is used to sort data in ascending order on a time attribute.

              +

              Precautions

              +

              Currently, only sorting by time attribute is supported.

              +

              Example

              +

              Sort data in ascending order on the time attribute.

              +
              SELECT *
+FROM Orders
+ORDER BY orderTime;
              +
              +

              Limit

              Function

              +

              This clause is used to constrain the number of rows returned.

              +

              Precautions

              +

              This clause is used in conjunction with ORDER BY to ensure that the results are deterministic.

              +

              Example

              +
              SELECT *
+FROM Orders
+ORDER BY orderTime
+LIMIT 3;
              +
              +
              +
              +
              +
              Parent topic: DML Snytax
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15079.html b/docs/dli/sqlreference/dli_08_15079.html new file mode 100644 index 000000000..4cc33b905 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15079.html @@ -0,0 +1,32 @@ + + +

              Top-N

              +

              Function

              Top-N queries ask for the N smallest or largest values ordered by columns. Both smallest and largest values sets are considered Top-N queries. Top-N queries are useful in cases where the need is to display only the N bottom-most or the N top- most records from batch/streaming table on a condition.

              +
              +

              Syntax

              SELECT [column_list]
+FROM (
+   SELECT [column_list],
+     ROW_NUMBER() OVER ([PARTITION BY col1[, col2...]]
+       ORDER BY col1 [asc|desc][, col2 [asc|desc]...]) AS rownum
+   FROM table_name)
+WHERE rownum <= N [AND conditions]
              +
              +

              Description

              • ROW_NUMBER(): Allocate a unique and consecutive number to each line starting from the first line in the current partition. Currently, we only support ROW_NUMBER as the over window function. In the future, we will support RANK() and DENSE_RANK().
              • PARTITION BY col1[, col2...]: Specifies the partition columns. Each partition will have a Top-N result.
              • ORDER BY col1 [asc|desc][, col2 [asc|desc]...]: Specifies the ordering columns. The ordering directions can be different on different columns.
              • WHERE rownum <= N: The rownum <= N is required for Flink to recognize this query is a Top-N query. The N represents the N smallest or largest records will be retained.
              • [AND conditions]: It is free to add other conditions in the where clause, but the other conditions can only be combined with rownum <= N using AND conjunction.
              +
              +

              Precautions

              • The TopN query is Result Updating.
              • Flink SQL will sort the input data stream according to the order key,
              • so if the top N records have been changed, the changed ones will be sent as retraction/update records to downstream.
              • If the top N records need to be stored in external storage, the result table should have the same unique key with the Top-N query.
              +
              +

              Example

              This is an example to get the top five products per category that have the maximum sales in realtime.

              +
              SELECT * 
+  FROM ( 
+     SELECT *,
+         ROW_NUMBER() OVER (PARTITION BY category ORDER BY sales DESC) as row_num
+     FROM ShopSales)
+  WHERE row_num <= 5;
              +
              +
              +
              +
              +
              Parent topic: DML Snytax
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15080.html b/docs/dli/sqlreference/dli_08_15080.html new file mode 100644 index 000000000..8b783308b --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15080.html @@ -0,0 +1,32 @@ + + +

              Deduplication

              +

              Function

              Deduplication removes rows that duplicate over a set of columns, keeping only the first one or the last one.

              +
              +

              Syntax

              SELECT [column_list]
+FROM (
+   SELECT [column_list],
+     ROW_NUMBER() OVER ([PARTITION BY col1[, col2...]]
+       ORDER BY time_attr [asc|desc]) AS rownum
+   FROM table_name)
+WHERE rownum = 1
              +
              +

              Description

              • ROW_NUMBER(): Assigns a unique, sequential number to each row, starting with one.
              • PARTITION BY col1[, col2...]: Specifies the partition columns, i.e. the deduplicate key.
              • ORDER BY time_attr [asc|desc]: Specifies the ordering column, it must be a time attribute. Currently Flink supports proctime only. Ascending (ASC) sorting keeps only the first row, while descending (DESC) sorting keeps only the last row.
              • WHERE rownum = 1: The rownum = 1 is required for Flink to recognize this query is deduplication.
              +
              +

              Precautions

              None

              +
              +

              Example

              The following examples show how to remove duplicate rows on order_id. The proctime is an event time attribute.

              +
              SELECT order_id, user, product, number
+  FROM (
+     SELECT *,
+         ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY proctime ASC) as row_num
+     FROM Orders)
+  WHERE row_num = 1;
              +
              +
              +
              +
              +
              Parent topic: DML Snytax
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15081.html b/docs/dli/sqlreference/dli_08_15081.html new file mode 100644 index 000000000..5d0c63bc2 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15081.html @@ -0,0 +1,21 @@ + + +

              Functions

              +
              + + diff --git a/docs/dli/sqlreference/dli_08_15082.html b/docs/dli/sqlreference/dli_08_15082.html new file mode 100644 index 000000000..5621c1eb3 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15082.html @@ -0,0 +1,178 @@ + + +

              UDFs

              +

              Overview

              DLI supports the following three types of user-defined functions (UDFs):

              +
              +
              • Regular UDF: takes in one or more input parameters and returns a single result.
              • User-defined table-generating function (UDTF): takes in one or more input parameters and returns multiple rows or columns.
              • User-defined aggregate function (UDAF): aggregates multiple records into one value.
              +

              Currently, UDF, UDTF, or UDAF custom functions cannot be written using Python.

              +
              +

              POM Dependency

              <dependency>
+    <groupId>org.apache.flink</groupId>
+    <artifactId>flink-table-common</artifactId>
+    <version>1.15.0</version>
+    <scope>provided</scope>
+</dependency>
              +
              +

              Using UDFs

              1. Encapsulate the implemented UDFs into a JAR package and upload the package to OBS.
              2. In the navigation pane of the DLI management console, choose Data Management > Package Management. On the displayed page, click Create and use the JAR package uploaded to OBS to create a package.
              3. In the left navigation, choose Job Management and click Flink Jobs. Locate the row where the target resides and click Edit in the Operation column to switch to the page where you can edit the job.
              4. Click the Running Parameters tab of your job, select the UDF JAR and click Save.
              5. Add the following statement to the SQL statements to use the functions:
              +
              +

              UDF

              The regular UDF must inherit the ScalarFunction function and implement the eval method. The open and close functions are optional.

              +
              +

              Example code

              +
              import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.functions.ScalarFunction;
+
+public class UdfScalarFunction extends ScalarFunction {
+    private int factor = 12;
+    public UdfScalarFunction() {
+        this.factor = 12;
+    }
+    /**
+   * (optional) Initialization
+     * @param context
+     */
+    @Override
+    public void open(FunctionContext context) {}
+    /**
+   * Custom logic
+     * @param s
+     * @return
+     */
+    public int eval(String s) {
+        return s.hashCode() * factor;
+    }
+    /**
+    * Optional
+     */
+    @Override   public void close() {}
+}
              +

              Example

              +
              1
+2
              CREATE FUNCTION udf_test AS 'com.company.udf.UdfScalarFunction';
+INSERT INTO sink_stream select udf_test(attr) FROM source_stream;
+
              +
              +

              UDTF

              The UDTF must inherit the TableFunction function and implement the eval method. The open and close functions are optional. If the UDTF needs to return multiple columns, you only need to declare the returned value as Tuple or Row. If Row is used, you need to overload the getResultType method to declare the returned field type.

              +
              +

              Example code

              +
              import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.functions.TableFunction;
+import org.apache.flink.types.Row;
+
+public class UdfTableFunction extends TableFunction<Row> {
+    /**
+   * (optional) Initialization
+     * @param context
+     */
+    @Override
+    public void open(FunctionContext context) {}
+    public void eval(String str, String split) {
+        for (String s : str.split(split)) {
+            Row row = new Row(2);
+            row.setField(0, s);
+            row.setField(1, s.length());
+            collect(row);
+        }
+    }
+    /**
+   * Declare the type returned by the function
+     * @return
+     */
+    @Override
+    public TypeInformation<Row> getResultType() {
+        return Types.ROW(Types.STRING, Types.INT);
+    }
+    /**
+    * Optional
+     */
+    @Override
+    public void close() {}
+}
              +

              Example

              +

              The UDTF supports CROSS JOIN and LEFT JOIN. When the UDTF is used, the LATERAL and TABLE keywords must be included.

              +
              • CROSS JOIN: does not output the data of a row in the left table if the UDTF does not output the result for the data of the row.
              • LEFT JOIN: outputs the data of a row in the left table even if the UDTF does not output the result for the data of the row, but pads null with UDTF-related fields.
              +
              1
+2
+3
+4
+5
+6
+7
              CREATE FUNCTION udtf_test AS 'com.company.udf.TableFunction';
+// CROSS JOIN
+INSERT INTO sink_stream select subValue, length FROM source_stream, LATERAL
+TABLE(udtf_test(attr, ',')) as T(subValue, length);
+// LEFT JOIN
+INSERT INTO sink_stream select subValue, length FROM source_stream LEFT JOIN LATERAL
+TABLE(udtf_test(attr, ',')) as T(subValue, length) ON TRUE;
+
              +
              +

              UDAF

              The UDAF must inherit the AggregateFunction function. You need to create an accumulator for storing the computing result, for example, WeightedAvgAccum in the following example code.

              +
              +

              Example code

              +
              public class WeightedAvgAccum {
+public long sum = 0;
+public int count = 0;
+}
              +

              +
              import org.apache.flink.table.functions.AggregateFunction;
+
+import java.util.Iterator;
+
+/**
+* The first type variable is the type returned by the aggregation function, and the second type variable is of the Accumulator type.
+ * Weighted Average user-defined aggregate function.
+ */
+public class UdfAggFunction extends AggregateFunction<Long, WeightedAvgAccum> {
+// Initialize the accumulator.
+    @Override
+    public WeightedAvgAccum createAccumulator() {
+        return new WeightedAvgAccum();
+    }
+// Return the intermediate computing value stored in the accumulator.
+    @Override
+    public Long getValue(WeightedAvgAccum acc) {
+        if (acc.count == 0) {
+            return null;
+        } else {
+            return acc.sum / acc.count;
+        }
+    }
+// Update the intermediate computing value according to the input.
+    public void accumulate(WeightedAvgAccum acc, long iValue) {
+        acc.sum += iValue;
+        acc.count += 1;
+    }
+// Perform the retraction operation, which is opposite to the accumulate operation.
+    public void retract(WeightedAvgAccum acc, long iValue) {
+        acc.sum -= iValue;
+        acc.count -= 1;
+    }
+// Combine multiple accumulator values.
+    public void merge(WeightedAvgAccum acc, Iterable<WeightedAvgAccum> it) {
+        Iterator<WeightedAvgAccum> iter = it.iterator();
+        while (iter.hasNext()) {
+            WeightedAvgAccum a = iter.next();
+            acc.count += a.count;
+            acc.sum += a.sum;
+        }
+    }
+// Reset the intermediate computing value.
+    public void resetAccumulator(WeightedAvgAccum acc) {
+        acc.count = 0;
+        acc.sum = 0L;
+    }
+}
              +

              Example

              +
              1
+2
              CREATE FUNCTION udaf_test AS 'com.company.udf.UdfAggFunction';
+INSERT INTO sink_stream SELECT udaf_test(attr2) FROM source_stream GROUP BY attr1;
+
              +
              +
              +
              +
              +
              Parent topic: Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15083.html b/docs/dli/sqlreference/dli_08_15083.html new file mode 100644 index 000000000..3734d15d8 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15083.html @@ -0,0 +1,57 @@ + + +

              Type Inference

              +

              Scenario

              Type inference summarizes the logic for validating input arguments and deriving data types for both the parameters and the result of a function. From a logical perspective, the planner needs information about expected types, precision, and scale. From a JVM perspective, the planner needs information about how internal data structures are represented as JVM objects when calling a user-defined function.

              +

              Flink's user-defined functions implement an automatic type inference extraction that derives data types from the function's class and its evaluation methods via reflection. However, this implicit reflective extraction approach is not always successful, for example, the Row type commonly used in UDTF cannot be extracted.

              +

              Flink 1.11 introduced a UDF registration interface and used a type inference approach, which does not support getResultType overload to declare the returned type in Flink 1.10. If you use this approach, the following exception will be thrown:

              +
              Caused by: org.apache.flink.table.api.ValidationException: Cannot extract a data type from a pure 'org.apache.flink.types.Row' class. Please use annotations to define field names and field types.
              +

              With Flink 1.15, the extraction process can be supported by annotating affected parameters, classes, or methods with @DataTypeHint and @FunctionHint.

              +
              +

              Code Samples

              The table ecosystem (similar to the SQL standard) is a strongly typed API. Therefore, both function parameters and return types must be mapped to a data type.

              +

              If more advanced type inference logic is required, an implementer can explicitly override the getTypeInference() method in every user-defined function.

              +

              However, the annotation approach is recommended because it keeps custom type inference logic close to the affected locations and falls back to the default behavior for the remaining implementation.

              +
              importorg.apache.flink.table.annotation.DataTypeHint;
+importorg.apache.flink.table.annotation.FunctionHint;
+importorg.apache.flink.table.functions.FunctionContext;
+importorg.apache.flink.table.functions.TableFunction;
+importorg.apache.flink.types.Row;
+publicclassUdfTableFunctionextendsTableFunction<Row>{
+    /**
+     * Initialization, which is optional
+     *@paramcontext
+     */
+    @Override
+     public void open(FunctionContextcontext) {  }
+
+    @FunctionHint(output=@DataTypeHint("ROW<s STRING, i INT>"))
+    publicvoideval(String str, String split) {
+        for (String s: str.split(split)) {
+            Row row=new Row(2);
+            row.setField(0, s);
+            row.setField(1, s.length());
+            collect(row);
+        }
+    }
+    /**
+    * The following is optional.
+    */
+   @Override
+   public void close() {}
+}
              +
              +

              Use Example

              The UDTF supports CROSS JOIN and LEFT JOIN. When the UDTF is used, the LATERAL and TABLE keywords must be included.

              +
              • CROSS JOIN: does not output the data of a row in the left table if the UDTF does not output the result for the data of the row.
              • LEFT JOIN: outputs the data of a row in the left table even if the UDTF does not output the result for the data of the row, but pads null with UDTF-related fields.
              +
              CREATE FUNCTION udtf_test AS 'com.company.udf.TableFunction';-- CROSS JOIN
+INSERT INTO sink_stream select subValue, length FROM source_stream, LATERAL
+TABLE(udtf_test(attr, ',')) as T(subValue, length);-- LEFT JOIN
+INSERT INTO sink_stream select subValue, length FROM source_stream LEFT JOIN
+LATERAL
+TABLE(udtf_test(attr, ',')) as T(subValue, length) ON TRUE;
              +
              +
              +
              +
              +
              Parent topic: Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15084.html b/docs/dli/sqlreference/dli_08_15084.html new file mode 100644 index 000000000..0cb86d2cd --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15084.html @@ -0,0 +1,184 @@ + + +

              Parameter Transfer

              +

              Scenario

              A UDF can be used in many jobs, and some parameter values vary with jobs. To easily modify the parameter values, you can set pipeline.global-job-parameters in the Runtime Configuration tab on the Flink OpenSource SQL editing page, and then get the parameter values in the UDF code and use the values as you need. You only need to change the parameter values in the runtime configuration tab to pass the new values to the UDF.

              +
              +

              Procedure

              Use the open(FunctionContext context) method in your UDF to pass parameters through a FunctionContext object. To pass parameters to a job, perform the following steps:

              +
              1. Add pipeline.global-job-parameters to Runtime Configuration on the Flink OpenSource SQL editing page. The format is as follows:
                pipeline.global-job-parameters=k1:v1,"k2:v1,v2",k3:"str:ing","k4:str""ing"
                +

                This configuration defines a map as shown in Table 1

                + +
                + + + + + + + + + + + + + + + + +
                Table 1 Examples for pipeline.global-job-parameters

                Key

                +

                Value

                +

                k1

                +

                v1

                +

                k2

                +

                v1,v2

                +

                k3

                +

                str:ing

                +

                k4

                +

                str""ing

                +
                +
                +
                • FunctionContext#getJobParameter obtains only the value of pipeline.global-job-parameters. You need to add all key-value pairs that will be used in the UDF to pipeline.global-job-parameters.
                • Keys and values are separated by colons (:). All key-values are connected by commas (,).
                • If the key or value contains commas (,), use double quotation marks (") to enclose key or value, for example, "v1,v2".
                • If the key or value contains colons (:), use double quotation marks (") to enclose the key or value, for example, "str:ing".
                • If the key or value contains a double quotation mark("), use another double quotation mark ("") to escape the first one, and use double quotation marks (") to enclose the key or value, for example, "str""ing".
                +
                +
              2. In your UDF code, use FunctionContext#getJobParameter to obtain the key-value pairs you set. The code example is as follows:
                context.getJobParameter("url","jdbc:mysql://xx.xx.xx.xx:3306/table");
+context.getJobParameter("driver","com.mysql.jdbc.Driver");
+context.getJobParameter("user","user");
+context.getJobParameter("password","password");
                +
              +
              +

              Code Samples

              The following sample UDF uses pipeline.global-job-parameters to pass parameters such as url, user, and password required for connecting to the database, obtains the udf_info table data, and combines this data with the stream data into JSON output.

              + +
              + + + + + + + +
              Table 2 udf_info

              key

              +

              value

              +

              class

              +

              class-4

              +
              +
              +

              SimpleJsonBuild.java

              +
              package udf;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.functions.ScalarFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+
+public class SimpleJsonBuild extends ScalarFunction {
+   private static final Logger LOG = LoggerFactory.getLogger(SimpleJsonBuild.class);
+   String remainedKey;
+   String remainedValue;
+
+   private Connection initConnection(Map<String, String> userParasMap) {
+       String url = userParasMap.get("url");
+       String driver = userParasMap.get("driver");
+       String user = userParasMap.get("user");
+       String password = userParasMap.get("password");
+       Connection conn = null;
+       try {
+           Class.forName(driver);
+           conn = DriverManager.getConnection(url, user, password);
+           LOG.info("connect successfully");
+      } catch (Exception e) {
+           LOG.error(String.valueOf(e));
+      }
+       return conn;
+  }
+
+   @Override
+   public void open(FunctionContext context) throws Exception {
+       Map<String, String> userParasMap = new HashMap<>();
+       Connection connection;
+       PreparedStatement pstmt;
+       ResultSet rs;
+
+       String url = context.getJobParameter("url","jdbc:mysql://xx.xx.xx.xx:3306/table");
+       String driver = context.getJobParameter("driver","com.mysql.jdbc.Driver");
+       String user = context.getJobParameter("user","user");
+       String password = context.getJobParameter("password","password");
+
+       userParasMap.put("url", url);
+       userParasMap.put("driver", driver);
+       userParasMap.put("user", user);
+       userParasMap.put("password", password);
+
+       connection = initConnection(userParasMap);
+       String sql = "select `key`, `value` from udf_info";
+       pstmt = connection.prepareStatement(sql);
+       rs = pstmt.executeQuery();
+
+       while (rs.next()) {
+           remainedKey = rs.getString(1);
+           remainedValue = rs.getString(2);
+      }
+  }
+
+   public String eval(String... params) throws IOException {
+       if (params != null && params.length != 0 && params.length % 2 <= 0) {
+           HashMap<String, String> hashMap = new HashMap();
+           for (int i = 0; i < params.length; i += 2) {
+               hashMap.put(params[i], params[i + 1]);
+               LOG.debug("now the key is " + params[i].toString() + "; now the value is " + params[i + 1].toString());
+          }
+           hashMap.put(remainedKey, remainedValue);
+           ObjectMapper mapper = new ObjectMapper();
+           String result = "{}";
+           try {
+               result = mapper.writeValueAsString(hashMap);
+          } catch (Exception ex) {
+               LOG.error("Get result failed." + ex.getMessage());
+          }
+           LOG.debug(result);
+           return result;
+      } else {
+           return "{}";
+      }
+  }
+
+   public static void main(String[] args) throws IOException {
+       SimpleJsonBuild  sjb = new SimpleJsonBuild();
+       System.out.println(sjb.eval("json1", "json2", "json3", "json4"));
+  }
+}
              +

              Add pipeline.global-job-parameters to Runtime Configuration on the Flink OpenSource SQL editing page. The format is as follows:

              +
              pipeline.global-job-parameters=url:'jdbc:mysql://x.x.x.x:xxxx/test',driver:com.mysql.jdbc.Driver,user:xxx,password:xxx
              +

              Flink OpenSource SQL

              +
              create function SimpleJsonBuild AS 'udf.SimpleJsonBuild';
+create table dataGenSource(user_id string, amount int) with (
+ 'connector' = 'datagen',
+ 'rows-per-second' = '1', --Generate a data record per second.
+ 'fields.user_id.kind' = 'random', --Specify a random generator for the user_id field.
+ 'fields.user_id.length' = '3' --Limit the length of user_id to 3.
+);
+create table printSink(message STRING) with ('connector' = 'print');
+insert into
+printSink
+SELECT
+SimpleJsonBuild("name", user_id, "age", cast(amount as string))
+from
+dataGenSource;
              +
              +

              Output

              On the Flin Jobs page, locate your job, and click More > FlinkUI in the Operation column. On the displayed page, click Task Managers > Stdout to view the job output.

              +

              +
              +
              +
              +
              +
              Parent topic: Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15085.html b/docs/dli/sqlreference/dli_08_15085.html new file mode 100644 index 000000000..229aed3a4 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15085.html @@ -0,0 +1,44 @@ + + +

              Built-In Functions

              +

              For details, see Built-in Functions.

              +
              + + diff --git a/docs/dli/sqlreference/dli_08_15086.html b/docs/dli/sqlreference/dli_08_15086.html new file mode 100644 index 000000000..7fb5bdccc --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15086.html @@ -0,0 +1,211 @@ + + +

              Comparison Functions

              +
              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Comparison functions

              SQL Function

              +

              Return Type

              +

              Description

              +

              value1 = value2

              +

              BOOLEAN

              +

              Returns TRUE if value1 equals value2;

              +

              returns UNKNOWN if either value1 or value2 is NULL.

              +

              value1 <> value2

              +

              BOOLEAN

              +

              Returns TRUE if value1 does not equal value2;

              +

              returns UNKNOWN if either value1 or value2 is NULL.

              +

              value1 > value2

              +

              BOOLEAN

              +

              Returns TRUE if value1 is greater than value2;

              +

              returns UNKNOWN if either value1 or value2 is NULL.

              +

              value1 >= value2

              +

              BOOLEAN

              +

              Returns TRUE if value1 is greater than or equal to value2;

              +

              returns UNKNOWN if either value1 or value2 is NULL.

              +

              value1 < value2

              +

              BOOLEAN

              +

              Returns TRUE if value1 is less than value2;

              +

              returns UNKNOWN if either value1 or value2 is NULL.

              +

              value1 <= value2

              +

              BOOLEAN

              +

              Returns TRUE if value1 is less than or equal to value2;

              +

              returns UNKNOWN if either value1 or value2 is NULL.

              +

              value IS NULL

              +

              BOOLEAN

              +

              Returns TRUE if value is NULL.

              +

              value IS NOT NULL

              +

              BOOLEAN

              +

              Returns TRUE if value is not NULL.

              +

              value1 IS DISTINCT FROM value2

              +

              BOOLEAN

              +

              Returns TRUE if value1 and value2 have different data types or values;

              +

              returns FALSE if they have the same data types and values.

              +

              Treats NULL as the same.

              +

              For example:

              +

              1 IS DISTINCT FROM NULL returns TRUE;

              +

              NULL IS DISTINCT FROM NULL returns FALSE.

              +

              value1 IS NOT DISTINCT FROM value2

              +

              BOOLEAN

              +

              Returns TRUE if they have the same data types and values;

              +

              returns FALSE if value1 and value2 have different data types or values.

              +

              Treats NULL as the same.

              +

              For example:

              +

              1 IS NOT DISTINCT FROM NULL returns FALSE;

              +

              NULL IS NOT DISTINCT FROM NULL returns TRUE.

              +

              value1 BETWEEN [ ASYMMETRIC | SYMMETRIC ] value2 AND value3

              +

              BOOLEAN

              +

              Returns TRUE if value1 is greater than or equal to value2 and less than or equal to value3, using the default or ASYMMETRIC keyword.

              +

              If SYMMETRIC is used, returns TRUE if value1 is inclusively between value2 and value3.

              +

              Returns FALSE or UNKNOWN if value2 or value3 is NULL.

              +

              For example:

              +
              • 12 BETWEEN 15 AND 12 returns FALSE;
              • 12 BETWEEN SYMMETRIC 15 AND 12 returns TRUE;
              • 12 BETWEEN 10 AND NULL returns UNKNOWN;
              • 12 BETWEEN NULL AND 10 returns FALSE;
              • 12 BETWEEN SYMMETRIC NULL AND 12 returns UNKNOWN.
              +

              value1 NOT BETWEEN [ ASYMMETRIC | SYMMETRIC ] value2 AND value3

              +

              BOOLEAN

              +

              Returns TRUE if value1 is less than value2 or greater than value3, using the default or ASYMMETRIC keyword.

              +

              If SYMMETRIC is used, returns TRUE if value1 is not between value2 and value3.

              +

              Returns TRUE or UNKNOWN if value2 or value3 is NULL.

              +

              For example:

              +
              • 12 NOT BETWEEN 15 AND 12 returns TRUE;
              • 12 NOT BETWEEN SYMMETRIC 15 AND 12 returns FALSE;
              • 12 NOT BETWEEN NULL AND 15 returns UNKNOWN;
              • 12 NOT BETWEEN 15 AND NULL returns TRUE;
              • 12 NOT BETWEEN SYMMETRIC 12 AND NULL returns UNKNOWN.
              +

              string1 LIKE string2 [ ESCAPE char ]

              +

              BOOLEAN

              +

              Returns TRUE if string1 matches string2;

              +

              returns UNKNOWN if either string1 or string2 is NULL.

              +

              Escape characters can be defined if needed, but they are not currently supported.

              +

              string1 NOT LIKE string2 [ ESCAPE char ]

              +

              BOOLEAN

              +

              Returns TRUE if string1 does not match string2;

              +

              returns UNKNOWN if either string1 or string2 is NULL.

              +

              Escape characters can be defined if needed, but they are not currently supported.

              +

              string1 SIMILAR TO string2 [ ESCAPE char ]

              +

              BOOLEAN

              +

              Returns TRUE if string1 matches the SQL regular expression string2;

              +

              returns UNKNOWN if either string1 or string2 is NULL.

              +

              Escape characters can be defined if needed, but they are not currently supported.

              +

              string1 NOT SIMILAR TO string2 [ ESCAPE char ]

              +

              BOOLEAN

              +

              Returns TRUE if string1 does not match the SQL regular expression string2;

              +

              returns UNKNOWN if either string1 or string2 is NULL.

              +

              Escape characters can be defined if needed, but they are not currently supported.

              +

              value1 IN (value2 [, value3]* )

              +

              BOOLEAN

              +

              Returns TRUE if value1 exists in the given list (value2, value3, …);

              +

              returns TRUE if the list contains NULL and value1 can be found, otherwise returns UNKNOWN.

              +

              Always returns UNKNOWN if value1 is NULL.

              +

              For example:

              +
              • 4 IN (1, 2, 3) returns FALSE;
              • 1 IN (1, 2, NULL) returns TRUE;
              • 4 IN (1, 2, NULL) returns UNKNOWN.
              +

              value1 NOT IN (value2 [, value3]* )

              +

              BOOLEAN

              +

              Returns TRUE if value1 does not exist in the given list (value2, value3, …);

              +

              returns FALSE if the list contains NULL and value1 can be found, otherwise returns UNKNOWN.

              +

              Always returns UNKNOWN if value1 is NULL.

              +

              For example:

              +
              • 4 NOT IN (1, 2, 3) returns TRUE;
              • 1 NOT IN (1, 2, NULL) returns FALSE;
              • 4 NOT IN (1, 2, NULL) returns UNKNOWN.
              +

              EXISTS (sub-query)

              +

              BOOLEAN

              +

              Returns TRUE if the subquery returns at least one row.

              +

              Only operations that can be overridden in join and grouping operations are supported. For streaming queries, this operation is rewritten in joins and grouping. The calculation of the query result required state may increase indefinitely based on the number of input rows.

              +

              Provide a query configuration with effective retention intervals to prevent excessive state.

              +

              value IN (sub-query)

              +

              BOOLEAN

              +

              Returns TRUE if value is equal to one row in the subquery result set.

              +

              value NOT IN (sub-query)

              +

              BOOLEAN

              +

              Returns TRUE if value is not contained in the rows returned by the subquery.

              +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15087.html b/docs/dli/sqlreference/dli_08_15087.html new file mode 100644 index 000000000..f7d3cc095 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15087.html @@ -0,0 +1,85 @@ + + +

              Logical Functions

              +
              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Logical functions

              SQL Function

              +

              Return Type

              +

              Description

              +

              boolean1 OR boolean2

              +

              BOOLEAN

              +

              Returns TRUE if boolean1 or boolean2 is TRUE. Supports three-valued logic. For example, true || Null(BOOLEAN) returns TRUE.

              +

              boolean1 AND boolean2

              +

              BOOLEAN

              +

              Returns TRUE if both boolean1 and boolean2 are TRUE. Supports three-valued logic. For example, true && Null(BOOLEAN) returns UNKNOWN.

              +

              NOT boolean

              +

              BOOLEAN

              +

              Returns TRUE if the boolean value is FALSE; returns FALSE if the boolean value is TRUE; returns UNKNOWN if the boolean value is UNKNOWN.

              +

              boolean IS FALSE

              +

              BOOLEAN

              +

              Returns TRUE if the boolean value is FALSE; returns FALSE if boolean is TRUE or UNKNOWN.

              +

              boolean IS NOT FALSE

              +

              BOOLEAN

              +

              Returns TRUE if boolean is TRUE or UNKNOWN; returns FALSE if boolean is FALSE.

              +

              boolean IS TRUE

              +

              BOOLEAN

              +

              Returns TRUE if boolean is TRUE; returns FALSE if boolean is FALSE or UNKNOWN.

              +

              boolean IS NOT TRUE

              +

              BOOLEAN

              +

              Returns TRUE if boolean is FALSE or UNKNOWN; returns FALSE if boolean is TRUE.

              +

              boolean IS UNKNOWN

              +

              BOOLEAN

              +

              Returns TRUE if the boolean value is UNKNOWN; returns FALSE if boolean is TRUE or FALSE.

              +

              boolean IS NOT UNKNOWN

              +

              BOOLEAN

              +

              Returns TRUE if boolean is TRUE or FALSE; returns FALSE if the boolean value is UNKNOWN.

              +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15088.html b/docs/dli/sqlreference/dli_08_15088.html new file mode 100644 index 000000000..f616afab7 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15088.html @@ -0,0 +1,241 @@ + + +

              Arithmetic Functions

              +
              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Arithmetic functions

              Operator

              +

              Description

              +

              + numeric

              +

              Returns a numeric.

              +

              - numeric

              +

              Returns the opposite of a numeric.

              +

              numeric1 + numeric2

              +

              Returns the sum of numeric1 and numeric2.

              +

              numeric1 - numeric2

              +

              Returns the difference between numeric1 and numeric2.

              +

              numeric1 * numberic2

              +

              Returns the product of numeric1 and numeric2.

              +

              numeric1 / numeric2

              +

              Returns the quotient of numeric1 divided by numeric2.

              +

              numeric1 % numeric2

              +

              Returns the remainder (modulus) of numeric1 divided by numeric2. The result is negative only if numeric1 is negative.

              +

              POWER(numeric1, numeric2)

              +

              Returns numeric1 raised to the power of numeric2.

              +

              ABS(numeric)

              +

              Returns the absolute value of numeric.

              +

              SQRT(numeric)

              +

              Returns the square root of numeric.

              +

              LN(numeric)

              +

              Returns the natural logarithm (base e) of numeric.

              +

              LOG10(numeric)

              +

              Returns the logarithm (base 10) of numeric.

              +

              LOG2(numeric)

              +

              Returns the logarithm (base 2) of numeric.

              +

              LOG(numeric2)

              +

              LOG(numeric1, numeric2)

              +

              When called with one argument, returns the natural logarithm of numeric2. When called with two arguments, returns the logarithm of numeric2 with base numeric1. Numeric2 must be greater than 0 and numeric1 must be greater than 1.

              +

              EXP(numeric)

              +

              Returns e raised to the power of numeric.

              +

              CEIL(numeric)

              +

              CEILING(numeric)

              +

              Rounds up and returns the smallest integer greater than or equal to numeric.

              +

              FLOOR(numeric)

              +

              Rounds down and returns the largest integer less than or equal to numeric.

              +

              SIN(numeric)

              +

              Returns the sine of numeric.

              +

              SINH(numeric)

              +

              Returns the hyperbolic sine of numeric. The return type is DOUBLE.

              +

              COS(numeric)

              +

              Returns the tangent of numeric.

              +

              TAN(numeric)

              +

              Calculates the tangent of given A.

              +

              TANH(numeric)

              +

              Returns the hyperbolic tangent of numeric. The return type is DOUBLE.

              +

              COT(numeric)

              +

              Returns the cotangent of numeric.

              +

              ASIN(numeric)

              +

              Returns the inverse sine of numeric.

              +

              ACOS(numeric)

              +

              Returns the inverse cosine of numeric.

              +

              ATAN(numeric)

              +

              Returns the inverse tangent of numeric.

              +

              ATAN2(numeric1, numeric2)

              +

              Returns the inverse tangent of the coordinate (numeric1, numeric2).

              +

              COSH(numeric)

              +

              Returns the hyperbolic cosine of numeric. The return type is DOUBLE.

              +

              DEGREES(numeric)

              +

              Returns the degree representation of the radian numeric.

              +

              RADIANS(numeric)

              +

              Returns the radian representation of the degree numeric.

              +

              SIGN(numeric)

              +

              Returns the sign of numeric.

              +

              ROUND(numeric, INT)

              +

              Returns the value of numeric rounded to INT decimal places.

              +

              PI()

              +

              Returns a value very close to pi.

              +

              E()

              +

              Returns a value very close to e.

              +

              RAND()

              +

              Returns a pseudo-random double-precision value within the range of [0.0, 1.0).

              +

              RAND(INT)

              +

              Returns a pseudo-random double-precision value within the range of [0.0, 1.0) with an initial seed of INT.

              +

              If two RAND functions have the same initial seed, they will return the same sequence of numbers.

              +

              RAND_INTEGER(INT)

              +

              Returns a pseudo-random integer within the range of [0, INT).

              +

              RAND_INTEGER(INT1, INT2)

              +

              Returns a pseudo-random integer within the range of [0, INT2) with an initial seed of INT1.

              +

              If two RAND_INTEGER functions have the same initial seed and boundary, they will return the same sequence of numbers.

              +

              UUID()

              +

              Returns a universally unique identifier (UUID) string based on RFC 4122 type 4 (pseudo-random generated).

              +

              For example, 3d3c68f7-f608-473f-b60c-b0c44ad4cc4e is generated using a cryptographically strong pseudo-random number generator.

              +

              BIN(INT)

              +

              Returns the string representation of INTEGER in binary format. If INTEGER is NULL, returns NULL.

              +

              For example, 4.bin() returns "100", and 12.bin() returns "1100".

              +

              HEX(numeric)

              +

              HEX(string)

              +

              Returns the string representation of the numeric value or STRING in hexadecimal format. If the parameter is NULL, returns NULL.

              +

              For example, the number 20 returns "14", the number 100 returns "64", and the string "hello,world" returns "68656C6C6F2C776F726C64".

              +

              TRUNCATE(numeric1, integer2)

              +

              Returns the number with integer2 decimal places truncated. If numeric1 or integer2 is NULL, returns NULL.

              +

              If integer2 is 0, the result has no decimal point or decimal part. integer2 can be negative, making the integer2 digits to the left of the decimal point zero.

              +

              This function can also be called with only one numeric1 parameter and without setting integer2.

              +

              If integer2 is not set, it defaults to 0. For example, 42.324.truncate(2) is 42.32, and 42.324.truncate() is 42.0.

              +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15089.html b/docs/dli/sqlreference/dli_08_15089.html new file mode 100644 index 000000000..d55924455 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15089.html @@ -0,0 +1,331 @@ + + +

              String Functions

              +

              String Functions

              DLI offers a wide range of string functions for processing and transforming string data. These functions include concatenation, case conversion, substring extraction, replacement, regex matching, encoding and decoding, format conversion, and more. Additionally, it supports string length calculation, position searching, padding, reversing, and even extracting values from JSON strings using the JSON_VAL function. These features are widely used in data cleansing, text processing, and data analysis scenarios, providing developers with powerful tool support.

              +

              For detailed string functions, see Table 1. For more information, see Apache Flink.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 String functions

              SQL Function

              +

              Description

              +

              string1 || string2

              +

              Returns the concatenation of STRING1 and STRING2.

              +

              CHAR_LENGTH(string) CHARACTER_LENGTH(string)

              +

              Returns the number of characters in a string.

              +

              UPPER(string)

              +

              Returns a string in uppercase.

              +

              LOWER(string)

              +

              Returns a string in lowercase.

              +

              POSITION(string1 IN string2)

              +

              Returns the position (starting from 1) of the first occurrence of STRING1 in STRING2.

              +

              Returns 0 if STRING1 is not found in STRING2.

              +

              TRIM([ BOTH | LEADING | TRAILING ] string1 FROM string2)

              +

              Returns the result of removing the string that starts/ends/starts and ends with STRING2 from STRING1. By default, both sides' spaces will be removed.

              +

              LTRIM(string)

              +

              Returns the string with left spaces removed from STRING.

              +

              For example, ' This is a test String.'.ltrim() returns 'This is a test String.'.

              +

              RTRIM(string)

              +

              Returns the string with right spaces removed from STRING.

              +

              For example, 'This is a test String. '.ltrim() returns 'This is a test String.'.

              +

              REPEAT(string, int)

              +

              Returns a string that is the concatenation of INT number of strings.

              +

              For example, REPEAT('This is a test String.', 2) returns "This is a test String.This is a test String.".

              +

              REGEXP_REPLACE(string1, string2, string3)

              +

              Returns a string where all substrings in STRING1 that match the regular expression STRING2 are replaced with STRING3.

              +

              For example, 'foobar'.regexpReplace('oo|ar', '') returns "fb".

              +

              OVERLAY(string1 PLACING string2 FROM integer1 [ FOR integer2 ])

              +

              Returns a string that replaces INT2 (STRING2's length by default) characters of STRING1 with STRING2 from position INT1.

              +

              For example, 'xxxxxtest'.overlay('xxxx', 6) returns "xxxxxxxxx", and 'xxxxxtest'.overlay('xxxx', 6, 2) returns "xxxxxxxxxst".

              +

              SUBSTRING(string FROM integer1 [ FOR integer2 ])

              +

              Returns a substring of STRING starting from position INT1 with length INT2 (default to the end).

              +

              REPLACE(string1, string2, string3)

              +

              Returns a new string where all occurrences of STRING2 in STRING1 are replaced with STRING3 (non-overlapping).

              +

              For example, 'hello world'.replace('world', 'flink') returns 'hello flink'; 'ababab'.replace('abab', 'z') returns 'zab'.

              +

              REGEXP_EXTRACT(string1, string2[, integer])

              +

              Splits the string STRING1 according to the regular expression rule STRING2 and returns the string at the specified position INTEGER1.

              +

              The regular expression match group index starts at 1, with 0 indicating the entire regular expression match. In addition, the regular expression match group index should not exceed the defined number of groups.

              +

              For example, REGEXP_EXTRACT('foothebar', 'foo(.*?)(bar)', 2) returns "bar".

              +

              INITCAP(string)

              +

              Returns a new string where the first character of each word is capitalized and the rest are lowercase. Here, a word is defined as a sequence of alphanumeric characters.

              +

              CONCAT(string1, string2, ...)

              +

              Returns a string that concatenates string1, string2, ..., together. If any parameter is NULL, NULL is returned.

              +

              For example, CONCAT('AA', 'BB', 'CC') returns "AABBCC".

              +

              CONCAT_WS(string1, string2, string3, ...)

              +

              Returns a string that concatenates STRING2, STRING3, ..., together with the separator STRING1.

              +

              A separator is added between each string to be concatenated.

              +

              If STRING1 is NULL, NULL is returned.

              +

              Compared to concat(), concat_ws() automatically skips NULL parameters.

              +

              For example, concat_ws('~', 'AA', Null(STRING), 'BB', '', 'CC') returns "AA~BB~~CC".

              +

              LPAD(string1, integer, string2)

              +

              Returns a new string where string2 is left-padded to the length of INT.

              +

              If the length of string1 is less than the value of INT, string1 is returned shortened to an integer character.

              +

              For example, LPAD('hi', 4, '??') returns "??hi"; LPAD('hi', 1, '??') returns "h".

              +

              RPAD(string1, integer, string2)

              +

              Returns a new string where string2 is right-padded to the length of INT.

              +

              If the length of string1 is less than the value of INT, returns a new string where string1 is shortened to a length of INT.

              +

              For example, RPAD('hi', 4, '??') returns "hi??"; RPAD('hi', 1, '??') returns "h".

              +

              FROM_BASE64(string)

              +

              Returns the result of decoding the base64-encoded string1. If the string is NULL, NULL is returned.

              +

              For example, FROM_BASE64('aGVsbG8gd29ybGQ=') returns "hello world".

              +

              TO_BASE64(string)

              +

              Returns the result of encoding the string to base64. If the string is NULL, NULL is returned.

              +

              For example, TO_BASE64('hello world') returns "aGVsbG8gd29ybGQ=".

              +

              ASCII(string)

              +

              Returns the numeric value of the first character in the string. If the string is NULL, NULL is returned.

              +

              For example, ascii('abc') returns 97, and ascii(CAST(NULL AS VARCHAR)) returns NULL.

              +

              CHR(integer)

              +

              Returns the ASCII character that corresponds to the binary value of the integer.

              +

              If the integer is greater than 255, we first take the modulo of the integer with 255 and return the CHR of the modulo.

              +

              If the integer is NULL, NULL is returned.

              +

              For example, chr(97) returns 'a', chr(353) returns 'a', and chr(CAST(NULL AS VARCHAR)) returns NULL.

              +

              DECODE(binary, string)

              +

              Decodes using the provided character set ('US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'). If any of the parameters are empty, the result will also be empty.

              +

              ENCODE(string1, string2)

              +

              Encodes using the provided character set ('US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'). If any of the parameters are empty, the result will also be empty.

              +

              INSTR(string1, string2)

              +

              Returns the position of the first occurrence of string2 in string1. Returns NULL if the value of any parameter is NULL.

              +

              LEFT(string, integer)

              +

              Returns the leftmost substring of the string with a length equal to the integer value. If the integer is negative, an empty string is returned. Returns NULL if the value of any parameter is NULL.

              +

              RIGHT(string, integer)

              +

              Returns the rightmost substring of the string with a length equal to the integer value. If the integer is negative, an empty string is returned. Returns NULL if the value of any parameter is NULL.

              +

              LOCATE(string1, string2[, integer])

              +

              Returns the position of the first occurrence of string1 after position integer in string2. If not found, returns 0. Returns NULL if either parameter is NULL.

              +

              PARSE_URL(string1, string2[, string3])

              +

              Returns a specified part from a URL. The valid values for string2 include "HOST", "PATH", "QUERY", "REF", "PROTOCOL", "AUTHORITY", "FILE", and "USERINFO".

              +

              Returns NULL if the value of any parameter is NULL.

              +

              For example, parse_url('http://facebook.com/path1/p.php?k1=v1&k2=v2#Ref1', 'HOST') returns 'facebook.com'.

              +

              You can also extract the value of a specific key in the QUERY by providing a keyword string3 as the third parameter.

              +

              For example, parse_url('http://facebook.com/path1/p.php?k1=v1&k2=v2#Ref1', 'QUERY', 'k1') returns 'v1'.

              +

              REGEXP(string1, string2)

              +

              Returns TRUE if any (possibly empty) substring of string1 matches the Java regular expression string2, otherwise it returns FALSE. Returns NULL if the value of any parameter is NULL.

              +

              REVERSE(string)

              +

              Returns the reversed string. If the string is NULL, returns NULL.

              +

              SPLIT_INDEX(string1, string2, integer1)

              +

              Splits string1 by the delimiter string2 and returns the integer-th (starting from zero) split string. If the integer is negative, returns NULL. Returns NULL if the value of any parameter is NULL.

              +

              STR_TO_MAP(string1[, string2, string3])

              +

              Splits string1 into key-value pairs using a separator and returns a map. string2 is the pair separator, and the default separator is a comma (,). string3 is the key-value separator, and the default separator is an equal sign (=).

              +

              Both separators are regular expressions, so special characters should be escaped beforehand, such as <([{\^-=$!|]})?*+.>.

              +

              SUBSTR(string[, integer1[, integer2]])

              +

              Returns a substring of a string starting from position integer1 with a length of integer2 (default to the end).

              +

              JSON_VAL(STRING json_string, STRING json_path)

              +

              Returns the value of the specified json_path from the json_string. For details about how to use the functions, see JSON_VAL Function.

              +
              NOTE:

              The following rules are listed in descending order of priority.

              +
              1. The two arguments json_string and json_path cannot be NULL.
              2. The value of json_string must be a valid JSON string. Otherwise, the function returns NULL.
              3. If json_string is an empty string, the function returns an empty string.
              4. If json_path is an empty string or the path does not exist, the function returns NULL.
              +
              +
              +
              +
              +

              JSON_VAL Function

              • Syntax
              +
              STRING JSON_VAL(STRING json_string, STRING json_path)
              + +
              + + + + + + + + + + + + + +
              Table 2 Parameters

              Parameter

              +

              Data Types

              +

              Description

              +

              json_string

              +

              STRING

              +

              JSON object to be parsed

              +

              json_path

              +

              STRING

              +

              Path expression for parsing the JSON string For the supported expressions, see Table 3.

              +
              +
              + +
              + + + + + + + + + + + + + + + + +
              Table 3 Expressions supported

              Expression

              +

              Description

              +

              $

              +

              Root node in the path

              +

              []

              +

              Access array elements

              +

              *

              +

              Array wildcard

              +

              .

              +

              Access child elements

              +
              +
              +
              • Example
                1. Test input data.
                  Test the data source kafka. The message content is as follows:
                  {"name":"James","age":24,"gender":"male","grade":{"math":95,"science":[80,85],"english":100}}
                  +
                  +
                2. Use JSON_VAL in SQL statements.
                  CREATE TABLE kafkaSource (
+  message string
+) WITH (
+  'connector' = 'kafka',
+  'topic-pattern' = '<yourSinkTopic>',
+  'properties.bootstrap.servers' = '<yourKafkaAddress1>:<yourKafkaPort>,<yourKafkaAddress2>:<yourKafkaPort>',
+  'properties.group.id' = '<yourGroupId>',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'csv',
+  'csv.field-delimiter' = '\u0001',    
+  'csv.quote-character' = ''''
+);
+
+
+CREATE TABLE printSink (
+  message1 STRING,
+  message2 STRING,
+  message3 STRING,
+  message4 STRING,
+  message5 STRING,    
+  message6 STRING
+) WITH (
+  'connector' = 'print'
+);
+insert into printSink select
+JSON_VAL(message,''),
+JSON_VAL(message,'$.name'),
+JSON_VAL(message,'$.grade.science'),
+JSON_VAL(message,'$.grade.science[*]'),
+JSON_VAL(message,'$.grade.science[1]'),
+JSON_VAL(message,'$.grade.dddd')
+from kafkaSource;
                  +
                3. Check the output of the out file of the taskmanager.
                  +I[null, James, [80,85], [80,85], 85, null]
                  +
                +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15090.html b/docs/dli/sqlreference/dli_08_15090.html new file mode 100644 index 000000000..5be8282c5 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15090.html @@ -0,0 +1,1807 @@ + + +

              Temporal Functions

              +

              Table 1 lists the time functions supported by Flink OpenSource SQL.

              +

              Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Temporal Functions

              Function

              +

              Return Type

              +

              Description

              +

              DATE string

              +

              DATE

              +

              Returns the SQL date parsed from a string in the format of "yyyy-MM-dd".

              +

              DATE_ADD

              +

              STRING

              +

              Resulting date after adding a certain number of days to a specified date. The data type is STRING.

              +

              DATE_SUB

              +

              STRING

              +

              Resulting date after subtracting a certain number of days from a specified date. The data type is STRING.

              +

              TIME string

              +

              TIME

              +

              Returns the SQL time parsed from a string in the format of "HH:mm:ss".

              +

              TIMESTAMP string

              +

              TIMESTAMP

              +

              Returns the SQL timestamp parsed from a string in the format of "yyyy-MM-dd HH:mm:ss[.SSS]".

              +

              INTERVAL string range

              +

              INTERVAL

              +

              Parses the SQL millisecond interval from a string in the format of "dd hh:mm:ss.fff" or the SQL month interval from a string in the format of "yyyy-mm".

              +

              The interval range can be DAY, MINUTE, DAY TO HOUR, or DAY TO SECOND, with the interval in milliseconds; YEAR or YEAR TO MONTH represents the interval in months.

              +

              For example, INTERVAL '10 00:00:00.004' DAY TO SECOND, INTERVAL '10' DAY, or INTERVAL '2-10' YEAR TO MONTH returns the interval.

              +

              CURRENT_DATE

              +

              DATE

              +

              Returns the current SQL date in the local time zone. In streaming mode, it is evaluated for each record. In batch processing mode, it is evaluated once at the beginning of the query and the same result is used for each row.

              +

              CURRENT_TIME

              +

              TIME

              +

              Returns the current SQL time in the local time zone, which is a synonym for LOCAL_TIME.

              +

              CURRENT_TIMESTAMP

              +

              TIMESTAMP

              +

              Returns the current SQL timestamp in the local time zone, with the return type of TIMESTAMP_LTZ(3). In streaming mode, it is evaluated for each record. In batch processing mode, it is evaluated once at the beginning of the query and the same result is used for each row.

              +

              LOCALTIME

              +

              TIME

              +

              Returns the current SQL time in the local time zone, with the return type of TIME(0). In streaming mode, it is evaluated for each record. In batch processing mode, it is evaluated once at the beginning of the query and the same result is used for each row.

              +

              LOCALTIMESTAMP

              +

              TIMESTAMP

              +

              Returns the current SQL timestamp in the local time zone, with the return type of TIMESTAMP(3). In streaming mode, it is evaluated for each record. In batch processing mode, it is evaluated once at the beginning of the query and the same result is used for each row.

              +

              NOW()

              +

              TIMESTAMP

              +

              Returns the current SQL timestamp in the local time zone, which is a synonym for CURRENT_TIMESTAMP.

              +

              CURRENT_ROW_TIMESTAMP()

              +

              TIMESTAMP_LTZ(3)

              +

              Returns the current SQL timestamp in the local time zone, with the return type of TIMESTAMP_LTZ(3). It is evaluated for each record, regardless of whether it is in batch processing mode or streaming mode.

              +

              EXTRACT(timeintervalunit FROM temporal)

              +

              BIGINT

              +

              Returns the long value extracted from the time interval unit part of the time.

              +

              For example, EXTRACT(DAY FROM DATE '2006-06-05') returns 5.

              +

              YEAR(date)

              +

              BIGINT

              +

              Returns the year from the SQL date, which is equivalent to EXTRACT(YEAR FROM date). For example, YEAR(DATE '1994-09-27') returns 1994.

              +

              QUARTER(date)

              +

              BIGINT

              +

              Returns the quarter of the year from the SQL date, which is an integer between 1 and 4, equivalent to EXTRACT(QUARTER FROM date).

              +

              For example, QUARTER(DATE '1994-09-27') returns 3.

              +

              MONTH(date)

              +

              BIGINT

              +

              +

              Returns the month of the year from the SQL date, which is an integer between 1 and 12, equivalent to EXTRACT(MONTH FROM date).

              +

              For example, MONTH(DATE '1994-09-27') returns 9.

              +

              WEEK(date)

              +

              BIGINT

              +

              Returns the week of the year from the SQL date, which is an integer between 1 and 53, equivalent to EXTRACT(WEEK FROM date).

              +

              For example, WEEK(DATE '1994-09-27') returns 39.

              +

              DAYOFYEAR(date)

              +

              BIGINT

              +

              Returns the day of the year from the SQL date, which is an integer between 1 and 366, equivalent to EXTRACT(DOY FROM date).

              +

              For example, DAYOFYEAR(DATE '1994-09-27') returns 270.

              +

              DAYOFMONTH(date)

              +

              BIGINT

              +

              Returns the day of the month from the SQL date, which is an integer between 1 and 31, equivalent to EXTRACT(DAY FROM date).

              +

              For example, DAYOFWEEK(DATE '1994-09-27') returns 3.

              +

              DAYOFWEEK(date)

              +

              BIGINT

              +

              Calculates which day of the week the current date is, with Sunday being 1.

              +

              For example, DAYOFWEEK(DATE'1994-09-27') returns 3.

              +

              HOUR(timestamp)

              +

              BIGINT

              +

              Returns the hour part of the hour unit from the SQL timestamp, which is an integer between 0 and 23, equivalent to EXTRACT(HOUR FROM timestamp).

              +

              For example, MINUTE(TIMESTAMP '1994-09-27 13:14:15') returns 14.

              +

              MINUTE(timestamp)

              +

              BIGINT

              +

              Returns the minute part of the minute unit from the SQL timestamp, which is an integer between 0 and 59, equivalent to EXTRACT(MINUTE FROM timestamp).

              +

              For example, MINUTE(TIMESTAMP '1994-09-27 13:14:15') returns 14.

              +

              SECOND(timestamp)

              +

              BIGINT

              +

              Returns the second part of the second unit from the SQL timestamp, which is an integer between 0 and 59, equivalent to EXTRACT(SECOND FROM timestamp).

              +

              For example, SECOND(TIMESTAMP '1994-09-27 13:14:15') returns 15.

              +

              FLOOR(timepoint TO timeintervalunit)

              +

              TIME

              +

              Returns the value of timepoint rounded down to the time interval unit timeintervalunit. For example, FLOOR(TIME '12:44:31' TO MINUTE) returns 12:44:00.

              +

              CEIL(timepoint TO timeintervalunit)

              +

              TIME

              +

              Return the value of timepoint rounded up to the time interval unit timeintervalunit.

              +

              For example, CEIL(TIME '12:44:31' TO MINUTE) returns 12:45:00.

              +

              (timepoint1, temporal1) OVERLAPS (timepoint2, temporal2)

              +

              BOOLEAN

              +

              Returns TRUE if the two time intervals defined by (timepoint1, temporal1) and (timepoint2, temporal2) overlap. The time value can be a time point or a time interval. For example, (TIME '2:55:00', INTERVAL '1' HOUR) OVERLAPS (TIME '3:30:00', INTERVAL '2' HOUR) returns TRUE;

              +

              (TIME '9:00:00', TIME '10:00:00') OVERLAPS (TIME '10:15:00', INTERVAL '3' HOUR) returns FALSE.

              +

              DATE_FORMAT(timestamp, string)

              +

              STRING

              +

              Converts the timestamp to a string value in the specified date format string. The format string is compatible with Java's SimpleDateFormat.

              +

              TIMESTAMPADD(timeintervalunit, interval, timepoint)

              +

              TIMESTAMP/DATE/TIME

              +

              Adds the result of combining interval with timeintervalunit to a timepoint that includes a date or datetime, and returns the resulting datetime.

              +

              For example, TIMESTAMPADD(WEEK, 1, DATE '2003-01-02') returns 2003-01-09.

              +

              TIMESTAMPDIFF(timepointunit, timepoint1, timepoint2)

              +

              INT

              +

              Returns the time interval between timepoint1 and timepoint2. The unit of the interval is given by the first parameter, which should be one of the following values: SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR.

              +

              CONVERT_TZ(string1, string2, string3)

              +

              TIMESTAMP

              +

              Convert the datetime string1 (with the default ISO timestamp format 'yyyy-MM-dd HH:mm:ss') from time zone string2 to the value in time zone string3. The format of time zone should be either an abbreviation such as PST, a full name such as Country A/City A, or a custom ID such as GMT-08:00.

              +

              For example, CONVERT_TZ('1970-01-01 00:00:00', 'UTC', 'Country A/City A') returns '1969-12-31 16:00:00'.

              +

              FROM_UNIXTIME(numeric[, string])

              +

              STRING

              +

              Returns the representation of the numeric parameter numberic in the string format (default is yyyy-MM-dd HH:mm:ss). Numeric is an internal timestamp value that represents the number of seconds since '1970-01-01 00:00:00' UTC, generated by the UNIX_TIMESTAMP() function. The return value is represented in the session time zone (specified in TableConfig).

              +

              For example, if in the UTC time zone, FROM_UNIXTIME(44) returns 1970-01-01 00:00:44, and if in the Asia/Tokyo time zone, it returns 1970-01-01 09:00:44.

              +

              UNIX_TIMESTAMP()

              +

              BIGINT

              +

              Gets the current Unix timestamp in seconds. This function is non-deterministic, meaning it will be recomputed for each record.

              +

              +

              UNIX_TIMESTAMP(string1[, string2])

              +

              BIGINT

              +

              Converts the datetime string1 in the format of string2 (default is 'yyyy-MM-dd HH:mm:ss') to a Unix timestamp in seconds, using the time zone specified in the table configuration.

              +

              +

              TO_DATE(string1[, string2])

              +

              DATE

              +

              Converts the string1 in the format of string2 (default is yyyy-MM-dd) to a date.

              +

              TO_TIMESTAMP_LTZ(numeric, precision)

              +

              TIMESTAMP_LTZ

              +

              Converts the epoch seconds or epoch milliseconds to TIMESTAMP_LTZ, with a valid precision of 0 or 3, where 0 represents TO_TIMESTAMP_LTZ(epochSeconds, 0) and 3 represents TO_TIMESTAMP_LTZ(epochMilliseconds, 3).

              +

              TO_TIMESTAMP(string1[, string2])

              +

              TIMESTAMP

              +

              Converts the string1 in the format of string2 (default is yyyy-MM-dd HH:mm:ss) in the UTC+0 time zone to a timestamp.

              +

              CURRENT_WATERMARK(rowtime)

              +

              -

              +

              Returns the current watermark of the given time column attribute rowtime. If there is no common watermark available from upstream operations in the pipeline, the function returns NULL. The return type of the function is inferred to match the provided time column attribute, but with an adjusted precision of 3. For example, if the time column attribute is TIMESTAMP_LTZ(9), the function returns TIMESTAMP_LTZ(3).

              +

              Note that this function can return NULL, which you may need to consider. For example, if you want to filter out late data, you can use:

              +
              WHERE   CURRENT_WATERMARK(ts) IS NULL   OR ts > CURRENT_WATERMARK(ts)
              +
              +
              +
              +

              DATE

              • Function

                Returns a SQL date parsed from string in form of yyyy-MM-dd.

                +
              • Description
                DATE DATE string
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                string

                +

                STRING

                +

                String in the SQL date format.

                +

                Note that the string must be in the yyyy-MM-dd format. Otherwise, an error will be reported.

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	DATE "2021-08-19" AS `result`
+FROM
+	testtable;
                  +
                • Test Result +
                  + + + + + +

                  result

                  +

                  2021-08-19

                  +
                  +
                  +
                +
              +
              +

              DATE_ADD

              • Function

                Returns the resulting date after adding a certain number of days to a specified date.

                +
              • Description
                DATE_ADD(string startdate, int days)
                +
              • Input parameters
                • startdate: specified date. The data type is TIMESTAMP or STRING.

                  The date format for the STRING type is yyyy-MM-dd HH:mm:ss.

                  +

                  The function supports the special case where this parameter value is NULL.

                  +
                  +
                • days: target number of days. The data type is INT.
                +
              • Return values

                Resulting date after adding a certain number of days to a specified date. The data type is STRING.

                +
              • Example
                Submit a Flink SQL statement.
                CREATE TABLE source (
+  time1 TIMESTAMP
+) WITH (
+  'connector' = 'datagen',
+  'rows-per-second' = '1'
+);
+create table Sink (
+  date1 string,
+  date2 string,
+  date3 string
+) with ('connector' = 'print');
+INSERT into
+  Sink
+select
+  DATE_ADD(time1, 30) as date1,
+  DATE_ADD('2017-09-15 00:00:00', 30) as date2,
+  DATE_ADD(cast(null as timestamp),30) as date3
+FROM source
                +
                +

                Test result

                + +
                + + + + + + + + + +

                date1 (string)

                +

                date2 (string)

                +

                date3 (string)

                +

                2024-06-28

                +

                2017-10-15

                +

                null

                +
                +
                +
              +
              +

              DATE_SUB

              • Function

                Returns the resulting date after subtracting a certain number of days from a specified date.

                +
              • Description
                DATE_SUB(string startdate, int days)
                +
              • Input parameters
                • startdate: specified date. The data type is TIMESTAMP or STRING.

                  The date format for the STRING type is yyyy-MM-dd HH:mm:ss.

                  +

                  The function supports the special case where this parameter value is NULL.

                  +
                  +
                • days: target number of days. The data type is INT.
                +
              • Return values

                Resulting date after subtracting a certain number of days from a specified date. The data type is STRING.

                +
              • Example
                Submit a Flink SQL statement.
                CREATE TABLE source (
+  time1 TIMESTAMP
+) WITH (
+  'connector' = 'datagen',
+  'rows-per-second' = '1'
+);
+create table Sink (
+  date1 string,
+  date2 string,
+  date3 string
+) with ('connector' = 'print');
+INSERT into
+  Sink
+select
+  DATE_SUB(time1,30) as date1,
+  DATE_SUB('2017-09-15 00:00:00', 30) as date2,
+  DATE_SUB(cast(null as timestamp),30) as date3
+FROM source
                +
                +

                Test result

                + +
                + + + + + + + + + +

                date1 (string)

                +

                date2 (string)

                +

                date3 (string)

                +

                2024-04-29

                +

                2017-08-16

                +

                null

                +
                +
                +
              +
              +

              TIME

              • Function

                Returns a SQL time parsed from string in form of HH:mm:ss[.fff].

                +
              • Description
                TIME TIME string
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                string

                +

                STRING

                +

                Time

                +

                Note that the string must be in the format of HH:mm:ss[.fff]. Otherwise, an error will be reported.

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	TIME "10:11:12" AS `result`,
+        TIME "10:11:12.032" AS `result2`
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + + + +

                  result

                  +

                  result2

                  +

                  10:11:12

                  +

                  10:11:12.032

                  +
                  +
                  +
                +
              +
              +

              TIMESTAMP

              • Function

                Converts the time string into timestamp. The time string format is yyyy-MM-dd HH:mm:ss[.fff]. The return value is of the TIMESTAMP(3) type.

                +
              • Description
                TIMESTAMP(3) TIMESTAMP string
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                string

                +

                STRING

                +

                Time

                +

                Note that the string must be in the format of yyyy-MM-dd HH:mm:ss[.fff]. Otherwise, an error will be reported.

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	TIMESTAMP "1997-04-25 13:14:15" AS `result`,
+        TIMESTAMP "1997-04-25 13:14:15.032" AS `result2`
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + + + +

                  result

                  +

                  result2

                  +

                  1997-04-25 13:14:15

                  +

                  1997-04-25 13:14:15.032

                  +
                  +
                  +
                +
              +
              +

              INTERVAL

              • Function

                Parses an interval string.

                +
              • Description
                INTERVAL INTERVAL string range
                +
              • Input parameters +
                + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                string

                +

                STRING

                +

                Timestamp string used together with the range parameter. The string is in either of the following two formats:

                +
                • yyyy-MM for SQL intervals of months. An interval range might be YEAR or YEAR TO MONTH for intervals of months.
                • dd hh:mm:ss.fff for SQL intervals of milliseconds. An interval range might be DAY, MINUTE, DAY TO HOUR, or DAY TO SECOND.
                +

                range

                +

                INTERVAL

                +

                Interval range. This parameter is used together with the string parameter.

                +

                Available values are as follows: YEAR, YEAR To Month, DAY, MINUTE, DAY TO HOUR and DAY TO SECOND.

                +
                +
                +
              • Example
                Test statement
                -- indicates that the interval is 10 days and 4 milliseconds.
+INTERVAL '10 00:00:00.004' DAY TO second
+-- The interval is 10 days.
+INTERVAL '10' 
+-- The interval is 2 years and 10 months.
+INTERVAL '2-10' YEAR TO MONTH
                +
                +
              +
              +

              CURRENT_DATE

              • Function

                Returns the current SQL time (yyyy-MM-dd) in the local time zone. The return value is of the DATE type.

                +
              • Description
                DATE CURRENT_DATE
                +
              • Input parameters

                None

                +
              • Example
                • Test statement
                  SELECT 
+	CURRENT_DATE AS `result`
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + +

                  result

                  +

                  2021-10-28

                  +
                  +
                  +
                +
              +
              +

              CURRENT_TIME

              • Function

                Returns the current SQL time (HH:mm:sss.fff) in the local time zone. The return value is of the TIME type.

                +
              • Description
                TIME CURRENT_TIME
                +
              • Input parameters

                None

                +
              • Example
                • Test statement
                  SELECT 
+	CURRENT_TIME AS `result`
+FROM
+	testtable;
                  +
                • Test Result +
                  + + + + + +

                  result

                  +

                  08:29:19.289

                  +
                  +
                  +
                +
              +
              +

              CURRENT_TIMESTAMP

              • Function

                Returns the current SQL timestamp in the local time zone. The return value is of the TIMESTAMP(3) type.

                +
              • Description
                TIMESTAMP(3) CURRENT_TIMESTAMP
                +
              • Input parameters

                None

                +
              • Example
                • Test statement
                  SELECT 
+	CURRENT_TIMESTAMP AS `result`
+FROM
+	testtable;
                  +
                • Test Result +
                  + + + + + +

                  result

                  +

                  2021-10-28 08:33:51.606

                  +
                  +
                  +
                +
              +
              +

              LOCALTIME

              • Function

                Returns the current SQL time in the local time zone. The return value is of the TIME type.

                +
              • Description
                TIME LOCALTIME
                +
              • Input parameters

                None

                +
              • Example
                • Test statement
                  SELECT 
+	LOCALTIME AS `result`
+FROM
+	testtable;
                  +
                • Test Result +
                  + + + + + +

                  result

                  +

                  16:39:37.706

                  +
                  +
                  +
                +
              +
              +

              LOCALTIMESTAMP

              • Function

                Returns the current SQL timestamp in the local time zone. The return value is of the TIMESTAMP(3) type.

                +
              • Description
                TIMESTAMP(3) LOCALTIMESTAMP
                +
              • Input parameters

                None

                +
              • Example
                • Test statement
                  SELECT 
+	LOCALTIMESTAMP AS `result`
+FROM
+	testtable;
                  +
                • Test Result +
                  + + + + + +

                  result

                  +

                  2021-10-28 16:43:17.625

                  +
                  +
                  +
                +
              +
              +

              EXTRACT

              • Function

                Returns a value extracted from the timeintervalunit part of temporal. The return value is of the BIGINT type.

                +
              • Description
                BIGINT EXTRACT(timeinteravlunit FROM temporal)
                +
              • Input parameters +
                + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                timeinteravlunit

                +

                TIMEUNIT

                +

                Time unit to be extracted from a time point or interval. The value can be YEAR, QUARTER, MONTH, WEEK, DAY, DOY, HOUR, MINUTE, SECOND.

                +

                temporal

                +

                DATE/TIME/TIMESTAMP/INTERVAL

                +

                Time point or interval

                +
                +
                +

                Do not specify a time unit that is not of any time points or intervals. Otherwise, the job fails to be submitted.

                +

                For example, an error message is displayed when the following statement is executed because YEAR cannot be extracted from TIME.

                +
                SELECT 
+	EXTRACT(YEAR FROM TIME '12:44:31' ) AS `result`
+FROM
+	testtable;
                +
                +
              • Example
                • Test statement
                  SELECT 
+	EXTRACT(YEAR FROM DATE '1997-04-25' ) AS `result`,
+        EXTRACT(MINUTE FROM TIME '12:44:31') AS `result2`,
+        EXTRACT(SECOND FROM TIMESTAMP '1997-04-25 13:14:15') AS `result3`,
+        EXTRACT(YEAR FROM INTERVAL '2-10' YEAR TO MONTH) AS `result4`,
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + + + + + + + +

                  result

                  +

                  result2

                  +

                  result3

                  +

                  result4

                  +

                  1997

                  +

                  44

                  +

                  15

                  +

                  2

                  +
                  +
                  +
                +
              +
              +

              YEAR

              • Function

                Returns the year from a SQL date date. The return value is of the BIGINT type.

                +
              • Description
                BIGINT YEAR(date)
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                date

                +

                DATE

                +

                SQL date

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	YEAR(DATE '1997-04-25' ) AS `result`
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + +

                  result

                  +

                  1997

                  +
                  +
                  +
                +
              +
              +

              QUARTER

              • Function

                Returns the quarter of a year (an integer between 1 and 4) from a SQL date date. The return value is of the BIGINT type.

                +
              • Description
                BIGINT QUARTER(date)
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                date

                +

                DATE

                +

                SQL date

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	QUARTER(DATE '1997-04-25' ) AS `result`
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + +

                  result

                  +

                  2

                  +
                  +
                  +
                +
              +
              +

              MONTH

              • Function

                Returns the month of a year (an integer between 1 and 12) from a SQL date date. The return value is of the BIGINT type.

                +
              • Description
                BIGINT MONTH(date)
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                date

                +

                DATE

                +

                SQL date

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	MONTH(DATE '1997-04-25' ) AS `result`
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + +

                  result

                  +

                  4

                  +
                  +
                  +
                +
              +
              +

              WEEK

              • Function

                Returns the week of a year from a SQL date date. The return value is of the BIGINT type.

                +
              • Description
                BIGINT WEEK(date)
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                date

                +

                DATE

                +

                SQL date

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	WEEK(DATE '1997-04-25' ) AS `result`
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + +

                  result

                  +

                  17

                  +
                  +
                  +
                +
              +
              +

              DAYOFYEAR

              • Function

                Returns the day of a year (an integer between 1 and 366) from SQL date date. The return value is of the BIGINT type.

                +
              • Description
                BIGINT DAYOFYEAR(date)
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                date

                +

                DATE

                +

                SQL date

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	DAYOFYEAR(DATE '1997-04-25' ) AS `result`
+FROM
+	testtable;
                  +
                • Test Result +
                  + + + + + +

                  result

                  +

                  115

                  +
                  +
                  +
                +
              +
              +

              DAYOFMONTH

              • Function

                Returns the day of a month (an integer between 1 and 31) from a SQL date date. The return value is of the BIGINT type.

                +
              • Description
                BIGINT DAYOFMONTH(date)
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                date

                +

                DATE

                +

                SQL date

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	DAYOFMONTH(DATE '1997-04-25' ) AS `result`
+FROM
+	testtable;
                  +
                • Test Result +
                  + + + + + +

                  result

                  +

                  25

                  +
                  +
                  +
                +
              +
              +

              DAYOFWEEK

              • Function

                Returns the day of a week (an integer between 1 and 7) from a SQL date date. The return value is of the BIGINT type.

                +

                Note that the start day of a week is Sunday.

                +
                +
              • Description
                BIGINT DAYOFWEEK(date)
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                date

                +

                DATE

                +

                SQL date

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	DAYOFWEEK(DATE '1997-04-25') AS `result`
+FROM
+	testtable;
                  +
                • Test Result +
                  + + + + + +

                  result

                  +

                  6

                  +
                  +
                  +
                +
              +
              +

              HOUR

              • Function

                Returns the hour of a day (an integer between 0 and 23) from SQL timestamp timestamp. The return value is of the BIGINT type.

                +
              • Description
                BIGINT HOUR(timestamp)
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                timestamp

                +

                TIMESTAMP

                +

                SQL timestamp

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	HOUR(TIMESTAMP '1997-04-25 10:11:12') AS `result`
+FROM
+	testtable;
                  +
                • Test Result +
                  + + + + + +

                  result

                  +

                  10

                  +
                  +
                  +
                +
              +
              +

              MINUTE

              • Function

                Returns the minute of an hour (an integer between 0 and 59) from a SQL timestamp. The return value is of the BIGINT type.

                +
              • Description
                BIGINT MINUTE(timestamp)
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                timestamp

                +

                TIMESTAMP

                +

                SQL timestamp

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	MINUTE(TIMESTAMP '1997-04-25 10:11:12') AS `result`
+FROM
+	testtable;
                  +
                • Test Result +
                  + + + + + +

                  result

                  +

                  11

                  +
                  +
                  +
                +
              +
              +

              SECOND

              • Function

                Returns the second of an hour (an integer between 0 and 59) from a SQL timestamp. The return value is of the BIGINT type.

                +
              • Description
                BIGINT SECOND(timestamp)
                +
              • Input parameters +
                + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                timestamp

                +

                TIMESTAMP

                +

                SQL timestamp

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	SECOND(TIMESTAMP '1997-04-25 10:11:12') AS `result`
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + +

                  result

                  +

                  12

                  +
                  +
                  +
                +
              +
              +

              FLOOR

              • Function

                Returns a value that rounds timepoint down to the time unit timeintervalunit.

                +
              • Description
                TIME/TIMESTAMP(3) FLOOR(timepoint TO timeintervalunit)
                +
              • Input parameters +
                + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                timepoint

                +

                TIMESTAMP/TIME

                +

                SQL time or SQL timestamp

                +

                timeintervalunit

                +

                TIMEUNIT

                +

                Time unit. The value can be YEAR, QUARTER, MONTH, WEEK, DAY, DOY, HOUR, MINUTE, or SECOND.

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	FLOOR(TIME '13:14:15' TO MINUTE) AS `result`
+        FLOOR(TIMESTAMP '1997-04-25 13:14:15' TO MINUTE) AS `result2`,
+        FLOOR(TIMESTAMP '1997-04-25 13:14:15' TO MINUTE) AS `result3`
+FROM	testtable;
                  +
                • Test result +
                  + + + + + + + + + +

                  message

                  +

                  message2

                  +

                  message3

                  +

                  13:14

                  +

                  13:14

                  +

                  1997-04-25T13:14

                  +
                  +
                  +
                +
              +
              +

              CEIL

              • Function

                Returns a value that rounds timepoint up to the time unit timeintervalunit.

                +
              • Description
                TIME/TIMESTAMP(3) CEIL(timepoint TO timeintervalunit)
                +
              • Input parameters +
                + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                timepoint

                +

                TIMESTAMP/TIME

                +

                SQL time or SQL timestamp

                +

                timeintervalunit

                +

                TIMEUNIT

                +

                Time unit. The value can be YEAR, QUARTER, MONTH, WEEK, DAY, DOY, HOUR, MINUTE, or SECOND.

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	CEIL(TIME '13:14:15' TO MINUTE) AS `result`
+        CEIL(TIMESTAMP '1997-04-25 13:14:15' TO MINUTE) AS `result2`,
+        CEIL(TIMESTAMP '1997-04-25 13:14:15' TO MINUTE) AS `result3`
+FROM	testtable;
                  +
                • Test Result +
                  + + + + + + + + + +

                  result

                  +

                  result2

                  +

                  result3

                  +

                  13:15

                  +

                  13:15

                  +

                  1997-04-25T13:15

                  +
                  +
                  +
                +
              +
              +

              OVERLAPS

              • Function

                Returns TRUE if two time intervals overlap; returns FALSE otherwise.

                +
              • Description
                BOOLEAN (timepoint1, temporal1) OVERLAPS (timepoint2, temporal2)
                +
              • Input parameters +
                + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                timepoint1/timepoint2

                +

                DATE/TIME/TIMESTAMP

                +

                Time point

                +

                temporal1/temporal2

                +

                DATE/TIME/TIMESTAMP/INTERVAL

                +

                Time point or interval

                +
                +
                +
                • (timepoint, temporal) is a closed interval.
                • The temporal can be of the DATE, TIME, TIMESTAMP, or INTERVAL type.
                  • When th temporal is DATE, TIME, or TIMESTAMP, (timepoint, temporal) indicates an interval between timepoint and temporal. The temporal can be earlier than the value of timepoint, for example, (DATE '1997-04-25', DATE '1997-04-23').
                  • When the temporal is INTERVAL, (timepoint, temporal) indicates an interval between timepoint and timepoint + temporal.
                  +
                • Ensure that (timepoint1, temporal1) and (timepoint2, temporal2) are intervals of the same data type.
                +
                +
              • Example
                • Test statement
                  SELECT 
+	(TIME '2:55:00', INTERVAL '1' HOUR) OVERLAPS (TIME '3:30:00', INTERVAL '2' HOUR) AS `result`,
+        (TIME '2:30:00', INTERVAL '1' HOUR) OVERLAPS (TIME '3:30:00', INTERVAL '2' HOUR) AS `result2`,
+	(TIME '2:30:00', INTERVAL '1' HOUR) OVERLAPS (TIME '3:31:00', INTERVAL '2' HOUR) AS `result3`,
+	(TIME '9:00:00', TIME '10:00:00') OVERLAPS (TIME '10:00:00', INTERVAL '3' HOUR) AS `result4`,
+	(TIMESTAMP '1997-04-25 12:00:00', TIMESTAMP '1997-04-25 12:20:00') OVERLAPS (TIMESTAMP '1997-04-25 13:00:00', INTERVAL '2' HOUR) AS `result5`,
+	(DATE '1997-04-23', INTERVAL '2' DAY) OVERLAPS (DATE '1997-04-25', INTERVAL '2' DAY) AS `result6`,
+	(DATE '1997-04-25', DATE '1997-04-23') OVERLAPS (DATE '1997-04-25', INTERVAL '2' DAY) AS `result7`
+FROM
+	testtable;
                  +
                • Test Result +
                  + + + + + + + + + + + + + + + + + +

                  result

                  +

                  result2

                  +

                  result3

                  +

                  result4

                  +

                  result5

                  +

                  result6

                  +

                  result7

                  +

                  true

                  +

                  true

                  +

                  false

                  +

                  true

                  +

                  false

                  +

                  true

                  +

                  true

                  +
                  +
                  +
                +
              +
              +

              DATE_FORMAT

              • Function

                Converts a timestamp to a value of string in the format specified by the date format string.

                +
              • Description
                STRING DATE_FORMAT(timestamp, dateformat)
                +
              • Input parameters +
                + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                timestamp

                +

                TIMESTAMP/STRING

                +

                Time point

                +

                dateformat

                +

                STRING

                +

                String in the date format

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	DATE_FORMAT(TIMESTAMP '1997-04-25 10:11:12', 'yyyy-MM-dd HH:mm:ss') AS `result`,
+        DATE_FORMAT(TIMESTAMP '1997-04-25 10:11:12', 'yyyy-MM-dd') AS `result2`,
+	DATE_FORMAT(TIMESTAMP '1997-04-25 10:11:12', 'yy/MM/dd HH:mm') AS `result3`,
+        DATE_FORMAT('1997-04-25 10:11:12', 'yyyy-MM-dd') AS `result4`
+FROM	testtable;
                  +
                • Test Result +
                  + + + + + + + + + + + +

                  result

                  +

                  result2

                  +

                  result3

                  +

                  result4

                  +

                  1997-04-25 10:11:12

                  +

                  1997-04-25

                  +

                  97/04/25 10:11

                  +

                  1997-04-25

                  +
                  +
                  +
                +
              +
              +

              TIMESTAMPADD

              • Function

                Returns the date and time by combining interval and timeintervalunit and adding the combination to timepoint.

                +

                The return value of TIMESTAMPADD is the value of timepoint. An exception is that if the input timepoint is of the TIMESTAMP type, the return value can be inserted into a table field of the DATE type.

                +
                +
              • Description
                TIMESTAMP(3)/DATE/TIME TIMESTAMPADD(timeintervalunit, interval, timepoint)
                +
              • Input parameters +
                + + + + + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                timeintervalunit

                +

                TIMEUNIT

                +

                Time unit.

                +

                interval

                +

                INT

                +

                Interval

                +

                timepoint

                +

                TIMESTAMP/DATE/TIME

                +

                Time point

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	TIMESTAMPADD(WEEK, 1, DATE '1997-04-25') AS `result`,
+        TIMESTAMPADD(QUARTER, 1, TIMESTAMP '1997-04-25 10:11:12') AS `result2`,
+	TIMESTAMPADD(SECOND, 2, TIME '10:11:12') AS `result3`
+FROM	testtable;
                  +
                • Test Result +
                  + + + + + + + + + +

                  result

                  +

                  result2

                  +

                  result3

                  +

                  1997-05-02

                  +
                  • If this field is inserted into a table field of the TIMESTAMP type, 1997-07-25T10:11:12 is returned.
                  +
                  • If this field is inserted into a table field of the TIMESTAMP type, 1997-07-25 is returned.
                  +

                  10:11:14

                  +
                  +
                  +
                +
              +
              +

              TIMESTAMPDIFF

              • Function

                Returns the (signed) number of timepointunit between timepoint1 and timepoint2. The unit for the interval is given by the first argument.

                +
              • Description
                INT TIMESTAMPDIFF(timepointunit, timepoint1, timepoint2)
                +
              • Input parameters +
                + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                timepointunit

                +

                TIMEUNIT

                +

                Time unit. The value can be SECOND, MINUTE, HOUR, DAY, MONTH or YEAR.

                +

                timepoint1/timepoint2

                +

                TIMESTAMP/DATE

                +

                Time point

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	TIMESTAMPDIFF(DAY, TIMESTAMP '1997-04-25 10:00:00', TIMESTAMP '1997-04-28 10:00:00') AS `result`,
+        TIMESTAMPDIFF(DAY, DATE '1997-04-25', DATE '1997-04-28') AS `result2`,
+	TIMESTAMPDIFF(DAY, TIMESTAMP '1997-04-27 10:00:20', TIMESTAMP '1997-04-25 10:00:00') AS `result3`
+FROM	testtable;
                  +
                • Test result +
                  + + + + + + + + + +

                  result

                  +

                  result2

                  +

                  result3

                  +

                  3

                  +

                  3

                  +

                  -2

                  +
                  +
                  +
                +
              +
              +

              CONVERT_TZ

              • Function

                Converts a datetime string1 (with default ISO timestamp format 'yyyy-MM-dd HH:mm:ss') from time zone string2 to time zone string3.

                +
              • Description
                STRING CONVERT_TZ(string1, string2, string3)
                +
              • Input parameters +
                + + + + + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                string1

                +

                STRING

                +

                SQL timestamp. If the value does not meet the format requirements, NULL is returned.

                +

                string2

                +

                STRING

                +

                Time zone before conversion. The format of time zone should be either an abbreviation such as PST, a full name such as Country A/City A, or a custom ID such as GMT-08:00.

                +

                string3

                +

                STRING

                +

                Time zone after conversion. The format of time zone should be either an abbreviation such as PST, a full name such as Country A/City A, or a custom ID such as GMT-08:00.

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	CONVERT_TZ(1970-01-01 00:00:00, UTC, Country A/City A) AS `result`,
+        CONVERT_TZ(1997-04-25 10:00:00, UTC, GMT-08:00) AS `result2`
+FROM	testtable;
                  +
                • Test Result +
                  + + + + + + + +

                  result

                  +

                  result2

                  +

                  1969-12-31 16:00:00

                  +

                  1997-04-25 02:00:00

                  +
                  +
                  +
                +
              +
              +

              FROM_UNIXTIME

              • Function

                Returns a representation of the numeric argument as a value in string format.

                +
              • Description
                STRING FROM_UNIXTIME(numeric[, string])
                +
              • Input parameters +
                + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                numeric

                +

                BIGINT

                +

                An internal timestamp representing the number of seconds since 1970-01-01 00:00:00 UTC. The value can be generated by the UNIX_TIMESTAMP() function.

                +

                string

                +

                STRING

                +

                Time. If this parameter is not specified, the default time format is yyyy-MM-dd HH:mm:ss format.

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	FROM_UNIXTIME(44) AS `result`,
+        FROM_UNIXTIME(44, 'yyyy:MM:dd') AS `result2`
+FROM	testtable;
                  +
                • Test Result +
                  + + + + + + + +

                  result

                  +

                  result2

                  +

                  1970-01-01 08:00:44

                  +

                  1970:01:01

                  +
                  +
                  +
                +
              +
              +

              UNIX_TIMESTAMP

              • Function

                Gets current Unix timestamp in seconds. The return value is of the BIGINT type.

                +
              • Description
                BIGINT UNIX_TIMESTAMP()
                +
              • Input parameters

                None

                +
              • Example
                • Test statement
                  SELECT 
+	UNIX_TIMESTAMP() AS `result`
+FROM
+	table;
                  +
                • Test result +
                  + + + + + +

                  result

                  +

                  1635401982

                  +
                  +
                  +
                +
              +
              +

              UNIX_TIMESTAMP(string1[, string2])

              • Function

                Converts date time string1 in format string2 to Unix timestamp (in seconds). The return value is of the BIGINT type.

                +
              • Description
                BIGINT UNIX_TIMESTAMP(string1[, string2])
                +
              • Input parameters +
                + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                string1

                +

                STRING

                +

                SQL timestamp string. An error is reported if the value does not comply with the string2 format.

                +

                string2

                +

                STRING

                +

                Time. If this parameter is not specified, the default time format is yyyy-MM-dd HH:mm:ss.

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	UNIX_TIMESTAMP('1997-04-25', 'yyyy-MM-dd') AS `result`,
+        UNIX_TIMESTAMP('1997-04-25 00:00:10', 'yyyy-MM-dd HH:mm:ss') AS `result2`,
+        UNIX_TIMESTAMP('1997-04-25 00:00:00') AS `result3`
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + + + + + +

                  result

                  +

                  result2

                  +

                  result3

                  +

                  861897600

                  +

                  861897610

                  +

                  861897600

                  +
                  +
                  +
                +
              +
              +

              TO_DATE

              • Function

                Converts a date string1 with format string2 to a date.

                +
              • Description
                DATE TO_DATE(string1[, string2])
                +
              • Input parameters +
                + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                string1

                +

                STRING

                +

                SQL timestamp string. If the value is not in the required format, an error is reported.

                +

                string2

                +

                STRING

                +

                Format. If this parameter is not specified, the default time format is yyyy-MM-dd.

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	TO_DATE('1997-04-25') AS `result`,
+        TO_DATE('1997:04:25', 'yyyy-MM-dd') AS `result2`,
+        TO_DATE('1997-04-25 00:00:00', 'yyyy-MM-dd HH:mm:ss') AS `result3`
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + + + + + +

                  result

                  +

                  result2

                  +

                  result3

                  +

                  1997-04-25

                  +

                  1997-04-25

                  +

                  1997-04-25

                  +
                  +
                  +
                +
              +
              +

              TO_TIMESTAMP

              • Function

                Converts date time string1 with format string2 to a timestamp.

                +
              • Description
                TIMESTAMP TO_TIMESTAMP(string1[, string2])
                +
              • Input parameters +
                + + + + + + + + + + + + + +

                Parameter

                +

                Data Type

                +

                Description

                +

                string1

                +

                STRING

                +

                SQL timestamp string. If the value is not in the required format, NULL is returned.

                +

                string2

                +

                STRING

                +

                Date format. If this parameter is not specified, the default format is yyyy-MM-dd HH:mm:ss.

                +
                +
                +
              • Example
                • Test statement
                  SELECT 
+	TO_TIMESTAMP('1997-04-25', 'yyyy-MM-dd') AS `result`,
+        TO_TIMESTAMP('1997-04-25 00:00:00') AS `result2`,
+        TO_TIMESTAMP('1997-04-25 00:00:00', 'yyyy-MM-dd HH:mm:ss') AS `result3`
+FROM
+	testtable;
                  +
                • Test result +
                  + + + + + + + + + +

                  result

                  +

                  result2

                  +

                  result3

                  +

                  1997-04-25 00:00

                  +

                  1997-04-25 00:00

                  +

                  1997-04-25 00:00

                  +
                  +
                  +
                +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15091.html b/docs/dli/sqlreference/dli_08_15091.html new file mode 100644 index 000000000..5c3218db1 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15091.html @@ -0,0 +1,88 @@ + + +

              Conditional Functions

              +

              Description

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Conditional functions

              Conditional Function

              +

              Description

              +

              CASE value

              +

              WHEN value1_1 [, value1_2 ]* THEN result1

              +

              [ WHEN value2_1 [, value2_2 ]* THEN result2 ]*

              +

              [ ELSE resultZ ]

              +

              END

              +

              Returns resultX when the first value in (valueX_1, valueX_2, ...) is included. If there is no matching value, returns result_z if provided, otherwise returns NULL.

              +

              CASE

              +

              WHEN condition1 THEN result1

              +

              [ WHEN condition2 THEN result2 ]*

              +

              [ ELSE resultZ ]

              +

              END

              +

              Returns resultX when the first condition X is met. If no conditions are met, returns result_z if provided, otherwise returns NULL.

              +

              NULLIF(value1, value2)

              +

              Returns NULL if value1 equals value2; otherwise, returns value1.

              +

              For example, NULLIF(5, 5) returns NULL; NULLIF(5, 0) returns 5.

              +

              COALESCE(value1, value2 [, value3 ]* )

              +

              Returns the first non-NULL value from value1, value2, ....

              +

              For example, COALESCE(3, 5, 3) returns 3.

              +

              IF(condition, true_value, false_value)

              +

              Returns true_value if the condition is met, otherwise returns false_value.

              +

              For example, IF(5 > 3, 5, 3) returns 5.

              +

              IFNULL(input, null_replacement)

              +

              Returns null_replacement if the input is NULL; otherwise, returns the input. This function returns a data type that is very clear about whether it is empty or not compared to COALESCE or CASE WHEN. The returned type is the common type of the two parameters, but can only be empty if null_replacement can be empty. This function allows nullable columns to be passed to functions or tables that use NOT NULL constraints.

              +

              For example, IFNULL(nullable_column, 5) will never return NULL.

              +

              IS_ALPHA(string)

              +

              Returns true if all characters in the string are letters; otherwise, returns false.

              +

              IS_DECIMAL(string)

              +

              Returns true if the string can be parsed as a valid number; otherwise, returns false.

              +

              IS_DIGIT(string)

              +

              Returns true if all characters in the string are numbers; otherwise, returns false.

              +

              GREATEST(value1[, value2]*)

              +

              Returns the maximum value of all input parameters. If the input parameters contain NULL, returns NULL.

              +

              LEAST(value1[, value2]*)

              +

              Returns the minimum value of all input parameters. If the input parameters contain NULL, returns NULL.

              +
              +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15092.html b/docs/dli/sqlreference/dli_08_15092.html new file mode 100644 index 000000000..9d751f00f --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15092.html @@ -0,0 +1,161 @@ + + +

              Type Conversion Functions

              +
              +
              + + + + + + + + + + +
              Table 1 Type conversion functions

              SQL Function

              +

              Description

              +

              CAST(value AS type)

              +

              Returns a new value that has been converted to the type.

              +

              For example, CAST('42' AS INT) returns 42;

              +

              CAST(NULL AS VARCHAR) returns NULL of type VARCHAR.

              +

              TYPEOF(input) | TYPEOF(input, force_serializable)

              +

              Returns a string representation of the data type of the input expression. By default, the returned string is a summary string that may omit some details for readability. If force_serializable is set to TRUE, the string representation can preserve the full data type that is stored in the catalog. Note that anonymous inline data types do not have a serializable string representation, and in this case, NULL is returned.

              +
              +
              +

              CAST Syntax Format

              CAST(value AS type)
              +
              +

              CAST Syntax Description

              This syntax is used to forcibly convert types.

              +
              +

              CAST Caveats

              If the input is NULL, NULL is returned.

              +
              +

              CAST Example 1: Converting the Amount Value to an Integer

              The following example converts the amount value to an integer.

              +
              insert into temp select cast(amount as INT) from source_stream;
              + +
              + + + + + + + + + + + + + + + + + + + + + +
              Table 2 Examples of CAST type conversion functions

              Example

              +

              Description

              +

              Example

              +

              cast(v1 as string)

              +

              Converts v1 to a string. The value of v1 can be of the numeric type or of the timestamp, date, or time type.

              +

              Table T1:

              +
              | content (INT)           |
+| -------------           |
+| 5                       |
              +

              Statement:

              +
              SELECT
+  cast(content as varchar)
+FROM
+  T1;
              +

              Result:

              +
              "5"
              +

              cast (v1 as int)

              +

              Converts v1 to the int type. The value of v1 can be a number or a character.

              +

              Table T1:

              +
              | content  (STRING)           |
+| -------------               |
+| "5"                         |
              +

              Statement:

              +
              SELECT
+  cast(content as int)
+FROM
+  T1;
              +

              Result:

              +
              5
              +

              cast(v1 as timestamp)

              +

              Converts v1 to the timestamp type. The value of v1 can be of the string, date, or time type.

              +

              Table T1:

              +
              | content  (STRING)          |
+| -------------              |
+| "2018-01-01 00:00:01"     |
              +

              Statement:

              +
              SELECT
+  cast(content as timestamp)
+FROM
+  T1;
              +

              Result:

              +
              1514736001000
              +

              cast(v1 as date)

              +

              Converts v1 to the date type. The value of v1 can be of the string or timestamp type.

              +

              Table T1:

              +
              | content  (TIMESTAMP)     |
+| -------------            |
+| 1514736001000            |
              +

              Statement:

              +
              SELECT
+  cast(content as date)
+FROM
+  T1;
              +

              Result:

              +
              "2018-01-01"
              +
              +
              +

              Flink jobs do not support the conversion of bigint to timestamp using CAST. You can convert it using to_timestamp.

              +
              +
              +

              CAST Example 2

              1. Create a Flink OpenSource SQL job by referring to Kafka and Print, enter the following job running script, and submit the job.
                When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold in the following script according to the actual situation.
                CREATE TABLE kafkaSource (
+  cast_int_to_string int, 
+  cast_String_to_int string,
+  case_string_to_timestamp string, 
+  case_timestamp_to_date timestamp
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  "format" = "json"
+);
+
+CREATE TABLE printSink (
+  cast_int_to_string string, 
+  cast_String_to_int int, 
+  case_string_to_timestamp timestamp, 
+  case_timestamp_to_date date
+) WITH (
+  'connector' = 'print'
+);
+
+insert into printSink select 
+  cast(cast_int_to_string as string),
+  cast(cast_String_to_int as int),
+  cast(case_string_to_timestamp as timestamp),  
+  cast(case_timestamp_to_date as date)
+from kafkaSource;
                +
                +
              2. Connect to the Kafka cluster and send the following test data to the Kafka topic:
                {"cast_int_to_string":"1", "cast_String_to_int": "1", "case_string_to_timestamp": "2022-04-02 15:00:00", "case_timestamp_to_date": "2022-04-02 15:00:00"}
                +
              3. View output.
                • Method 1:
                  1. Log in to the DLI management console and choose Job Management > Flink Streaming Jobs.
                  2. Locate the row that contains the target Flink job, and choose More & > FlinkUI in the Operation column.
                  3. On the Flink UI, choose Task Managers, click the task name, and select Stdout to view the job run logs.
                  +
                • Method 2: If you select Save Job Log on the Running Parameters tab before submitting the job, perform the following operations:
                  1. Log in to the DLI management console and choose Job Management > Flink Streaming Jobs.
                  2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the corresponding log based on the job running date.
                  3. Go to the folder of the corresponding date, find the folder whose name contains taskmanager, download the taskmanager.out file, and view the result log.
                  +
                +
                The query result is as follows:
                +I(1,1,2022-04-02T15:00,2022-04-02)
                +
                +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15093.html b/docs/dli/sqlreference/dli_08_15093.html new file mode 100644 index 000000000..c5958627e --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15093.html @@ -0,0 +1,46 @@ + + +

              Collection Functions

              +

              Description

              +
              + + + + + + + + + + + + + + + + + + + +
              Table 1 Collection functions

              Collection Function

              +

              Description

              +

              CARDINALITY(array)

              +

              Returns the number of elements in the array.

              +

              array '[' INT ']'

              +

              Returns the element at the INT position in the array. The index starts at 1.

              +

              ELEMENT(array)

              +

              Returns the unique element in the array (with a base of 1); returns NULL if the array is empty; throws an exception if there are multiple elements in the array.

              +

              CARDINALITY(map)

              +

              Returns the number of entries in the map.

              +

              map '[' value ']'

              +

              Returns the value corresponding to the specified key in the map.

              +
              +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15094.html b/docs/dli/sqlreference/dli_08_15094.html new file mode 100644 index 000000000..dba9c3582 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15094.html @@ -0,0 +1,229 @@ + + +

              JSON Functions

              +

              JSON functions use JSON path expressions described in the SQL standard ISO/IEC TR 19075-6. Their syntax is inspired by ECMAScript and adopts many of its features, but is neither a subset nor a superset of it.

              +

              There are two types of path expressions: lax mode and strict mode. When omitted, it defaults to strict mode. Strict mode is intended to check data from a schema perspective and will throw an error when data does not conform to the path expression. However, functions like JSON_VALUE allow for defining fallback behavior when encountering errors. Lax mode, on the other hand, will convert errors to an empty sequence.

              +

              The special character $ represents the root node in a JSON path. Paths can access properties ($.a), array elements ($.a[0].b), or all elements in an array ($.a[*].b).

              +

              Known limitations: not all features of lax mode are currently supported correctly.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 JSON functions

              SQL Function

              +

              Description

              +

              IS JSON [ { VALUE | SCALAR | ARRAY | OBJECT } ]

              +

              Determines whether a given string is a valid JSON string.

              +

              Specifying an optional type parameter will impose constraints on the allowed types of JSON objects. If the string is a valid JSON but not of that type, it returns false. The default value is VALUE.

              +
              -- TRUE
+'1' IS JSON
+'[]' IS JSON
+'{}' IS JSON
+
+-- TRUE
+'"abc"' IS JSON
+-- FALSE
+'abc' IS JSON
+NULL IS JSON
+
+-- TRUE
+'1' IS JSON SCALAR
+-- FALSE
+'1' IS JSON ARRAY
+-- FALSE
+'1' IS JSON OBJECT
+
+-- FALSE
+'{}' IS JSON SCALAR
+-- FALSE
+'{}' IS JSON ARRAY
+-- TRUE
+'{}' IS JSON OBJECT
              +

              JSON_EXISTS(jsonValue, path [ { TRUE | FALSE | UNKNOWN | ERROR } ON ERROR ])

              +

              Determines whether a JSON string satisfies a given path search condition.

              +

              If error behavior is ignored, FALSE ON ERROR is the default value.

              +
              -- TRUE
+SELECT JSON_EXISTS('{"a": true}', '$.a');
+-- FALSE
+SELECT JSON_EXISTS('{"a": true}', '$.b');
+-- TRUE
+SELECT JSON_EXISTS('{"a": [{ "b": 1 }]}',
+  '$.a[0].b');
+
+-- TRUE
+SELECT JSON_EXISTS('{"a": true}',
+  'strict $.b' TRUE ON ERROR);
+-- FALSE
+SELECT JSON_EXISTS('{"a": true}',
+  'strict $.b' FALSE ON ERROR);
              +

              JSON_STRING(value)

              +

              Serializes a value to JSON.

              +

              This function returns a JSON string containing the serialized value. If the value is NULL, the function returns NULL.

              +
              -- NULL
+JSON_STRING(CAST(NULL AS INT))
+
+-- '1'
+JSON_STRING(1)
+-- 'true'
+JSON_STRING(TRUE)
+-- '"Hello, World!"'
+JSON_STRING('Hello, World!')
+-- '[1,2]'
+JSON_STRING(ARRAY[1, 2])
              +

              JSON_VALUE(jsonValue, path [RETURNING <dataType>] [ { NULL | ERROR | DEFAULT <defaultExpr> } ON EMPTY ] [ { NULL | ERROR | DEFAULT <defaultExpr> } ON ERROR ])

              +

              Extracts a scalar from a JSON string.

              +

              This method searches for the given path expression in the JSON string and returns the value if it is a scalar. If it is not a scalar value, it cannot be returned. By default, the value is returned as a STRING type. The returnType can be used to select a different type, supporting the following types:

              +

              VARCHAR/STRING

              +

              BOOLEAN

              +

              INTEGER

              +

              DOUBLE

              +

              For empty path expressions or errors, it can be defined to return null, throw an error, or return a defined default value. If omitted, the default value is NULL ON EMPTY or NULL ON ERROR. The default value can be a literal or an expression. If the default value itself causes an error, it will execute the error behavior of ON EMPTY and ON ERROR.

              +
              -- "true"
+JSON_VALUE('{"a": true}', '$.a')
+
+-- TRUE
+JSON_VALUE('{"a": true}', '$.a' RETURNING BOOLEAN)
+
+-- "false"
+JSON_VALUE('{"a": true}', 'lax $.b'
+    DEFAULT FALSE ON EMPTY)
+
+-- "false"
+JSON_VALUE('{"a": true}', 'strict $.b'
+    DEFAULT FALSE ON ERROR)
+
+-- 0.998D
+JSON_VALUE('{"a.b": [0.998,0.996]}','$.["a.b"][0]' 
+    RETURNING DOUBLE)
              +

              JSON_QUERY(jsonValue, path [ { WITHOUT | WITH CONDITIONAL | WITH UNCONDITIONAL } [ ARRAY ] WRAPPER ] [ { NULL | EMPTY ARRAY | EMPTY OBJECT | ERROR } ON EMPTY ] [ { NULL | EMPTY ARRAY | EMPTY OBJECT | ERROR } ON ERROR ])

              +

              Extracts a JSON value from a JSON string.

              +

              The result is always returned as a STRING. The RETURNING clause is currently not supported.

              +

              The wrappingBehavior determines whether the extracted value should be wrapped in an array unconditionally or only if the value itself is not an array.

              +

              The onEmpty and onError determine the behavior when the path expression is empty or throws an error. By default, null is returned in both cases. Other options are to use an empty array, an empty object, or throw an error.

              +
              -- '{ "b": 1 }'
+JSON_QUERY('{ "a": { "b": 1 } }', '$.a')
+-- '[1, 2]'
+JSON_QUERY('[1, 2]', '$')
+-- NULL
+JSON_QUERY(CAST(NULL AS STRING), '$')
+-- '["c1","c2"]'
+JSON_QUERY('{"a":[{"c":"c1"},{"c":"c2"}]}',
+    'lax $.a[*].c')
+
+-- Wrap result into an array
+-- '[{}]'
+JSON_QUERY('{}', '$' WITH CONDITIONAL ARRAY WRAPPER)
+-- '[1, 2]'
+JSON_QUERY('[1, 2]', '$' WITH CONDITIONAL ARRAY WRAPPER)
+-- '[[1, 2]]'
+JSON_QUERY('[1, 2]', '$' WITH UNCONDITIONAL ARRAY WRAPPER)
+
+-- Scalars must be wrapped to be returned
+-- NULL
+JSON_QUERY(1, '$')
+-- '[1]'
+JSON_QUERY(1, '$' WITH CONDITIONAL ARRAY WRAPPER)
+
+-- Behavior if path expression is empty / there is an error
+-- '{}'
+JSON_QUERY('{}', 'lax $.invalid' EMPTY OBJECT ON EMPTY)
+-- '[]'
+JSON_QUERY('{}', 'strict $.invalid' EMPTY ARRAY ON ERROR)
              +

              JSON_OBJECT([[KEY] key VALUE value]* [ { NULL | ABSENT } ON NULL ])

              +

              Builds a JSON object string from a list of key-value pairs.

              +

              Note that the keys must be non-null string literals, while the values can be any expression.

              +

              The function returns a JSON string. The ON NULL behavior defines how to handle NULL values. If omitted, the default is NULL ON NULL.

              +

              Values created from another JSON constructor (JSON_OBJECT, JSON_ARRAY) will be inserted directly instead of being inserted as a string. This allows for building nested JSON structures.

              +
              -- '{}'
+JSON_OBJECT()
+
+-- '{"K1":"V1","K2":"V2"}'
+JSON_OBJECT('K1' VALUE 'V1', 'K2' VALUE 'V2')
+
+-- Expressions as values
+JSON_OBJECT('orderNo' VALUE orders.orderId)
+
+-- ON NULL
+JSON_OBJECT(KEY 'K1' VALUE CAST(NULL AS STRING) NULL ON NULL)   -- '{"K1":null}'
+JSON_OBJECT(KEY 'K1' VALUE CAST(NULL AS STRING) ABSENT ON NULL) -- '{}'
+
+-- '{"K1":{"K2":"V"}}'
+JSON_OBJECT(
+  KEY 'K1'
+  VALUE JSON_OBJECT(
+    KEY 'K2'
+    VALUE 'V'
+  )
+)
              +

              JSON_OBJECTAGG([KEY] key VALUE value [ { NULL | ABSENT } ON NULL ])

              +

              Builds a JSON object string by aggregating key-value expressions into a single JSON object.

              +

              The key expression must return a non-null string. The value expression can be anything, including other JSON functions. If the value is NULL, the ON NULL behavior defines what to do. If omitted, the default is NULL ON NULL.

              +

              Note that keys must be unique. If a key appears multiple times, an error will be thrown.

              +

              This feature is currently not supported in OVER windows.

              +
              -- '{"Apple":2,"Banana":17,"Orange":0}'
+SELECT
+  JSON_OBJECTAGG(KEY product VALUE cnt)
+FROM orders
              +

              JSON_ARRAY([value]* [ { NULL | ABSENT } ON NULL ])

              +

              Builds a JSON array string from a list of values.

              +

              The function returns a JSON string. These values can be any expression. The ON NULL behavior defines how to handle NULL values. If omitted, the default is ABSENT ON NULL.

              +

              Elements created from another JSON constructor (JSON_OBJECT, JSON_ARRAY) will be inserted directly instead of being inserted as a string. This allows for building nested JSON structures.

              +
              -- '[]'
+JSON_ARRAY()
+-- '[1,"2"]'
+JSON_ARRAY(1, '2')
+
+-- Expressions as values
+JSON_ARRAY(orders.orderId)
+
+-- ON NULL
+JSON_ARRAY(CAST(NULL AS STRING) NULL ON NULL) -- '[null]'
+JSON_ARRAY(CAST(NULL AS STRING) ABSENT ON NULL) -- '[]'
+
+-- '[[1]]'
+JSON_ARRAY(JSON_ARRAY(1))
              +

              JSON_ARRAYAGG(items [ { NULL | ABSENT } ON NULL ])

              +

              Builds a JSON object string by aggregating elements into an array.

              +

              The element expression can be anything, including other JSON functions. If the value is NULL, the ON NULL behavior defines what to do. If omitted, the default is ABSENT ON NULL.

              +

              This feature is currently not supported in OVER windows, unbounded session windows, or hop windows.

              +
              -- '["Apple","Banana","Orange"]'
+SELECT
+  JSON_ARRAYAGG(product)
+FROM orders
              +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15095.html b/docs/dli/sqlreference/dli_08_15095.html new file mode 100644 index 000000000..b201dc8a9 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15095.html @@ -0,0 +1,36 @@ + + +

              Value Construction Functions

              +

              Description

              +
              + + + + + + + + + + + + + +
              Table 1 Value construction functions

              Value Construction Function

              +

              Description

              +

              -- implicit constructor with parenthesis (value1 [, value2]*)

              +

              Returns a row created from a list of values (value1, value2, ...). The implicit row constructor supports any expression as a field, but requires at least two fields. The explicit row constructor can handle any number of fields, but currently does not support all types of field expressions well.

              +

              ARRAY '[' value1 [, value2 ]* ']'

              +

              Returns an array created from a list of values (value1, value2, ...).

              +

              MAP '[' value1, value2 [, value3, value4 ]* ']'

              +

              Returns a map created from a list of key-value pairs ((value1, value2), (value3, value4), ...).

              +
              +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15096.html b/docs/dli/sqlreference/dli_08_15096.html new file mode 100644 index 000000000..c3078c190 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15096.html @@ -0,0 +1,30 @@ + + +

              Value Retrieval Functions

              +
              +
              + + + + + + + + + + +
              Table 1 Value retrieval functions

              SQL Function

              +

              Description

              +

              tableName.compositeType.field

              +

              Returns the value of a field from a Flink composite type (e.g. Tuple, POJO) by name.

              +

              tableName.compositeType.*

              +

              Returns the flattened representation of a Flink composite type (e.g. Tuple, POJO), converting each of its immediate subtypes into a separate field. In most cases, the fields in the flattened representation have similar names to the original fields, but with a $ separator (e.g. mypojo$mytuple$f0).

              +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15097.html b/docs/dli/sqlreference/dli_08_15097.html new file mode 100644 index 000000000..f2b11dd63 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15097.html @@ -0,0 +1,30 @@ + + +

              Grouping Functions

              +
              +
              + + + + + + + + + + +
              Table 1 Grouping functions

              SQL Function

              +

              Description

              +

              GROUP_ID()

              +

              Returns an integer that uniquely identifies the combination of grouping keys.

              +

              GROUPING(expression1 [, expression2]* ) | GROUPING_ID(expression1 [, expression2]* )

              +

              Returns a bit vector for the given grouping expression.

              +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15098.html b/docs/dli/sqlreference/dli_08_15098.html new file mode 100644 index 000000000..76a405797 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15098.html @@ -0,0 +1,55 @@ + + +

              Hash Functions

              +
              +
              + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Hash functions

              Hash Function

              +

              Description

              +

              MD5(string)

              +

              Returns the MD5 hash value of the string as a 32-digit hexadecimal string and returns NULL if the string is NULL.

              +

              SHA1(string)

              +

              Returns the SHA-1 hash value of the string as a 40-digit hexadecimal string and returns NULL if the string is NULL.

              +

              SHA224(string)

              +

              Returns the SHA-224 hash value of the string as a 56-digit hexadecimal string and returns NULL if the string is NULL.

              +

              SHA256(string)

              +

              Returns the SHA-256 hash value of the string as a 64-digit hexadecimal string and returns NULL if the string is NULL.

              +

              SHA384(string)

              +

              Returns the SHA-384 hash value of the string as a 96-digit hexadecimal string and returns NULL if the string is NULL.

              +

              SHA512(string)

              +

              Returns the SHA-512 hash value of the string as a 128-digit hexadecimal string and returns NULL if the string is NULL.

              +

              SHA2(string, hashLength)

              +

              Uses the SHA-2 series hash function (SHA-224, SHA-256, SHA-384, or SHA-512) to return the hash value. The first parameter is the string to be hashed, and the second parameter hashLength is the length of the result in bits (224, 256, 384, or 512). Returns NULL if string or hashLength is NULL.

              +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15099.html b/docs/dli/sqlreference/dli_08_15099.html new file mode 100644 index 000000000..866a6b3ea --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15099.html @@ -0,0 +1,121 @@ + + +

              Aggregate Functions

              +

              Aggregate functions process all rows as input and produce a single aggregate value as the output.

              + +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Aggregate functions

              Function

              +

              Description

              +

              COUNT([ ALL ] expression | DISTINCT expression1 [, expression2]*)

              +

              By default or with the keyword ALL, returns the number of input rows where the expression is not NULL. Using DISTINCT calculates the count after removing duplicates.

              +

              COUNT(*) | COUNT(1)

              +

              Returns the number of input rows.

              +

              AVG([ ALL | DISTINCT ] expression)

              +

              By default or with the keyword ALL, returns the average value (arithmetic mean) of the expression across all input rows. Using DISTINCT calculates the average after removing duplicates.

              +

              SUM([ ALL | DISTINCT ] expression)

              +

              By default or with the keyword ALL, returns the sum of the expression across all input rows. Using DISTINCT calculates the sum after removing duplicates.

              +

              MAX([ ALL | DISTINCT ] expression)

              +

              By default or with the keyword ALL, returns the maximum value of the expression across all input rows. Using DISTINCT calculates the maximum after removing duplicates.

              +

              MIN([ ALL | DISTINCT ] expression )

              +

              By default or with the keyword ALL, returns the minimum value of the expression across all input rows. Using DISTINCT calculates the minimum after removing duplicates.

              +

              STDDEV_POP([ ALL | DISTINCT ] expression)

              +

              By default or with the keyword ALL, returns the population standard deviation of the expression across all input rows. Using DISTINCT calculates the standard deviation after removing duplicates.

              +

              STDDEV_SAMP([ ALL | DISTINCT ] expression)

              +

              By default or with the keyword ALL, returns the sample standard deviation of the expression across all input rows. Using DISTINCT calculates the standard deviation after removing duplicates.

              +

              VAR_POP([ ALL | DISTINCT ] expression)

              +

              By default or with the keyword ALL, returns the population variance (square of the population standard deviation) of the expression across all input rows. Using DISTINCT calculates the variance after removing duplicates.

              +

              VAR_SAMP([ ALL | DISTINCT ] expression)

              +

              By default or with the keyword ALL, returns the sample variance (square of the sample standard deviation) of the expression across all input rows. Using DISTINCT calculates the variance after removing duplicates.

              +

              COLLECT([ ALL | DISTINCT ] expression)

              +

              By default or with the keyword ALL, returns multiple sets of expressions across all input rows. NULL values are ignored. Using DISTINCT calculates the sets after removing duplicates.

              +

              VARIANCE([ ALL | DISTINCT ] expression)

              +

              A synonym for VAR_SAMP().

              +

              RANK()

              +

              Returns the rank of a value within a set of values. The result is 1 plus the number of rows preceding or equal to the current row in the ordering of the partition. The rank may not be consecutive in the sequence.

              +

              DENSE_RANK()

              +

              Returns the rank of a value within a set of values. The result is one plus the previously assigned rank value. Unlike the rank function, dense_rank does not leave gaps in the ranking sequence.

              +

              ROW_NUMBER()

              +

              Assigns a unique sequential number to each row within a window partition based on the ordering of rows by rows. ROW_NUMBER is similar to RANK. ROW_NUMBER numbers all rows sequentially (for example, 1, 2, 3, 4, 5). RANK provides the same sequence value for equal rows (for example, 1, 2, 2, 4, 5).

              +

              LEAD(expression [, offset] [, default])

              +

              Returns the value of the expression at the offset-th row after the current row in the window. The default value of offset is 1, and the default value of default is NULL.

              +

              LAG(expression [, offset] [, default])

              +

              Returns the value of the expression at the offset-th row before the current row in the window. The default value of offset is 1, and the default value of default is NULL.

              +

              FIRST_VALUE(expression)

              +

              Returns the first value in a set of ordered values.

              +

              LAST_VALUE(expression)

              +

              Returns the last value in a set of ordered values.

              +

              LISTAGG(expression [, separator])

              +

              Concatenates the values of string expressions and places a separator value between them. The default separator value is , if no separator is added at the end of the string.

              +
              +
              +
              +
              +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15101.html b/docs/dli/sqlreference/dli_08_15101.html new file mode 100644 index 000000000..22260e015 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15101.html @@ -0,0 +1,15 @@ + + +

              Table-Valued Functions

              +
              +
              + + +
              +
              Parent topic: Built-In Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15102.html b/docs/dli/sqlreference/dli_08_15102.html new file mode 100644 index 000000000..46d5974ce --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15102.html @@ -0,0 +1,149 @@ + + +

              string_split

              +

              The string_split function splits a target string into substrings based on the specified separator and returns a substring list.

              +

              Description

              string_split(target, separator)
              + +
              + + + + + + + + + + + + + +
              Table 1 string_split parameters

              Parameter

              +

              Data Types

              +

              Description

              +

              target

              +

              STRING

              +

              Target string to be processed

              +
              NOTE:
              • If target is NULL, an empty line is returned.
              • If target contains two or more consecutive separators, an empty substring is returned.
              • If target does not contain a specified separator, the original string passed to target is returned.
              +
              +

              separator

              +

              VARCHAR

              +

              Separator. Currently, only single-character separators are supported.

              +
              +
              +
              +

              Example

              1. Create a Flink OpenSource SQL job by referring to Kafka and Print, enter the following job running script, and submit the job.
                When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.
                CREATE TABLE kafkaSource (
+  target STRING,    
+  separator  VARCHAR
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE printSink (
+  target STRING,    
+  item STRING
+) WITH (
+  'connector' = 'print'
+);
+insert into printSink select target,  item from kafkaSource, lateral table(string_split(target, separator)) as T(item);
                +
                +
              2. Connect to the Kafka cluster and send the following test data to the Kafka topic:
                {"target":"test-flink","separator":"-"}
+{"target":"flink","separator":"-"}
+{"target":"one-two-ww-three","separator":"-"}
                +

                The data is as follows:

                + +
                + + + + + + + + + + + + + +
                Table 2 Test table data

                target (STRING)

                +

                separator (VARCHAR)

                +

                test-flink

                +

                -

                +

                flink

                +

                -

                +

                one-two-ww-three

                +

                -

                +
                +
                +
              3. View output.
                • Method 1:
                  1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                  2. Locate the row that contains the target Flink job, and choose More > FlinkUI in the Operation column.
                  3. On the Flink UI, choose Task Managers, click the task name, and select Stdout to view job logs.
                  +
                • Method 2: If you select Save Job Log on the Running Parameters tab before submitting the job, perform the following operations:
                  1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                  2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                  3. Go to the folder of the date, find the folder whose name contains taskmanager, download the taskmanager.out file, and view result logs.
                  +
                +
                The query result is as follows:
                +I(test-flink,test)
++I(test-flink,flink)
++I(flink,flink)
++I(one-two-ww-three,one)
++I(one-two-ww-three,two)
++I(one-two-ww-three,ww)
++I(one-two-ww-three,three)
                +
                +

                The output data is as follows:

                + +
                + + + + + + + + + + + + + + + + + + + + + + + + + +
                Table 3 Result table data

                target (STRING)

                +

                item (STRING)

                +

                test-flink

                +

                test

                +

                test-flink

                +

                flink

                +

                flink

                +

                flink

                +

                one-two-ww-three

                +

                one

                +

                one-two-ww-three

                +

                two

                +

                one-two-ww-three

                +

                ww

                +

                one-two-ww-three

                +

                three

                +
                +
                +
              +
              +
              +
              +
              +
              Parent topic: Table-Valued Functions
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15103.html b/docs/dli/sqlreference/dli_08_15103.html new file mode 100644 index 000000000..226a25e1d --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15103.html @@ -0,0 +1,21 @@ + + +

              GaussDB(DWS)

              +
              + + diff --git a/docs/dli/sqlreference/dli_08_15104.html b/docs/dli/sqlreference/dli_08_15104.html new file mode 100644 index 000000000..479ae2c79 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15104.html @@ -0,0 +1,287 @@ + + +

              GaussDB(DWS) Source Table (Not Recommended)

              +

              Function

              DLI reads data of Flink jobs from GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex types and deliver space information services, multi-version concurrent control (MVCC), and high concurrency. It applies to location applications, financial insurance, and e-Commerce.

              +

              GaussDB(DWS) is an online data processing database based on the cloud infrastructure and platform and helps you mine and analyze massive sets of data.

              +

              You are advised to use GaussDB(DWS) self-developed GaussDB(DWS) connector.

              +
              +
              +

              Prerequisites

              • You have created a GaussDB(DWS) cluster.

                For details about how to create a GaussDB(DWS) cluster, see Creating a Cluster in the Data Warehouse Service Management Guide.

                +
              • You have created a GaussDB(DWS) database table.
              • An enhanced datasource connection has been created for DLI to connect to GaussDB(DWS) clusters, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required. +
              +
              +

              Precautions

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • Fields in the with parameter can only be enclosed in single quotes.
              +
              +

              Syntax

               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
              create table dwsSource (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (','PRIMARY KEY (attr_name, ...) NOT ENFORCED)
+  (',' watermark for rowtime_column_name as watermark-strategy_expression)
+)
+with (
+  'connector' = 'gaussdb',
+  'url' = '',
+  'table-name' = '',
+  'username' = '',
+  'password' = ''
+);
+
              +
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameter description

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector to be used. Set this parameter to gaussdb.

              +

              url

              +

              Yes

              +

              None

              +

              String

              +

              JDBC connection address. Set the IP address in this parameter to the internal IP address of GaussDB(DWS).

              +

              If you use the gsjdbc4 driver, set the value in jdbc:postgresql://${ip}:${port}/${dbName} format.

              +

              If you use the gsjdbc200 driver, set the value in jdbc:gaussdb://${ip}:${port}/${dbName} format.

              +

              table-name

              +

              Yes

              +

              None

              +

              String

              +

              Name of the GaussDB(DWS) table to be operated. If the GaussDB(DWS) table is in a schema, refer to the description of GaussDB(DWS) table in a schema.

              +

              driver

              +

              No

              +

              org.postgresql.Driver

              +

              String

              +

              JDBC connection driver. The default value is org.postgresql.Driver.

              +

              username

              +

              No

              +

              None

              +

              String

              +

              Username for GaussDB(DWS) database authentication. This parameter must be configured in pair with password.

              +

              password

              +

              No

              +

              None

              +

              String

              +

              Password for GaussDB(DWS) database authentication. This parameter must be configured in pair with username.

              +

              scan.partition.column

              +

              No

              +

              None

              +

              String

              +

              Name of the column used to partition the input.

              +

              Note: This parameter must be used together with scan.partition.lower-bound, scan.partition.upper-bound, and scan.partition.num.

              +

              scan.partition.lower-bound

              +

              No

              +

              None

              +

              Integer

              +

              Lower bound of values to be fetched for the first partition.

              +

              This parameter must be used together with scan.partition.column, scan.partition.upper-bound, and scan.partition.num.

              +

              scan.partition.upper-bound

              +

              No

              +

              None

              +

              Integer

              +

              Upper bound of values to be fetched for the last partition.

              +

              This parameter must be used together with scan.partition.column, scan.partition.lower-bound, and scan.partition.num.

              +

              scan.partition.num

              +

              No

              +

              None

              +

              Integer

              +

              Number of partitions to be created.

              +

              This parameter must be used together with scan.partition.column, scan.partition.upper-bound, and scan.partition.upper-bound.

              +

              scan.fetch-size

              +

              No

              +

              0

              +

              Integer

              +

              Number of rows fetched from the database each time. The default value is 0, indicating that the number of rows is not limited.

              +
              +
              +
              +

              Example

              In this example, data is read from the GaussDB(DWS) data source and written to the Print result table. The procedure is as follows:

              +
              1. Create a table named dws_order in GaussDB(DWS).
                create table public.dws_order(
+  order_id VARCHAR,
+  order_channel VARCHAR,
+  order_time VARCHAR,
+  pay_amount FLOAT8,
+  real_pay FLOAT8,
+  pay_time VARCHAR,
+  user_id VARCHAR,
+  user_name VARCHAR,
+  area_id VARCHAR);
                +
                Insert data into the dws_order table.
                insert into public.dws_order
+  (order_id,
+  order_channel,
+  order_time,
+  pay_amount,
+  real_pay,
+  pay_time,
+  user_id,
+  user_name,
+  area_id) values
+  ('202103241000000001', 'webShop', '2021-03-24 10:00:00', '100.00', '100.00', '2021-03-24 10:02:03', '0001', 'Alice', '330106'),
+  ('202103251202020001', 'miniAppShop', '2021-03-25 12:02:02', '60.00', '60.00', '2021-03-25 12:03:00', '0002', 'Bob', '330110');
                +
                +
              2. Create an enhanced datasource connection in the VPC and subnet where GaussDB(DWS) locates, and bind the connection to the required Flink elastic resource pool.
              3. Set GaussDB(DWS) security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the GaussDB(DWS) address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              4. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. The job script uses the GaussDB(DWS) data source and the Print result table.
                When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.
                CREATE TABLE dwsSource (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'gaussdb',
+  'url' = 'jdbc:postgresql://DWSIP:DWSPort/DWSdbName',
+  'table-name' = 'dws_order',
+  'driver' = 'org.postgresql.Driver',
+  'username' = 'DWSUserName',
+  'password' = 'DWSPassword'
+);
+
+CREATE TABLE printSink (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'print'
+);
+
+insert into printSink select * from dwsSource;
                +
                +
              5. Perform the following operations to view the data result in the taskmanager.out file:
                1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                3. Go to the folder of the date, find the folder whose name contains taskmanager, download the taskmanager.out file, and view result logs.
                +

                The data result is as follows:

                +
                +I(202103241000000001,webShop,2021-03-24 10:00:00,100.0,100.0,2021-03-24 10:02:03,0001,Alice,330106)
++I(202103251202020001,miniAppShop,2021-03-25 12:02:02,60.0,60.0,2021-03-25 12:03:00,0002,Bob,330110)
                +
              +
              +

              FAQ

              • Q: What should I do if the job execution fails and the log contains the following error information?
                java.io.IOException: unable to open JDBC writer
+...
+Caused by: org.postgresql.util.PSQLException: The connection attempt failed.
+...
+Caused by: java.net.SocketTimeoutException: connect timed out
                +
                A: The datasource connection is not bound or the binding fails. +
                +
              +
              +
              • Q: How can I configure a GaussDB(DWS) table that is in a schema?

                A: The following provides an example of configuring the dws_order table in the dbuser2 schema:

                +
                CREATE TABLE dwsSource (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'gaussdb',
+  'url' = 'jdbc:postgresql://DWSIP:DWSPort/DWSdbName',
+  'table-name' = 'dbuser2.dws_order',
+  'driver' = 'org.postgresql.Driver',
+  'username' = 'DWSUserName',
+  'password' = 'DWSPassword'
+);
                +
              +
              +
              +
              +
              Parent topic: GaussDB(DWS)
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15105.html b/docs/dli/sqlreference/dli_08_15105.html new file mode 100644 index 000000000..2cba88bfb --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15105.html @@ -0,0 +1,329 @@ + + +

              GaussDB(DWS) Result Table (Not Recommended)

              +

              Function

              DLI outputs the Flink job output data to GaussDB(DWS). GaussDB(DWS) database kernel is compliant with PostgreSQL. The PostgreSQL database can store data of more complex types and deliver space information services, multi-version concurrent control (MVCC), and high concurrency. It applies to location applications, financial insurance, and e-Commerce.

              +

              GaussDB(DWS) is an online data processing database based on the cloud infrastructure and platform and helps you mine and analyze massive sets of data.

              +

              You are advised to use GaussDB(DWS) self-developed GaussDB(DWS) connector.

              +
              +
              +

              Prerequisites

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • You have created a GaussDB(DWS) cluster. For details about how to create a GaussDB(DWS) cluster, see Creating a Cluster in the Data Warehouse Service Management Guide.
              • You have created a GaussDB(DWS) database table.
              • An enhanced datasource connection has been created for DLI to connect to GaussDB(DWS) clusters, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required. +
              +
              +

              Precautions

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • Fields in the with parameter can only be enclosed in single quotes.
              +
              • To use the upsert mode, you must define the primary key for both the GaussDB(DWS) result table and the GaussDB(DWS) table connected to the result table.
              • If tables with the same name exist in different GaussDB(DWS) schemas, you need to specify the schemas in the Flink open source SQL statements.
              • If you use the gsjdbc4 driver for connection, set driver to org.postgresql.Driver. You can omit this parameter because the gsjdbc4 driver is the default one.
                For example, run the following statements to use the gsjdbc4 driver to write data to GaussDB(DWS) in upsert mode:
                 1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
                create table dwsSink(
+  car_id STRING,
+  car_owner STRING,
+  car_brand STRING,
+  car_speed INT
+) with (
+  'connector' = 'gaussdb',
+  'url' = 'jdbc:postgresql://DwsAddress:DwsPort/DwsDatabase',
+  'table-name' = 'car_info',
+  'username' = 'DwsUserName',
+  'password' = 'DwsPasswrod',
+  'write.mode' = 'upsert'
+);
+
                +
                +
                +
              +
              +

              Syntax

              Do not set all attributes in a GaussDB(DWS) result table to PRIMARY KEY.

              +
              +
               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
              create table dwsSink (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+  (','PRIMARY KEY (attr_name, ...) NOT ENFORCED)
+)
+with (
+  'connector' = 'gaussdb',
+  'url' = '',
+  'table-name' = '',
+  'driver' = '',
+  'username' = '',
+  'password' = ''
+);
+
              +
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameter description

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Type

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector to be used. Set this parameter to gaussdb.

              +

              url

              +

              Yes

              +

              None

              +

              String

              +

              JDBC connection address.

              +

              If you use the gsjdbc4 driver, set the value in jdbc:postgresql://${ip}:${port}/${dbName} format.

              +

              If you use the gsjdbc200 driver, set the value in jdbc:gaussdb://${ip}:${port}/${dbName} format.

              +

              table-name

              +

              Yes

              +

              None

              +

              String

              +

              Name of the table to be operated. If the GaussDB(DWS) table is in a schema, the format is schema\".\"Table name. For details, see FAQ.

              +

              driver

              +

              No

              +

              org.postgresql.Driver

              +

              String

              +

              JDBC connection driver. The default value is org.postgresql.Driver.

              +

              username

              +

              No

              +

              None

              +

              String

              +

              Username for GaussDB(DWS) database authentication. This parameter must be configured in pair with password.

              +

              password

              +

              No

              +

              None

              +

              String

              +

              Password for GaussDB(DWS) database authentication. This parameter must be configured in pair with username.

              +

              write.mode

              +

              No

              +

              None

              +

              String

              +

              Data write mode. The value can be copy, insert, or upsert. The default value is upsert.

              +

              This parameter must be configured depending on primary key.

              +
              • If primary key is not configured, data can be appended in copy and insert modes.
              • If primary key is configured, all the three modes are available.
              +

              Note: GaussDB(DWS) does not support the update of distribution columns. The primary keys of columns to be updated must cover all distribution columns defined in the GaussDB(DWS) table.

              +

              sink.buffer-flush.max-rows

              +

              No

              +

              100

              +

              Integer

              +

              Maximum number of rows to buffer for each write request.

              +

              It can improve the performance of writing data, but may increase the latency.

              +

              You can set this parameter to 0 to disable it.

              +

              sink.buffer-flush.interval

              +

              No

              +

              1s

              +

              Duration

              +

              Interval for refreshing the buffer, during which data is refreshed by asynchronous threads.

              +

              It can improve the performance of writing data to the database, but may increase the latency.

              +

              You can set this parameter to 0 to disable it.

              +

              Note: If sink.buffer-flush.max-size and sink.buffer-flush.max-rows are both set to 0 and the buffer refresh interval is configured, the buffer is asynchronously refreshed.

              +

              The format is {length value}{time unit label}, for example, 123ms, 321s. The supported time units include d, h, min, s, and ms (default unit).

              +

              sink.max-retries

              +

              No

              +

              3

              +

              Integer

              +

              Maximum number of write retries.

              +

              write.escape-string-value

              +

              No

              +

              false

              +

              Boolean

              +

              Whether to escape values of the string type. This parameter is used only when write.mode is set to copy.

              +

              key-by-before-sink

              +

              No

              +

              false

              +

              Boolean

              +

              Whether to partition by the specified primary key before the sink operator

              +

              This parameter aims to solve the problem of interlocking between two subtasks when they acquire row locks based on the primary key from GaussDB(DWS), multiple concurrent writes occur, and write.mode is upsert. This happens when a batch of data written to the sink by multiple subtasks has more than one record with the same primary key, and the order of these records with the same primary key is inconsistent.

              +
              +
              +
              +

              Example

              In this example, data is read from the Kafka data source and written to the GaussDB(DWS) result table in insert mode. The procedure is as follows:

              +
              1. Create an enhanced datasource connection in the VPC and subnet where GaussDB(DWS) and Kafka locate, and bind the connection to the required Flink elastic resource pool.
              2. Set GaussDB(DWS) and Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the GaussDB(DWS) and Kafka address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.
              3. Connect to the GaussDB(DWS) database and create a table named dws_order.
                create table public.dws_order(
+  order_id VARCHAR,
+  order_channel VARCHAR,
+  order_time VARCHAR,
+  pay_amount FLOAT8,
+  real_pay FLOAT8,
+  pay_time VARCHAR,
+  user_id VARCHAR,
+  user_name VARCHAR,
+  area_id VARCHAR);
                +
              4. Create a Flink OpenSource SQL job. Enter the following job script and submit the job. The job script uses the Kafka data source and the GaussDB(DWS) result table.
                When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Change the values of the parameters in bold as needed in the following script.
                CREATE TABLE kafkaSource (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'GroupId',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+CREATE TABLE dwsSink (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string
+) WITH (
+  'connector' = 'gaussdb',
+  'url' = 'jdbc:postgresql://DWSAddress:DWSPort/DWSdbName',
+  'table-name' = 'dws_order',
+  'driver' = 'org.postgresql.Driver',
+  'username' = 'DWSUserName',
+  'password' = 'DWSPassword',
+  'write.mode' = 'insert'
+);
+
+insert into dwsSink select * from kafkaSource;
                +
                +
              5. Connect to the Kafka cluster and enter the following test data to Kafka:
                {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
                +
              6. Run the following SQL statement in GaussDB(DWS) to view the data result:
                 select * from dws_order
                +
                The data result is as follows:
                202103241000000001	webShop	2021-03-24 10:00:00	100.0	100.0	2021-03-24 10:02:03	0001	Alice	330106
                +
                +
              +
              +

              FAQ

              • Q: What should I do if the Flink job execution fails and the log contains the following error information?
                java.io.IOException: unable to open JDBC writer
+...
+Caused by: org.postgresql.util.PSQLException: The connection attempt failed.
+...
+Caused by: java.net.SocketTimeoutException: connect timed out
                +
                A: The datasource connection is not bound or the binding fails. +
                +
              +
              • Q: How can I configure a GaussDB(DWS) table that is in a schema?
                A: When GaussDB(DWS) table test is in schema ads_game_sdk_base, refer to the 'table-name' parameter setting in the following example:
                CREATE TABLE ads_rpt_game_sdk_realtime_ada_reg_user_pay_mm (
+  ddate DATE,
+  dmin TIMESTAMP(3),
+  game_appkey VARCHAR,
+  channel_id VARCHAR,
+  pay_user_num_1m bigint,
+  pay_amt_1m bigint,
+  PRIMARY KEY (ddate, dmin, game_appkey, channel_id) NOT ENFORCED
+) WITH (
+  'connector' = 'gaussdb',
+  'url' = 'jdbc:postgresql://<yourDwsAddress>:<yourDwsPort>/dws_bigdata_db',
+  'table-name' = 'ads_game_sdk_base.test',
+  'username' = '<yourUsername>',
+  'password' = '<yourPassword>',
+  'write.mode' = 'upsert'
+);
                +
                +
              +
              +
              • Q: What can I do if a job is running properly but there is no data in GaussDB(DWS)?
                A: Check the following items:
                • Check whether the JobManager and TaskManager logs contain error information. To view logs, perform the following steps:
                  1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                  2. Click the name of the corresponding Flink job, choose Run Log, click OBS Bucket, and locate the folder of the log you want to view according to the date.
                  3. Go to the folder of the date, find the folder whose name contains taskmanager or jobmanager, download the taskmanager.out or jobmanager.out file, and view result logs.
                  +
                • Check whether the datasource connection is correctly bound and whether a security group rule allows access of the queue.
                • Check whether the GaussDB(DWS) table to which data is to be written exists in multiple schemas. If it does, specify the schemas in the Flink job.
                +
                +
              +
              +
              +
              +
              Parent topic: GaussDB(DWS)
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15106.html b/docs/dli/sqlreference/dli_08_15106.html new file mode 100644 index 000000000..12dd70e65 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15106.html @@ -0,0 +1,359 @@ + + +

              GaussDB(DWS) Dimension Table (Not Recommended)

              +

              Function

              Create a GaussDB(DWS) table to connect to source streams for wide table generation.

              +

              You are advised to use GaussDB(DWS) self-developed GaussDB(DWS) connector.

              +
              +
              +

              Prerequisites

              • Ensure that you have created a GaussDB(DWS) cluster using your account.
              • A DWS database table has been created.
              • An enhanced datasource connection has been created for DLI to connect to DWS clusters, so that jobs can run on the dedicated queue of DLI and you can set the security group rules as required. +
              +
              +

              Precautions

              • When you create a Flink OpenSource SQL job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs.
              • Storing authentication credentials such as usernames and passwords in code or plaintext poses significant security risks. It is recommended using DEW to manage credentials instead. Storing encrypted credentials in configuration files or environment variables and decrypting them when needed ensures security. For details, see .
              • Fields in the with parameter can only be enclosed in single quotes.
              +
              +

              Syntax

               1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
              create table dwsSource (
+  attr_name attr_type 
+  (',' attr_name attr_type)* 
+)
+with (
+  'connector' = 'gaussdb',
+  'url' = '',
+  'table-name' = '',
+  'username' = '',
+  'password' = ''
+);
+
              +
              +
              +

              Parameters

              +
              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              Table 1 Parameter description

              Parameter

              +

              Mandatory

              +

              Default Value

              +

              Data Types

              +

              Description

              +

              connector

              +

              Yes

              +

              None

              +

              String

              +

              Connector type. Set this parameter to gaussdb.

              +

              url

              +

              Yes

              +

              None

              +

              String

              +

              JDBC connection address.

              +

              If you use the gsjdbc4 driver, set the value in jdbc:postgresql://${ip}:${port}/${dbName} format.

              +

              If you use the gsjdbc200 driver, set the value in jdbc:gaussdb://${ip}:${port}/${dbName} format.

              +

              table-name

              +

              Yes

              +

              None

              +

              String

              +

              Name of the table where the data will be read from the database

              +

              driver

              +

              No

              +

              None

              +

              String

              +

              JDBC connection driver. The default value is org.postgresql.Driver.

              +

              username

              +

              No

              +

              None

              +

              String

              +

              Database authentication user name. This parameter must be configured in pair with password.

              +

              password

              +

              No

              +

              None

              +

              String

              +

              Database authentication password. This parameter must be configured in pair with username.

              +

              scan.partition.column

              +

              No

              +

              None

              +

              String

              +

              Name of the column used to partition the input

              +

              This parameter must be set when scan.partition.lower-bound, scan.partition.upper-bound, and scan.partition.num are all configured, and should not be set when other three parameters are not.

              +

              scan.partition.lower-bound

              +

              No

              +

              None

              +

              Integer

              +

              Lower bound of values to be fetched for the first partition

              +

              This parameter must be set when scan.partition.column, scan.partition.upper-bound, and scan.partition.num are all configured, and should not be set when other three parameters are not.

              +

              scan.partition.upper-bound

              +

              No

              +

              None

              +

              Integer

              +

              Upper bound of values to be fetched for the last partition

              +

              This parameter must be set when scan.partition.column, scan.partition.lower-bound, and scan.partition.num are all configured, and should not be set when other three parameters are not.

              +

              scan.partition.num

              +

              No

              +

              None

              +

              Integer

              +

              Number of partitions to be created

              +

              This parameter must be set when scan.partition.column, scan.partition.upper-bound, and scan.partition.upper-bound are all configured, and should not be set when other three parameters are not.

              +

              scan.fetch-size

              +

              No

              +

              0

              +

              Integer

              +

              Number of rows fetched from the database each time. The default value 0 indicates that the number of rows is not limited.

              +

              scan.auto-commit

              +

              No

              +

              true

              +

              Boolean

              +

              Automatic commit flag.

              +

              It determines whether each statement is committed in a transaction automatically.

              +

              lookup.cache.max-rows

              +

              No

              +

              None

              +

              Integer

              +

              Maximum number of cached rows in a dimension table. When the rows exceed this value, the data that is added first will be marked as expired.

              +

              Lookup cache is disabled by default.

              +

              lookup.cache.ttl

              +

              No

              +

              +

              None

              +

              Duration

              +

              Maximum time to live (TTL) of for every rows in lookup cache. Caches exceeding the TTL will be expired. The format is {length value}{time unit label}, for example, 123ms, 321s. The supported time units include d, h, min, s, and ms (default unit).

              +

              Lookup cache is disabled by default.

              +

              lookup.max-retries

              +

              No

              +

              3

              +

              Integer

              +

              Maximum retry times if lookup database failed.

              +
              +
              +
              +

              Example

              Read data from a Kafka source table, use a GaussDB(DWS) table as the dimension table. Write wide table information generated by the source and dimension tables to a Kafka result table. The procedure is as follows:

              +
              1. Create an enhanced datasource connection in the VPC and subnet where DWS and Kafka locate, and bind the connection to the required Flink elastic resource pool.
              2. Set GaussDB(DWS) and Kafka security groups and add inbound rules to allow access from the Flink queue. Test the connectivity using the DWS and Kafka address. If the connection passes the test, it is bound to the queue.
              3. Connect to the GaussDB(DWS) database instance, create a table as a dimension table, and name the table area_info. Example SQL statements are as follows:
                create table public.area_info(
+  area_id VARCHAR,
+  area_province_name VARCHAR,
+  area_city_name VARCHAR,
+  area_county_name VARCHAR,
+  area_street_name VARCHAR,
+  region_name VARCHAR);
                +
              4. Connect to the database and run the following statement to insert test data into the dimension table area_info:
                  insert into area_info
+  (area_id, area_province_name, area_city_name, area_county_name, area_street_name, region_name) 
+  values
+  ('330102', 'a1', 'b1', 'c1', 'd1', 'e1'),
+  ('330106', 'a1', 'b1', 'c2', 'd2', 'e1'),
+  ('330108', 'a1', 'b1', 'c3', 'd3', 'e1'),
+  ('330110', 'a1', 'b1', 'c4', 'd4', 'e1');
                +
              5. Create a Flink OpenSource SQL job Enter the following job script and submit the job. The job script uses Kafka as the data source and a GaussDB(DWS) table as the dimension table. Data is output to a Kafka result table.
                When you create a job, set Flink Version to 1.15 in the Running Parameters tab. Select Save Job Log, and specify the OBS bucket for saving job logs. Set the values of the parameters in bold in the following script as needed.
                CREATE TABLE orders (
+  order_id string,
+  order_channel string,
+  order_time string,
+  pay_amount double,
+  real_pay double,
+  pay_time string,
+  user_id string,
+  user_name string,
+  area_id string,
+  proctime as Proctime()
+) WITH (
+  'connector' = 'kafka',
+  'topic' = 'KafkaSourceTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'properties.group.id' = 'dws-order',
+  'scan.startup.mode' = 'latest-offset',
+  'format' = 'json'
+);
+
+-- Create an address dimension table
+create table area_info (
+    area_id string, 
+    area_province_name string, 
+    area_city_name string, 
+    area_county_name string,
+    area_street_name string, 
+    region_name string 
+) WITH (
+  'connector' = 'gaussdb',
+  'driver' = 'org.postgresql.Driver',
+  'url' = 'jdbc:postgresql://DwsAddress:DwsPort/DwsDbName',
+  'table-name' = 'area_info',
+  'username' = 'DwsUserName',
+  'password' = 'DwsPassword',
+  'lookup.cache.max-rows' = '10000',
+  'lookup.cache.ttl' = '2h'
+);
+
+-- Generate a wide table based on the address dimension table containing detailed order information.
+create table order_detail(
+    order_id string,
+    order_channel string,
+    order_time string,
+    pay_amount double,
+    real_pay double,
+    pay_time string,
+    user_id string,
+    user_name string,
+    area_id string,
+    area_province_name string,
+    area_city_name string,
+    area_county_name string,
+    area_street_name string,
+    region_name string
+) with (
+  'connector' = 'kafka',
+  'topic' = 'KafkaSinkTopic',
+  'properties.bootstrap.servers' = 'KafkaAddress1:KafkaPort,KafkaAddress2:KafkaPort',
+  'format' = 'json'
+);
+
+insert into order_detail
+    select orders.order_id, orders.order_channel, orders.order_time, orders.pay_amount, orders.real_pay, orders.pay_time, orders.user_id, orders.user_name,
+           area.area_id, area.area_province_name, area.area_city_name, area.area_county_name,
+           area.area_street_name, area.region_name  from orders
+    left join area_info for system_time as of orders.proctime as area on orders.area_id = area.area_id;
                +
                +
              6. Connect to the Kafka cluster and insert the following test data into the source topic in Kafka:
                {"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
+
+{"order_id":"202103251202020001", "order_channel":"miniAppShop", "order_time":"2021-03-25 12:02:02", "pay_amount":"60.00", "real_pay":"60.00", "pay_time":"2021-03-25 12:03:00", "user_id":"0002", "user_name":"Bob", "area_id":"330110"}
+
+{"order_id":"202103251505050001", "order_channel":"qqShop", "order_time":"2021-03-25 15:05:05", "pay_amount":"500.00", "real_pay":"400.00", "pay_time":"2021-03-25 15:10:00", "user_id":"0003", "user_name":"Cindy", "area_id":"330108"}
                +
              7. Connect to the Kafka cluster and read data from the sink topic of Kafka. The result is as follows:
                {"order_id":"202103241606060001","order_channel":"appShop","order_time":"2021-03-24 16:06:06","pay_amount":200.0,"real_pay":180.0,"pay_time":"2021-03-24 16:10:06","user_id":"0001","user_name":"Alice","area_id":"330106","area_province_name":"a1","area_city_name":"b1","area_county_name":"c2","area_street_name":"d2","region_name":"e1"}
+
+{"order_id":"202103251202020001","order_channel":"miniAppShop","order_time":"2021-03-25 12:02:02","pay_amount":60.0,"real_pay":60.0,"pay_time":"2021-03-25 12:03:00","user_id":"0002","user_name":"Bob","area_id":"330110","area_province_name":"a1","area_city_name":"b1","area_county_name":"c4","area_street_name":"d4","region_name":"e1"}
+
+{"order_id":"202103251505050001","order_channel":"qqShop","order_time":"2021-03-25 15:05:05","pay_amount":500.0,"real_pay":400.0,"pay_time":"2021-03-25 15:10:00","user_id":"0003","user_name":"Cindy","area_id":"330108","area_province_name":"a1","area_city_name":"b1","area_county_name":"c3","area_street_name":"d3","region_name":"e1"}
                +
              +
              +

              FAQs

              • Q: What should I do if Flink job logs contain the following error information?
                java.io.IOException: unable to open JDBC writer
+...
+Caused by: org.postgresql.util.PSQLException: The connection attempt failed.
+...
+Caused by: java.net.SocketTimeoutException: connect timed out
                +
                A: The datasource connection is not bound or the binding fails. +
                +
              +
              • Q: How can I configure a GaussDB(DWS) table that is in a schema?
                A: In the following example configures the area_info table in the dbuser2 schema.
                -- Create an address dimension table
+create table area_info (
+    area_id string, 
+    area_province_name string,
+    area_city_name string,
+    area_county_name string,
+    area_street_name string, 
+    region_name string 
+) WITH (
+ 'connector' = 'gaussdb',
+  'driver' = 'org.postgresql.Driver',
+  'url' = 'jdbc:postgresql://DwsAddress:DwsPort/DwsDbname',
+  'table-name' = 'dbuser2.area_info',
+  'username' = 'DwsUserName',
+  'password' = 'DwsPassword',
+  'lookup.cache.max-rows' = '10000',
+  'lookup.cache.ttl' = '2h'
+);
                +
                +
              +
              +
              +
              +
              +
              Parent topic: GaussDB(DWS)
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15107.html b/docs/dli/sqlreference/dli_08_15107.html new file mode 100644 index 000000000..01ec3d629 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15107.html @@ -0,0 +1,170 @@ + + +

              INSERT INTO

              +

              This section describes how to use the INSERT INTO statement to write job results to a sink table.

              +

              Writing Data to a Sink Table

              • Syntax
                1
+2
                  INSERT INTO your_sink
+  SELECT ... FROM your_source WHERE ...
+
                +
                +
              • Example
                In this example, two tables my_source and my_sink are defined, and the INSERT INTO statement is used to select data from the source table and insert the data to the sink table.
                 1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
                -- Use the datagen connector to create the source table my_source.
+CREATE TABLE my_source (
+  name VARCHAR,
+  age BIGINT
+) WITH (
+  'connector' = 'datagen');
+
+-- Use the JDBC connector to create the sink table my_sink.
+CREATE TABLE my_sink (
+  name VARCHAR,
+  age BIGINT
+) WITH (
+  'connector' = 'jdbc',
+  'url' = 'jdbc:mysql://xxx/your-database',
+  'table-name' = 'your-table',
+  'username' = 'your-username',
+  'password' = 'your-password'
+);
+
+-- Run the INSERT INTO statement to select data from the my_source table and insert the data into the my_sink table.
+INSERT INTO my_sink
+SELECT name, age
+FROM my_source;
+
                +
                +
                +
              +
              +

              Writing Data to Multiple Sink Tables

              EXECUTE STATEMENT SET BEGIN... END; is a required statement for writing data to multiple sink tables. It is used to define multiple data insertion operations in the same job.

              +

              EXECUTE STATEMENT SET BEGIN... END; is required when data is written to multiple sink tables.

              +
              +
              • Syntax
                 1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
                EXECUTE STATEMENT SET BEGIN
+-- First DML statement
+  INSERT INTO your_sink1
+  SELECT ... FROM your_source WHERE ...;
+
+-- Second DML statement
+  INSERT INTO your_sink2
+  SELECT ... FROM your_source WHERE ...
+
+...
+END;
+
                +
                +
              • Example
                In this example, the source table datagen_source and sink tables print_sinkA and print_sinkB are defined. EXECUTE STATEMENT is used to execute two INSERT INTO statements to write the converted data to two different sinks.
                 1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
                -- Use the datagen connector to create the source table datagen_source.
+CREATE TABLE datagen_source (
+  name VARCHAR,
+  age BIGINT
+) WITH (
+  'connector' = 'datagen'
+);
+
+-- Use the print connector to create the result tables print_sinkA and print_sinkB.
+
+CREATE TABLE print_sinkA(
+  name VARCHAR,
+  age BIGINT
+) WITH (
+  'connector' = 'print' 
+);
+
+CREATE TABLE print_sinkB(
+  name VARCHAR,
+  age BIGINT
+) WITH (
+  'connector' = 'print' 
+);
+
+-- Use EXECUTE STATEMENT SET BEGIN to execute two INSERT INTO statements.
+-- The first INSERT INTO statement converts the data in the datagen_source table as needed and writes the converted data to print_sinkA.
+-- The second INSERT INTO statement converts data as needed and writes the converted data to print_sinkB.
+EXECUTE  STATEMENT SET BEGIN
+INSERT INTO print_sinkA 
+  SELECT UPPER(name), min(age) 
+  FROM datagen_source 
+  GROUP BY UPPER(name);
+INSERT INTO print_sinkB 
+  SELECT LOWER(name), max(age) 
+  FROM datagen_source 
+  GROUP BY LOWER(name);
+END;
+
                +
                +
                +
              +
              +
              +
              +
              +
              Parent topic: DML Snytax
              +
              +
              + diff --git a/docs/dli/sqlreference/dli_08_15108.html b/docs/dli/sqlreference/dli_08_15108.html new file mode 100644 index 000000000..f8cfe3e90 --- /dev/null +++ b/docs/dli/sqlreference/dli_08_15108.html @@ -0,0 +1,16 @@ + + +

              Flink OpenSource SQL 1.15 Usage

              +

              When switching from Flink 1.12 to Flink 1.15 for job execution, keep in mind the following considerations when utilizing Flink OpenSource SQL 1.15:

              +
              • Flink SQL utilizes a SQL client submission method. To configure this submission method in Flink 1.15, you need to use the SET 'key'='value' command in your SQL script. This is different from the optimization parameters used in Flink 1.12. For details about the syntax, see SQL Client Configuration.
              • The following Flink connectors are added to Flink 1.15: Doris Connector and Hive Connector. For details, see Overview.
              • In Flink 1.15, you need to configure a custom agency on the tenant plane and configure agency information in the job. The permissions included an agency should be configured based on the specific service scenario requirements of the job.
              • Methods to manage credentials for Flink 1.15 jobs:
                • You are advised to use DEW to manage access credentials, such as passwords and keys, in Flink OpenSource SQL.
                • Manage fixed AKs/SKs used by Flink Jar jobs to access OBS, temporary AKs/SKs used by Flink Jar jobs to obtain agencies, and temporary AKs/SKs used by Flink SQL UDFs to obtain agencies.
                +
              • There are differences in the way Flink 1.15 Jar reads custom configuration files compared to Flink 1.12.
              • The Flink 1.15 Jar program uses a child-first reverse class loading mechanism. By setting the parent.first.classloader.jars parameter to include the names of the desired jars, for example, test1.jar,test2.jar, certain dependency packages can be loaded by the parent class loader.
              • For the built-in JAR file list of Flink 1.15 Jar, obtain information about Flink 1.15 dependency packages from Flink job logs.
                1. Check the logs of a Flink job.
                  1. Log in to the DLI management console. In the navigation pane on the left, choose Job Management > Flink Jobs.
                  2. Click the name of the desired job. On the displayed page, click the Run Log tab.
                  3. Check the latest run logs. For more logs, check the OBS bucket where the job logs are stored.
                  +
                2. Search for dependency information in the logs.

                  Search for Classpath: in the logs to check the dependencies.

                  +
                +
              • Flink 1.15 no longer supports DLI package management. To upload dependency packages and files, select the OBS path directly when editing the job.
              +
              + + diff --git a/docs/dli/sqlreference/dli_spark_avg.html b/docs/dli/sqlreference/dli_spark_avg.html index 6051825af..dcdf92be0 100644 --- a/docs/dli/sqlreference/dli_spark_avg.html +++ b/docs/dli/sqlreference/dli_spark_avg.html @@ -37,7 +37,7 @@ 
              _c0        
 100.0
            -
            • Calculates the average inventory of all items in each warehouse when used with group by. An example command is as follows:
              select warehourseId, avg(items) from warehourse group by warehourseId;
              +
              • Calculates the average inventory of all items in each warehouse when used with group by. An example command is as follows:
                select warehouseId, avg(items) from warehouse group by warehouseId;

                The command output is as follows:

                warehouseId _c1
 city1    155       
diff --git a/docs/dli/sqlreference/dli_spark_bround.html b/docs/dli/sqlreference/dli_spark_bround.html
index 6775ee5b4..75e55c631 100644
--- a/docs/dli/sqlreference/dli_spark_bround.html
+++ b/docs/dli/sqlreference/dli_spark_bround.html
@@ -45,14 +45,14 @@
 
                 
                If the value of a or d is NULL, NULL is returned.
            -

            Example Code

            The value 1 is returned.

            -
            select bin(1);
            -

            The value NULL is returned.

            -
            select bin(null);
            -

            The value 1000 is returned.

            -
            select bin(8);
            -

            The value 1000 is returned.

            -
            select bin(8.123456);
            +

            Example Code

            The value 123.4 is returned.

            +
            select bround(123.45,1);
            +

            The value 123.6 is returned.

            +
            select bround(123.55,1);
            +

            The value NULL is returned.

            +
            select bround(null);
            +

            The value 123.457 is returned.

            +
            select bround(123.456789,3.123456);
            diff --git a/docs/dli/sqlreference/dli_spark_corr.html b/docs/dli/sqlreference/dli_spark_corr.html index 0a9759394..e6b979f93 100644 --- a/docs/dli/sqlreference/dli_spark_corr.html +++ b/docs/dli/sqlreference/dli_spark_corr.html @@ -43,7 +43,7 @@

            The command output is as follows:

            _c0        
 1.242355
            -
          • When used with group by, it groups all offerings by warehouse (warehourseId) and returns the correlation coefficient between the inventory (items) and the price of offerings within each group. An example command is as follows:
            select warehourseId, corr(items,price) from warehourse group by warehourseId;
            +
          • When used with group by, it groups all offerings by warehouse (warehouseId) and returns the correlation coefficient between the inventory (items) and the price of offerings within each group. An example command is as follows:
            select warehouseId, corr(items,price) from warehouse group by warehouseId;

            The command output is as follows:

            warehouseId _c1        
 city1   0.43124   
diff --git a/docs/dli/sqlreference/dli_spark_cos.html b/docs/dli/sqlreference/dli_spark_cos.html
index f7c505a9c..897e9efae 100644
--- a/docs/dli/sqlreference/dli_spark_cos.html
+++ b/docs/dli/sqlreference/dli_spark_cos.html
@@ -33,7 +33,7 @@
 
             
            If the value of a is NULL, NULL is returned.
            
 
            
 
-
            Example Code
            The value -0.99999999999999999986 is returned.
            
+
            Example Code
            The value -0.9999999999999986 is returned.
            
 
            select cos(3.1415926);
            
 
            The value NULL is returned.
            
 
            select cos(null);
            
diff --git a/docs/dli/sqlreference/dli_spark_covar_pop.html b/docs/dli/sqlreference/dli_spark_covar_pop.html
index 24fd7a830..610110b6a 100644
--- a/docs/dli/sqlreference/dli_spark_covar_pop.html
+++ b/docs/dli/sqlreference/dli_spark_covar_pop.html
@@ -37,7 +37,7 @@
 
            The command output is as follows:
            
 
            _c0        
 1.242355   
            
-
          • When used with group by, it groups all offerings by warehouse (warehourseId) and returns the covariance between the inventory (items) and the price of offerings within each group. An example command is as follows:
            select warehourseId, covar_pop(items, price) from warehourse group by warehourseId;
            +
          • When used with group by, it groups all offerings by warehouse (warehouseId) and returns the covariance between the inventory (items) and the price of offerings within each group. An example command is as follows:
            select warehouseId, covar_pop(items, price) from warehouse group by warehouseId;

            The command output is as follows:

            warehouseId _c1        
 city1    1.13124   
diff --git a/docs/dli/sqlreference/dli_spark_covar_samp.html b/docs/dli/sqlreference/dli_spark_covar_samp.html
index 1343b1c07..4e1dff96c 100644
--- a/docs/dli/sqlreference/dli_spark_covar_samp.html
+++ b/docs/dli/sqlreference/dli_spark_covar_samp.html
@@ -37,7 +37,7 @@
 
            The command output is as follows:
            
 
            _c0        
 1.242355   
            
-
          • When used with group by, it groups all offerings by warehouse (warehourseId) and returns the sample covariance between the inventory (items) and the price of offerings within each group. An example command is as follows:
            select warehourseId, covar_samp(items,price) from warehourse group by warehourseId;
            +
          • When used with group by, it groups all offerings by warehouse (warehouseId) and returns the sample covariance between the inventory (items) and the price of offerings within each group. An example command is as follows:
            select warehouseId, covar_samp(items,price) from warehouse group by warehouseId;

            The command output is as follows:

            warehouseId _c1        
 city1    1.03124   
diff --git a/docs/dli/sqlreference/dli_spark_current_date.html b/docs/dli/sqlreference/dli_spark_current_date.html
index b3a369325..08a49bea0 100644
--- a/docs/dli/sqlreference/dli_spark_current_date.html
+++ b/docs/dli/sqlreference/dli_spark_current_date.html
@@ -2,7 +2,7 @@
 
 
            current_date
            
 
            This function is used to return the current date, in the yyyy-mm-dd format.
            
-
            Similar function: getdate. The getdate function is used to return the current system time, in the yyyy-mm-dd hh:mi:ss format.
            
+
            Similar function: getdate. The getdate function is used to return the current system time, in the yyyy-mm-dd hh:mi:ss format.
            
 
            Syntax
            current_date()
            
 
            
 
            Parameters
            None
            
diff --git a/docs/dli/sqlreference/dli_spark_date_add.html b/docs/dli/sqlreference/dli_spark_date_add.html
index cbe296c36..afc74fa05 100644
--- a/docs/dli/sqlreference/dli_spark_date_add.html
+++ b/docs/dli/sqlreference/dli_spark_date_add.html
@@ -2,8 +2,8 @@
 
 
            date_add
            
 
            This function is used to calculate the number of days in which start_date is increased by days.
            
-
            To obtain the date with a specified change range based on the current date, use this function together with the current_date or getdate function.
            
-
            Note that the logic of this function is opposite to that of the date_sub function.
            
+
            To obtain the date with a specified change range based on the current date, use this function together with the current_date or getdate function.
            
+
            Note that the logic of this function is opposite to that of the date_sub function.
            
 
            Syntax
            date_add(string startdate, int days)
            
 
            
 
            Parameters
            
diff --git a/docs/dli/sqlreference/dli_spark_date_format.html b/docs/dli/sqlreference/dli_spark_date_format.html
index d9ff497ab..93a9bbc4f 100644
--- a/docs/dli/sqlreference/dli_spark_date_format.html
+++ b/docs/dli/sqlreference/dli_spark_date_format.html
@@ -36,7 +36,7 @@
 
 
            Format, based on which the date is converted
            
 
            The value is a combination of the time unit (year, month, day, hour, minute, and second) and any character.
            
-
            • YYYY or yyyy indicates the year.
            • MM indicates the month.
            • mm indicates the minute.
            • dd indicates the day.
            • HH indicates the 24-hour clock.
            • hh indicates the 12-hour clock.
            • mi indicates the minute.
            • ss indicates the second.
            • SSS indicates millisecond.
            
+
            • yyyy indicates the year.
            • MM indicates the month.
            • dd indicates the day.
            • HH indicates the 24-hour clock.
            • hh indicates the 12-hour clock.
            • mm indicates the minute.
            • ss indicates the second.
            
 
 
 
@@ -47,10 +47,7 @@
 
             
            • If the value of date is not of the DATE or STRING type, the error message "data type mismatch" is displayed.
            • If the value of date is of the DATE or STRING type but is not in one of the supported formats, NULL is returned.
            • If the value of date is NULL, NULL is returned.
            • If the value of format is NULL, NULL is returned.
            
 
            
 
            
-
            Example Code
            Assume that the current time is 2023-08-14 16:59.
            
-
            The value 2023-08-14 04:59:15.997 is returned.
            
-
            select date_format(from_utc_timestamp(getdate(), 'UTC'),'yyyy-MM-dd hh:mm:ss.SSS');
            
-
            The value 2023-08-14 is returned.
            
+
            Example Code
            The value 2023-08-14 is returned.
            
 
            select date_format('2023-08-14','yyyy-MM-dd');
            
 
            The value 2023-08 is returned.
            
 
            select date_format('2023-08-14','yyyy-MM')
            
diff --git a/docs/dli/sqlreference/dli_spark_date_sub.html b/docs/dli/sqlreference/dli_spark_date_sub.html
index a836a5487..ee80f3821 100644
--- a/docs/dli/sqlreference/dli_spark_date_sub.html
+++ b/docs/dli/sqlreference/dli_spark_date_sub.html
@@ -2,8 +2,8 @@
 
 
            date_sub
            
 
            This function is used to calculate the number of days in which start_date is subtracted by days.
            
-
            To obtain the date with a specified change range based on the current date, use this function together with the current_date or getdate function.
            
-
            Note that the logic of this function is opposite to that of the date_add function.
            
+
            To obtain the date with a specified change range based on the current date, use this function together with the current_date or getdate function.
            
+
            Note that the logic of this function is opposite to that of the date_add function.
            
 
            Syntax
            date_sub(string startdate, int days)
            
 
            
 
            Parameters
            
@@ -37,7 +37,7 @@
 
            BIGINT
            
 
 
            Number of days to be reduced
            
-
            • If the value is greater than 0, the number of days is increased.
            • If the value of days is less than 0, the number of days is subtracted.
            • If the value is 0, the date does not change.
            • If the value is NULL, NULL is returned.
            
+
            • When days is greater than 0, the specified number of days is subtracted from startdate.
            • When days is less than 0, the specified number of days is added to startdate.
            • When days equals 0, that means adding 0 days, so the date does not change.
            • When the value is NULL, NULL is returned.
            
 
 
 
@@ -52,7 +52,7 @@
 
            select date_sub('2023-08-14 17:00:00', 2);
            
 
            The value 2023-08-15 is returned after one day is added.
            
 
            select date_sub(date'2023-08-14', -1);
            
-
            If the current time is 2023-08-14 17:00:00, 2022-08-13 is returned.
            
+
            If the current time is 2023-08-14 17:00:00, 2023-08-13 is returned.
            
 
            select date_sub(getdate(),1);
            
 
            The value NULL is returned.
            
 
            select date_sub('2023-08-14 17:00:00', null);
            
diff --git a/docs/dli/sqlreference/dli_spark_dateadd.html b/docs/dli/sqlreference/dli_spark_dateadd.html
index 20f5c8dae..8a7829dc2 100644
--- a/docs/dli/sqlreference/dli_spark_dateadd.html
+++ b/docs/dli/sqlreference/dli_spark_dateadd.html
@@ -2,7 +2,7 @@
 
 
            dateadd
            
 
            This function is used to change a date based on datepart and delta.
            
-
            To obtain the date with a specified change range based on the current date, use this function together with the current_date or getdate function.
            
+
            To obtain the date with a specified change range based on the current date, use this function together with the current_date or getdate function.
            
 
            Syntax
            dateadd(string date, bigint delta, string datepart)
            
 
            
 
            Parameters
            
@@ -46,7 +46,7 @@
 
 
            Unit, based on which the date is modified
            
 
            This parameter supports the following extended date formats: year, month or mon, day, and hour.
            
-
            • YYYY or yyyy indicates the year.
            • MM indicates the month.
            • mm indicates the minute.
            • dd indicates the day.
            • HH indicates the 24-hour clock.
            • hh indicates the 12-hour clock.
            • mi indicates the minute.
            • ss indicates the second.
            • SSS indicates the millisecond.
            
+
            • yyyy indicates the year.
            • MM indicates the month.
            • dd indicates the day.
            • hh indicates the hour.
            • mi indicates the minute.
            • ss indicates the second.
            
 
 
 
diff --git a/docs/dli/sqlreference/dli_spark_datediff.html b/docs/dli/sqlreference/dli_spark_datediff.html
index d492346f4..b46d39c59 100644
--- a/docs/dli/sqlreference/dli_spark_datediff.html
+++ b/docs/dli/sqlreference/dli_spark_datediff.html
@@ -2,7 +2,7 @@
 
 
            datediff
            
 
            This function is used to calculate the difference between date1 and date2.
            
-
            Similar function: datediff1. The datediff1 function is used to calculate the difference between date1 and date2 and return the difference in a specified datepart.
            
+
            Similar function: datediff1. The datediff1 function is used to calculate the difference between date1 and date2 and return the difference in a specified datepart.
            
 
            Syntax
            datediff(string date1, string date2)
            
 
            
 
            Parameters
            
diff --git a/docs/dli/sqlreference/dli_spark_datediff1.html b/docs/dli/sqlreference/dli_spark_datediff1.html
index 73c2f1d88..2fe589015 100644
--- a/docs/dli/sqlreference/dli_spark_datediff1.html
+++ b/docs/dli/sqlreference/dli_spark_datediff1.html
@@ -2,7 +2,7 @@
 
 
            datediff1
            
 
            This function is used to calculate the difference between date1 and date2 and return the difference in a specified datepart.
            
-
            Similar function: datediff. The datediff function is used to calculate the difference between date1 and date2 but does not return the difference in a specified datepart.
            
+
            Similar function: datediff. The datediff function is used to calculate the difference between date1 and date2 but does not return the difference in a specified datepart.
            
 
            Syntax
            datediff1(string date1, string date2, string datepart)
            
 
            
 
            Parameters
            
@@ -50,7 +50,7 @@
 
 
            Unit of the time to be returned
            
 
            This parameter supports the following extended date formats: year, month or mon, day, and hour.
            
-
            • YYYY or yyyy indicates the year.
            • MM indicates the month.
            • mm indicates the minute.
            • dd indicates the day.
            • HH indicates the 24-hour clock.
            • hh indicates the 12-hour clock.
            • mi indicates the minute.
            • ss indicates the second.
            • SSS indicates millisecond.
            
+
            • yyyy indicates the year.
            • MM indicates the month.
            • dd indicates the day.
            • hh indicates the hour.
            • mi indicates the minute.
            • ss indicates the second.
            
 
 
 
diff --git a/docs/dli/sqlreference/dli_spark_datepart.html b/docs/dli/sqlreference/dli_spark_datepart.html
index 40fda2569..c3a4f77fd 100644
--- a/docs/dli/sqlreference/dli_spark_datepart.html
+++ b/docs/dli/sqlreference/dli_spark_datepart.html
@@ -36,7 +36,7 @@
 
 
            Time unit of the value to be returned
            
 
            This parameter supports the following extended date formats: year, month or mon, day, and hour.
            
-
            • YYYY or yyyy indicates the year.
            • MM indicates the month.
            • mm indicates the minute.
            • dd indicates the day.
            • HH indicates the 24-hour clock.
            • hh indicates the 12-hour clock.
            • mi indicates the minute.
            • ss indicates the second.
            • SSS indicates millisecond.
            
+
            • yyyy indicates the year.
            • MM indicates the month.
            • dd indicates the day.
            • hh indicates the hour.
            • mi indicates the minute.
            • ss indicates the second.
            
 
 
 
@@ -51,8 +51,8 @@
 
            select datepart(date '2023-08-14 17:00:00', 'yyyy');
            
 
            The value 2023 is returned.
            
 
            select datepart('2023-08-14 17:00:00', 'yyyy');
            
-
            The value 0 is returned.
            
-
            select datepart(date '2023-08-14 17:00:00', 'mi');
            
+
            The value 59 is returned.
            
+
            select datepart('2023-08-14 17:59:59', 'mi')
            
 
            The value NULL is returned.
            
 
            select datepart(date '2023-08-14', null);
            
 
            
diff --git a/docs/dli/sqlreference/dli_spark_datetrunc.html b/docs/dli/sqlreference/dli_spark_datetrunc.html
index c5eb5eb77..22708264d 100644
--- a/docs/dli/sqlreference/dli_spark_datetrunc.html
+++ b/docs/dli/sqlreference/dli_spark_datetrunc.html
@@ -37,7 +37,7 @@
 
 
            Time unit of the value to be returned
            
 
            This parameter supports the following extended date formats: year, month or mon, day, and hour.
            
-
            • YYYY or yyyy indicates the year.
            • MM indicates the month.
            • mm indicates the minute.
            • dd indicates the day.
            • HH indicates the 24-hour clock.
            • hh indicates the 12-hour clock.
            • mi indicates the minute.
            • ss indicates the second.
            • SSS indicates millisecond.
            
+
            • yyyy indicates the year.
            • MM indicates the month.
            • dd indicates the day.
            • hh indicates the hour.
            • mi indicates the minute.
            • ss indicates the second.
            
 
 
 
@@ -57,8 +57,8 @@
 
            select datetrunc('2023-08-14 17:00:00', 'DD');
            
 
            The value 2023-01-01 is returned.
            
 
            select datetrunc('2023-08-14', 'yyyy');
            
-
            The value 2023-08-14 is returned.
            
-
            select datetrunc('2023-08-14 17:00:00', 'HH');
            
+
            The value 2023-08-14 17:00:00 is returned.
            
+
            select datetrunc('2023-08-14 17:11:11', 'hh');
            
 
            The value NULL is returned.
            
 
            select datetrunc('2023-08-14', null);
            
 
            
diff --git a/docs/dli/sqlreference/dli_spark_get_json_object.html b/docs/dli/sqlreference/dli_spark_get_json_object.html
index f602fc32a..9238114bc 100644
--- a/docs/dli/sqlreference/dli_spark_get_json_object.html
+++ b/docs/dli/sqlreference/dli_spark_get_json_object.html
@@ -60,14 +60,14 @@
          • Extracts information from a JSON object with a period (.). An example command is as follows:

            Create a table.

            create table json_table (id string, json string);

            Insert data into the table. The key contains a period (.).

            -
            insert into table json_table (id, json) values ("1", "{\"China.hangzhou\":{\"region\":{\"rid\":6}}}");
            +
            insert into table json_table (id, json) values ("1", "{\"city1\":{\"region\":{\"rid\":6}}}");

            Insert data into the table. The key does not contain a period (.).

            -
            insert into table json_table (id, json) values ("2", "{\"China_hangzhou\":{\"region\":{\"rid\":7}}}");
            -

            Obtain the value of rid. If the key is China.hangzhou, 6 is returned. Only [''] can be used for parsing because a period (.) is included.

            -
            select get_json_object(json, "$['China.hangzhou'].region['id']") from json_table where id =1;
            -

            Obtain the value of rid. If the key is China_hangzhou, 7 is returned. You can use either of the following methods:

            -
            select get_json_object(json, "$['China_hangzhou'].region['id']") from json_table where id =2; 
-select get_json_object(json, "$.China_hangzhou.region['id']") from json_table where id =2;
            +
            insert into table json_table (id, json) values ("2", "{\"city1\":{\"region\":{\"rid\":7}}}");
            +

            Obtain the value of rid. If the key is city1, 6 is returned. Only [''] can be used for parsing because a period (.) is included.

            +
            select get_json_object(json, "$['city1'].region['id']") from json_table where id =1;
            +

            Obtain the value of rid. If the key is city1, 7 is returned. You can use either of the following methods:

            +
            select get_json_object(json, "$['city1'].region['id']") from json_table where id =2; 
+select get_json_object(json, "$.city1.region['id']") from json_table where id =2;
          • The json parameter is either empty or has an invalid format. An example command is as follows:

            The value NULL is returned.

            select get_json_object('','$.array[2]');

            The value NULL is returned.

            diff --git a/docs/dli/sqlreference/dli_spark_getdate.html b/docs/dli/sqlreference/dli_spark_getdate.html index 3fa14f537..f4f7792be 100644 --- a/docs/dli/sqlreference/dli_spark_getdate.html +++ b/docs/dli/sqlreference/dli_spark_getdate.html @@ -2,7 +2,7 @@

            getdate

            This function is used to return the current system time, in the yyyy-mm-dd hh:mi:ss format.

            -

            Similar function: current_date. The current_date function is used to return the current date, in the yyyy-mm-dd format.

            +

            Similar function: current_date. The current_date function is used to return the current date, in the yyyy-mm-dd format.

            Syntax

            getdate()

            Parameters

            None

            diff --git a/docs/dli/sqlreference/dli_spark_instr.html b/docs/dli/sqlreference/dli_spark_instr.html index 2ccf0b682..63be79ea5 100644 --- a/docs/dli/sqlreference/dli_spark_instr.html +++ b/docs/dli/sqlreference/dli_spark_instr.html @@ -3,7 +3,7 @@

            instr

            This function is used to return the index of substr that appears earliest in str.

            It returns NULL if either of the arguments are NULL and returns 0 if substr does not exist in str. Note that the first character in str has index 1.

            -

            Similar function: instr1. The instr1 function is used to calculate the position of the substring str2 in the string str1. The instr1 function allows you to specify the start search position and the number of matching times.

            +

            Similar function: instr1. The instr1 function is used to calculate the position of the substring str2 in the string str1. The instr1 function allows you to specify the start search position and the number of matching times.

            Syntax

            instr(string <str>, string <substr>)

            Parameters

            diff --git a/docs/dli/sqlreference/dli_spark_instr1.html b/docs/dli/sqlreference/dli_spark_instr1.html index fd63a4fd0..f26d6787d 100644 --- a/docs/dli/sqlreference/dli_spark_instr1.html +++ b/docs/dli/sqlreference/dli_spark_instr1.html @@ -2,7 +2,7 @@

            instr1

            This function is used to return the position of substring str2 in string str1.

            -

            Similar function: instr. The instr function is used to return the index of substr that appears earliest in str. However, instr does not support specifying the start search position and matching times.

            +

            Similar function: instr. The instr function is used to return the index of substr that appears earliest in str. However, instr does not support specifying the start search position and matching times.

            Syntax

            instr1(string <str1>, string <str2>[, bigint <start_position>[, bigint <nth_appearance>]])

            Parameters

            diff --git a/docs/dli/sqlreference/dli_spark_isdate.html b/docs/dli/sqlreference/dli_spark_isdate.html index 11c48e65c..77dca2e35 100644 --- a/docs/dli/sqlreference/dli_spark_isdate.html +++ b/docs/dli/sqlreference/dli_spark_isdate.html @@ -37,7 +37,7 @@

            Format of the date to be converted

            Constant of the STRING type. Extended date formats are not supported.

            The value is a combination of the time unit (year, month, day, hour, minute, and second) and any character.

            -
            • YYYY or yyyy indicates the year.
            • MM indicates the month.
            • mm indicates the minute.
            • dd indicates the day.
            • HH indicates the 24-hour clock.
            • hh indicates the 12-hour clock.
            • mi indicates the minute.
            • ss indicates the second.
            • SSS indicates millisecond.
            +
            • yyyy indicates the year.
            • mm indicates the month.
            • dd indicates the day.
            • hh indicates the hour.
            • mi indicates the minute.
            • ss indicates the second.
            diff --git a/docs/dli/sqlreference/dli_spark_last_day.html b/docs/dli/sqlreference/dli_spark_last_day.html index b21606a72..d00947a16 100644 --- a/docs/dli/sqlreference/dli_spark_last_day.html +++ b/docs/dli/sqlreference/dli_spark_last_day.html @@ -2,7 +2,7 @@

            last_day

            This function is used to return the last day of the month a date belongs to.

            -

            Similar function: lastday. The lastday function is used to return the last day of the month a date belongs to. The hour, minute, and second part is 00:00:00.

            +

            Similar function: lastday. The lastday function is used to return the last day of the month a date belongs to. The hour, minute, and second part is 00:00:00.

            Syntax

            last_day(string date)

            Parameters

            diff --git a/docs/dli/sqlreference/dli_spark_lastday.html b/docs/dli/sqlreference/dli_spark_lastday.html index 0fb0972c5..e156df8bf 100644 --- a/docs/dli/sqlreference/dli_spark_lastday.html +++ b/docs/dli/sqlreference/dli_spark_lastday.html @@ -2,7 +2,7 @@

            lastday

            This function is used to return the last day of the month a date belongs to. The hour, minute, and second part is 00:00:00.

            -

            Similar function: last_day. The last_day function is used to return the last day of the month a date belongs to. The return value is in the yyyy-mm-dd format.

            +

            Similar function: last_day. The last_day function is used to return the last day of the month a date belongs to. The return value is in the yyyy-mm-dd format.

            Syntax

            lastday(string date)

            Parameters

            diff --git a/docs/dli/sqlreference/dli_spark_length.html b/docs/dli/sqlreference/dli_spark_length.html index 1a63e5764..bf4dc7e10 100644 --- a/docs/dli/sqlreference/dli_spark_length.html +++ b/docs/dli/sqlreference/dli_spark_length.html @@ -2,7 +2,7 @@

            length

            This function is used to return the length of a string.

            -

            Similar function: lengthb. The lengthb function is used to return the length of string str in bytes and return a value of the STRING type.

            +

            Similar function: lengthb. The lengthb function is used to return the length of string str in bytes and return a value of the STRING type.

            Syntax

            length(string <str>)

            Parameters

            diff --git a/docs/dli/sqlreference/dli_spark_lengthb.html b/docs/dli/sqlreference/dli_spark_lengthb.html index 4093b0333..21a7848c0 100644 --- a/docs/dli/sqlreference/dli_spark_lengthb.html +++ b/docs/dli/sqlreference/dli_spark_lengthb.html @@ -2,7 +2,7 @@

            lengthb

            This function is used to return the length of a specified string in bytes.

            -

            Similar function: length. The length function is used to return the length of a string and return a value of the BIGINT type.

            +

            Similar function: length. The length function is used to return the length of a string and return a value of the BIGINT type.

            Syntax

            lengthb(string <str>)

            Parameters

            diff --git a/docs/dli/sqlreference/dli_spark_ltrim.html b/docs/dli/sqlreference/dli_spark_ltrim.html index 810faf9e3..c83477f94 100644 --- a/docs/dli/sqlreference/dli_spark_ltrim.html +++ b/docs/dli/sqlreference/dli_spark_ltrim.html @@ -4,7 +4,7 @@

            This function is used to remove characters from the left of str.

            • If trimChars is not specified, spaces are removed by default.
            • If trimChars is specified, the function removes the longest possible substring from the left end of str that consists of characters in the trimChars set.

            Similar functions:

            -
            • rtrim. This function is used to remove characters from the right of str.
            • trim. This function is used to remove characters from the left and right of str.
            +
            • rtrim. This function is used to remove characters from the right of str.
            • trim. This function is used to remove characters from the left and right of str.

            Syntax

            ltrim([<trimChars>,] string <str>)

            Parameters

            diff --git a/docs/dli/sqlreference/dli_spark_max.html b/docs/dli/sqlreference/dli_spark_max.html index 76c01f397..971e43c59 100644 --- a/docs/dli/sqlreference/dli_spark_max.html +++ b/docs/dli/sqlreference/dli_spark_max.html @@ -37,7 +37,7 @@

            The command output is as follows:

            _c0        
 900
            -
          • When used with group by, it returns the maximum inventory of each warehouse. An example command is as follows:
            select warehourseId, max(items) from warehouse group by warehourseId;
            +
          • When used with group by, it returns the maximum inventory of each warehouse. An example command is as follows:
            select warehouseId, max(items) from warehouse group by warehouseId;

            The command output is as follows:

            warehouseId _c1        
 city1    200        
diff --git a/docs/dli/sqlreference/dli_spark_max_pt.html b/docs/dli/sqlreference/dli_spark_max_pt.html
index e3c3bb356..b7321c77d 100644
--- a/docs/dli/sqlreference/dli_spark_max_pt.html
+++ b/docs/dli/sqlreference/dli_spark_max_pt.html
@@ -29,7 +29,7 @@
 
 
 
            Return Values
            The return value is of the STRING type.
            
-
             
            • The value of the largest level-1 partition is returned.
            • If a partition is added to a table using the ALTER TABLE command, but it does not contain any data, it will not be included in the returned values.
            
+
             
            • The value of the largest level-1 partition is returned.
            • If a partition is added to a table using the ALTER TABLE command, but it does not contain any data, it will not be included in the returned values.
            
 
            
 
            
 
            Example Code
            For example, table1 is a partitioned table with partitions of 20120801 and 20120802, both of which contain data, and the max_pt value in the following statement will be 20120802. The DLI SQL statement will read data from the partition with pt = 20120802.
            
diff --git a/docs/dli/sqlreference/dli_spark_min.html b/docs/dli/sqlreference/dli_spark_min.html
index 5693a1fb3..cb0a525f2 100644
--- a/docs/dli/sqlreference/dli_spark_min.html
+++ b/docs/dli/sqlreference/dli_spark_min.html
@@ -37,7 +37,7 @@
 
            The command output is as follows:
            
 
            _c0        
 600        
            
-
          • When used with group by, it returns the minimum inventory of each warehouse. An example command is as follows:
            select warehourseId, min(items) from warehouse group by warehourseId;
            +
          • When used with group by, it returns the minimum inventory of each warehouse. An example command is as follows:
            select warehouseId, min(items) from warehouse group by warehouseId;

            The command output is as follows:

            warehouseId _c1        
 city1      15         
diff --git a/docs/dli/sqlreference/dli_spark_percentile.html b/docs/dli/sqlreference/dli_spark_percentile.html
index b0d1d575e..48220cc0e 100644
--- a/docs/dli/sqlreference/dli_spark_percentile.html
+++ b/docs/dli/sqlreference/dli_spark_percentile.html
@@ -35,16 +35,22 @@
 
             
            The value should be between 0 and 1. Otherwise, NULL is returned.
            
 
            
 
-
            Example Code
            • Calculates the 0.5 percentile of all offering inventories (items). An example command is as follows:
              select stddev_samp(items,0.5) from warehouse; 
              
+
              Example Code
              • Calculates the 0.5 percentile of all offering inventories (items). An example command is as follows:
                select percentile(items,0.5) from warehouse; 
                
 
                The command output is as follows:
                
-
                _c0        
-500.6      
                
-
              • When used with group by, it groups all offerings by warehouse (warehourseId) and returns the 0.5 percentile of the offering inventory (items) in the same group. An example command is as follows:
                select warehourseId, stddev_samp(items, 0.5) from warehourse group by warehourseId; 
                
+
                +------------+
+| _c0        |
++------------+
+| 500.6      |
++------------+
                
+
              • When used with group by, it groups all offerings by warehouse (warehouseId) and returns the 0.5 percentile of the offering inventory (items) in the same group. An example command is as follows:
                select warehouseId, percentile(items, 0.5) from warehouse group by warehouseId; 
                
 
                The command output is as follows:
                
-
                warehouseId _c1        
-city1    499.6      
-city2    354.8      
-city3    565.7      
                
+
                +------------+------------+
+| warehouseId| _c1        |
++------------+------------+
+| city1    | 499.6      |
+| city2    | 354.8      |
+| city3    | 565.7      |
++------------+------------+
                
 
              
 
              
 
            
diff --git a/docs/dli/sqlreference/dli_spark_percentile_approx.html b/docs/dli/sqlreference/dli_spark_percentile_approx.html
index 19f6b16bb..e06908fd7 100644
--- a/docs/dli/sqlreference/dli_spark_percentile_approx.html
+++ b/docs/dli/sqlreference/dli_spark_percentile_approx.html
@@ -40,16 +40,22 @@
 
 
            Return Values
            The return value is of the DOUBLE type.
            
 
            
-
            Example Code
            • Calculates the 0.5 percentile of all offering inventories (items), with an accuracy of 100. An example command is as follows:
              select stddev_samp(items,0.5, 100) from warehouse;
              
+
              Example Code
              • Calculates the 0.5 percentile of all offering inventories (items), with an accuracy of 100. An example command is as follows:
                select PERCENTILE_APPROX(items,0.5,100) from warehouse;
                
 
                The command output is as follows:
                
-
                _c0        
-500        
                
-
              • When used with group by, it groups all offerings by warehouse (warehourseId) and returns the 0.5 percentile of the offering inventory (items) in the same group, with an accuracy of 100. An example command is as follows:
                select warehourseId, stddev_samp(items, 0.5, 100) from warehourse group by warehourseId; 
                
+
                +------------+
+| _c0        |
++------------+
+| 521        |
++------------+
                
+
              • When used with group by, it groups all offerings by warehouse (warehouseId) and returns the 0.5 percentile of the offering inventory (items) in the same group, with an accuracy of 100. An example command is as follows:
                select warehouseId, PERCENTILE_APPROX(items, 0.5, 100) from warehouse group by warehouseId; 
                
 
                The command output is as follows:
                
-
                warehouseId _c1        
-city1     499        
-city2     354        
-city3     565        
                
+
                +------------+------------+
+| warehouseId| _c1        |
++------------+------------+
+| city1    | 499     |
+| city2    | 354     |
+| city3    | 565     |
++------------+------------+
                
 
              
 
              
 
            
diff --git a/docs/dli/sqlreference/dli_spark_printf.html b/docs/dli/sqlreference/dli_spark_printf.html
index d4eef30a0..c43712b2e 100644
--- a/docs/dli/sqlreference/dli_spark_printf.html
+++ b/docs/dli/sqlreference/dli_spark_printf.html
@@ -40,8 +40,8 @@
 
            Return Values
            The return value is of the STRING type.
            
 
            The value is returned after the parameters that filled in Obj are specified for format.
            
 
            
-
            Example Code
            The string name: Zhang San, age: 20, gender: female, place of origin: city 1 is returned.
            
-
            SELECT printf('Name: %s, Age: %d, Gender: %s, Place of origin: %s', "Zhang San", 20, "Female", "City 1");
            
+
            Example Code
            The string name: user1, age: 20, gender: female, place of origin: city 1 is returned.
            
+
            SELECT printf('Name: %s, Age: %d, Gender: %s, Place of origin: %s', "user1", 20, "Female", "City 1");
            
 
            
 
            
 
            
diff --git a/docs/dli/sqlreference/dli_spark_regexp_replace.html b/docs/dli/sqlreference/dli_spark_regexp_replace.html
index a2683e558..b9c2fc426 100644
--- a/docs/dli/sqlreference/dli_spark_regexp_replace.html
+++ b/docs/dli/sqlreference/dli_spark_regexp_replace.html
@@ -1,9 +1,9 @@
 
 
 
            regexp_replace
            
-
            This function has slight variations in its functionality depending on the version of Spark being used.
            • For Spark 2.4.5 or earlier: Replaces the substring that matches pattern in the string source with the specified string replace_string and returns the result string.
            • For Spark 3.3.1: Replaces the substring that matches the pattern for the occurrence time in the source string and the substring that matches the pattern later with the specified string replace_string and returns the result string.
            
+
            This function has slight variations in its functionality depending on the version of Spark being used.
            • For Spark 2.4.5 or earlier: Replaces the substring that matches pattern in the string source with the specified string replace_string and returns the result string.
            • For Spark 3.1.1: Replaces the substring that matches the pattern for the occurrence time in the source string and the substring that matches the pattern later with the specified string replace_string and returns the result string.
            
 
            
-
            Similar function: regexp_replace1. The regexp_replace1 function is used to replace the substring that matches pattern for the occurrence time in the source string with the specified string replace_string and return the result string. However, the egexp_replace1 function applies only to Spark 2.4.5 or earlier.
            
+
            Similar function: regexp_replace1. The regexp_replace1 function is used to replace the substring that matches pattern for the occurrence time in the source string with the specified string replace_string and return the result string. However, the egexp_replace1 function applies only to Spark 2.4.5 or earlier.
            
 
            The regexp_replace1 function is applicable to Spark 2.4.5. It supports specifying the occurrence parameter, whereas the regexp_replace function does not support it.
            
 
            Syntax
            • Spark 2.4.5 or earlier
              regexp_replace(string <source>, string <pattern>, string <replace_string>)
              
 
            
diff --git a/docs/dli/sqlreference/dli_spark_regexp_replace1.html b/docs/dli/sqlreference/dli_spark_regexp_replace1.html
index a91a8d71b..2157d7767 100644
--- a/docs/dli/sqlreference/dli_spark_regexp_replace1.html
+++ b/docs/dli/sqlreference/dli_spark_regexp_replace1.html
@@ -4,7 +4,7 @@
 
            This function is used to replace the substring that matches pattern for the occurrence time in the source string with the specified string replace_string and return the result string.
            
 
             
            This function applies only to Spark 2.4.5 or earlier.
            
 
            
-
            Similar function: regexp_replace. The regexp_replace function has slightly different functionalities for different versions of Spark. For details, see regexp_replace.
            
+
            Similar function: regexp_replace. The regexp_replace function has slightly different functionalities for different versions of Spark. For details, see regexp_replace.
            
 
            Syntax
            regexp_replace1(string <source>, string <pattern>, string <replace_string>[, bigint <occurrence>])
            
 
            
 
            Parameters
            
@@ -63,15 +63,15 @@
 
            
 
            
 
            Example Code
            The value 2222 is returned.
            
-
            select regexp_replace('abcd', '[a-z]', '2');
            
+
            select regexp_replace1('abcd', '[a-z]', '2');
            
 
            The value 2bcd is returned.
            
-
            select regexp_replace('abcd', '[a-z]', '2', 1);
            
+
            select regexp_replace1('abcd', '[a-z]', '2', 1);
            
 
            The value a2cd is returned.
            
-
            select regexp_replace('abcd', '[a-z]', '2', 2);
            
+
            select regexp_replace1('abcd', '[a-z]', '2', 2);
            
 
            The value ab2d is returned.
            
-
            select regexp_replace('abcd', '[a-z]', '2', 3);
            
+
            select regexp_replace1('abcd', '[a-z]', '2', 3);
            
 
            The value abc2 is returned.
            
-
            select regexp_replace('abcd', '[a-z]', '2', 4);
            
+
            select regexp_replace1('abcd', '[a-z]', '2', 4);
            
 
            
 
            
 
            
diff --git a/docs/dli/sqlreference/dli_spark_rtrim.html b/docs/dli/sqlreference/dli_spark_rtrim.html
index bbee8c35a..7842b65f6 100644
--- a/docs/dli/sqlreference/dli_spark_rtrim.html
+++ b/docs/dli/sqlreference/dli_spark_rtrim.html
@@ -4,7 +4,7 @@
 
            This function is used to remove characters from the right of str.
            
 
            • If trimChars is not specified, spaces are removed by default.
            • If trimChars is specified, the function removes the longest possible substring from the right end of str that consists of characters in the trimChars set.
            
 
            Similar functions:
            
-
            • ltrim. This function is used to remove characters from the left of str.
            • trim. This function is used to remove characters from the left and right of str.
            
+
            • ltrim. This function is used to remove characters from the left of str.
            • trim. This function is used to remove characters from the left and right of str.
            
 
            Syntax
            rtrim([<trimChars>, ]string <str>)
            
 
            or
            
 
            rtrim(trailing [<trimChars>] from <str>)
            
diff --git a/docs/dli/sqlreference/dli_spark_stddev_pop.html b/docs/dli/sqlreference/dli_spark_stddev_pop.html
index 36d847905..615e6073d 100644
--- a/docs/dli/sqlreference/dli_spark_stddev_pop.html
+++ b/docs/dli/sqlreference/dli_spark_stddev_pop.html
@@ -30,7 +30,7 @@
 
            The command output is as follows:
            
 
            _c0        
 1.342355   
            
-
          • When used with group by, it groups all offerings by warehouse (warehourseId) and returns the deviation of the offering inventory (items) in the same group. An example command is as follows:
            select warehourseId, stddev_pop(items) from warehourse group by warehourseId;
            +
          • When used with group by, it groups all offerings by warehouse (warehouseId) and returns the deviation of the offering inventory (items) in the same group. An example command is as follows:
            select warehouseId, stddev_pop(items) from warehouse group by warehouseId;

            The command output is as follows:

            warehouseId _c1        
 city1    1.23124   
diff --git a/docs/dli/sqlreference/dli_spark_stddev_samp.html b/docs/dli/sqlreference/dli_spark_stddev_samp.html
index 3619013dd..fa9d20198 100644
--- a/docs/dli/sqlreference/dli_spark_stddev_samp.html
+++ b/docs/dli/sqlreference/dli_spark_stddev_samp.html
@@ -26,16 +26,22 @@
 
 
            Return Values
            The return value is of the DOUBLE type.
            
 
            
-
            Example Code
            • Calculates the sample covariance between the inventory (items) and the price of all offerings. An example command is as follows:
              select covar_samp(items,price) from warehouse; 
              
+
              Example Code
              • Calculates the sample deviation of all offering inventories (items). An example command is as follows:
                select stddev_samp(items) from warehouse; 
                
 
                The command output is as follows:
                
-
                _c0        
-1.242355   
                
-
              • When used with group by, it groups all offerings by warehouse (warehourseId) and returns the sample covariance between the inventory (items) and the price of offerings within each group. An example command is as follows:
                select warehourseId, covar_samp(items,price) from warehourse group by warehourseId; 
                
+
                +------------+
+| _c0        |
++------------+
+| 1.342355   |
++------------+
                
+
              • When used with group by, it groups all offerings by warehouse (warehouseId) and returns the sample deviation of the offering inventory (items) in the same group. An example command is as follows:
                select warehouseId, stddev_samp(items) from warehouse group by warehouseId; 
                
 
                The command output is as follows:
                
-
                warehouseId _c1        
-city1    1.03124   
-city2    1.03344   
-city3    1.33425   
                
+
                +------------+------------+
+| warehouseId| _c1        |
++------------+------------+
+| city1    | 1.23124   |
+| city2    | 1.23344   |
+| city3    | 1.43425   |
++------------+------------+
                
 
              
 
              
 
            
diff --git a/docs/dli/sqlreference/dli_spark_tan.html b/docs/dli/sqlreference/dli_spark_tan.html
index ab0054462..f9728aa89 100644
--- a/docs/dli/sqlreference/dli_spark_tan.html
+++ b/docs/dli/sqlreference/dli_spark_tan.html
@@ -33,7 +33,7 @@
 
             
            If the value of a is NULL, NULL is returned.
            
 
            
 
-
            Example Code
            The value 0.99999999999999999999 is returned.
            
+
            Example Code
            The value 0.9999999999999999 is returned.
            
 
            select tan(pi()/4);
            
 
            The value NULL is returned.
            
 
            select tan(null);
            
diff --git a/docs/dli/sqlreference/dli_spark_to_char.html b/docs/dli/sqlreference/dli_spark_to_char.html
index 50237fa77..abed007e2 100644
--- a/docs/dli/sqlreference/dli_spark_to_char.html
+++ b/docs/dli/sqlreference/dli_spark_to_char.html
@@ -37,7 +37,7 @@
 
            Format of the date to be converted
            
 
            Constant of the STRING type. Extended date formats are not supported.
            
 
            The value is a combination of the time unit (year, month, day, hour, minute, and second) and any character.
            
-
            • YYYY or yyyy indicates the year.
            • MM indicates the month.
            • mm indicates the minute.
            • dd indicates the day.
            • HH indicates the 24-hour clock.
            • hh indicates the 12-hour clock.
            • mi indicates the minute.
            • ss indicates the second.
            • SSS indicates millisecond.
            
+
            • yyyy indicates the year.
            • mm indicates the month.
            • dd indicates the day.
            • hh indicates the hour.
            • mi indicates the minute.
            • ss indicates the second.
            
 
 
 
diff --git a/docs/dli/sqlreference/dli_spark_to_date.html b/docs/dli/sqlreference/dli_spark_to_date.html
index 32f6945dc..8b6838fe5 100644
--- a/docs/dli/sqlreference/dli_spark_to_date.html
+++ b/docs/dli/sqlreference/dli_spark_to_date.html
@@ -2,7 +2,7 @@
 
 
            to_date
            
 
            This function is used to return the year, month, and day in a time.
            
-
            Similar function: to_date1. The to_date1 function is used to convert a string in a specified format to a date value. The date format can be specified.
            
+
            Similar function: to_date1. The to_date1 function is used to convert a string in a specified format to a date value. The date format can be specified.
            
 
            Syntax
            to_date(string timestamp)
            
 
            
 
            Parameters
            
diff --git a/docs/dli/sqlreference/dli_spark_to_date1.html b/docs/dli/sqlreference/dli_spark_to_date1.html
index a0e1bdc13..75b864f34 100644
--- a/docs/dli/sqlreference/dli_spark_to_date1.html
+++ b/docs/dli/sqlreference/dli_spark_to_date1.html
@@ -2,7 +2,7 @@
 
 
            to_date1
            
 
            This function is used to convert a string in a specified format to a date value.
            
-
            Similar function: to_date. The to_date function is used to return the year, month, and day in a time. The date format cannot be specified.
            
+
            Similar function: to_date. The to_date function is used to return the year, month, and day in a time. The date format cannot be specified.
            
 
            Syntax
            to_date1(string date, string format)
            
 
            
 
            Parameters
            
@@ -36,7 +36,7 @@
 
            Format of the date to be converted
            
 
            Constant of the STRING type. Extended date formats are not supported.
            
 
            The value is a combination of the time unit (year, month, day, hour, minute, and second) and any character.
            
-
            • YYYY or yyyy indicates the year.
            • MM indicates the month.
            • mm indicates the minute.
            • dd indicates the day.
            • HH indicates the 24-hour clock.
            • hh indicates the 12-hour clock.
            • mi indicates the minute.
            • ss indicates the second.
            • SSS indicates the millisecond.
            
+
            • yyyy indicates the year.
            • mm indicates the month.
            • dd indicates the day.
            • hh indicates the hour.
            • mi indicates the minute.
            • ss indicates the second.
            
 
 
 
@@ -47,12 +47,12 @@
 
             
            • If the value of date is not of the DATE or STRING type, the error message "data type mismatch" is displayed.
            • If the value of date is of the DATE or STRING type but is not in one of the supported formats, NULL is returned.
            • If the value of date is NULL, NULL is returned.
            • If the value of format is NULL, a date in the yyyy-mm-dd format is returned.
            
 
            
 
            
-
            Example Code
            The value NULL is returned.
            
-
            select to_date1('2023-08-16 10:54:36','yyyy-mm-dd');
            
+
            Example Code
            The value 2023-08-16 10:54:36 is returned.
            
+
            select to_date1('2023-08-16 10:54:36','yyyy-mm-dd hh:mi:ss');
            
 
            The value 2023-08-16 00:00:00 is returned.
            
 
            select to_date1('2023-08-16','yyyy-mm-dd');
            
 
            The value NULL is returned.
            
-
            select to_date(null);
            
+
            select to_date1(null);
            
 
            The value 2023-08-16 is returned.
            
 
            select to_date1('2023-08-16 10:54:36');
            
 
            
diff --git a/docs/dli/sqlreference/dli_spark_trim.html b/docs/dli/sqlreference/dli_spark_trim.html
index 48262235b..243dfb358 100644
--- a/docs/dli/sqlreference/dli_spark_trim.html
+++ b/docs/dli/sqlreference/dli_spark_trim.html
@@ -4,7 +4,7 @@
 
            This function is used to remove characters from the left and right of str.
            
 
            • If trimChars is not specified, spaces are removed by default.
            • If trimChars is specified, the function removes the longest possible substring from both the left and right ends of str that consists of characters in the trimChars set.
            
 
            Similar functions:
            
-
            • ltrim. This function is used to remove characters from the left of str.
            • rtrim. This function is used to remove characters from the right of str.
            
+
            • ltrim. This function is used to remove characters from the left of str.
            • rtrim. This function is used to remove characters from the right of str.
            
 
            Syntax
            trim([<trimChars>,]string <str>)
            
 
            or
            
 
            trim([BOTH] [<trimChars>] from <str>)
            
@@ -52,7 +52,7 @@
 
            It is equivalent to the following statement:
            
 
            select trim(both from ' yxabcxx '); 
 select trim(from ' yxabcxx ');
            
-
          • Removes all substrings from both the left and right ends of the string yxabcxx that consist of characters in the set xy.

            The function returns Txyom, as any substring starting with x or y from both the left and right ends is removed.

            +
          • Removes all substrings from both the left and right ends of the string yxabcxx that consist of characters in the set xy.

            The function returns abc, as any substring starting with x or y from both the left and right ends is removed.

            select trim('xy', 'yxabcxx');

            It is equivalent to the following statement:

            select trim(both 'xy' from 'yxabcxx');
            diff --git a/docs/dli/sqlreference/dli_spark_trunc.html b/docs/dli/sqlreference/dli_spark_trunc.html index e03d53398..81564f912 100644 --- a/docs/dli/sqlreference/dli_spark_trunc.html +++ b/docs/dli/sqlreference/dli_spark_trunc.html @@ -35,7 +35,7 @@

            Format of the date to be converted

            The value is a combination of the time unit (year, month, day, hour, minute, and second) and any character.

            -
            • YYYY or yyyy indicates the year.
            • MM indicates the month.
            • mm indicates the minute.
            • dd indicates the day.
            • HH indicates the 24-hour clock.
            • hh indicates the 12-hour clock.
            • mi indicates the minute.
            • ss indicates the second.
            • SSS indicates the millisecond.
            +
            • yyyy indicates the year.
            • MM indicates the month.
            @@ -43,7 +43,6 @@

            Return Values

            The return value is of the DATE type, in the yyyy-mm-dd format.

            -

            • If the value of date is not of the DATE or STRING type, the error message "data type mismatch" is displayed.
            • If the value of date is of the DATE or STRING type but is not in one of the supported formats, NULL is returned.
            • If the value of date is NULL, NULL is returned.
            • If the value of format is NULL, NULL is returned.
            diff --git a/docs/dli/sqlreference/dli_spark_unix_timestamp.html b/docs/dli/sqlreference/dli_spark_unix_timestamp.html index c29c4ac8b..3b9bbcc5b 100644 --- a/docs/dli/sqlreference/dli_spark_unix_timestamp.html +++ b/docs/dli/sqlreference/dli_spark_unix_timestamp.html @@ -34,9 +34,9 @@

            STRING

            Format to be converted

            -

            If this parameter is left blank, the default format yyyy-mm-dd hh:mm:ss is used.

            +

            If this parameter is left blank, the default format yyyy-MM-dd hh:mm:ss is used.

            The value is a combination of the time unit (year, month, day, hour, minute, and second) and any character.

            -
            • YYYY or yyyy indicates the year.
            • MM indicates the month.
            • mm indicates the minute.
            • dd indicates the day.
            • HH indicates the 24-hour clock.
            • hh indicates the 12-hour clock.
            • mi indicates the minute.
            • ss indicates the second.
            • SSS indicates the millisecond.
            +
            • yyyy indicates the year.
            • MM indicates the month.
            • dd indicates the day.
            • hh indicates the hour.
            • mi indicates the minute.
            • ss indicates the second.
            diff --git a/docs/dli/sqlreference/dli_spark_variance_var_pop.html b/docs/dli/sqlreference/dli_spark_variance_var_pop.html index fb15430a4..d9bc56eb7 100644 --- a/docs/dli/sqlreference/dli_spark_variance_var_pop.html +++ b/docs/dli/sqlreference/dli_spark_variance_var_pop.html @@ -34,9 +34,9 @@ select var_pop(items) from warehouse;

            The command output is as follows:

            _c0        
 203.42352
            -
          • When used with group by, it groups all offerings by warehouse (warehourseId) and returns the variance of the offering inventory (items) in the same group. An example command is as follows:
            select warehourseId, variance(items) from warehourse group by warehourseId; 
+
          • When used with group by, it groups all offerings by warehouse (warehouseId) and returns the variance of the offering inventory (items) in the same group. An example command is as follows:
            select warehouseId, variance(items) from warehouse group by warehouseId; 
 -- It is equivalent to the following statement:
-select warehourseId, var_pop(items) from warehourse group by warehourseId;
            +select warehouseId, var_pop(items) from warehouse group by warehouseId;

            The command output is as follows:

            1
 2
@@ -46,7 +46,6 @@ select warehourseId, var_pop(items) from warehourse group by warehourseId;
            city2 17.23344 city3 12.43425
            -
          diff --git a/docs/dli/sqlreference/dli_spark_war_samp.html b/docs/dli/sqlreference/dli_spark_war_samp.html index 719e86330..dda291c43 100644 --- a/docs/dli/sqlreference/dli_spark_war_samp.html +++ b/docs/dli/sqlreference/dli_spark_war_samp.html @@ -31,7 +31,7 @@

          The command output is as follows:

          _c0        
 294.342355
          -
        • When used with group by, it groups all offerings by warehouse (warehourseId) and returns the sample variance of the offering inventory (items) in the same group. An example command is as follows:
          select warehourseId, var_samp(items) from warehourse group by warehourseId;
          +
        • When used with group by, it groups all offerings by warehouse (warehouseId) and returns the sample variance of the offering inventory (items) in the same group. An example command is as follows:
          select warehouseId, var_samp(items) from warehouse group by warehouseId;

          The command output is as follows:

          warehouseId _c1        
 city1    18.23124   
diff --git a/docs/dli/sqlreference/en-us_image_0000001742720709.gif b/docs/dli/sqlreference/en-us_image_0000001742720709.gif
new file mode 100644
index 000000000..93ecc2048
Binary files /dev/null and b/docs/dli/sqlreference/en-us_image_0000001742720709.gif differ
diff --git a/docs/dli/sqlreference/en-us_image_0000001827630114.png b/docs/dli/sqlreference/en-us_image_0000001827630114.png
new file mode 100644
index 000000000..952ff376d
Binary files /dev/null and b/docs/dli/sqlreference/en-us_image_0000001827630114.png differ
diff --git a/docs/dli/sqlreference/en-us_image_0000001870733085.png b/docs/dli/sqlreference/en-us_image_0000001870733085.png
new file mode 100644
index 000000000..c4a12fdaa
Binary files /dev/null and b/docs/dli/sqlreference/en-us_image_0000001870733085.png differ
diff --git a/docs/dli/sqlreference/en-us_image_0000001874189597.png b/docs/dli/sqlreference/en-us_image_0000001874189597.png
new file mode 100644
index 000000000..572361a00
Binary files /dev/null and b/docs/dli/sqlreference/en-us_image_0000001874189597.png differ
diff --git a/docs/dli/sqlreference/en-us_topic_0000001571023676.html b/docs/dli/sqlreference/en-us_topic_0000001571023676.html
deleted file mode 100644
index 12ca2fb69..000000000
--- a/docs/dli/sqlreference/en-us_topic_0000001571023676.html
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
-
          Table Lifecycle Management
          
-
          
-
-
diff --git a/docs/dli/sqlreference/en-us_topic_0000001621263317.html b/docs/dli/sqlreference/en-us_topic_0000001621263317.html
deleted file mode 100644
index 277f97a15..000000000
--- a/docs/dli/sqlreference/en-us_topic_0000001621263317.html
+++ /dev/null
@@ -1,129 +0,0 @@
-
-
-
          Specifying the Lifecycle of a Table When Creating the Table
          
-
          Function
          DLI provides table lifecycle management to allow you to specify the lifecycle of a table when creating the table. DLI determines whether to reclaim a table based on the table's last modification time and its lifecycle. By setting the lifecycle of a table, you can better manage a large number of tables, automatically delete data tables that are no longer used for a long time, and simplify the process of reclaiming data tables. Additionally, data restoration settings are supported to prevent data loss caused by misoperations.
          
-
          
-
          Table Reclamation Rules
          • When creating a table, use TBLPROPERTIES to specify the lifecycle of the table.
            • Non-partitioned table
              If the table is not a partitioned table, the system determines whether to reclaim the table after the lifecycle time based on the last modification time of the table.
              
-
            • Partitioned table
              If the table is a partitioned table, the system determines whether the partition needs to be reclaimed based on the last modification time (LAST_ACCESS_TIME) of the partition. After the last partition of a partitioned table is reclaimed, the table is not deleted.
              
-
              Only table-level lifecycle management is supported for partitioned tables.
              
-
            
-
          • Lifecycle reclamation starts at a specified time every day to scan all partitions.
            Lifecycle reclamation starts at a specified time every day. Reclamation only occurs if the last modification time of the table data (LAST_ACCESS_TIME) detected when scanning complete partitions exceeds the time specified by the lifecycle.
            
-
            Assume that the lifecycle of a partitioned table is one day and the last modification time of the partitioned data is 15:00 on May 20, 2023. If the table is scanned before 15:00 on May 20, 2023 (less than one day), the partitions in the table will not be reclaimed. If the last data modification time (LAST_ACCESS_TIME) of a table partition exceeds the time specified by the lifecycle during reclamation scan on May 20, 2023, the partition will be reclaimed.
            
-
          • The lifecycle function periodically reclaims tables or partitions, which are reclaimed irregularly every day depending on the level of busyness of the service. It cannot ensure that a table or partition will be reclaimed immediately after its lifecycle expires.
          • After a table is deleted, all properties of the table, including the lifecycle, will be deleted. After a table with the same name is created again, the lifecycle of the table will be determined by the new property.
          
-
          
-
          Constraints and Limitations
          • Before using the lifecycle function, log in to the DLI console, choose Global Configuration > Service Authorization, select Tenant Administrator(Project-level), and click Update on the Assign Agency Permissions page.
          • The table lifecycle function currently only supports creating tables and versioning tables using Hive and Datasource syntax.
          • The unit of the lifecycle is in days. The value should be a positive integer.
          • The lifecycle can be set only at the table level. The lifecycle specified for a partitioned table applies to all partitions of the table.
          • After the lifecycle is set, DLI and OBS tables will support data backup. The backup directory for OBS tables needs to be set manually. The backup directory must be in the parallel file system and in the same bucket as the original table directory. It cannot have the same directory or subdirectory name as the original table.
          
-
          
-
          Syntax
          • Creating a DLI table using the Datasource syntax
            CREATE TABLE table_name(name string, id int) 
-USING parquet 
-TBLPROPERTIES( "dli.lifecycle.days"=1 );
            
-
          • Creating a DLI table using the Hive syntax
            CREATE TABLE table_name(name string, id int) 
-stored as parquet 
-TBLPROPERTIES( "dli.lifecycle.days"=1 );
            
-
          • Creating an OBS table using the Datasource syntax
            CREATE TABLE table_name(name string, id int) 
-USING parquet 
-OPTIONS (path "obs://dli-test/table_name")
-TBLPROPERTIES( "dli.lifecycle.days"=1, "external.table.purge"='true', "dli.lifecycle.trash.dir"='obs://dli-test/Lifecycle-Trash' );
            
-
          • Creating an OBS table using the Hive syntax
            1
-2
-3
-4
            CREATE TABLE table_name(name string, id int) 
-STORED AS parquet 
-LOCATION 'obs://dli-test/table_name'
-TBLPROPERTIES( "dli.lifecycle.days"=1, "external.table.purge"='true', "dli.lifecycle.trash.dir"='obs://dli-test/Lifecycle-Trash' );
-
            
-
-
            
-
          
-
          
-
          Keywords
          • TBLPROPERTIES: Table properties, which can be used to extend the lifecycle of a table.
          • OPTIONS: path of the new table, which is applicable to OBS tables created using the Datasource syntax.
          • LOCATION: path of the new table, which is applicable to OBS tables created using the Hive syntax.
          
-
          
-
          Parameters
          
-
          
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
          Table 1 Parameters
          Parameter
          
-
          Mandatory
          
-
          Description
          
-
          table_name
          
-
          Yes
          
-
          Name of the table whose lifecycle needs to be set
          
-
          dli.lifecycle.days
          
-
          Yes
          
-
          Lifecycle duration. The value must be a positive integer, in days.
          
-
          external.table.purge
          
-
          No
          
-
          This parameter is available only for OBS tables.
          
-
          Whether to clear data in the path when deleting a table or partition. The data is not cleared by default.
          
-
          When this parameter is set to true:
          
-
          • After a file is deleted from a non-partitioned OBS table, the table directory is also deleted.
          • The custom partition data in the partitioned OBS table is also deleted.
          
-
          dli.lifecycle.trash.dir
          
-
          No
          
-
          This parameter is available only for OBS tables.
          
-
          When external.table.purge is set to true, the backup directory will be deleted. By default, backup data is deleted seven days later.
          
-
          
-
          
-
          
-
          Example
          • Create the test_datasource_lifecycle table using the Datasource syntax. The lifecycle is set to 100 days.
            1
-2
-3
            CREATE TABLE test_datasource_lifecycle(id int) 
-USING parquet 
-TBLPROPERTIES( "dli.lifecycle.days"=100);
-
            
-
-
            
-
          • Create the test_hive_lifecycle table using the Hive syntax. The lifecycle is set to 100 days.
            1
-2
-3
            CREATE TABLE test_hive_lifecycle(id int) 
-stored as parquet 
-TBLPROPERTIES( "dli.lifecycle.days"=100);
-
            
-
-
            
-
          • Create the test_datasource_lifecycle_obs table using the Datasource syntax. The lifecycle is set to 100 days. When the lifecycle expires, data is deleted by default and backed up to the obs://dli-test/ directory.
            1
-2
-3
-4
            CREATE TABLE test_datasource_lifecycle_obs(name string, id int) 
-USING parquet 
-OPTIONS (path "obs://dli-test/xxx")
-TBLPROPERTIES( "dli.lifecycle.days"=100, "external.table.purge"='true', "dli.lifecycle.trash.dir"='obs://dli-test/Lifecycle-Trash' );
-
            
-
-
            
-
          • Create the test_hive_lifecycle_obs table using the Hive syntax. The lifecycle is set to 100 days. When the lifecycle expires, data is deleted by default and backed up to the obs://dli-test/ directory.
            1
-2
-3
-4
            CREATE TABLE test_hive_lifecycle_obs(name string, id int) 
-STORED AS parquet 
-LOCATION 'obs://dli-test/xxx'
-TBLPROPERTIES( "dli.lifecycle.days"=100, "external.table.purge"='true', "dli.lifecycle.trash.dir"='obs://dli-test/Lifecycle-Trash' );
-
            
-
-
            
-
          
-
          
-
          
-
          
-
          
-
          Parent topic: Table Lifecycle Management
          
-
          
-
          
-
diff --git a/docs/dli/sqlreference/en-us_topic_0000001621382957.html b/docs/dli/sqlreference/en-us_topic_0000001621382957.html
deleted file mode 100644
index 983df631d..000000000
--- a/docs/dli/sqlreference/en-us_topic_0000001621382957.html
+++ /dev/null
@@ -1,56 +0,0 @@
-
-
-
          Modifying the Lifecycle of a Table
          
-
          Function
          This section describes how to modify the lifecycle of an existing partitioned or non-partitioned table.
          
-
          When the lifecycle function is enabled for the first time, the system scans tables or partitions, scans table data files in the path, and updates LAST_ACCESS_TIME of tables or partitions. The time required depends on the number of partitions and files.
          
-
          
-
          Constraints and Limitations
          • The table lifecycle function currently only supports creating tables and versioning tables using Hive and Datasource syntax.
          • The unit of the lifecycle is in days. The value should be a positive integer.
          • The lifecycle can be set only at the table level. The lifecycle specified for a partitioned table applies to all partitions of the table.
          
-
          
-
          Syntax
          ALTER TABLE table_name
-SET TBLPROPERTIES("dli.lifecycle.days"='N')
          
-
          
-
          Keywords
          TBLPROPERTIES: Table properties, which can be used to extend the lifecycle of a table.
          
-
          
-
          Parameters
          
-
          
-
-
-
-
-
-
-
-
-
-
-
-
-
-
          Table 1 Parameters
          Parameter
          
-
          Mandatory
          
-
          Description
          
-
          table_name
          
-
          Yes
          
-
          Name of the table whose lifecycle needs to be modified
          
-
          dli.lifecycle.days
          
-
          Yes
          
-
          Lifecycle duration after the modification. The value must be a positive integer, in days.
          
-
          
-
          
-
          
-
          Example
          • Example 1: Enable the lifecycle function for the test_lifecycle_exists table and set the lifecycle to 50 days.
            alter table test_lifecycle_exists
-SET TBLPROPERTIES("dli.lifecycle.days"='50');
            
-
          • Example 2: Enable the lifecycle function for an existing partitioned or non-partitioned table for which lifecycle is not set, for example, for the test_lifecycle_exists table, and set the lifecycle to 50 days.
            alter table test_lifecycle_exists
-SET TBLPROPERTIES(
-    "dli.lifecycle.days"='50',
-    "dli.table.lifecycle.status"='enable'
-);
            
-
          
-
          
-
          
-
          
-
          
-
          Parent topic: Table Lifecycle Management
          
-
          
-
          
-
diff --git a/docs/dli/sqlreference/en-us_topic_0000001621542965.html b/docs/dli/sqlreference/en-us_topic_0000001621542965.html
deleted file mode 100644
index 2be083214..000000000
--- a/docs/dli/sqlreference/en-us_topic_0000001621542965.html
+++ /dev/null
@@ -1,85 +0,0 @@
-
-
-
          Disabling or Restoring the Lifecycle of a Table
          
-
          Function
          This section describes how to disable or restore the lifecycle of a specified table or partition.
          
-
          You can disable or restore the lifecycle of a table in either of the following scenarios:
          1. If the lifecycle function has been enabled for a table or partitioned table, the system allows you to disable or restore the lifecycle of the table by changing the value of dli.table.lifecycle.status.
          2. If the lifecycle function is not enabled for a table or partitioned table, the system will add the dli.table.lifecycle.status property to allow you to disable or restore the lifecycle function of the table.
          
-
          
-
          
-
          Constraints and Limitations
          • The table lifecycle function currently only supports creating tables and versioning tables using Hive and Datasource syntax.
          • The unit of the lifecycle is in days. The value should be a positive integer.
          • The lifecycle can be set only at the table level. The lifecycle specified for a partitioned table applies to all partitions of the table.
          
-
          
-
          Syntax
          • This syntax can be used to disable or restore the lifecycle of a table at the table level.
            1
            ALTER TABLE table_name SET TBLPROPERTIES("dli.table.lifecycle.status"={enable|disable});
-
            
-
-
            
-
          
-
          • This syntax can be used to disable or restore the lifecycle of a specified table at the table or partition table level.
            1
            ALTER TABLE table_name [pt_spec] LIFECYCLE {enable|disable};
-
            
-
-
            
-
          
-
          
-
          Keywords
          TBLPROPERTIES: Table properties, which can be used to extend the lifecycle of a table.
          
-
          
-
          Parameters
          
-
          
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
          Table 1 Parameters
          Parameter
          
-
          Mandatory
          
-
          Description
          
-
          table_name
          
-
          Yes
          
-
          Name of the table whose lifecycle is to be disabled or restored
          
-
          pt_spec
          
-
          No
          
-
          Partition information of the table whose lifecycle is to be disabled or restored. The format is partition_col1=col1_value1, partition_col2=col2_value1.... For a table with multi-level partitions, all partition values must be specified.
          
-
          enable
          
-
          No
          
-
          Restores the lifecycle function of a table or a specified partition.
          • The table and its partitions participate in lifecycle reclamation again. By default, the lifecycle configuration of the current table and its partitions is used.
          • Before enabling the table lifecycle function, it is recommended to modify the lifecycle configuration of the table and its partitions. This will help prevent any accidental data reclamation caused by the previous configuration once the table lifecycle function is enabled.
          
-
          
-
          disable
          
-
          No
          
-
          Disables the lifecycle function of a table or a specified partition.
          
-
          • Prevents a table and all its partitions from being reclaimed by the lifecycle. It takes priority over restoring the lifecycle of a table and its partitions. That is, when the lifecycle function of a table or a specified partition is disabled, the partition information of the table whose lifecycle is to be disabled or restored is invalid.
          • After the lifecycle function of a table is disabled, the lifecycle configuration of the table and the enable and disable flags of its partitions are retained.
          • After the lifecycle function of a table is disabled, the lifecycle configuration of the table and partitioned table can still be modified.
          
-
          
-
          
-
          
-
          Example
          • Example 1: Disable the lifecycle function of the test_lifecycle table.
            1
            alter table test_lifecycle SET TBLPROPERTIES("dli.table.lifecycle.status"='disable');
-
            
-
-
            
-
          
-
          • Example 2: Disable the lifecycle function for the partition whose time is 20230520 in the test_lifecycle table.
            1
            alter table test_lifecycle partition (dt='20230520') LIFECYCLE 'disable';
-
            
-
-
            
-
             
            • After the lifecycle function of a partitioned table is disabled, the lifecycle function of all partitions within the table will also be disabled.
            
-
            
-
          
-
          
-
          
-
          
-
          
-
          Parent topic: Table Lifecycle Management
          
-
          
-
          
-
diff --git a/docs/dli/sqlreference/en-us_topic_0000001873107668.html b/docs/dli/sqlreference/en-us_topic_0000001873107668.html
new file mode 100644
index 000000000..2022815f3
--- /dev/null
+++ b/docs/dli/sqlreference/en-us_topic_0000001873107668.html
@@ -0,0 +1,93 @@
+
+
+
          Reusing Results of Subqueries
          
+
          Function
          To improve query performance, caching the results of a subquery and reusing them in different parts of the query can be done to avoid redundant calculations of the same subquery during the execution of a SELECT statement.
          
+
          
+
          Syntax
          1
+2
+3
+4
+5
+6
+7
+8
          WITH cte_name AS (
+    SELECT ...
+    FROM table_name
+	WHERE ...
+)
+SELECT ...
+FROM cte_name
+WHERE ...
+
          
+
          
+
          
+
          Keywords
          
+
          
+
+
+
+
+
+
+
+
+
+
+
          Table 1 Keywords
          Parameter
          
+
          Description
          
+
          cte_name
          
+
          Custom subquery name
          
+
          table_name
          
+
          Name of the table where subquery commands are executed
          
+
          
+
          
+
          
+
          Example
          • Example 1: Define the sales_data subquery based on the sales table, query all items with sales amounts greater than 100, and retrieve all data from the subquery.
            1
+2
+3
+4
+5
+6
            WITH sales_data AS (
+  SELECT product_name, sales_amount
+  FROM sales
+  WHERE sales_amount > 100
+)
+SELECT * FROM sales_data;
+
            
+
            
+
          • Example 2: Define two subqueries, sales_data1 and sales_data2, based on the sales table, where sales_data1 retrieves all items with sales amounts greater than 100 and sales_data2 retrieves all items with sales amounts less than 20. Finally, merge the data from both subqueries.
             1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
            WITH sales_data1 AS (
+  SELECT product_name, sales_amount
+  FROM sales
+  WHERE sales_amount > 100
+),
+sales_data2 AS (
+  SELECT product_name, sales_amount
+  FROM sales
+  WHERE sales_amount < 20
+)
+SELECT * FROM sales_data1
+UNION ALL
+SELECT * FROM sales_data2;
+
            
+
            
+
          
+
          
+
          
+
          
+
          
+
          Parent topic: Spark SQL Syntax Reference
          
+
          
+
          
+
diff --git a/docs/dli/sqlreference/en-us_topic_0000002172127506.html b/docs/dli/sqlreference/en-us_topic_0000002172127506.html
new file mode 100644
index 000000000..2de622dcc
--- /dev/null
+++ b/docs/dli/sqlreference/en-us_topic_0000002172127506.html
@@ -0,0 +1,24 @@
+
+
+
          Flink OpenSource SQL 1.15 Syntax Reference
          
+
          
+
          
+
+